diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml
index af674592bb9..510457ca46e 100644
--- a/.github/workflows/bazel.yml
+++ b/.github/workflows/bazel.yml
@@ -7,6 +7,7 @@ on:
       - master
       - develop
       - bazel
+
 jobs:
   build:
     name: "bazel-compile (${{ matrix.os }})"
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index 81db2a656cb..b249072f8db 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -10,10 +10,10 @@ jobs:
     steps:
       - uses: actions/checkout@master
       - name: Set up JDK 8
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v4
         with:
           java-version: "8"
-          distribution: "adopt"
+          distribution: "corretto"
           cache: "maven"
       - name: Generate coverage report
         run: mvn -B test -T 2C --file pom.xml
@@ -22,3 +22,4 @@ jobs:
         with:
           fail_ci_if_error: true
           verbose: true
+          token: cf0cba0a-22f8-4580-89ab-4f1dec3bda6f
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
new file mode 100644
index 00000000000..ad5bcbd6c25
--- /dev/null
+++ b/.github/workflows/integration-test.yml
@@ -0,0 +1,45 @@
+name: Run Integration Tests
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+  push:
+    branches: [master, develop]
+
+jobs:
+  it-test:
+    name: "maven-compile (${{ matrix.os }}, JDK-${{ matrix.jdk }})"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        jdk: [8]
+    steps:
+      - name: Cache Maven Repos
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up JDK ${{ matrix.jdk }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.jdk }}
+          distribution: "corretto"
+          cache: "maven"
+
+      - name: Run integration tests with Maven
+        run: mvn clean verify -Pit-test -Pskip-unit-tests
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: always()
+        with:
+          report_paths: 'test/target/failsafe-reports/TEST-*.xml'
+          annotate_only: true
+          include_passed: true
+          detailed_summary: true
diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml
index 75bf91eb18f..4eacd65b846 100644
--- a/.github/workflows/maven.yaml
+++ b/.github/workflows/maven.yaml
@@ -4,6 +4,7 @@ on:
     types: [opened, reopened, synchronize]
   push:
     branches: [master, develop, bazel]
+
 jobs:
   java_build:
     name: "maven-compile (${{ matrix.os }}, JDK-${{ matrix.jdk }})"
@@ -18,10 +19,28 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Set up JDK ${{ matrix.jdk }}
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.jdk }}
-          distribution: "adopt"
+          # See https://github.com/actions/setup-java?tab=readme-ov-file#supported-distributions
+          # AdoptOpenJDK got moved to Eclipse Temurin and won't be updated anymore.
+          distribution: "corretto"
           cache: "maven"
       - name: Build with Maven
         run: mvn -B package --file pom.xml
+
+      - name: Upload Auth JVM crash logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: jvm-crash-logs
+          path: /Users/runner/work/rocketmq/rocketmq/auth/hs_err_pid*.log
+          retention-days: 1
+
+      - name: Upload broker JVM crash logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: jvm-crash-logs
+          path: /Users/runner/work/rocketmq/rocketmq/broker/hs_err_pid*.log
+          retention-days: 1
diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml
index ef2db755d00..99d7309fd0c 100644
--- a/.github/workflows/pr-ci.yml
+++ b/.github/workflows/pr-ci.yml
@@ -21,7 +21,7 @@ jobs:
       - name: Build distribution tar
         run: |
           mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         name: Upload distribution tar
         with:
           name: rocketmq
@@ -30,7 +30,7 @@ jobs:
         run: |
           mkdir -p ./pr
           echo ${{ github.event.number }} > ./pr/NR
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: pr
           path: pr/
diff --git a/.github/workflows/pr-e2e-test.yml b/.github/workflows/pr-e2e-test.yml
index d0371e31135..0bc74a65ad5 100644
--- a/.github/workflows/pr-e2e-test.yml
+++ b/.github/workflows/pr-e2e-test.yml
@@ -13,10 +13,11 @@ env:
 
 jobs:
   docker:
-    runs-on: ubuntu-latest
     if: >
+      github.repository == 'apache/rocketmq' &&
       github.event.workflow_run.event == 'pull_request' &&
       github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-latest
     timeout-minutes: 30
     strategy:
       matrix:
@@ -24,18 +25,18 @@ jobs:
         java-version: ["8"]
     steps:
       - name: 'Download artifact'
-        uses: actions/github-script@v3.1.0
+        uses: actions/github-script@v6
         with:
           script: |
-            var artifacts = await github.actions.listWorkflowRunArtifacts({
+            let artifacts = await github.rest.actions.listWorkflowRunArtifacts({
                owner: context.repo.owner,
                repo: context.repo.repo,
                run_id: ${{github.event.workflow_run.id }},
             });
-            var matchArtifactRmq = artifacts.data.artifacts.filter((artifact) => {
+            let matchArtifactRmq = artifacts.data.artifacts.filter((artifact) => {
               return artifact.name == "rocketmq"
             })[0];
-            var download = await github.actions.downloadArtifact({
+            let download = await github.rest.actions.downloadArtifact({
                owner: context.repo.owner,
                repo: context.repo.repo,
                artifact_id: matchArtifactRmq.id,
@@ -67,14 +68,16 @@ jobs:
           mkdir versionlist
           touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`"
           sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO}
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         name: Upload distribution tar
         with:
           name: versionlist
           path: rocketmq-docker/image-build-ci/versionlist/*
   
   list-version:
-    if: always()
+    if: >
+      github.repository == 'apache/rocketmq' &&
+      always()
     name: List version
     needs: [docker]
     runs-on: ubuntu-latest
@@ -82,7 +85,7 @@ jobs:
     outputs:
       version-json: ${{ steps.show_versions.outputs.version-json }}
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         name: Download versionlist
         with:
           name: versionlist
@@ -93,6 +96,7 @@ jobs:
           a=(`ls versionlist`)
           printf '%s\n' "${a[@]}" | jq -R . | jq -s .
           echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT
+
   deploy:
     if: ${{ success() }}
     name: Deploy RocketMQ
@@ -155,7 +159,7 @@ jobs:
           annotate_only: true
           include_passed: true
           detailed_summary: true
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         name: Upload test log
         with:
@@ -184,6 +188,9 @@ jobs:
           test-cmd: |
             cd ../common &&  mvn -Prelease -DskipTests clean package -U
             cd ../rocketmq-admintools && source bin/env.sh
+            LATEST_GO_VERSION=$(curl -s https://go.dev/VERSION?m=text | awk 'NR==1')
+            wget "https://go.dev/dl/${LATEST_GO_VERSION}.linux-amd64.tar.gz" && \
+            rm -rf /usr/local/go && tar -C /usr/local -xzf ${LATEST_GO_VERSION}.linux-amd64.tar.gz
             cd ../golang && go get -u github.com/apache/rocketmq-clients/golang && gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./mqgotest/... -timeout 2m  -v
           job-id: 0
       - name: Publish Test Report
@@ -194,7 +201,7 @@ jobs:
           annotate_only: true
           include_passed: true
           detailed_summary: true
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         name: Upload test log
         with:
@@ -230,7 +237,7 @@ jobs:
           annotate_only: true
           include_passed: true
           detailed_summary: true
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         name: Upload test log
         with:
@@ -253,5 +260,4 @@ jobs:
           action: "clean"
           ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
           test-version: "${{ matrix.version }}"
-          job-id: ${{ strategy.job-index }}
-          
+          job-id: ${{ strategy.job-index }}
\ No newline at end of file
diff --git a/.github/workflows/push-ci.yml b/.github/workflows/push-ci.yml
index ad29a57c8a8..6de8595724f 100644
--- a/.github/workflows/push-ci.yml
+++ b/.github/workflows/push-ci.yml
@@ -15,6 +15,7 @@ env:
 
 jobs:
   dist-tar:
+    if: github.repository == 'apache/rocketmq'
     name: Build dist tar
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -30,7 +31,7 @@ jobs:
       - name: Build distribution tar
         run: |
           mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         name: Upload distribution tar
         with:
           name: rocketmq
@@ -52,7 +53,7 @@ jobs:
           repository: apache/rocketmq-docker.git
           ref: master
           path: rocketmq-docker
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         name: Download distribution tar
         with:
           name: rocketmq
@@ -71,15 +72,16 @@ jobs:
           mkdir versionlist
           touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`"
           sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO}
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         name: Upload distribution tar
         with:
           name: versionlist
           path: rocketmq-docker/image-build-ci/versionlist/*
 
-  
   list-version:
-    if: always()
+    if: >
+      github.repository == 'apache/rocketmq' && 
+      always()
     name: List version
     needs: [docker]
     runs-on: ubuntu-latest
@@ -87,7 +89,7 @@ jobs:
     outputs:
       version-json: ${{ steps.show_versions.outputs.version-json }}
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         name: Download versionlist
         with:
           name: versionlist
@@ -98,9 +100,10 @@ jobs:
           a=(`ls versionlist`)
           printf '%s\n' "${a[@]}" | jq -R . | jq -s .
           echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT
-  deploy:
+
+  deploy-e2e:
     if: ${{ success() }}
-    name: Deploy RocketMQ
+    name: Deploy RocketMQ For E2E
     needs: [list-version,docker]
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -131,10 +134,45 @@ jobs:
               image:
                 repository: ${{env.DOCKER_REPO}}
                 tag: ${{ matrix.version }}
+
+  deploy-benchmark:
+    if: ${{ success() }}
+    name: Deploy RocketMQ For Benchmarking
+    needs: [list-version,docker]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
+    steps:
+      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288
+        name: Deploy rocketmq
+        with:
+          action: "deploy"
+          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
+          test-version: "${{ matrix.version }}"
+          chart-git: "https://ghproxy.com/https://github.com/apache/rocketmq-docker.git"
+          chart-branch: "master"
+          chart-path: "./rocketmq-k8s-helm"
+          job-id: "001-${{ strategy.job-index }}"
+          helm-values: |
+            nameserver:
+              image:
+                repository: ${{env.DOCKER_REPO}}
+                tag: ${{ matrix.version }}
+            broker:
+              image:
+                repository: ${{env.DOCKER_REPO}}
+                tag: ${{ matrix.version }}
+            proxy:
+              image:
+                repository: ${{env.DOCKER_REPO}}
+                tag: ${{ matrix.version }}
+
   test-e2e-grpc-java:
     if: ${{ success() }}
     name: Test E2E grpc java
-    needs: [list-version, deploy]
+    needs: [list-version, deploy-e2e]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
@@ -160,7 +198,7 @@ jobs:
           annotate_only: true
           include_passed: true
           detailed_summary: true
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         name: Upload test log
         with:
@@ -170,7 +208,7 @@ jobs:
   test-e2e-golang:
     if: ${{ success() }}
     name: Test E2E golang
-    needs: [list-version, deploy]
+    needs: [list-version, deploy-e2e]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
@@ -189,6 +227,9 @@ jobs:
           test-cmd: |
             cd ../common &&  mvn -Prelease -DskipTests clean package -U
             cd ../rocketmq-admintools && source bin/env.sh
+            LATEST_GO_VERSION=$(curl -s https://go.dev/VERSION?m=text | awk 'NR==1')
+            wget "https://go.dev/dl/${LATEST_GO_VERSION}.linux-amd64.tar.gz" && \
+            rm -rf /usr/local/go && tar -C /usr/local -xzf ${LATEST_GO_VERSION}.linux-amd64.tar.gz
             cd ../golang && go get -u github.com/apache/rocketmq-clients/golang && gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./mqgotest/... -timeout 2m  -v
           job-id: 0
       - name: Publish Test Report
@@ -199,7 +240,7 @@ jobs:
           annotate_only: true
           include_passed: true
           detailed_summary: true
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         name: Upload test log
         with:
@@ -209,7 +250,7 @@ jobs:
   test-e2e-remoting-java:
     if: ${{ success() }}
     name: Test E2E remoting java
-    needs: [ list-version, deploy ]
+    needs: [ list-version, deploy-e2e ]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
@@ -235,17 +276,45 @@ jobs:
           annotate_only: true
           include_passed: true
           detailed_summary: true
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         name: Upload test log
         with:
           name: test-e2e-remoting-java-log.txt
           path: testlog.txt
 
-  clean:
+  benchmark-test:
+    if: ${{ success() }}
+    runs-on: ubuntu-latest
+    name: Performance benchmark test
+    needs: [ list-version, deploy-benchmark ]
+    timeout-minutes: 60
+    steps:
+      - uses: apache/rocketmq-test-tool/benchmark-runner@ce372e5f3906ca1891e4918b05be14608eae608e
+        name: Performance benchmark
+        with:
+          action: "performance-benchmark"
+          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
+          job-id: "001-${{ strategy.job-index }}"
+          # The time to run the test, 15 minutes
+          test-time: "900"
+          # Some thresholds set in advance
+          min-send-tps-threshold: "12000"
+          max-rt-ms-threshold: "500"
+          avg-rt-ms-threshold: "10"
+          max-2c-rt-ms-threshold: "150"
+          avg-2c-rt-ms-threshold: "10"
+      - name: Upload test report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-report
+          path: benchmark/
+
+  clean-e2e:
     if: always()
-    name: Clean
-    needs: [list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java]
+    name: Clean E2E
+    needs: [ list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java ]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
@@ -260,3 +329,20 @@ jobs:
           test-version: "${{ matrix.version }}"
           job-id: ${{ strategy.job-index }}
           
+  clean-benchmark:
+    if: always()
+    name: Clean Benchmarking
+    needs: [ list-version, benchmark-test ]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
+    steps:
+      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288
+        name: clean
+        with:
+          action: "clean"
+          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
+          test-version: "${{ matrix.version }}"
+          job-id: "001-${{ strategy.job-index }}"
\ No newline at end of file
diff --git a/.github/workflows/rerun-workflow.yml b/.github/workflows/rerun-workflow.yml
new file mode 100644
index 00000000000..6c319505d2c
--- /dev/null
+++ b/.github/workflows/rerun-workflow.yml
@@ -0,0 +1,22 @@
+name: Rerun workflow
+on:
+  workflow_run:
+    workflows: ["Build and Run Tests by Maven" , "Build and Run Tests by Bazel"]
+    types:
+      - completed
+
+permissions:
+  actions: write
+
+jobs:
+  rerun:
+    if: github.event.workflow_run.conclusion == 'failure' && fromJSON(github.event.workflow_run.run_attempt) < 3
+    runs-on: ubuntu-latest
+    steps:
+      - name: rerun ${{  github.event.workflow_run.id }}
+        env:
+          GH_REPO:  ${{ github.repository }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh run watch ${{ github.event.workflow_run.id }} > /dev/null 2>&1
+          gh run rerun ${{ github.event.workflow_run.id }} --failed
\ No newline at end of file
diff --git a/.github/workflows/snapshot-automation.yml b/.github/workflows/snapshot-automation.yml
index 88f5f4e0ccb..9b297583b12 100644
--- a/.github/workflows/snapshot-automation.yml
+++ b/.github/workflows/snapshot-automation.yml
@@ -36,6 +36,7 @@ env:
 
 jobs:
   dist-tar:
+    if: github.repository == 'apache/rocketmq'
     name: Build dist tar
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -58,9 +59,9 @@ jobs:
         with:
           ref: ${{ github.event.inputs.branch }}
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
-          distribution: "temurin"
+          distribution: "corretto"
           java-version: "8"
           cache: "maven"
       - name: Build distribution tar
@@ -68,7 +69,7 @@ jobs:
           MAVEN_SETTINGS: ${{ github.workspace }}/.github/asf-deploy-settings.xml
         run: |
           mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         name: Upload distribution tar
         with:
           name: rocketmq
@@ -90,7 +91,7 @@ jobs:
           repository: apache/rocketmq-docker.git
           ref: master
           path: rocketmq-docker
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         name: Download distribution tar
         with:
           name: rocketmq
@@ -109,14 +110,16 @@ jobs:
           mkdir versionlist
           touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`"
           sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO}
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         name: Upload distribution tar
         with:
           name: versionlist
           path: rocketmq-docker/image-build-ci/versionlist/*
 
   list-version:
-    if: always()
+    if: >
+      github.repository == 'apache/rocketmq' && 
+      always()
     name: List version
     needs: [ docker-build ]
     runs-on: ubuntu-latest
@@ -124,7 +127,7 @@ jobs:
     outputs:
       version-json: ${{ steps.show_versions.outputs.version-json }}
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         name: Download versionlist
         with:
           name: versionlist
@@ -199,7 +202,7 @@ jobs:
           annotate_only: true
           include_passed: true
           detailed_summary: true
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         name: Upload test log
         with:
@@ -237,10 +240,10 @@ jobs:
           ref: develop
           persist-credentials: false
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 8
-          distribution: "temurin"
+          distribution: "corretto"
           cache: "maven"
       - name: Update default pom version
         if: github.event.inputs.rocketmq_version == ''
diff --git a/.gitignore b/.gitignore
index c20f4bf7685..4ee76210738 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,4 +17,5 @@ bazel-out
 bazel-bin
 bazel-rocketmq
 bazel-testlogs
-.vscode
\ No newline at end of file
+.vscode
+MODULE.bazel.lock
\ No newline at end of file
diff --git a/BUILD.bazel b/BUILD.bazel
index 358527c3149..ba33a9e6123 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -44,5 +44,6 @@ java_library(
         "@maven//:org_awaitility_awaitility",
         "@maven//:org_openjdk_jmh_jmh_core",
         "@maven//:org_openjdk_jmh_jmh_generator_annprocess",
+        "@maven//:org_mockito_mockito_junit_jupiter",
     ],
 )
diff --git a/MODULE.bazel b/MODULE.bazel
new file mode 100644
index 00000000000..15fc5c6e3a6
--- /dev/null
+++ b/MODULE.bazel
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+###############################################################################
+# Bazel now uses Bzlmod by default to manage external dependencies.
+# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel.
+#
+# For more details, please check https://github.com/bazelbuild/bazel/issues/18958
+###############################################################################
diff --git a/NOTICE b/NOTICE
index 85b0032cd80..6e7ed4a0f20 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
 Apache RocketMQ
-Copyright 2016-2024 The Apache Software Foundation
+Copyright 2016-2025 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/README.md b/README.md
index 5aaa2ba73c3..1c82b34f92d 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
 [![Maven Central][maven-central-image]][maven-central-url]
 [![Release][release-image]][release-url]
 [![License][license-image]][license-url]
-[![Average Time to Resolve An Issue][percentage-of-issues-still-open-image]][pencentage-of-issues-still-open-url]
+[![Average Time to Resolve An Issue][percentage-of-issues-still-open-image]][percentage-of-issues-still-open-url]
 [![Percentage of Issues Still Open][average-time-to-resolve-an-issue-image]][average-time-to-resolve-an-issue-url]
 [![Twitter Follow][twitter-follow-image]][twitter-follow-url]
 
@@ -49,21 +49,21 @@ $ java -version
 java version "1.8.0_121"
 ```
 
-For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip) to download the 5.1.4 RocketMQ binary release,
+For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.3.2/rocketmq-all-5.3.2-bin-release.zip) to download the 5.3.2 RocketMQ binary release,
 unpack it to your local disk, such as `D:\rocketmq`.
 For macOS and Linux users, execute following commands:
 
 ```shell
 # Download release from the Apache mirror
-$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip
+$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.3.2/rocketmq-all-5.3.2-bin-release.zip
 
 # Unpack the release
-$ unzip rocketmq-all-5.1.4-bin-release.zip
+$ unzip rocketmq-all-5.3.2-bin-release.zip
 ```
 
 Prepare a terminal and change to the extracted `bin` directory:
 ```shell
-$ cd rocketmq-all-5.1.4-bin-release/bin
+$ cd rocketmq-all-5.3.2-bin-release/bin
 ```
 
 **1) Start NameServer**
@@ -125,7 +125,7 @@ $ docker run -it --net=host apache/rocketmq ./mqnamesrv
 **2) Start Broker**
 
 ```shell
-$ docker run -it --net=host --mount source=/tmp/store,target=/home/rocketmq/store apache/rocketmq ./mqbroker -n localhost:9876
+$ docker run -it --net=host --mount type=bind,source=/tmp/store,target=/home/rocketmq/store apache/rocketmq ./mqbroker -n localhost:9876
 ```
 
 ### Run RocketMQ in Kubernetes
@@ -183,7 +183,7 @@ name-service         1/1     107m
 * [RocketMQ Connect](https://github.com/apache/rocketmq-connect): A tool for scalably and reliably streaming data between Apache RocketMQ and other systems.
 * [RocketMQ MQTT](https://github.com/apache/rocketmq-mqtt): A new MQTT protocol architecture model, based on which Apache RocketMQ can better support messages from terminals such as IoT devices and Mobile APP.
 * [RocketMQ EventBridge](https://github.com/apache/rocketmq-eventbridge): EventBridge make it easier to build a event-driven application.
-* [RocketMQ Incubating Community Projects](https://github.com/apache/rocketmq-externals): Icubator community projects of Apache RocketMQ, including [logappender](https://github.com/apache/rocketmq-externals/tree/master/logappender), [rocketmq-ansible](https://github.com/apache/rocketmq-externals/tree/master/rocketmq-ansible), [rocketmq-beats-integration](https://github.com/apache/rocketmq-externals/tree/master/rocketmq-beats-integration), [rocketmq-cloudevents-binding](https://github.com/apache/rocketmq-externals/tree/master/rocketmq-cloudevents-binding), etc.
+* [RocketMQ Incubating Community Projects](https://github.com/apache/rocketmq-externals): Incubator community projects of Apache RocketMQ, including [logappender](https://github.com/apache/rocketmq-externals/tree/master/logappender), [rocketmq-ansible](https://github.com/apache/rocketmq-externals/tree/master/rocketmq-ansible), [rocketmq-beats-integration](https://github.com/apache/rocketmq-externals/tree/master/rocketmq-beats-integration), [rocketmq-cloudevents-binding](https://github.com/apache/rocketmq-externals/tree/master/rocketmq-cloudevents-binding), etc.
 * [RocketMQ Site](https://github.com/apache/rocketmq-site): The repository for Apache RocketMQ website.
 * [RocketMQ E2E](https://github.com/apache/rocketmq-e2e): A project for testing Apache RocketMQ, including end-to-end, performance, compatibility tests.
 
@@ -245,6 +245,6 @@ services.
 [average-time-to-resolve-an-issue-image]: http://isitmaintained.com/badge/resolution/apache/rocketmq.svg
 [average-time-to-resolve-an-issue-url]: http://isitmaintained.com/project/apache/rocketmq
 [percentage-of-issues-still-open-image]: http://isitmaintained.com/badge/open/apache/rocketmq.svg
-[pencentage-of-issues-still-open-url]: http://isitmaintained.com/project/apache/rocketmq
+[percentage-of-issues-still-open-url]: http://isitmaintained.com/project/apache/rocketmq
 [twitter-follow-image]: https://img.shields.io/twitter/follow/ApacheRocketMQ?style=social
 [twitter-follow-url]: https://twitter.com/intent/follow?screen_name=ApacheRocketMQ
diff --git a/WORKSPACE b/WORKSPACE
index 16f66b73255..9125a67f88b 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -41,6 +41,7 @@ maven_install(
     artifacts = [
         "junit:junit:4.13.2",
         "com.alibaba:fastjson:1.2.76",
+        "com.alibaba.fastjson2:fastjson2:2.0.43",
         "org.hamcrest:hamcrest-library:1.3",
         "io.netty:netty-all:4.1.65.Final",
         "org.assertj:assertj-core:3.22.0",
@@ -70,7 +71,7 @@ maven_install(
         "org.bouncycastle:bcpkix-jdk15on:1.69",
         "com.google.code.gson:gson:2.8.9",
         "com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru:1.4.2",
-        "org.apache.rocketmq:rocketmq-proto:2.0.3",
+        "org.apache.rocketmq:rocketmq-proto:2.0.4",
         "com.google.protobuf:protobuf-java:3.20.1",
         "com.google.protobuf:protobuf-java-util:3.20.1",
         "com.conversantmedia:disruptor:1.2.10",
@@ -110,6 +111,9 @@ maven_install(
         "com.alipay.sofa:jraft-core:1.3.14",
         "com.alipay.sofa:hessian:3.3.6",
         "io.netty:netty-tcnative-boringssl-static:2.0.48.Final",
+        "org.mockito:mockito-junit-jupiter:4.11.0",
+        "com.alibaba.fastjson2:fastjson2:2.0.43",
+        "org.junit.jupiter:junit-jupiter-api:5.9.1",
     ],
     fetch_sources = True,
     repositories = [
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java b/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java
deleted file mode 100644
index 315184c6150..00000000000
--- a/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.acl;
-
-import com.google.protobuf.GeneratedMessageV3;
-import java.util.List;
-import java.util.Map;
-import org.apache.rocketmq.acl.common.AuthenticationHeader;
-import org.apache.rocketmq.common.AclConfig;
-import org.apache.rocketmq.common.PlainAccessConfig;
-import org.apache.rocketmq.remoting.protocol.DataVersion;
-import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-
-public interface AccessValidator {
-
-    /**
-     * Parse to get the AccessResource(user, resource, needed permission)
-     *
-     * @param request
-     * @param remoteAddr
-     * @return Plain access resource result,include access key,signature and some other access attributes.
-     */
-    AccessResource parse(RemotingCommand request, String remoteAddr);
-
-    /**
-     * Parse to get the AccessResource from gRPC protocol
-     * @param messageV3
-     * @param header
-     * @return Plain access resource
-     */
-    AccessResource parse(GeneratedMessageV3 messageV3, AuthenticationHeader header);
-
-    /**
-     * Validate the access resource.
-     *
-     * @param accessResource
-     */
-    void validate(AccessResource accessResource);
-
-    /**
-     * Update the access resource config
-     *
-     * @param plainAccessConfig
-     * @return
-     */
-    boolean updateAccessConfig(PlainAccessConfig plainAccessConfig);
-
-    /**
-     * Delete the access resource config
-     *
-     * @return
-     */
-    boolean deleteAccessConfig(String accessKey);
-
-    /**
-     * Get the access resource config version information
-     *
-     * @return
-     */
-    @Deprecated
-    String getAclConfigVersion();
-
-    /**
-     * Update globalWhiteRemoteAddresses in acl yaml config file
-     *
-     * @return
-     */
-    boolean updateGlobalWhiteAddrsConfig(List<String> globalWhiteAddrsList);
-
-    boolean updateGlobalWhiteAddrsConfig(List<String> globalWhiteAddrsList, String aclFileFullPath);
-
-    /**
-     * get broker cluster acl config information
-     *
-     * @return
-     */
-    AclConfig getAllAclConfig();
-
-    /**
-     * get all access resource config version information
-     *
-     * @return
-     */
-    Map<String, DataVersion> getAllAclConfigVersion();
-}
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java
deleted file mode 100644
index d129c66d1c8..00000000000
--- a/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.acl.common;
-
-public class AclConstants {
-
-    public static final String CONFIG_GLOBAL_WHITE_ADDRS = "globalWhiteRemoteAddresses";
-
-    public static final String CONFIG_ACCOUNTS = "accounts";
-
-    public static final String CONFIG_ACCESS_KEY = "accessKey";
-
-    public static final String CONFIG_SECRET_KEY = "secretKey";
-
-    public static final String CONFIG_WHITE_ADDR = "whiteRemoteAddress";
-
-    public static final String CONFIG_ADMIN_ROLE = "admin";
-
-    public static final String CONFIG_DEFAULT_TOPIC_PERM = "defaultTopicPerm";
-
-    public static final String CONFIG_DEFAULT_GROUP_PERM = "defaultGroupPerm";
-
-    public static final String CONFIG_TOPIC_PERMS = "topicPerms";
-
-    public static final String CONFIG_GROUP_PERMS = "groupPerms";
-
-    public static final String CONFIG_DATA_VERSION = "dataVersion";
-
-    public static final String CONFIG_COUNTER = "counter";
-
-    public static final String CONFIG_TIME_STAMP = "timestamp";
-
-    public static final String PUB = "PUB";
-
-    public static final String SUB = "SUB";
-
-    public static final String DENY = "DENY";
-
-    public static final String PUB_SUB = "PUB|SUB";
-
-    public static final String SUB_PUB = "SUB|PUB";
-
-    public static final int ACCESS_KEY_MIN_LENGTH = 6;
-
-    public static final int SECRET_KEY_MIN_LENGTH = 6;
-}
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AuthenticationHeader.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AuthenticationHeader.java
deleted file mode 100644
index 5b00c00c787..00000000000
--- a/acl/src/main/java/org/apache/rocketmq/acl/common/AuthenticationHeader.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.acl.common;
-
-import com.google.common.base.MoreObjects;
-
-public class AuthenticationHeader {
-    private String remoteAddress;
-    private String tenantId;
-    private String namespace;
-    private String authorization;
-    private String datetime;
-    private String sessionToken;
-    private String requestId;
-    private String language;
-    private String clientVersion;
-    private String protocol;
-    private int requestCode;
-
-    AuthenticationHeader(final String remoteAddress, final String tenantId, final String namespace,
-        final String authorization, final String datetime, final String sessionToken, final String requestId,
-        final String language, final String clientVersion, final String protocol, final int requestCode) {
-        this.remoteAddress = remoteAddress;
-        this.tenantId = tenantId;
-        this.namespace = namespace;
-        this.authorization = authorization;
-        this.datetime = datetime;
-        this.sessionToken = sessionToken;
-        this.requestId = requestId;
-        this.language = language;
-        this.clientVersion = clientVersion;
-        this.protocol = protocol;
-        this.requestCode = requestCode;
-    }
-
-    public static class MetadataHeaderBuilder {
-        private String remoteAddress;
-        private String tenantId;
-        private String namespace;
-        private String authorization;
-        private String datetime;
-        private String sessionToken;
-        private String requestId;
-        private String language;
-        private String clientVersion;
-        private String protocol;
-        private int requestCode;
-
-        MetadataHeaderBuilder() {
-        }
-
-        public AuthenticationHeader.MetadataHeaderBuilder remoteAddress(final String remoteAddress) {
-            this.remoteAddress = remoteAddress;
-            return this;
-        }
-
-        public AuthenticationHeader.MetadataHeaderBuilder tenantId(final String tenantId) {
-            this.tenantId = tenantId;
-            return this;
-        }
-
-        public AuthenticationHeader.MetadataHeaderBuilder namespace(final String namespace) {
-            this.namespace = namespace;
-            return this;
-        }
-
-        public AuthenticationHeader.MetadataHeaderBuilder authorization(final String authorization) {
-            this.authorization = authorization;
-            return this;
-        }
-
-        public AuthenticationHeader.MetadataHeaderBuilder datetime(final String datetime) {
-            this.datetime = datetime;
-            return this;
-        }
-
-        public AuthenticationHeader.MetadataHeaderBuilder sessionToken(final String sessionToken) {
-            this.sessionToken = sessionToken;
-            return this;
-        }
-
-        public AuthenticationHeader.MetadataHeaderBuilder requestId(final String requestId) {
-            this.requestId = requestId;
-            return this;
-        }
-
-        public AuthenticationHeader.MetadataHeaderBuilder language(final String language) {
-            this.language = language;
-            return this;
-        }
-
-        public AuthenticationHeader.MetadataHeaderBuilder clientVersion(final String clientVersion) {
-            this.clientVersion = clientVersion;
-            return this;
-        }
-
-        public AuthenticationHeader.MetadataHeaderBuilder protocol(final String protocol) {
-            this.protocol = protocol;
-            return this;
-        }
-
-        public AuthenticationHeader.MetadataHeaderBuilder requestCode(final int requestCode) {
-            this.requestCode = requestCode;
-            return this;
-        }
-
-        public AuthenticationHeader build() {
-            return new AuthenticationHeader(this.remoteAddress, this.tenantId, this.namespace, this.authorization,
-                this.datetime, this.sessionToken, this.requestId, this.language, this.clientVersion, this.protocol,
-                this.requestCode);
-        }
-    }
-
-    public static AuthenticationHeader.MetadataHeaderBuilder builder() {
-        return new AuthenticationHeader.MetadataHeaderBuilder();
-    }
-
-    public String getRemoteAddress() {
-        return this.remoteAddress;
-    }
-
-    public String getTenantId() {
-        return this.tenantId;
-    }
-
-    public String getNamespace() {
-        return this.namespace;
-    }
-
-    public String getAuthorization() {
-        return this.authorization;
-    }
-
-    public String getDatetime() {
-        return this.datetime;
-    }
-
-    public String getSessionToken() {
-        return this.sessionToken;
-    }
-
-    public String getRequestId() {
-        return this.requestId;
-    }
-
-    public String getLanguage() {
-        return this.language;
-    }
-
-    public String getClientVersion() {
-        return this.clientVersion;
-    }
-
-    public String getProtocol() {
-        return this.protocol;
-    }
-
-    public int getRequestCode() {
-        return this.requestCode;
-    }
-
-    public void setRemoteAddress(final String remoteAddress) {
-        this.remoteAddress = remoteAddress;
-    }
-
-    public void setTenantId(final String tenantId) {
-        this.tenantId = tenantId;
-    }
-
-    public void setNamespace(final String namespace) {
-        this.namespace = namespace;
-    }
-
-    public void setAuthorization(final String authorization) {
-        this.authorization = authorization;
-    }
-
-    public void setDatetime(final String datetime) {
-        this.datetime = datetime;
-    }
-
-    public void setSessionToken(final String sessionToken) {
-        this.sessionToken = sessionToken;
-    }
-
-    public void setRequestId(final String requestId) {
-        this.requestId = requestId;
-    }
-
-    public void setLanguage(final String language) {
-        this.language = language;
-    }
-
-    public void setClientVersion(final String clientVersion) {
-        this.clientVersion = clientVersion;
-    }
-
-    public void setProtocol(final String protocol) {
-        this.protocol = protocol;
-    }
-
-    public void setRequestCode(int requestCode) {
-        this.requestCode = requestCode;
-    }
-
-    @Override
-    public String toString() {
-        return MoreObjects.toStringHelper(this)
-            .add("remoteAddress", remoteAddress)
-            .add("tenantId", tenantId)
-            .add("namespace", namespace)
-            .add("authorization", authorization)
-            .add("datetime", datetime)
-            .add("sessionToken", sessionToken)
-            .add("requestId", requestId)
-            .add("language", language)
-            .add("clientVersion", clientVersion)
-            .add("protocol", protocol)
-            .add("requestCode", requestCode)
-            .toString();
-    }
-}
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AuthorizationHeader.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AuthorizationHeader.java
deleted file mode 100644
index eb75aa3bee9..00000000000
--- a/acl/src/main/java/org/apache/rocketmq/acl/common/AuthorizationHeader.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.acl.common;
-
-import com.google.common.base.MoreObjects;
-import org.apache.commons.codec.DecoderException;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Hex;
-
-public class AuthorizationHeader {
-    private static final String HEADER_SEPARATOR = " ";
-    private static final String CREDENTIALS_SEPARATOR = "/";
-    private static final int AUTH_HEADER_KV_LENGTH = 2;
-    private static final String CREDENTIAL = "Credential";
-    private static final String SIGNED_HEADERS = "SignedHeaders";
-    private static final String SIGNATURE = "Signature";
-    private String method;
-    private String accessKey;
-    private String[] signedHeaders;
-    private String signature;
-
-    /**
-     * Parse authorization from gRPC header.
-     *
-     * @param header gRPC header string.
-     * @throws Exception exception.
-     */
-    public AuthorizationHeader(String header) throws DecoderException {
-        String[] result = header.split(HEADER_SEPARATOR, 2);
-        if (result.length != 2) {
-            throw new DecoderException("authorization header is incorrect");
-        }
-        this.method = result[0];
-        String[] keyValues = result[1].split(",");
-        for (String keyValue : keyValues) {
-            String[] kv = keyValue.trim().split("=", 2);
-            int kvLength = kv.length;
-            if (kv.length != AUTH_HEADER_KV_LENGTH) {
-                throw new DecoderException("authorization keyValues length is incorrect, actual length=" + kvLength);
-            }
-            String authItem = kv[0];
-            if (CREDENTIAL.equals(authItem)) {
-                String[] credential = kv[1].split(CREDENTIALS_SEPARATOR);
-                int credentialActualLength = credential.length;
-                if (credentialActualLength == 0) {
-                    throw new DecoderException("authorization credential length is incorrect, actual length=" + credentialActualLength);
-                }
-                this.accessKey = credential[0];
-                continue;
-            }
-            if (SIGNED_HEADERS.equals(authItem)) {
-                this.signedHeaders = kv[1].split(";");
-                continue;
-            }
-            if (SIGNATURE.equals(authItem)) {
-                this.signature = this.hexToBase64(kv[1]);
-            }
-        }
-    }
-
-    public String hexToBase64(String input) throws DecoderException {
-        byte[] bytes = Hex.decodeHex(input);
-        return Base64.encodeBase64String(bytes);
-    }
-
-    public String getMethod() {
-        return this.method;
-    }
-
-    public String getAccessKey() {
-        return this.accessKey;
-    }
-
-    public String[] getSignedHeaders() {
-        return this.signedHeaders;
-    }
-
-    public String getSignature() {
-        return this.signature;
-    }
-
-    public void setMethod(final String method) {
-        this.method = method;
-    }
-
-    public void setAccessKey(final String accessKey) {
-        this.accessKey = accessKey;
-    }
-
-    public void setSignedHeaders(final String[] signedHeaders) {
-        this.signedHeaders = signedHeaders;
-    }
-
-    public void setSignature(final String signature) {
-        this.signature = signature;
-    }
-
-    @Override
-    public String toString() {
-        return MoreObjects.toStringHelper(this)
-            .add("method", method)
-            .add("accessKey", accessKey)
-            .add("signedHeaders", signedHeaders)
-            .add("signature", signature)
-            .toString();
-    }
-}
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java b/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java
deleted file mode 100644
index 38649b08327..00000000000
--- a/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.acl.common;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.acl.plain.PlainAccessResource;
-import org.apache.rocketmq.remoting.protocol.RequestCode;
-
-public class Permission {
-
-    public static final byte DENY = 1;
-    public static final byte ANY = 1 << 1;
-    public static final byte PUB = 1 << 2;
-    public static final byte SUB = 1 << 3;
-
-    public static final Set<Integer> ADMIN_CODE = new HashSet<>();
-
-    static {
-        // UPDATE_AND_CREATE_TOPIC
-        ADMIN_CODE.add(RequestCode.UPDATE_AND_CREATE_TOPIC);
-        // UPDATE_BROKER_CONFIG
-        ADMIN_CODE.add(RequestCode.UPDATE_BROKER_CONFIG);
-        // DELETE_TOPIC_IN_BROKER
-        ADMIN_CODE.add(RequestCode.DELETE_TOPIC_IN_BROKER);
-        // UPDATE_AND_CREATE_SUBSCRIPTIONGROUP
-        ADMIN_CODE.add(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP);
-        // DELETE_SUBSCRIPTIONGROUP
-        ADMIN_CODE.add(RequestCode.DELETE_SUBSCRIPTIONGROUP);
-    }
-
-    public static boolean checkPermission(byte neededPerm, byte ownedPerm) {
-        if ((ownedPerm & DENY) > 0) {
-            return false;
-        }
-        if ((neededPerm & ANY) > 0) {
-            return (ownedPerm & PUB) > 0 || (ownedPerm & SUB) > 0;
-        }
-        return (neededPerm & ownedPerm) > 0;
-    }
-
-    public static byte parsePermFromString(String permString) {
-        if (permString == null) {
-            return Permission.DENY;
-        }
-        switch (permString.trim()) {
-            case AclConstants.PUB:
-                return Permission.PUB;
-            case AclConstants.SUB:
-                return Permission.SUB;
-            case AclConstants.PUB_SUB:
-            case AclConstants.SUB_PUB:
-                return Permission.PUB | Permission.SUB;
-            case AclConstants.DENY:
-                return Permission.DENY;
-            default:
-                return Permission.DENY;
-        }
-    }
-
-    public static void parseResourcePerms(PlainAccessResource plainAccessResource, Boolean isTopic,
-        List<String> resources) {
-        if (resources == null || resources.isEmpty()) {
-            return;
-        }
-        for (String resource : resources) {
-            String[] items = StringUtils.split(resource, "=");
-            if (items.length == 2) {
-                plainAccessResource.addResourceAndPerm(isTopic ? items[0].trim() : PlainAccessResource.getRetryTopic(items[0].trim()), parsePermFromString(items[1].trim()));
-            } else {
-                throw new AclException(String.format("Parse resource permission failed for %s:%s", isTopic ? "topic" : "group", resource));
-            }
-        }
-    }
-
-    public static void checkResourcePerms(List<String> resources) {
-        if (resources == null || resources.isEmpty()) {
-            return;
-        }
-
-        for (String resource : resources) {
-            String[] items = StringUtils.split(resource, "=");
-            if (items.length != 2) {
-                throw new AclException(String.format("Parse Resource format error for %s.\n" +
-                    "The expected resource format is 'Res=Perm'. For example: topicA=SUB", resource));
-            }
-
-            if (!AclConstants.DENY.equals(items[1].trim()) && Permission.DENY == Permission.parsePermFromString(items[1].trim())) {
-                throw new AclException(String.format("Parse resource permission error for %s.\n" +
-                    "The expected permissions are 'SUB' or 'PUB' or 'SUB|PUB' or 'PUB|SUB'.", resource));
-            }
-        }
-    }
-
-    public static boolean needAdminPerm(Integer code) {
-        return ADMIN_CODE.contains(code);
-    }
-}
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java
deleted file mode 100644
index 1e185afff6a..00000000000
--- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java
+++ /dev/null
@@ -1,469 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.acl.plain;
-
-import apache.rocketmq.v2.AckMessageRequest;
-import apache.rocketmq.v2.ChangeInvisibleDurationRequest;
-import apache.rocketmq.v2.ClientType;
-import apache.rocketmq.v2.EndTransactionRequest;
-import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest;
-import apache.rocketmq.v2.HeartbeatRequest;
-import apache.rocketmq.v2.Message;
-import apache.rocketmq.v2.NotifyClientTerminationRequest;
-import apache.rocketmq.v2.QueryAssignmentRequest;
-import apache.rocketmq.v2.QueryRouteRequest;
-import apache.rocketmq.v2.ReceiveMessageRequest;
-import apache.rocketmq.v2.Resource;
-import apache.rocketmq.v2.SendMessageRequest;
-import apache.rocketmq.v2.Subscription;
-import apache.rocketmq.v2.SubscriptionEntry;
-import apache.rocketmq.v2.TelemetryCommand;
-import com.google.protobuf.GeneratedMessageV3;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import org.apache.commons.codec.DecoderException;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.rocketmq.acl.AccessResource;
-import org.apache.rocketmq.acl.common.AclException;
-import org.apache.rocketmq.acl.common.AclUtils;
-import org.apache.rocketmq.acl.common.AuthenticationHeader;
-import org.apache.rocketmq.acl.common.AuthorizationHeader;
-import org.apache.rocketmq.acl.common.Permission;
-import org.apache.rocketmq.acl.common.SessionCredentials;
-import org.apache.rocketmq.common.KeyBuilder;
-import org.apache.rocketmq.common.MQVersion;
-import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.common.PlainAccessConfig;
-import org.apache.rocketmq.remoting.common.RemotingHelper;
-import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
-import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-import org.apache.rocketmq.remoting.protocol.RequestCode;
-import org.apache.rocketmq.remoting.protocol.ResponseCode;
-import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
-import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;
-import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;
-import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
-
-public class PlainAccessResource implements AccessResource {
-
-    // Identify the user
-    private String accessKey;
-
-    private String secretKey;
-
-    private String whiteRemoteAddress;
-
-    private boolean admin;
-
-    private byte defaultTopicPerm = 1;
-
-    private byte defaultGroupPerm = 1;
-
-    private Map<String, Byte> resourcePermMap;
-
-    private RemoteAddressStrategy remoteAddressStrategy;
-
-    private int requestCode;
-
-    // The content to calculate the content
-    private byte[] content;
-
-    private String signature;
-
-    private String secretToken;
-
-    private String recognition;
-
-    public PlainAccessResource() {
-    }
-
-    public static PlainAccessResource parse(RemotingCommand request, String remoteAddr) {
-        PlainAccessResource accessResource = new PlainAccessResource();
-        if (remoteAddr != null && remoteAddr.contains(":")) {
-            accessResource.setWhiteRemoteAddress(remoteAddr.substring(0, remoteAddr.lastIndexOf(':')));
-        } else {
-            accessResource.setWhiteRemoteAddress(remoteAddr);
-        }
-
-        accessResource.setRequestCode(request.getCode());
-
-        if (request.getExtFields() == null) {
-            // If request's extFields is null,then return accessResource directly(users can use whiteAddress pattern)
-            // The following logic codes depend on the request's extFields not to be null.
-            return accessResource;
-        }
-        accessResource.setAccessKey(request.getExtFields().get(SessionCredentials.ACCESS_KEY));
-        accessResource.setSignature(request.getExtFields().get(SessionCredentials.SIGNATURE));
-        accessResource.setSecretToken(request.getExtFields().get(SessionCredentials.SECURITY_TOKEN));
-
-        try {
-            switch (request.getCode()) {
-                case RequestCode.SEND_MESSAGE:
-                    final String topic = request.getExtFields().get("topic");
-                    if (PlainAccessResource.isRetryTopic(topic)) {
-                        accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("group")), Permission.SUB);
-                    } else {
-                        accessResource.addResourceAndPerm(topic, Permission.PUB);
-                    }
-                    break;
-                case RequestCode.SEND_MESSAGE_V2:
-                case RequestCode.SEND_BATCH_MESSAGE:
-                    final String topicV2 = request.getExtFields().get("b");
-                    if (PlainAccessResource.isRetryTopic(topicV2)) {
-                        accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("a")), Permission.SUB);
-                    } else {
-                        accessResource.addResourceAndPerm(topicV2, Permission.PUB);
-                    }
-                    break;
-                case RequestCode.CONSUMER_SEND_MSG_BACK:
-                    accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("group")), Permission.SUB);
-                    break;
-                case RequestCode.PULL_MESSAGE:
-                    accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.SUB);
-                    accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("consumerGroup")), Permission.SUB);
-                    break;
-                case RequestCode.QUERY_MESSAGE:
-                    accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.SUB);
-                    break;
-                case RequestCode.HEART_BEAT:
-                    HeartbeatData heartbeatData = HeartbeatData.decode(request.getBody(), HeartbeatData.class);
-                    for (ConsumerData data : heartbeatData.getConsumerDataSet()) {
-                        accessResource.addResourceAndPerm(getRetryTopic(data.getGroupName()), Permission.SUB);
-                        for (SubscriptionData subscriptionData : data.getSubscriptionDataSet()) {
-                            accessResource.addResourceAndPerm(subscriptionData.getTopic(), Permission.SUB);
-                        }
-                    }
-                    break;
-                case RequestCode.UNREGISTER_CLIENT:
-                    final UnregisterClientRequestHeader unregisterClientRequestHeader =
-                        (UnregisterClientRequestHeader) request
-                            .decodeCommandCustomHeader(UnregisterClientRequestHeader.class);
-                    accessResource.addResourceAndPerm(getRetryTopic(unregisterClientRequestHeader.getConsumerGroup()), Permission.SUB);
-                    break;
-                case RequestCode.GET_CONSUMER_LIST_BY_GROUP:
-                    final GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader =
-                        (GetConsumerListByGroupRequestHeader) request
-                            .decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class);
-                    accessResource.addResourceAndPerm(getRetryTopic(getConsumerListByGroupRequestHeader.getConsumerGroup()), Permission.SUB);
-                    break;
-                case RequestCode.UPDATE_CONSUMER_OFFSET:
-                    final UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader =
-                        (UpdateConsumerOffsetRequestHeader) request
-                            .decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class);
-                    accessResource.addResourceAndPerm(getRetryTopic(updateConsumerOffsetRequestHeader.getConsumerGroup()), Permission.SUB);
-                    accessResource.addResourceAndPerm(updateConsumerOffsetRequestHeader.getTopic(), Permission.SUB);
-                    break;
-                default:
-                    break;
-
-            }
-        } catch (Throwable t) {
-            throw new AclException(t.getMessage(), t);
-        }
-
-        // Content
-        SortedMap<String, String> map = new TreeMap<>();
-        for (Map.Entry<String, String> entry : request.getExtFields().entrySet()) {
-            if (request.getVersion() <= MQVersion.Version.V4_9_3.ordinal() &&
-                    MixAll.UNIQUE_MSG_QUERY_FLAG.equals(entry.getKey())) {
-                continue;
-            }
-            if (!SessionCredentials.SIGNATURE.equals(entry.getKey())) {
-                map.put(entry.getKey(), entry.getValue());
-            }
-        }
-        accessResource.setContent(AclUtils.combineRequestContent(request, map));
-        return accessResource;
-    }
-
-    public static PlainAccessResource parse(GeneratedMessageV3 messageV3, AuthenticationHeader header) {
-        PlainAccessResource accessResource = new PlainAccessResource();
-        String remoteAddress = header.getRemoteAddress();
-        if (remoteAddress != null && remoteAddress.contains(":")) {
-            accessResource.setWhiteRemoteAddress(RemotingHelper.parseHostFromAddress(remoteAddress));
-        } else {
-            accessResource.setWhiteRemoteAddress(remoteAddress);
-        }
-        try {
-            AuthorizationHeader authorizationHeader = new AuthorizationHeader(header.getAuthorization());
-            accessResource.setAccessKey(authorizationHeader.getAccessKey());
-            accessResource.setSignature(authorizationHeader.getSignature());
-        } catch (DecoderException e) {
-            throw new AclException(e.getMessage(), e);
-        }
-        accessResource.setSecretToken(header.getSessionToken());
-        accessResource.setRequestCode(header.getRequestCode());
-        accessResource.setContent(header.getDatetime().getBytes(StandardCharsets.UTF_8));
-
-        try {
-            String rpcFullName = messageV3.getDescriptorForType().getFullName();
-            if (HeartbeatRequest.getDescriptor().getFullName().equals(rpcFullName)) {
-                HeartbeatRequest request = (HeartbeatRequest) messageV3;
-                if (ClientType.PUSH_CONSUMER.equals(request.getClientType())
-                    || ClientType.SIMPLE_CONSUMER.equals(request.getClientType())) {
-                    if (!request.hasGroup()) {
-                        throw new AclException("Consumer heartbeat doesn't have group");
-                    } else {
-                        accessResource.addGroupResourceAndPerm(request.getGroup(), Permission.SUB);
-                    }
-                }
-            } else if (SendMessageRequest.getDescriptor().getFullName().equals(rpcFullName)) {
-                SendMessageRequest request = (SendMessageRequest) messageV3;
-                if (request.getMessagesCount() <= 0) {
-                    throw new AclException("SendMessageRequest, messageCount is zero", ResponseCode.MESSAGE_ILLEGAL);
-                }
-                Resource topic = request.getMessages(0).getTopic();
-                for (Message message : request.getMessagesList()) {
-                    if (!message.getTopic().equals(topic)) {
-                        throw new AclException("SendMessageRequest, messages' topic is not consistent", ResponseCode.MESSAGE_ILLEGAL);
-                    }
-                }
-                accessResource.addResourceAndPerm(topic, Permission.PUB);
-            } else if (ReceiveMessageRequest.getDescriptor().getFullName().equals(rpcFullName)) {
-                ReceiveMessageRequest request = (ReceiveMessageRequest) messageV3;
-                accessResource.addGroupResourceAndPerm(request.getGroup(), Permission.SUB);
-                accessResource.addResourceAndPerm(request.getMessageQueue().getTopic(), Permission.SUB);
-            } else if (AckMessageRequest.getDescriptor().getFullName().equals(rpcFullName)) {
-                AckMessageRequest request = (AckMessageRequest) messageV3;
-                accessResource.addGroupResourceAndPerm(request.getGroup(), Permission.SUB);
-                accessResource.addResourceAndPerm(request.getTopic(), Permission.SUB);
-            } else if (ForwardMessageToDeadLetterQueueRequest.getDescriptor().getFullName().equals(rpcFullName)) {
-                ForwardMessageToDeadLetterQueueRequest request = (ForwardMessageToDeadLetterQueueRequest) messageV3;
-                accessResource.addGroupResourceAndPerm(request.getGroup(), Permission.SUB);
-                accessResource.addResourceAndPerm(request.getTopic(), Permission.SUB);
-            } else if (EndTransactionRequest.getDescriptor().getFullName().equals(rpcFullName)) {
-                EndTransactionRequest request = (EndTransactionRequest) messageV3;
-                accessResource.addResourceAndPerm(request.getTopic(), Permission.PUB);
-            } else if (TelemetryCommand.getDescriptor().getFullName().equals(rpcFullName)) {
-                TelemetryCommand command = (TelemetryCommand) messageV3;
-                if (command.getCommandCase() == TelemetryCommand.CommandCase.SETTINGS) {
-                    if (command.getSettings().hasPublishing()) {
-                        List<Resource> topicList = command.getSettings().getPublishing().getTopicsList();
-                        for (Resource topic : topicList) {
-                            accessResource.addResourceAndPerm(topic, Permission.PUB);
-                        }
-                    }
-                    if (command.getSettings().hasSubscription()) {
-                        Subscription subscription = command.getSettings().getSubscription();
-                        accessResource.addGroupResourceAndPerm(subscription.getGroup(), Permission.SUB);
-                        for (SubscriptionEntry entry : subscription.getSubscriptionsList()) {
-                            accessResource.addResourceAndPerm(entry.getTopic(), Permission.SUB);
-                        }
-                    }
-                    if (!command.getSettings().hasPublishing() && !command.getSettings().hasSubscription()) {
-                        throw new AclException("settings command doesn't have publishing or subscription");
-                    }
-                }
-            } else if (NotifyClientTerminationRequest.getDescriptor().getFullName().equals(rpcFullName)) {
-                NotifyClientTerminationRequest request = (NotifyClientTerminationRequest) messageV3;
-                accessResource.addGroupResourceAndPerm(request.getGroup(), Permission.SUB);
-            } else if (QueryRouteRequest.getDescriptor().getFullName().equals(rpcFullName)) {
-                QueryRouteRequest request = (QueryRouteRequest) messageV3;
-                accessResource.addResourceAndPerm(request.getTopic(), Permission.ANY);
-            } else if (QueryAssignmentRequest.getDescriptor().getFullName().equals(rpcFullName)) {
-                QueryAssignmentRequest request = (QueryAssignmentRequest) messageV3;
-                accessResource.addGroupResourceAndPerm(request.getGroup(), Permission.SUB);
-                accessResource.addResourceAndPerm(request.getTopic(), Permission.SUB);
-            } else if (ChangeInvisibleDurationRequest.getDescriptor().getFullName().equals(rpcFullName)) {
-                ChangeInvisibleDurationRequest request = (ChangeInvisibleDurationRequest) messageV3;
-                accessResource.addGroupResourceAndPerm(request.getGroup(), Permission.SUB);
-                accessResource.addResourceAndPerm(request.getTopic(), Permission.SUB);
-            }
-        } catch (Throwable t) {
-            throw new AclException(t.getMessage(), t);
-        }
-        return accessResource;
-    }
-
-    private void addResourceAndPerm(Resource resource, byte permission) {
-        String resourceName = NamespaceUtil.wrapNamespace(resource.getResourceNamespace(), resource.getName());
-        addResourceAndPerm(resourceName, permission);
-    }
-
-    private void addGroupResourceAndPerm(Resource resource, byte permission) {
-        String resourceName = NamespaceUtil.wrapNamespace(resource.getResourceNamespace(), resource.getName());
-        addResourceAndPerm(getRetryTopic(resourceName), permission);
-    }
-
-    public static PlainAccessResource build(PlainAccessConfig plainAccessConfig, RemoteAddressStrategy remoteAddressStrategy) {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setAccessKey(plainAccessConfig.getAccessKey());
-        plainAccessResource.setSecretKey(plainAccessConfig.getSecretKey());
-        plainAccessResource.setWhiteRemoteAddress(plainAccessConfig.getWhiteRemoteAddress());
-
-        plainAccessResource.setAdmin(plainAccessConfig.isAdmin());
-
-        plainAccessResource.setDefaultGroupPerm(Permission.parsePermFromString(plainAccessConfig.getDefaultGroupPerm()));
-        plainAccessResource.setDefaultTopicPerm(Permission.parsePermFromString(plainAccessConfig.getDefaultTopicPerm()));
-
-        Permission.parseResourcePerms(plainAccessResource, false, plainAccessConfig.getGroupPerms());
-        Permission.parseResourcePerms(plainAccessResource, true, plainAccessConfig.getTopicPerms());
-
-        plainAccessResource.setRemoteAddressStrategy(remoteAddressStrategy);
-        return plainAccessResource;
-    }
-
-    public static boolean isRetryTopic(String topic) {
-        return null != topic && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX);
-    }
-
-    public static String printStr(String resource, boolean isGroup) {
-        if (resource == null) {
-            return null;
-        }
-        if (isGroup) {
-            return String.format("%s:%s", "group", getGroupFromRetryTopic(resource));
-        } else {
-            return String.format("%s:%s", "topic", resource);
-        }
-    }
-
-    public static String getGroupFromRetryTopic(String retryTopic) {
-        if (retryTopic == null) {
-            return null;
-        }
-        return KeyBuilder.parseGroup(retryTopic);
-    }
-
-    public static String getRetryTopic(String group) {
-        if (group == null) {
-            return null;
-        }
-        return MixAll.getRetryTopic(group);
-    }
-
-    public void addResourceAndPerm(String resource, byte perm) {
-        if (resource == null) {
-            return;
-        }
-        if (resourcePermMap == null) {
-            resourcePermMap = new HashMap<>();
-        }
-        resourcePermMap.put(resource, perm);
-    }
-
-    public String getAccessKey() {
-        return accessKey;
-    }
-
-    public void setAccessKey(String accessKey) {
-        this.accessKey = accessKey;
-    }
-
-    public String getSecretKey() {
-        return secretKey;
-    }
-
-    public void setSecretKey(String secretKey) {
-        this.secretKey = secretKey;
-    }
-
-    public String getWhiteRemoteAddress() {
-        return whiteRemoteAddress;
-    }
-
-    public void setWhiteRemoteAddress(String whiteRemoteAddress) {
-        this.whiteRemoteAddress = whiteRemoteAddress;
-    }
-
-    public boolean isAdmin() {
-        return admin;
-    }
-
-    public void setAdmin(boolean admin) {
-        this.admin = admin;
-    }
-
-    public byte getDefaultTopicPerm() {
-        return defaultTopicPerm;
-    }
-
-    public void setDefaultTopicPerm(byte defaultTopicPerm) {
-        this.defaultTopicPerm = defaultTopicPerm;
-    }
-
-    public byte getDefaultGroupPerm() {
-        return defaultGroupPerm;
-    }
-
-    public void setDefaultGroupPerm(byte defaultGroupPerm) {
-        this.defaultGroupPerm = defaultGroupPerm;
-    }
-
-    public Map<String, Byte> getResourcePermMap() {
-        return resourcePermMap;
-    }
-
-    public String getRecognition() {
-        return recognition;
-    }
-
-    public void setRecognition(String recognition) {
-        this.recognition = recognition;
-    }
-
-    public int getRequestCode() {
-        return requestCode;
-    }
-
-    public void setRequestCode(int requestCode) {
-        this.requestCode = requestCode;
-    }
-
-    public String getSecretToken() {
-        return secretToken;
-    }
-
-    public void setSecretToken(String secretToken) {
-        this.secretToken = secretToken;
-    }
-
-    public RemoteAddressStrategy getRemoteAddressStrategy() {
-        return remoteAddressStrategy;
-    }
-
-    public void setRemoteAddressStrategy(RemoteAddressStrategy remoteAddressStrategy) {
-        this.remoteAddressStrategy = remoteAddressStrategy;
-    }
-
-    public String getSignature() {
-        return signature;
-    }
-
-    public void setSignature(String signature) {
-        this.signature = signature;
-    }
-
-    @Override
-    public String toString() {
-        return ToStringBuilder.reflectionToString(this);
-    }
-
-    public byte[] getContent() {
-        return content;
-    }
-
-    public void setContent(byte[] content) {
-        this.content = content;
-    }
-}
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java
deleted file mode 100644
index a7015eaca73..00000000000
--- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.acl.plain;
-
-import com.google.protobuf.GeneratedMessageV3;
-import java.util.List;
-import java.util.Map;
-import org.apache.rocketmq.acl.AccessResource;
-import org.apache.rocketmq.acl.AccessValidator;
-import org.apache.rocketmq.acl.common.AuthenticationHeader;
-import org.apache.rocketmq.common.AclConfig;
-import org.apache.rocketmq.common.PlainAccessConfig;
-import org.apache.rocketmq.remoting.protocol.DataVersion;
-import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-
-public class PlainAccessValidator implements AccessValidator {
-
-    private PlainPermissionManager aclPlugEngine;
-
-    public PlainAccessValidator() {
-        aclPlugEngine = new PlainPermissionManager();
-    }
-
-    @Override
-    public AccessResource parse(RemotingCommand request, String remoteAddr) {
-        return PlainAccessResource.parse(request, remoteAddr);
-    }
-
-    @Override
-    public AccessResource parse(GeneratedMessageV3 messageV3, AuthenticationHeader header) {
-        return PlainAccessResource.parse(messageV3, header);
-    }
-
-    @Override
-    public void validate(AccessResource accessResource) {
-        aclPlugEngine.validate((PlainAccessResource) accessResource);
-    }
-
-    @Override
-    public boolean updateAccessConfig(PlainAccessConfig plainAccessConfig) {
-        return aclPlugEngine.updateAccessConfig(plainAccessConfig);
-    }
-
-    @Override
-    public boolean deleteAccessConfig(String accessKey) {
-        return aclPlugEngine.deleteAccessConfig(accessKey);
-    }
-
-    @Override
-    public String getAclConfigVersion() {
-        return aclPlugEngine.getAclConfigDataVersion();
-    }
-
-    @Override
-    public boolean updateGlobalWhiteAddrsConfig(List<String> globalWhiteAddrsList) {
-        return aclPlugEngine.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList);
-    }
-
-    @Override
-    public boolean updateGlobalWhiteAddrsConfig(List<String> globalWhiteAddrsList, String aclFileFullPath) {
-        return aclPlugEngine.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList, aclFileFullPath);
-    }
-
-    @Override
-    public AclConfig getAllAclConfig() {
-        return aclPlugEngine.getAllAclConfig();
-    }
-
-    @Override
-    public Map<String, DataVersion> getAllAclConfigVersion() {
-        return aclPlugEngine.getDataVersionMap();
-    }
-}
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionChecker.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionChecker.java
deleted file mode 100644
index 8e6c317b237..00000000000
--- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionChecker.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.acl.plain;
-
-import java.util.Map;
-import org.apache.rocketmq.acl.AccessResource;
-import org.apache.rocketmq.acl.PermissionChecker;
-import org.apache.rocketmq.acl.common.AclException;
-import org.apache.rocketmq.acl.common.Permission;
-
-public class PlainPermissionChecker implements PermissionChecker {
-    public void check(AccessResource checkedAccess, AccessResource ownedAccess) {
-        PlainAccessResource checkedPlainAccess = (PlainAccessResource) checkedAccess;
-        PlainAccessResource ownedPlainAccess = (PlainAccessResource) ownedAccess;
-
-        if (ownedPlainAccess.isAdmin()) {
-            // admin user don't need verification
-            return;
-        }
-        if (Permission.needAdminPerm(checkedPlainAccess.getRequestCode())) {
-            throw new AclException(String.format("Need admin permission for request code=%d, but accessKey=%s is not", checkedPlainAccess.getRequestCode(), ownedPlainAccess.getAccessKey()));
-        }
-
-        Map<String, Byte> needCheckedPermMap = checkedPlainAccess.getResourcePermMap();
-        Map<String, Byte> ownedPermMap = ownedPlainAccess.getResourcePermMap();
-
-        if (needCheckedPermMap == null) {
-            // If the needCheckedPermMap is null,then return
-            return;
-        }
-
-        for (Map.Entry<String, Byte> needCheckedEntry : needCheckedPermMap.entrySet()) {
-            String resource = needCheckedEntry.getKey();
-            Byte neededPerm = needCheckedEntry.getValue();
-            boolean isGroup = PlainAccessResource.isRetryTopic(resource);
-
-            if (ownedPermMap == null || !ownedPermMap.containsKey(resource)) {
-                // Check the default perm
-                byte ownedPerm = isGroup ? ownedPlainAccess.getDefaultGroupPerm() :
-                    ownedPlainAccess.getDefaultTopicPerm();
-                if (!Permission.checkPermission(neededPerm, ownedPerm)) {
-                    throw new AclException(String.format("No default permission for %s", PlainAccessResource.printStr(resource, isGroup)));
-                }
-                continue;
-            }
-            if (!Permission.checkPermission(neededPerm, ownedPermMap.get(resource))) {
-                throw new AclException(String.format("No permission for %s", PlainAccessResource.printStr(resource, isGroup)));
-            }
-        }
-    }
-}
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java
deleted file mode 100644
index 345aed06c5a..00000000000
--- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java
+++ /dev/null
@@ -1,648 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.acl.plain;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.FileAlreadyExistsException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.acl.PermissionChecker;
-import org.apache.rocketmq.acl.common.AclConstants;
-import org.apache.rocketmq.acl.common.AclException;
-import org.apache.rocketmq.acl.common.AclUtils;
-import org.apache.rocketmq.acl.common.Permission;
-import org.apache.rocketmq.common.AclConfig;
-import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.common.PlainAccessConfig;
-import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.common.topic.TopicValidator;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.remoting.protocol.DataVersion;
-import org.apache.rocketmq.srvutil.AclFileWatchService;
-
-public class PlainPermissionManager {
-
-    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
-
-    private String fileHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY,
-        System.getenv(MixAll.ROCKETMQ_HOME_ENV));
-
-    private String defaultAclDir;
-
-    private String defaultAclFile;
-
-    private Map<String/** fileFullPath **/, Map<String/** AccessKey **/, PlainAccessResource>> aclPlainAccessResourceMap = new HashMap<>();
-
-    private Map<String/** AccessKey **/, String/** fileFullPath **/> accessKeyTable = new HashMap<>();
-
-    private List<RemoteAddressStrategy> globalWhiteRemoteAddressStrategy = new ArrayList<>();
-
-    private RemoteAddressStrategyFactory remoteAddressStrategyFactory = new RemoteAddressStrategyFactory();
-
-    private Map<String/** fileFullPath **/, List<RemoteAddressStrategy>> globalWhiteRemoteAddressStrategyMap = new HashMap<>();
-
-    private boolean isWatchStart;
-
-    private Map<String/** fileFullPath **/, DataVersion> dataVersionMap = new HashMap<>();
-
-    @Deprecated
-    private final DataVersion dataVersion = new DataVersion();
-
-    private List<String> fileList = new ArrayList<>();
-
-    private final PermissionChecker permissionChecker = new PlainPermissionChecker();
-
-    public PlainPermissionManager() {
-        this.defaultAclDir = MixAll.dealFilePath(fileHome + File.separator + "conf" + File.separator + "acl");
-        this.defaultAclFile = MixAll.dealFilePath(fileHome + File.separator + System.getProperty("rocketmq.acl.plain.file", "conf" + File.separator + "plain_acl.yml"));
-        load();
-        watch();
-    }
-
-    public List<String> getAllAclFiles(String path) {
-        if (!new File(path).exists()) {
-            log.info("The default acl dir {} is not exist", path);
-            return new ArrayList<>();
-        }
-        List<String> allAclFileFullPath = new ArrayList<>();
-        File file = new File(path);
-        File[] files = file.listFiles();
-        for (int i = 0; i < files.length; i++) {
-            String fileName = files[i].getAbsolutePath();
-            File f = new File(fileName);
-            if (fileName.equals(fileHome + MixAll.ACL_CONF_TOOLS_FILE)) {
-                continue;
-            } else if (fileName.endsWith(".yml") || fileName.endsWith(".yaml")) {
-                allAclFileFullPath.add(fileName);
-            } else if (f.isDirectory()) {
-                allAclFileFullPath.addAll(getAllAclFiles(fileName));
-            }
-        }
-        return allAclFileFullPath;
-    }
-
-    public void load() {
-        if (fileHome == null || fileHome.isEmpty()) {
-            return;
-        }
-
-        Map<String, Map<String, PlainAccessResource>> aclPlainAccessResourceMap = new HashMap<>();
-        Map<String, String> accessKeyTable = new HashMap<>();
-        List<RemoteAddressStrategy> globalWhiteRemoteAddressStrategy = new ArrayList<>();
-        Map<String, List<RemoteAddressStrategy>> globalWhiteRemoteAddressStrategyMap = new HashMap<>();
-        Map<String, DataVersion> dataVersionMap = new HashMap<>();
-
-        assureAclConfigFilesExist();
-
-        fileList = getAllAclFiles(defaultAclDir);
-        if (new File(defaultAclFile).exists() && !fileList.contains(defaultAclFile)) {
-            fileList.add(defaultAclFile);
-        }
-
-        for (int i = 0; i < fileList.size(); i++) {
-            final String currentFile = MixAll.dealFilePath(fileList.get(i));
-            PlainAccessData plainAclConfData = AclUtils.getYamlDataObject(currentFile,
-                PlainAccessData.class);
-            if (plainAclConfData == null) {
-                log.warn("No data in file {}", currentFile);
-                continue;
-            }
-            log.info("Broker plain acl conf data is : {}", plainAclConfData.toString());
-
-            List<RemoteAddressStrategy> globalWhiteRemoteAddressStrategyList = new ArrayList<>();
-            List<String> globalWhiteRemoteAddressesList = plainAclConfData.getGlobalWhiteRemoteAddresses();
-            if (globalWhiteRemoteAddressesList != null && !globalWhiteRemoteAddressesList.isEmpty()) {
-                for (int j = 0; j < globalWhiteRemoteAddressesList.size(); j++) {
-                    globalWhiteRemoteAddressStrategyList.add(remoteAddressStrategyFactory.
-                        getRemoteAddressStrategy(globalWhiteRemoteAddressesList.get(j)));
-                }
-            }
-            if (globalWhiteRemoteAddressStrategyList.size() > 0) {
-                globalWhiteRemoteAddressStrategyMap.put(currentFile, globalWhiteRemoteAddressStrategyList);
-                globalWhiteRemoteAddressStrategy.addAll(globalWhiteRemoteAddressStrategyList);
-            }
-
-            List<PlainAccessConfig> accounts = plainAclConfData.getAccounts();
-            Map<String, PlainAccessResource> plainAccessResourceMap = new HashMap<>();
-            if (accounts != null && !accounts.isEmpty()) {
-                for (PlainAccessConfig plainAccessConfig : accounts) {
-                    PlainAccessResource plainAccessResource = buildPlainAccessResource(plainAccessConfig);
-                    //AccessKey can not be defined in multiple ACL files
-                    if (accessKeyTable.get(plainAccessResource.getAccessKey()) == null) {
-                        plainAccessResourceMap.put(plainAccessResource.getAccessKey(), plainAccessResource);
-                        accessKeyTable.put(plainAccessResource.getAccessKey(), currentFile);
-                    } else {
-                        log.warn("The accessKey {} is repeated in multiple ACL files", plainAccessResource.getAccessKey());
-                    }
-                }
-            }
-            if (plainAccessResourceMap.size() > 0) {
-                aclPlainAccessResourceMap.put(currentFile, plainAccessResourceMap);
-            }
-
-            List<PlainAccessData.DataVersion> dataVersions = plainAclConfData.getDataVersion();
-            DataVersion dataVersion = new DataVersion();
-            if (dataVersions != null && !dataVersions.isEmpty()) {
-                DataVersion firstElement = new DataVersion();
-                firstElement.setCounter(new AtomicLong(dataVersions.get(0).getCounter()));
-                firstElement.setTimestamp(dataVersions.get(0).getTimestamp());
-                dataVersion.assignNewOne(firstElement);
-            }
-            dataVersionMap.put(currentFile, dataVersion);
-        }
-
-        if (dataVersionMap.containsKey(defaultAclFile)) {
-            this.dataVersion.assignNewOne(dataVersionMap.get(defaultAclFile));
-        }
-        this.dataVersionMap = dataVersionMap;
-        this.globalWhiteRemoteAddressStrategyMap = globalWhiteRemoteAddressStrategyMap;
-        this.globalWhiteRemoteAddressStrategy = globalWhiteRemoteAddressStrategy;
-        this.aclPlainAccessResourceMap = aclPlainAccessResourceMap;
-        this.accessKeyTable = accessKeyTable;
-    }
-
-    /**
-     * Currently GlobalWhiteAddress is defined in {@link #defaultAclFile}, so make sure it exists.
-     */
-    private void assureAclConfigFilesExist() {
-        final Path defaultAclFilePath = Paths.get(this.defaultAclFile);
-        if (!Files.exists(defaultAclFilePath)) {
-            try {
-                Files.createFile(defaultAclFilePath);
-            } catch (FileAlreadyExistsException e) {
-                // Maybe created by other threads
-            } catch (IOException e) {
-                log.error("Error in creating " + this.defaultAclFile, e);
-                throw new AclException(e.getMessage());
-            }
-        }
-    }
-
-    public void load(String aclFilePath) {
-        aclFilePath = MixAll.dealFilePath(aclFilePath);
-        Map<String, PlainAccessResource> plainAccessResourceMap = new HashMap<>();
-        List<RemoteAddressStrategy> globalWhiteRemoteAddressStrategy = new ArrayList<>();
-
-        PlainAccessData plainAclConfData = AclUtils.getYamlDataObject(aclFilePath,
-            PlainAccessData.class);
-        if (plainAclConfData == null) {
-            log.warn("No data in {}, skip it", aclFilePath);
-            return;
-        }
-        log.info("Broker plain acl conf data is : {}", plainAclConfData.toString());
-        List<String> globalWhiteRemoteAddressesList = plainAclConfData.getGlobalWhiteRemoteAddresses();
-        if (globalWhiteRemoteAddressesList != null && !globalWhiteRemoteAddressesList.isEmpty()) {
-            for (int i = 0; i < globalWhiteRemoteAddressesList.size(); i++) {
-                globalWhiteRemoteAddressStrategy.add(remoteAddressStrategyFactory.
-                    getRemoteAddressStrategy(globalWhiteRemoteAddressesList.get(i)));
-            }
-        }
-
-        this.globalWhiteRemoteAddressStrategy.addAll(globalWhiteRemoteAddressStrategy);
-        if (this.globalWhiteRemoteAddressStrategyMap.get(aclFilePath) != null) {
-            List<RemoteAddressStrategy> remoteAddressStrategyList = this.globalWhiteRemoteAddressStrategyMap.get(aclFilePath);
-            for (int i = 0; i < remoteAddressStrategyList.size(); i++) {
-                this.globalWhiteRemoteAddressStrategy.remove(remoteAddressStrategyList.get(i));
-            }
-            this.globalWhiteRemoteAddressStrategyMap.put(aclFilePath, globalWhiteRemoteAddressStrategy);
-        }
-
-        List<PlainAccessConfig> accounts = plainAclConfData.getAccounts();
-        if (accounts != null && !accounts.isEmpty()) {
-            for (PlainAccessConfig plainAccessConfig : accounts) {
-                PlainAccessResource plainAccessResource = buildPlainAccessResource(plainAccessConfig);
-                //AccessKey can not be defined in multiple ACL files
-                String oldPath = this.accessKeyTable.get(plainAccessResource.getAccessKey());
-                if (oldPath == null || aclFilePath.equals(oldPath)) {
-                    plainAccessResourceMap.put(plainAccessResource.getAccessKey(), plainAccessResource);
-                    this.accessKeyTable.put(plainAccessResource.getAccessKey(), aclFilePath);
-                } else {
-                    log.warn("The accessKey {} is repeated in multiple ACL files", plainAccessResource.getAccessKey());
-                }
-            }
-        }
-
-        // For loading dataversion part just
-        List<PlainAccessData.DataVersion> dataVersions = plainAclConfData.getDataVersion();
-        DataVersion dataVersion = new DataVersion();
-        if (dataVersions != null && !dataVersions.isEmpty()) {
-            DataVersion firstElement = new DataVersion();
-            firstElement.setCounter(new AtomicLong(dataVersions.get(0).getCounter()));
-            firstElement.setTimestamp(dataVersions.get(0).getTimestamp());
-            dataVersion.assignNewOne(firstElement);
-        }
-
-        this.aclPlainAccessResourceMap.put(aclFilePath, plainAccessResourceMap);
-        this.dataVersionMap.put(aclFilePath, dataVersion);
-        if (aclFilePath.equals(defaultAclFile)) {
-            this.dataVersion.assignNewOne(dataVersion);
-        }
-    }
-
-    @Deprecated
-    public String getAclConfigDataVersion() {
-        return this.dataVersion.toJson();
-    }
-
-    public Map<String, DataVersion> getDataVersionMap() {
-        return this.dataVersionMap;
-    }
-
-    public PlainAccessData updateAclConfigFileVersion(String aclFileName, PlainAccessData updateAclConfigMap) {
-
-        List<PlainAccessData.DataVersion> dataVersions = updateAclConfigMap.getDataVersion();
-        DataVersion dataVersion = new DataVersion();
-        if (dataVersions != null) {
-            if (dataVersions.size() > 0) {
-                dataVersion.setTimestamp(dataVersions.get(0).getTimestamp());
-                dataVersion.setCounter(new AtomicLong(dataVersions.get(0).getCounter()));
-            }
-        }
-        dataVersion.nextVersion();
-        List<PlainAccessData.DataVersion> versionElement = new ArrayList<>();
-        PlainAccessData.DataVersion dataVersionNew = new PlainAccessData.DataVersion();
-        dataVersionNew.setTimestamp(dataVersion.getTimestamp());
-        dataVersionNew.setCounter(dataVersion.getCounter().get());
-        versionElement.add(dataVersionNew);
-        updateAclConfigMap.setDataVersion(versionElement);
-
-        dataVersionMap.put(aclFileName, dataVersion);
-
-        return updateAclConfigMap;
-    }
-
-    public boolean updateAccessConfig(PlainAccessConfig plainAccessConfig) {
-
-        if (plainAccessConfig == null) {
-            log.error("Parameter value plainAccessConfig is null,Please check your parameter");
-            throw new AclException("Parameter value plainAccessConfig is null, Please check your parameter");
-        }
-        checkPlainAccessConfig(plainAccessConfig);
-
-        Permission.checkResourcePerms(plainAccessConfig.getTopicPerms());
-        Permission.checkResourcePerms(plainAccessConfig.getGroupPerms());
-
-        if (accessKeyTable.containsKey(plainAccessConfig.getAccessKey())) {
-            PlainAccessConfig updateAccountMap = null;
-            String aclFileName = accessKeyTable.get(plainAccessConfig.getAccessKey());
-            PlainAccessData aclAccessConfigMap = AclUtils.getYamlDataObject(aclFileName, PlainAccessData.class);
-            List<PlainAccessConfig> accounts = aclAccessConfigMap.getAccounts();
-            if (null != accounts) {
-                for (PlainAccessConfig account : accounts) {
-                    if (account.getAccessKey().equals(plainAccessConfig.getAccessKey())) {
-                        // Update acl access config elements
-                        accounts.remove(account);
-                        updateAccountMap = createAclAccessConfigMap(account, plainAccessConfig);
-                        accounts.add(updateAccountMap);
-                        aclAccessConfigMap.setAccounts(accounts);
-                        break;
-                    }
-                }
-            } else {
-                // Maybe deleted in file, add it back
-                accounts = new LinkedList<>();
-                updateAccountMap = createAclAccessConfigMap(null, plainAccessConfig);
-                accounts.add(updateAccountMap);
-                aclAccessConfigMap.setAccounts(accounts);
-            }
-            Map<String, PlainAccessResource> accountMap = aclPlainAccessResourceMap.get(aclFileName);
-            if (accountMap == null) {
-                accountMap = new HashMap<>(1);
-                accountMap.put(plainAccessConfig.getAccessKey(), buildPlainAccessResource(plainAccessConfig));
-            } else if (accountMap.size() == 0) {
-                accountMap.put(plainAccessConfig.getAccessKey(), buildPlainAccessResource(plainAccessConfig));
-            } else {
-                for (Map.Entry<String, PlainAccessResource> entry : accountMap.entrySet()) {
-                    if (entry.getValue().getAccessKey().equals(plainAccessConfig.getAccessKey())) {
-                        PlainAccessResource plainAccessResource = buildPlainAccessResource(plainAccessConfig);
-                        accountMap.put(entry.getKey(), plainAccessResource);
-                        break;
-                    }
-                }
-            }
-            aclPlainAccessResourceMap.put(aclFileName, accountMap);
-            return AclUtils.writeDataObject(aclFileName, updateAclConfigFileVersion(aclFileName, aclAccessConfigMap));
-        } else {
-            String fileName = MixAll.dealFilePath(defaultAclFile);
-            //Create acl access config elements on the default acl file
-            if (aclPlainAccessResourceMap.get(defaultAclFile) == null || aclPlainAccessResourceMap.get(defaultAclFile).size() == 0) {
-                try {
-                    File defaultAclFile = new File(fileName);
-                    if (!defaultAclFile.exists()) {
-                        defaultAclFile.createNewFile();
-                    }
-                } catch (IOException e) {
-                    log.warn("create default acl file has exception when update accessConfig. ", e);
-                }
-            }
-            PlainAccessData aclAccessConfigMap = AclUtils.getYamlDataObject(defaultAclFile, PlainAccessData.class);
-            if (aclAccessConfigMap == null) {
-                aclAccessConfigMap = new PlainAccessData();
-            }
-            List<PlainAccessConfig> accounts = aclAccessConfigMap.getAccounts();
-            // When no accounts defined
-            if (null == accounts) {
-                accounts = new ArrayList<>();
-            }
-            accounts.add(createAclAccessConfigMap(null, plainAccessConfig));
-            aclAccessConfigMap.setAccounts(accounts);
-            accessKeyTable.put(plainAccessConfig.getAccessKey(), fileName);
-            if (aclPlainAccessResourceMap.get(fileName) == null) {
-                Map<String, PlainAccessResource> plainAccessResourceMap = new HashMap<>(1);
-                plainAccessResourceMap.put(plainAccessConfig.getAccessKey(), buildPlainAccessResource(plainAccessConfig));
-                aclPlainAccessResourceMap.put(fileName, plainAccessResourceMap);
-            } else {
-                Map<String, PlainAccessResource> plainAccessResourceMap = aclPlainAccessResourceMap.get(fileName);
-                plainAccessResourceMap.put(plainAccessConfig.getAccessKey(), buildPlainAccessResource(plainAccessConfig));
-                aclPlainAccessResourceMap.put(fileName, plainAccessResourceMap);
-            }
-            return AclUtils.writeDataObject(defaultAclFile, updateAclConfigFileVersion(defaultAclFile, aclAccessConfigMap));
-        }
-    }
-
-    public PlainAccessConfig createAclAccessConfigMap(PlainAccessConfig existedAccountMap,
-        PlainAccessConfig plainAccessConfig) {
-
-        PlainAccessConfig newAccountsMap = null;
-        if (existedAccountMap == null) {
-            newAccountsMap = new PlainAccessConfig();
-        } else {
-            newAccountsMap = existedAccountMap;
-        }
-
-        if (StringUtils.isEmpty(plainAccessConfig.getAccessKey()) ||
-            plainAccessConfig.getAccessKey().length() <= AclConstants.ACCESS_KEY_MIN_LENGTH) {
-            throw new AclException(String.format(
-                "The accessKey=%s cannot be null and length should longer than 6",
-                plainAccessConfig.getAccessKey()));
-        }
-        newAccountsMap.setAccessKey(plainAccessConfig.getAccessKey());
-
-        if (!StringUtils.isEmpty(plainAccessConfig.getSecretKey())) {
-            if (plainAccessConfig.getSecretKey().length() <= AclConstants.SECRET_KEY_MIN_LENGTH) {
-                throw new AclException(String.format(
-                    "The secretKey=%s value length should longer than 6",
-                    plainAccessConfig.getSecretKey()));
-            }
-            newAccountsMap.setSecretKey(plainAccessConfig.getSecretKey());
-        }
-        if (plainAccessConfig.getWhiteRemoteAddress() != null) {
-            newAccountsMap.setWhiteRemoteAddress(plainAccessConfig.getWhiteRemoteAddress());
-        }
-        if (!StringUtils.isEmpty(String.valueOf(plainAccessConfig.isAdmin()))) {
-            newAccountsMap.setAdmin(plainAccessConfig.isAdmin());
-        }
-        if (!StringUtils.isEmpty(plainAccessConfig.getDefaultTopicPerm())) {
-            newAccountsMap.setDefaultTopicPerm(plainAccessConfig.getDefaultTopicPerm());
-        }
-        if (!StringUtils.isEmpty(plainAccessConfig.getDefaultGroupPerm())) {
-            newAccountsMap.setDefaultGroupPerm(plainAccessConfig.getDefaultGroupPerm());
-        }
-        if (plainAccessConfig.getTopicPerms() != null) {
-            newAccountsMap.setTopicPerms(plainAccessConfig.getTopicPerms());
-        }
-        if (plainAccessConfig.getGroupPerms() != null) {
-            newAccountsMap.setGroupPerms(plainAccessConfig.getGroupPerms());
-        }
-
-        return newAccountsMap;
-    }
-
-    public boolean deleteAccessConfig(String accessKey) {
-        if (StringUtils.isEmpty(accessKey)) {
-            log.error("Parameter value accessKey is null or empty String,Please check your parameter");
-            return false;
-        }
-
-        if (accessKeyTable.containsKey(accessKey)) {
-            String aclFileName = accessKeyTable.get(accessKey);
-            PlainAccessData aclAccessConfigData = AclUtils.getYamlDataObject(aclFileName,
-                PlainAccessData.class);
-            if (aclAccessConfigData == null) {
-                log.warn("No data found in {} when deleting access config of {}", aclFileName, accessKey);
-                return true;
-            }
-            List<PlainAccessConfig> accounts = aclAccessConfigData.getAccounts();
-            Iterator<PlainAccessConfig> itemIterator = accounts.iterator();
-            while (itemIterator.hasNext()) {
-                if (itemIterator.next().getAccessKey().equals(accessKey)) {
-                    // Delete the related acl config element
-                    itemIterator.remove();
-                    accessKeyTable.remove(accessKey);
-                    aclAccessConfigData.setAccounts(accounts);
-                    return AclUtils.writeDataObject(aclFileName, updateAclConfigFileVersion(aclFileName, aclAccessConfigData));
-                }
-            }
-        }
-        return false;
-    }
-
-    public boolean updateGlobalWhiteAddrsConfig(List<String> globalWhiteAddrsList) {
-        return this.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList, this.defaultAclFile);
-    }
-
-    public boolean updateGlobalWhiteAddrsConfig(List<String> globalWhiteAddrsList, String fileName) {
-        if (fileName == null || fileName.equals("")) {
-            fileName = this.defaultAclFile;
-        }
-
-        if (globalWhiteAddrsList == null) {
-            log.error("Parameter value globalWhiteAddrsList is null,Please check your parameter");
-            return false;
-        }
-
-        File file = new File(fileName);
-        if (!file.exists() || file.isDirectory()) {
-            log.error("Parameter value " + fileName + " is not exist or is a directory, please check your parameter");
-            return false;
-        }
-
-        if (!file.getAbsolutePath().startsWith(fileHome)) {
-            log.error("Parameter value " + fileName + " is not in the directory rocketmq.home.dir " + fileHome);
-            return false;
-        }
-
-        if (!fileName.endsWith(".yml") && fileName.endsWith(".yaml")) {
-            log.error("Parameter value " + fileName + " is not a ACL configuration file");
-            return false;
-        }
-
-        PlainAccessData aclAccessConfigMap = AclUtils.getYamlDataObject(fileName, PlainAccessData.class);
-        if (aclAccessConfigMap == null) {
-            aclAccessConfigMap = new PlainAccessData();
-            log.info("No data in {}, create a new aclAccessConfigMap", fileName);
-        }
-        // Update globalWhiteRemoteAddr element in memory map firstly
-        aclAccessConfigMap.setGlobalWhiteRemoteAddresses(new ArrayList<>(globalWhiteAddrsList));
-        return AclUtils.writeDataObject(fileName, updateAclConfigFileVersion(fileName, aclAccessConfigMap));
-
-    }
-
-    public AclConfig getAllAclConfig() {
-        AclConfig aclConfig = new AclConfig();
-        List<PlainAccessConfig> configs = new ArrayList<>();
-        List<String> whiteAddrs = new ArrayList<>();
-        Set<String> accessKeySets = new HashSet<>();
-
-        for (int i = 0; i < fileList.size(); i++) {
-            String path = fileList.get(i);
-            PlainAccessData plainAclConfData = AclUtils.getYamlDataObject(path,
-                PlainAccessData.class);
-            if (plainAclConfData == null) {
-                continue;
-            }
-            List<String> globalWhiteAddrs = plainAclConfData.getGlobalWhiteRemoteAddresses();
-            if (globalWhiteAddrs != null && !globalWhiteAddrs.isEmpty()) {
-                whiteAddrs.addAll(globalWhiteAddrs);
-            }
-
-            List<PlainAccessConfig> plainAccessConfigs = plainAclConfData.getAccounts();
-            if (plainAccessConfigs != null && !plainAccessConfigs.isEmpty()) {
-                for (int j = 0; j < plainAccessConfigs.size(); j++) {
-                    if (!accessKeySets.contains(plainAccessConfigs.get(j).getAccessKey())) {
-                        accessKeySets.add(plainAccessConfigs.get(j).getAccessKey());
-                        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-                        plainAccessConfig.setGroupPerms(plainAccessConfigs.get(j).getGroupPerms());
-                        plainAccessConfig.setDefaultTopicPerm(plainAccessConfigs.get(j).getDefaultTopicPerm());
-                        plainAccessConfig.setDefaultGroupPerm(plainAccessConfigs.get(j).getDefaultGroupPerm());
-                        plainAccessConfig.setAccessKey(plainAccessConfigs.get(j).getAccessKey());
-                        plainAccessConfig.setSecretKey(plainAccessConfigs.get(j).getSecretKey());
-                        plainAccessConfig.setAdmin(plainAccessConfigs.get(j).isAdmin());
-                        plainAccessConfig.setTopicPerms(plainAccessConfigs.get(j).getTopicPerms());
-                        plainAccessConfig.setWhiteRemoteAddress(plainAccessConfigs.get(j).getWhiteRemoteAddress());
-                        configs.add(plainAccessConfig);
-                    }
-                }
-            }
-        }
-        aclConfig.setPlainAccessConfigs(configs);
-        aclConfig.setGlobalWhiteAddrs(whiteAddrs);
-        return aclConfig;
-    }
-
-    private void watch() {
-        try {
-            AclFileWatchService aclFileWatchService = new AclFileWatchService(defaultAclDir, defaultAclFile, new AclFileWatchService.Listener() {
-                @Override
-                public void onFileChanged(String aclFileName) {
-                    load(aclFileName);
-                }
-
-                @Override
-                public void onFileNumChanged(String path) {
-                    load();
-                }
-            });
-            aclFileWatchService.start();
-            log.info("Succeed to start AclFileWatchService");
-            this.isWatchStart = true;
-        } catch (Exception e) {
-            log.error("Failed to start AclWatcherService", e);
-        }
-
-    }
-
-    void checkPerm(PlainAccessResource needCheckedAccess, PlainAccessResource ownedAccess) {
-        permissionChecker.check(needCheckedAccess, ownedAccess);
-    }
-
-    void clearPermissionInfo() {
-        this.aclPlainAccessResourceMap.clear();
-        this.accessKeyTable.clear();
-        this.globalWhiteRemoteAddressStrategy.clear();
-    }
-
-    public void checkPlainAccessConfig(PlainAccessConfig plainAccessConfig) throws AclException {
-        if (plainAccessConfig.getAccessKey() == null
-            || plainAccessConfig.getSecretKey() == null
-            || plainAccessConfig.getAccessKey().length() <= AclConstants.ACCESS_KEY_MIN_LENGTH
-            || plainAccessConfig.getSecretKey().length() <= AclConstants.SECRET_KEY_MIN_LENGTH) {
-            throw new AclException(String.format(
-                "The accessKey=%s and secretKey=%s cannot be null and length should longer than 6",
-                plainAccessConfig.getAccessKey(), plainAccessConfig.getSecretKey()));
-        }
-    }
-
-    public PlainAccessResource buildPlainAccessResource(PlainAccessConfig plainAccessConfig) throws AclException {
-        checkPlainAccessConfig(plainAccessConfig);
-        return PlainAccessResource.build(plainAccessConfig, remoteAddressStrategyFactory.
-            getRemoteAddressStrategy(plainAccessConfig.getWhiteRemoteAddress()));
-    }
-
-    public void validate(PlainAccessResource plainAccessResource) {
-
-        // Check the global white remote addr
-        for (RemoteAddressStrategy remoteAddressStrategy : globalWhiteRemoteAddressStrategy) {
-            if (remoteAddressStrategy.match(plainAccessResource)) {
-                return;
-            }
-        }
-
-        if (plainAccessResource.getAccessKey() == null) {
-            throw new AclException("No accessKey is configured");
-        }
-
-        if (!accessKeyTable.containsKey(plainAccessResource.getAccessKey())) {
-            throw new AclException(String.format("No acl config for %s", plainAccessResource.getAccessKey()));
-        }
-
-        // Check the white addr for accessKey
-        String aclFileName = accessKeyTable.get(plainAccessResource.getAccessKey());
-        PlainAccessResource ownedAccess = aclPlainAccessResourceMap.getOrDefault(aclFileName, new HashMap<>()).get(plainAccessResource.getAccessKey());
-        if (ownedAccess == null) {
-            throw new AclException(String.format("No PlainAccessResource for accessKey=%s", plainAccessResource.getAccessKey()));
-        }
-        if (ownedAccess.getRemoteAddressStrategy().match(plainAccessResource)) {
-            return;
-        }
-
-        // Check the signature
-        String signature = AclUtils.calSignature(plainAccessResource.getContent(), ownedAccess.getSecretKey());
-        if (!signature.equals(plainAccessResource.getSignature())) {
-            throw new AclException(String.format("Check signature failed for accessKey=%s", plainAccessResource.getAccessKey()));
-        }
-
-        //Skip the topic RMQ_SYS_TRACE_TOPIC permission check,if the topic RMQ_SYS_TRACE_TOPIC is used for message trace
-        Map<String, Byte> resourcePermMap = plainAccessResource.getResourcePermMap();
-        if (resourcePermMap != null) {
-            Byte permission = resourcePermMap.get(TopicValidator.RMQ_SYS_TRACE_TOPIC);
-            if (permission != null && permission == Permission.PUB) {
-                return;
-            }
-        }
-
-        // Check perm of each resource
-        checkPerm(plainAccessResource, ownedAccess);
-    }
-
-    public boolean isWatchStart() {
-        return isWatchStart;
-    }
-}
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java
deleted file mode 100644
index fb4151e5366..00000000000
--- a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.acl.plain;
-
-import java.util.HashSet;
-import java.util.Set;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.validator.routines.InetAddressValidator;
-import org.apache.rocketmq.acl.common.AclException;
-import org.apache.rocketmq.acl.common.AclUtils;
-import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-
-public class RemoteAddressStrategyFactory {
-
-    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
-
-    public static final NullRemoteAddressStrategy NULL_NET_ADDRESS_STRATEGY = new NullRemoteAddressStrategy();
-
-    public static final BlankRemoteAddressStrategy BLANK_NET_ADDRESS_STRATEGY = new BlankRemoteAddressStrategy();
-
-    public RemoteAddressStrategy getRemoteAddressStrategy(PlainAccessResource plainAccessResource) {
-        return getRemoteAddressStrategy(plainAccessResource.getWhiteRemoteAddress());
-    }
-
-    public RemoteAddressStrategy getRemoteAddressStrategy(String remoteAddr) {
-        if (StringUtils.isBlank(remoteAddr)) {
-            return BLANK_NET_ADDRESS_STRATEGY;
-        }
-        if ("*".equals(remoteAddr) || "*.*.*.*".equals(remoteAddr) || "*:*:*:*:*:*:*:*".equals(remoteAddr)) {
-            return NULL_NET_ADDRESS_STRATEGY;
-        }
-        if (remoteAddr.endsWith("}")) {
-            if (AclUtils.isColon(remoteAddr)) {
-                String[] strArray = StringUtils.split(remoteAddr, ":");
-                String last = strArray[strArray.length - 1];
-                if (!last.startsWith("{")) {
-                    throw new AclException(String.format("MultipleRemoteAddressStrategy netAddress examine scope Exception netAddress: %s", remoteAddr));
-                }
-                return new MultipleRemoteAddressStrategy(AclUtils.getAddresses(remoteAddr, last));
-            } else {
-                String[] strArray = StringUtils.split(remoteAddr, ".");
-                // However a right IP String provided by user,it always can be divided into 4 parts by '.'.
-                if (strArray.length < 4) {
-                    throw new AclException(String.format("MultipleRemoteAddressStrategy has got a/some wrong format IP(s): %s ", remoteAddr));
-                }
-                String lastStr = strArray[strArray.length - 1];
-                if (!lastStr.startsWith("{")) {
-                    throw new AclException(String.format("MultipleRemoteAddressStrategy netAddress examine scope Exception netAddress: %s", remoteAddr));
-                }
-                return new MultipleRemoteAddressStrategy(AclUtils.getAddresses(remoteAddr, lastStr));
-            }
-        } else if (AclUtils.isComma(remoteAddr)) {
-            return new MultipleRemoteAddressStrategy(StringUtils.split(remoteAddr, ","));
-        } else if (AclUtils.isAsterisk(remoteAddr) || AclUtils.isMinus(remoteAddr)) {
-            return new RangeRemoteAddressStrategy(remoteAddr);
-        }
-        return new OneRemoteAddressStrategy(remoteAddr);
-
-    }
-
-    public static class NullRemoteAddressStrategy implements RemoteAddressStrategy {
-        @Override
-        public boolean match(PlainAccessResource plainAccessResource) {
-            return true;
-        }
-
-    }
-
-    public static class BlankRemoteAddressStrategy implements RemoteAddressStrategy {
-        @Override
-        public boolean match(PlainAccessResource plainAccessResource) {
-            return false;
-        }
-
-    }
-
-    public static class MultipleRemoteAddressStrategy implements RemoteAddressStrategy {
-
-        private final Set<String> multipleSet = new HashSet<>();
-
-        public MultipleRemoteAddressStrategy(String[] strArray) {
-            InetAddressValidator validator = InetAddressValidator.getInstance();
-            for (String netAddress : strArray) {
-                if (validator.isValidInet4Address(netAddress)) {
-                    multipleSet.add(netAddress);
-                } else if (validator.isValidInet6Address(netAddress)) {
-                    multipleSet.add(AclUtils.expandIP(netAddress, 8));
-                } else {
-                    throw new AclException(String.format("NetAddress examine Exception netAddress is %s", netAddress));
-                }
-            }
-        }
-
-        @Override
-        public boolean match(PlainAccessResource plainAccessResource) {
-            InetAddressValidator validator = InetAddressValidator.getInstance();
-            String whiteRemoteAddress = plainAccessResource.getWhiteRemoteAddress();
-            if (validator.isValidInet6Address(whiteRemoteAddress)) {
-                whiteRemoteAddress = AclUtils.expandIP(whiteRemoteAddress, 8);
-            }
-            return multipleSet.contains(whiteRemoteAddress);
-        }
-
-    }
-
-    public static class OneRemoteAddressStrategy implements RemoteAddressStrategy {
-
-        private String netAddress;
-
-        public OneRemoteAddressStrategy(String netAddress) {
-            this.netAddress = netAddress;
-            InetAddressValidator validator = InetAddressValidator.getInstance();
-            if (!(validator.isValidInet4Address(netAddress) || validator.isValidInet6Address(
-                netAddress))) {
-                throw new AclException(String.format("NetAddress examine Exception netAddress is %s",
-                    netAddress));
-            }
-        }
-
-        @Override
-        public boolean match(PlainAccessResource plainAccessResource) {
-            String writeRemoteAddress = AclUtils.expandIP(plainAccessResource.getWhiteRemoteAddress(), 8).toUpperCase();
-            return AclUtils.expandIP(netAddress, 8).toUpperCase().equals(writeRemoteAddress);
-        }
-
-    }
-
-    public static class RangeRemoteAddressStrategy implements RemoteAddressStrategy {
-
-        private String head;
-
-        private int start;
-
-        private int end;
-
-        private int index;
-
-        public RangeRemoteAddressStrategy(String remoteAddr) {
-//            IPv6 Address
-            if (AclUtils.isColon(remoteAddr)) {
-                AclUtils.IPv6AddressCheck(remoteAddr);
-                String[] strArray = StringUtils.split(remoteAddr, ":");
-                for (int i = 1; i < strArray.length; i++) {
-                    if (ipv6Analysis(strArray, i)) {
-                        AclUtils.verify(remoteAddr, index - 1);
-                        String preAddress = AclUtils.v6ipProcess(remoteAddr);
-                        this.index = StringUtils.split(preAddress, ":").length;
-                        this.head = preAddress;
-                        break;
-                    }
-                }
-            } else {
-                String[] strArray = StringUtils.split(remoteAddr, ".");
-                if (analysis(strArray, 1) || analysis(strArray, 2) || analysis(strArray, 3)) {
-                    AclUtils.verify(remoteAddr, index - 1);
-                    StringBuilder sb = new StringBuilder();
-                    for (int j = 0; j < index; j++) {
-                        sb.append(strArray[j].trim()).append(".");
-                    }
-                    this.head = sb.toString();
-                }
-            }
-        }
-
-        private boolean analysis(String[] strArray, int index) {
-            String value = strArray[index].trim();
-            this.index = index;
-            if ("*".equals(value)) {
-                setValue(0, 255);
-            } else if (AclUtils.isMinus(value)) {
-                if (value.indexOf("-") == 0) {
-                    throw new AclException(String.format("RangeRemoteAddressStrategy netAddress examine scope Exception value %s ", value));
-
-                }
-                String[] valueArray = StringUtils.split(value, "-");
-                this.start = Integer.parseInt(valueArray[0]);
-                this.end = Integer.parseInt(valueArray[1]);
-                if (!(AclUtils.isScope(end) && AclUtils.isScope(start) && start <= end)) {
-                    throw new AclException(String.format("RangeRemoteAddressStrategy netAddress examine scope Exception start is %s , end is %s", start, end));
-                }
-            }
-            return this.end > 0;
-        }
-
-        private boolean ipv6Analysis(String[] strArray, int index) {
-            String value = strArray[index].trim();
-            this.index = index;
-            if ("*".equals(value)) {
-                int min = Integer.parseInt("0", 16);
-                int max = Integer.parseInt("ffff", 16);
-                setValue(min, max);
-            } else if (AclUtils.isMinus(value)) {
-                if (value.indexOf("-") == 0) {
-                    throw new AclException(String.format("RangeRemoteAddressStrategy netAddress examine scope Exception value %s ", value));
-                }
-                String[] valueArray = StringUtils.split(value, "-");
-                this.start = Integer.parseInt(valueArray[0], 16);
-                this.end = Integer.parseInt(valueArray[1], 16);
-                if (!(AclUtils.isIPv6Scope(end) && AclUtils.isIPv6Scope(start) && start <= end)) {
-                    throw new AclException(String.format("RangeRemoteAddressStrategy netAddress examine scope Exception start is %s , end is %s", start, end));
-                }
-            }
-            return this.end > 0;
-        }
-
-        private void setValue(int start, int end) {
-            this.start = start;
-            this.end = end;
-        }
-
-        @Override
-        public boolean match(PlainAccessResource plainAccessResource) {
-            String netAddress = plainAccessResource.getWhiteRemoteAddress();
-            InetAddressValidator validator = InetAddressValidator.getInstance();
-            if (validator.isValidInet4Address(netAddress)) {
-                if (netAddress.startsWith(this.head)) {
-                    String value;
-                    if (index == 3) {
-                        value = netAddress.substring(this.head.length());
-                    } else if (index == 2) {
-                        value = netAddress.substring(this.head.length(), netAddress.lastIndexOf('.'));
-                    } else {
-                        value = netAddress.substring(this.head.length(), netAddress.lastIndexOf('.', netAddress.lastIndexOf('.') - 1));
-                    }
-                    Integer address = Integer.valueOf(value);
-                    return address >= this.start && address <= this.end;
-                }
-            } else if (validator.isValidInet6Address(netAddress)) {
-                netAddress = AclUtils.expandIP(netAddress, 8).toUpperCase();
-                if (netAddress.startsWith(this.head)) {
-                    String value = netAddress.substring(5 * index, 5 * index + 4);
-                    Integer address = Integer.parseInt(value, 16);
-                    return address >= this.start && address <= this.end;
-                }
-            }
-            return false;
-        }
-    }
-}
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/RemotingClientAccessTest.java b/acl/src/test/java/org/apache/rocketmq/acl/RemotingClientAccessTest.java
deleted file mode 100644
index 88c5e09a94c..00000000000
--- a/acl/src/test/java/org/apache/rocketmq/acl/RemotingClientAccessTest.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.acl;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import org.apache.rocketmq.acl.common.AclClientRPCHook;
-import org.apache.rocketmq.acl.common.AclException;
-import org.apache.rocketmq.acl.common.SessionCredentials;
-import org.apache.rocketmq.acl.plain.AclTestHelper;
-import org.apache.rocketmq.acl.plain.PlainAccessResource;
-import org.apache.rocketmq.acl.plain.PlainAccessValidator;
-import org.apache.rocketmq.remoting.exception.RemotingCommandException;
-import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-import org.apache.rocketmq.remoting.protocol.RequestCode;
-import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class RemotingClientAccessTest {
-
-    private PlainAccessValidator plainAccessValidator;
-    private AclClientRPCHook aclClient;
-    private SessionCredentials sessionCredentials;
-
-    private File confHome;
-
-    private String clientAddress = "10.7.1.3";
-
-    @Before
-    public void init() throws IOException {
-        String folder = "access_acl_conf";
-        confHome = AclTestHelper.copyResources(folder, true);
-        System.setProperty("rocketmq.home.dir", confHome.getAbsolutePath());
-        System.setProperty("rocketmq.acl.plain.file", "/access_acl_conf/acl/plain_acl.yml".replace("/", File.separator));
-
-        plainAccessValidator = new PlainAccessValidator();
-        sessionCredentials = new SessionCredentials();
-        sessionCredentials.setAccessKey("rocketmq3");
-        sessionCredentials.setSecretKey("12345678");
-        aclClient = new AclClientRPCHook(sessionCredentials);
-    }
-
-    @After
-    public void cleanUp() {
-        AclTestHelper.recursiveDelete(confHome);
-    }
-
-    @Test(expected = AclException.class)
-    public void testProduceDenyTopic() {
-        SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
-        messageRequestHeader.setTopic("topicD");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
-        aclClient.doBeforeRequest(clientAddress, remotingCommand);
-
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), clientAddress);
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void testProduceAuthorizedTopic() {
-        SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
-        messageRequestHeader.setTopic("topicA");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
-        aclClient.doBeforeRequest(clientAddress, remotingCommand);
-
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), clientAddress);
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-
-    @Test(expected = AclException.class)
-    public void testConsumeDenyTopic() {
-        PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();
-        pullMessageRequestHeader.setTopic("topicD");
-        pullMessageRequestHeader.setConsumerGroup("groupB");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, pullMessageRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-            Assert.fail("Should not throw IOException");
-        }
-
-    }
-
-    @Test
-    public void testConsumeAuthorizedTopic() {
-        PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();
-        pullMessageRequestHeader.setTopic("topicB");
-        pullMessageRequestHeader.setConsumerGroup("groupB");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, pullMessageRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test(expected = AclException.class)
-    public void testConsumeInDeniedGroup() {
-        PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();
-        pullMessageRequestHeader.setTopic("topicB");
-        pullMessageRequestHeader.setConsumerGroup("groupD");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, pullMessageRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void testConsumeInAuthorizedGroup() {
-        PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();
-        pullMessageRequestHeader.setTopic("topicB");
-        pullMessageRequestHeader.setConsumerGroup("groupB");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, pullMessageRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-}
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java b/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java
deleted file mode 100644
index 8fd8052c8a4..00000000000
--- a/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.acl.common;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import org.apache.rocketmq.acl.plain.PlainAccessResource;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class PermissionTest {
-
-    @Test
-    public void fromStringGetPermissionTest() {
-        byte perm = Permission.parsePermFromString("PUB");
-        Assert.assertEquals(perm, Permission.PUB);
-
-        perm = Permission.parsePermFromString("SUB");
-        Assert.assertEquals(perm, Permission.SUB);
-
-        perm = Permission.parsePermFromString("PUB|SUB");
-        Assert.assertEquals(perm, Permission.PUB | Permission.SUB);
-
-        perm = Permission.parsePermFromString("SUB|PUB");
-        Assert.assertEquals(perm, Permission.PUB | Permission.SUB);
-
-        perm = Permission.parsePermFromString("DENY");
-        Assert.assertEquals(perm, Permission.DENY);
-
-        perm = Permission.parsePermFromString("1");
-        Assert.assertEquals(perm, Permission.DENY);
-
-        perm = Permission.parsePermFromString(null);
-        Assert.assertEquals(perm, Permission.DENY);
-
-    }
-
-    @Test
-    public void checkPermissionTest() {
-        boolean boo = Permission.checkPermission(Permission.DENY, Permission.DENY);
-        Assert.assertFalse(boo);
-
-        boo = Permission.checkPermission(Permission.PUB, Permission.PUB);
-        Assert.assertTrue(boo);
-
-        boo = Permission.checkPermission(Permission.SUB, Permission.SUB);
-        Assert.assertTrue(boo);
-
-        boo = Permission.checkPermission(Permission.PUB, (byte) (Permission.PUB | Permission.SUB));
-        Assert.assertTrue(boo);
-
-        boo = Permission.checkPermission(Permission.SUB, (byte) (Permission.PUB | Permission.SUB));
-        Assert.assertTrue(boo);
-
-        boo = Permission.checkPermission(Permission.ANY, (byte) (Permission.PUB | Permission.SUB));
-        Assert.assertTrue(boo);
-
-        boo = Permission.checkPermission(Permission.ANY, Permission.SUB);
-        Assert.assertTrue(boo);
-
-        boo = Permission.checkPermission(Permission.ANY, Permission.PUB);
-        Assert.assertTrue(boo);
-
-        boo = Permission.checkPermission(Permission.DENY, Permission.ANY);
-        Assert.assertFalse(boo);
-
-        boo = Permission.checkPermission(Permission.DENY, Permission.PUB);
-        Assert.assertFalse(boo);
-
-        boo = Permission.checkPermission(Permission.DENY, Permission.SUB);
-        Assert.assertFalse(boo);
-
-    }
-
-    @Test(expected = AclException.class)
-    public void setTopicPermTest() {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        Map<String, Byte> resourcePermMap = plainAccessResource.getResourcePermMap();
-
-        Permission.parseResourcePerms(plainAccessResource, false, null);
-        Assert.assertNull(resourcePermMap);
-
-        List<String> groups = new ArrayList<>();
-        Permission.parseResourcePerms(plainAccessResource, false, groups);
-        Assert.assertNull(resourcePermMap);
-
-        groups.add("groupA=DENY");
-        groups.add("groupB=PUB|SUB");
-        groups.add("groupC=PUB");
-        Permission.parseResourcePerms(plainAccessResource, false, groups);
-        resourcePermMap = plainAccessResource.getResourcePermMap();
-
-        byte perm = resourcePermMap.get(PlainAccessResource.getRetryTopic("groupA"));
-        Assert.assertEquals(perm, Permission.DENY);
-
-        perm = resourcePermMap.get(PlainAccessResource.getRetryTopic("groupB"));
-        Assert.assertEquals(perm,Permission.PUB | Permission.SUB);
-
-        perm = resourcePermMap.get(PlainAccessResource.getRetryTopic("groupC"));
-        Assert.assertEquals(perm, Permission.PUB);
-
-        List<String> topics = new ArrayList<>();
-        topics.add("topicA=DENY");
-        topics.add("topicB=PUB|SUB");
-        topics.add("topicC=PUB");
-
-        Permission.parseResourcePerms(plainAccessResource, true, topics);
-
-        perm = resourcePermMap.get("topicA");
-        Assert.assertEquals(perm, Permission.DENY);
-
-        perm = resourcePermMap.get("topicB");
-        Assert.assertEquals(perm, Permission.PUB | Permission.SUB);
-
-        perm = resourcePermMap.get("topicC");
-        Assert.assertEquals(perm, Permission.PUB);
-
-        List<String> erron = new ArrayList<>();
-        erron.add("");
-        Permission.parseResourcePerms(plainAccessResource, false, erron);
-    }
-
-    @Test
-    public void checkAdminCodeTest() {
-        Set<Integer> code = new HashSet<>();
-        code.add(17);
-        code.add(25);
-        code.add(215);
-        code.add(200);
-        code.add(207);
-
-        for (int i = 0; i < 400; i++) {
-            boolean boo = Permission.needAdminPerm(i);
-            if (boo) {
-                Assert.assertTrue(code.contains(i));
-            }
-        }
-    }
-
-    @Test
-    public void AclExceptionTest() {
-        AclException aclException = new AclException("CAL_SIGNATURE_FAILED",10015);
-        AclException aclExceptionWithMessage = new AclException("CAL_SIGNATURE_FAILED",10015,"CAL_SIGNATURE_FAILED Exception");
-        Assert.assertEquals(aclException.getCode(),10015);
-        Assert.assertEquals(aclExceptionWithMessage.getStatus(),"CAL_SIGNATURE_FAILED");
-        aclException.setCode(10016);
-        Assert.assertEquals(aclException.getCode(),10016);
-        aclException.setStatus("netAddress examine scope Exception netAddress");
-        Assert.assertEquals(aclException.getStatus(),"netAddress examine scope Exception netAddress");
-    }
-
-    @Test
-    public void checkResourcePermsNormalTest() {
-        Permission.checkResourcePerms(null);
-        Permission.checkResourcePerms(new ArrayList<>());
-        Permission.checkResourcePerms(Arrays.asList("topicA=PUB"));
-        Permission.checkResourcePerms(Arrays.asList("topicA=PUB", "topicB=SUB", "topicC=PUB|SUB"));
-    }
-
-    @Test(expected = AclException.class)
-    public void checkResourcePermsExceptionTest1() {
-        Permission.checkResourcePerms(Arrays.asList("topicA"));
-    }
-
-    @Test(expected = AclException.class)
-    public void checkResourcePermsExceptionTest2() {
-        Permission.checkResourcePerms(Arrays.asList("topicA="));
-    }
-
-    @Test(expected = AclException.class)
-    public void checkResourcePermsExceptionTest3() {
-        Permission.checkResourcePerms(Arrays.asList("topicA=DENY1"));
-    }
-}
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/AclTestHelper.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/AclTestHelper.java
deleted file mode 100644
index a1bfc53ed51..00000000000
--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/AclTestHelper.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.acl.plain;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.util.UUID;
-import java.util.Iterator;
-import org.junit.Assert;
-import org.springframework.core.io.Resource;
-import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
-
-public final class AclTestHelper {
-    private AclTestHelper() {
-    }
-
-    private static void copyTo(String path, InputStream src, File dstDir, String flag, boolean into)
-        throws IOException {
-        Preconditions.checkNotNull(flag);
-        Iterator<String> iterator = Splitter.on(File.separatorChar).split(path).iterator();
-        boolean found = false;
-        File dir = dstDir;
-        while (iterator.hasNext()) {
-            String current = iterator.next();
-            if (!found && flag.equals(current)) {
-                found = true;
-                if (into) {
-                    dir = new File(dir, flag);
-                    if (!dir.exists()) {
-                        Assert.assertTrue(dir.mkdirs());
-                    }
-                }
-                continue;
-            }
-
-            if (found) {
-                if (!iterator.hasNext()) {
-                    dir = new File(dir, current);
-                } else {
-                    dir = new File(dir, current);
-                    if (!dir.exists()) {
-                        Assert.assertTrue(dir.mkdir());
-                    }
-                }
-            }
-        }
-
-        Assert.assertTrue(dir.createNewFile());
-        byte[] buffer = new byte[4096];
-        BufferedInputStream bis = new BufferedInputStream(src);
-        int len = 0;
-        try (BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(dir.toPath()))) {
-            while ((len = bis.read(buffer)) > 0) {
-                bos.write(buffer, 0, len);
-            }
-        }
-    }
-
-    public static void recursiveDelete(File file) {
-        if (file.isFile()) {
-            file.delete();
-        } else {
-            File[] files = file.listFiles();
-            if (null != files) {
-                for (File f : files) {
-                    recursiveDelete(f);
-                }
-            }
-            file.delete();
-        }
-    }
-
-    public static File copyResources(String folder) throws IOException {
-        return copyResources(folder, false);
-    }
-
-    public static File copyResources(String folder, boolean into) throws IOException {
-        File home = new File(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString().replace('-', '_'));
-        if (!home.exists()) {
-            Assert.assertTrue(home.mkdirs());
-        }
-        PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(AclTestHelper.class.getClassLoader());
-        Resource[] resources = resolver.getResources(String.format("classpath:%s/**/*", folder));
-        for (Resource resource : resources) {
-            if (!resource.isReadable()) {
-                continue;
-            }
-            String description = resource.getDescription();
-            int start = description.indexOf('[');
-            int end = description.lastIndexOf(']');
-            String path = description.substring(start + 1, end);
-            try (InputStream inputStream = resource.getInputStream()) {
-                copyTo(path, inputStream, home, folder, into);
-            }
-        }
-        return home;
-    }
-}
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java
deleted file mode 100644
index 5193457146d..00000000000
--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java
+++ /dev/null
@@ -1,311 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.acl.plain;
-
-import org.apache.rocketmq.acl.common.AclClientRPCHook;
-import org.apache.rocketmq.acl.common.AclConstants;
-import org.apache.rocketmq.acl.common.AclException;
-import org.apache.rocketmq.acl.common.AclUtils;
-import org.apache.rocketmq.acl.common.SessionCredentials;
-import org.apache.rocketmq.common.AclConfig;
-import org.apache.rocketmq.common.PlainAccessConfig;
-import org.apache.rocketmq.remoting.exception.RemotingCommandException;
-import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-import org.apache.rocketmq.remoting.protocol.RequestCode;
-import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * <p> In this class, we'll test the following scenarios, each containing several consecutive operations on ACL,
- * <p> like updating and deleting ACL, changing config files and checking validations.
- * <p> Case 1: Only conf/plain_acl.yml exists;
- * <p> Case 2: Only conf/acl/plain_acl.yml exists;
- * <p> Case 3: Both conf/plain_acl.yml and conf/acl/plain_acl.yml exists.
- */
-public class PlainAccessControlFlowTest {
-    public static final String DEFAULT_TOPIC = "topic-acl";
-
-    public static final String DEFAULT_GROUP = "GID_acl";
-
-    public static final String DEFAULT_PRODUCER_AK = "ak11111";
-    public static final String DEFAULT_PRODUCER_SK = "1234567";
-
-    public static final String DEFAULT_CONSUMER_SK = "7654321";
-    public static final String DEFAULT_CONSUMER_AK = "ak22222";
-
-    public static final String DEFAULT_GLOBAL_WHITE_ADDR = "172.16.123.123";
-    public static final List<String> DEFAULT_GLOBAL_WHITE_ADDRS_LIST = Collections.singletonList(DEFAULT_GLOBAL_WHITE_ADDR);
-
-    @Test
-    public void testEmptyAclFolderCase() throws NoSuchFieldException, IllegalAccessException,
-        IOException {
-        String folder = "empty_acl_folder_conf";
-        File home = AclTestHelper.copyResources(folder);
-        System.setProperty("rocketmq.home.dir", home.getAbsolutePath());
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        checkDefaultAclFileExists();
-        testValidationAfterConsecutiveUpdates(plainAccessValidator);
-        testValidationAfterConfigFileChanged(plainAccessValidator);
-        AclTestHelper.recursiveDelete(home);
-    }
-
-    @Test
-    public void testOnlyAclFolderCase() throws NoSuchFieldException, IllegalAccessException, IOException {
-        String folder = "only_acl_folder_conf";
-        File home = AclTestHelper.copyResources(folder);
-        System.setProperty("rocketmq.home.dir", home.getAbsolutePath());
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        checkDefaultAclFileExists();
-        testValidationAfterConsecutiveUpdates(plainAccessValidator);
-        testValidationAfterConfigFileChanged(plainAccessValidator);
-        AclTestHelper.recursiveDelete(home);
-    }
-
-    @Test
-    public void testBothAclFileAndFolderCase() throws NoSuchFieldException, IllegalAccessException,
-        IOException {
-        String folder = "both_acl_file_folder_conf";
-        File root = AclTestHelper.copyResources(folder);
-        System.setProperty("rocketmq.home.dir", root.getAbsolutePath());
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        checkDefaultAclFileExists();
-        testValidationAfterConsecutiveUpdates(plainAccessValidator);
-        testValidationAfterConfigFileChanged(plainAccessValidator);
-        AclTestHelper.recursiveDelete(root);
-    }
-
-    private void testValidationAfterConfigFileChanged(
-        PlainAccessValidator plainAccessValidator) throws NoSuchFieldException, IllegalAccessException {
-        PlainAccessConfig producerAccessConfig = generateProducerAccessConfig();
-        PlainAccessConfig consumerAccessConfig = generateConsumerAccessConfig();
-        List<PlainAccessConfig> plainAccessConfigList = new LinkedList<>();
-        plainAccessConfigList.add(producerAccessConfig);
-        plainAccessConfigList.add(consumerAccessConfig);
-        PlainAccessData ymlMap = new PlainAccessData();
-        ymlMap.setAccounts(plainAccessConfigList);
-
-        // write prepared PlainAccessConfigs to file
-        final String aclConfigFile = System.getProperty("rocketmq.home.dir") + File.separator + "conf/plain_acl.yml";
-        AclUtils.writeDataObject(aclConfigFile, ymlMap);
-
-        loadConfigFile(plainAccessValidator, aclConfigFile);
-
-        // check if added successfully
-        final AclConfig allAclConfig = plainAccessValidator.getAllAclConfig();
-        final List<PlainAccessConfig> plainAccessConfigs = allAclConfig.getPlainAccessConfigs();
-        checkPlainAccessConfig(producerAccessConfig, plainAccessConfigs);
-        checkPlainAccessConfig(consumerAccessConfig, plainAccessConfigs);
-
-        //delete consumer account
-        plainAccessConfigList.remove(consumerAccessConfig);
-        AclUtils.writeDataObject(aclConfigFile, ymlMap);
-
-        loadConfigFile(plainAccessValidator, aclConfigFile);
-
-        // sending messages will be successful using prepared credentials
-        SessionCredentials producerCredential = new SessionCredentials(DEFAULT_PRODUCER_AK, DEFAULT_PRODUCER_SK);
-        AclClientRPCHook producerHook = new AclClientRPCHook(producerCredential);
-        validateSendMessage(RequestCode.SEND_MESSAGE, DEFAULT_TOPIC, producerHook, "", plainAccessValidator);
-        validateSendMessage(RequestCode.SEND_MESSAGE_V2, DEFAULT_TOPIC, producerHook, "", plainAccessValidator);
-
-        // consuming messages will be failed for account has been deleted
-        SessionCredentials consumerCredential = new SessionCredentials(DEFAULT_CONSUMER_AK, DEFAULT_CONSUMER_SK);
-        AclClientRPCHook consumerHook = new AclClientRPCHook(consumerCredential);
-        boolean isConsumeFailed = false;
-        try {
-            validatePullMessage(DEFAULT_TOPIC, DEFAULT_GROUP, consumerHook, "", plainAccessValidator);
-        } catch (AclException e) {
-            isConsumeFailed = true;
-        }
-        Assert.assertTrue("Message should not be consumed after account deleted", isConsumeFailed);
-
-    }
-
-    private void testValidationAfterConsecutiveUpdates(
-        PlainAccessValidator plainAccessValidator) throws NoSuchFieldException, IllegalAccessException {
-        PlainAccessConfig producerAccessConfig = generateProducerAccessConfig();
-        plainAccessValidator.updateAccessConfig(producerAccessConfig);
-
-        PlainAccessConfig consumerAccessConfig = generateConsumerAccessConfig();
-        plainAccessValidator.updateAccessConfig(consumerAccessConfig);
-
-        plainAccessValidator.updateGlobalWhiteAddrsConfig(DEFAULT_GLOBAL_WHITE_ADDRS_LIST, null);
-
-        // check if the above config updated successfully
-        final AclConfig allAclConfig = plainAccessValidator.getAllAclConfig();
-        final List<PlainAccessConfig> plainAccessConfigs = allAclConfig.getPlainAccessConfigs();
-        checkPlainAccessConfig(producerAccessConfig, plainAccessConfigs);
-        checkPlainAccessConfig(consumerAccessConfig, plainAccessConfigs);
-
-        Assert.assertEquals(DEFAULT_GLOBAL_WHITE_ADDRS_LIST, allAclConfig.getGlobalWhiteAddrs());
-
-        // check sending and consuming messages
-        SessionCredentials producerCredential = new SessionCredentials(DEFAULT_PRODUCER_AK, DEFAULT_PRODUCER_SK);
-        AclClientRPCHook producerHook = new AclClientRPCHook(producerCredential);
-        validateSendMessage(RequestCode.SEND_MESSAGE, DEFAULT_TOPIC, producerHook, "", plainAccessValidator);
-        validateSendMessage(RequestCode.SEND_MESSAGE_V2, DEFAULT_TOPIC, producerHook, "", plainAccessValidator);
-
-        SessionCredentials consumerCredential = new SessionCredentials(DEFAULT_CONSUMER_AK, DEFAULT_CONSUMER_SK);
-        AclClientRPCHook consumerHook = new AclClientRPCHook(consumerCredential);
-        validatePullMessage(DEFAULT_TOPIC, DEFAULT_GROUP, consumerHook, "", plainAccessValidator);
-
-        // load from file
-        loadConfigFile(plainAccessValidator,
-            System.getProperty("rocketmq.home.dir") + File.separator + "conf/plain_acl.yml");
-        SessionCredentials unmatchedCredential = new SessionCredentials("non_exists_sk", "non_exists_sk");
-        AclClientRPCHook dummyHook = new AclClientRPCHook(unmatchedCredential);
-        validateSendMessage(RequestCode.SEND_MESSAGE, DEFAULT_TOPIC, dummyHook, DEFAULT_GLOBAL_WHITE_ADDR, plainAccessValidator);
-        validateSendMessage(RequestCode.SEND_MESSAGE_V2, DEFAULT_TOPIC, dummyHook, DEFAULT_GLOBAL_WHITE_ADDR, plainAccessValidator);
-        validatePullMessage(DEFAULT_TOPIC, DEFAULT_GROUP, dummyHook, DEFAULT_GLOBAL_WHITE_ADDR, plainAccessValidator);
-
-        //recheck after reloading
-        validateSendMessage(RequestCode.SEND_MESSAGE, DEFAULT_TOPIC, producerHook, "", plainAccessValidator);
-        validateSendMessage(RequestCode.SEND_MESSAGE_V2, DEFAULT_TOPIC, producerHook, "", plainAccessValidator);
-        validatePullMessage(DEFAULT_TOPIC, DEFAULT_GROUP, consumerHook, "", plainAccessValidator);
-
-    }
-
-    private void loadConfigFile(PlainAccessValidator plainAccessValidator,
-        String configFileName) throws NoSuchFieldException, IllegalAccessException {
-        Class<PlainAccessValidator> clazz = PlainAccessValidator.class;
-        Field f = clazz.getDeclaredField("aclPlugEngine");
-        f.setAccessible(true);
-        PlainPermissionManager aclPlugEngine = (PlainPermissionManager) f.get(plainAccessValidator);
-        aclPlugEngine.load(configFileName);
-    }
-
-    private PlainAccessConfig generateConsumerAccessConfig() {
-        PlainAccessConfig plainAccessConfig2 = new PlainAccessConfig();
-        plainAccessConfig2.setAccessKey(DEFAULT_CONSUMER_AK);
-        plainAccessConfig2.setSecretKey(DEFAULT_CONSUMER_SK);
-        plainAccessConfig2.setAdmin(false);
-        plainAccessConfig2.setDefaultTopicPerm(AclConstants.DENY);
-        plainAccessConfig2.setDefaultGroupPerm(AclConstants.DENY);
-        plainAccessConfig2.setTopicPerms(Collections.singletonList(DEFAULT_TOPIC + "=" + AclConstants.SUB));
-        plainAccessConfig2.setGroupPerms(Collections.singletonList(DEFAULT_GROUP + "=" + AclConstants.SUB));
-        return plainAccessConfig2;
-    }
-
-    private PlainAccessConfig generateProducerAccessConfig() {
-        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-        plainAccessConfig.setAccessKey(DEFAULT_PRODUCER_AK);
-        plainAccessConfig.setSecretKey(DEFAULT_PRODUCER_SK);
-        plainAccessConfig.setAdmin(false);
-        plainAccessConfig.setDefaultTopicPerm(AclConstants.DENY);
-        plainAccessConfig.setDefaultGroupPerm(AclConstants.DENY);
-        plainAccessConfig.setTopicPerms(Collections.singletonList(DEFAULT_TOPIC + "=" + AclConstants.PUB));
-        return plainAccessConfig;
-    }
-
-    public void validatePullMessage(String topic,
-        String group,
-        AclClientRPCHook aclClientRPCHook,
-        String remoteAddr,
-        PlainAccessValidator plainAccessValidator) {
-        PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();
-        pullMessageRequestHeader.setTopic(topic);
-        pullMessageRequestHeader.setConsumerGroup(group);
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE,
-            pullMessageRequestHeader);
-        aclClientRPCHook.doBeforeRequest(remoteAddr, remotingCommand);
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(
-                RemotingCommand.decode(buf), remoteAddr);
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-            Assert.fail("Should not throw RemotingCommandException");
-        }
-    }
-
-    public void validateSendMessage(int requestCode,
-        String topic,
-        AclClientRPCHook aclClientRPCHook,
-        String remoteAddr,
-        PlainAccessValidator plainAccessValidator) {
-        SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
-        messageRequestHeader.setTopic(topic);
-        RemotingCommand remotingCommand;
-        if (RequestCode.SEND_MESSAGE == requestCode) {
-            remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
-        } else {
-            remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2,
-                SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(messageRequestHeader));
-        }
-
-        aclClientRPCHook.doBeforeRequest(remoteAddr, remotingCommand);
-
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(
-                RemotingCommand.decode(buf), remoteAddr);
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-            Assert.fail("Should not throw RemotingCommandException");
-        }
-    }
-
-    private void checkPlainAccessConfig(final PlainAccessConfig plainAccessConfig,
-        final List<PlainAccessConfig> plainAccessConfigs) {
-        for (PlainAccessConfig config : plainAccessConfigs) {
-            if (config.getAccessKey().equals(plainAccessConfig.getAccessKey())) {
-                Assert.assertEquals(plainAccessConfig.getSecretKey(), config.getSecretKey());
-                Assert.assertEquals(plainAccessConfig.isAdmin(), config.isAdmin());
-                Assert.assertEquals(plainAccessConfig.getDefaultGroupPerm(), config.getDefaultGroupPerm());
-                Assert.assertEquals(plainAccessConfig.getDefaultGroupPerm(), config.getDefaultGroupPerm());
-                Assert.assertEquals(plainAccessConfig.getWhiteRemoteAddress(), config.getWhiteRemoteAddress());
-                if (null != plainAccessConfig.getTopicPerms()) {
-                    Assert.assertNotNull(config.getTopicPerms());
-                    Assert.assertTrue(config.getTopicPerms().containsAll(plainAccessConfig.getTopicPerms()));
-                }
-                if (null != plainAccessConfig.getGroupPerms()) {
-                    Assert.assertNotNull(config.getGroupPerms());
-                    Assert.assertTrue(config.getGroupPerms().containsAll(plainAccessConfig.getGroupPerms()));
-                }
-            }
-        }
-    }
-
-    private void checkDefaultAclFileExists() {
-        boolean isExists = Files.exists(Paths.get(System.getProperty("rocketmq.home.dir")
-            + File.separator + "conf" + File.separator + "plain_acl.yml"));
-        Assert.assertTrue("default acl config file should exist", isExists);
-    }
-
-}
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java
deleted file mode 100644
index ef0cffbdcc8..00000000000
--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java
+++ /dev/null
@@ -1,1112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.acl.plain;
-
-import com.google.common.base.Joiner;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import org.apache.rocketmq.acl.common.AclClientRPCHook;
-import org.apache.rocketmq.acl.common.AclConstants;
-import org.apache.rocketmq.acl.common.AclException;
-import org.apache.rocketmq.acl.common.AclUtils;
-import org.apache.rocketmq.acl.common.SessionCredentials;
-import org.apache.rocketmq.common.AclConfig;
-import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.common.PlainAccessConfig;
-import org.apache.rocketmq.remoting.exception.RemotingCommandException;
-import org.apache.rocketmq.remoting.protocol.DataVersion;
-import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-import org.apache.rocketmq.remoting.protocol.RequestCode;
-import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;
-import org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
-import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;
-import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;
-import org.apache.rocketmq.remoting.protocol.heartbeat.ProducerData;
-import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class PlainAccessValidatorTest {
-
-    private PlainAccessValidator plainAccessValidator;
-    private AclClientRPCHook aclClient;
-    private SessionCredentials sessionCredentials;
-
-    private File confHome;
-
-    @Before
-    public void init() throws IOException {
-        String folder = "conf";
-        confHome = AclTestHelper.copyResources(folder, true);
-        System.setProperty("rocketmq.home.dir", confHome.getAbsolutePath());
-        plainAccessValidator = new PlainAccessValidator();
-        sessionCredentials = new SessionCredentials();
-        sessionCredentials.setAccessKey("RocketMQ");
-        sessionCredentials.setSecretKey("12345678");
-        sessionCredentials.setSecurityToken("87654321");
-        aclClient = new AclClientRPCHook(sessionCredentials);
-    }
-
-    @After
-    public void cleanUp() {
-        AclTestHelper.recursiveDelete(confHome);
-    }
-
-    @Test
-    public void contentTest() {
-        SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
-        messageRequestHeader.setTopic("topicA");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "127.0.0.1");
-            String signature = AclUtils.calSignature(accessResource.getContent(), sessionCredentials.getSecretKey());
-
-            Assert.assertEquals(accessResource.getSignature(), signature);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-
-    }
-
-    @Test
-    public void validateTest() {
-        SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
-        messageRequestHeader.setTopic("topicB");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-
-    }
-
-    @Test
-    public void validateSendMessageTest() {
-        SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
-        messageRequestHeader.setTopic("topicB");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void validateSendMessageToRetryTopicTest() {
-        SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
-        messageRequestHeader.setTopic(MixAll.getRetryTopic("groupB"));
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void validateSendMessageV2Test() {
-        SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
-        messageRequestHeader.setTopic("topicB");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(messageRequestHeader));
-        aclClient.doBeforeRequest("", remotingCommand);
-
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void validateSendMessageV2ToRetryTopicTest() {
-        SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
-        messageRequestHeader.setTopic(MixAll.getRetryTopic("groupC"));
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(messageRequestHeader));
-        aclClient.doBeforeRequest("", remotingCommand);
-
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6:9876");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void validateForAdminCommandWithOutAclRPCHook() {
-        RemotingCommand consumerOffsetAdminRequest = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_CONSUMER_OFFSET, null);
-        plainAccessValidator.parse(consumerOffsetAdminRequest, "192.168.0.1:9876");
-
-        RemotingCommand subscriptionGroupAdminRequest = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG, null);
-        plainAccessValidator.parse(subscriptionGroupAdminRequest, "192.168.0.1:9876");
-
-        RemotingCommand delayOffsetAdminRequest = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_DELAY_OFFSET, null);
-        plainAccessValidator.parse(delayOffsetAdminRequest, "192.168.0.1:9876");
-
-        RemotingCommand allTopicConfigAdminRequest = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, null);
-        plainAccessValidator.parse(allTopicConfigAdminRequest, "192.168.0.1:9876");
-
-    }
-
-    @Test
-    public void validatePullMessageTest() {
-        PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();
-        pullMessageRequestHeader.setTopic("topicC");
-        pullMessageRequestHeader.setConsumerGroup("groupC");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, pullMessageRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void validateConsumeMessageBackTest() {
-        ConsumerSendMsgBackRequestHeader consumerSendMsgBackRequestHeader = new ConsumerSendMsgBackRequestHeader();
-        consumerSendMsgBackRequestHeader.setOriginTopic("topicC");
-        consumerSendMsgBackRequestHeader.setGroup("groupC");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, consumerSendMsgBackRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void validateQueryMessageTest() {
-        QueryMessageRequestHeader queryMessageRequestHeader = new QueryMessageRequestHeader();
-        queryMessageRequestHeader.setTopic("topicC");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, queryMessageRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void validateQueryMessageByKeyTest() {
-        QueryMessageRequestHeader queryMessageRequestHeader = new QueryMessageRequestHeader();
-        queryMessageRequestHeader.setTopic("topicC");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, queryMessageRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-        remotingCommand.addExtField(MixAll.UNIQUE_MSG_QUERY_FLAG, "false");
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.1.1:9876");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void validateHeartBeatTest() {
-        HeartbeatData heartbeatData = new HeartbeatData();
-        Set<ProducerData> producerDataSet = new HashSet<>();
-        Set<ConsumerData> consumerDataSet = new HashSet<>();
-        Set<SubscriptionData> subscriptionDataSet = new HashSet<>();
-        ProducerData producerData = new ProducerData();
-        producerData.setGroupName("groupB");
-        ConsumerData consumerData = new ConsumerData();
-        consumerData.setGroupName("groupC");
-        SubscriptionData subscriptionData = new SubscriptionData();
-        subscriptionData.setTopic("topicC");
-        producerDataSet.add(producerData);
-        consumerDataSet.add(consumerData);
-        subscriptionDataSet.add(subscriptionData);
-        consumerData.setSubscriptionDataSet(subscriptionDataSet);
-        heartbeatData.setProducerDataSet(producerDataSet);
-        heartbeatData.setConsumerDataSet(consumerDataSet);
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, null);
-        remotingCommand.setBody(heartbeatData.encode());
-        aclClient.doBeforeRequest("", remotingCommand);
-        ByteBuffer buf = remotingCommand.encode();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void validateUnRegisterClientTest() {
-        UnregisterClientRequestHeader unregisterClientRequestHeader = new UnregisterClientRequestHeader();
-        unregisterClientRequestHeader.setConsumerGroup("groupB");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT, unregisterClientRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void validateGetConsumerListByGroupTest() {
-        GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader = new GetConsumerListByGroupRequestHeader();
-        getConsumerListByGroupRequestHeader.setConsumerGroup("groupB");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, getConsumerListByGroupRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void validateUpdateConsumerOffSetTest() {
-        UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader = new UpdateConsumerOffsetRequestHeader();
-        updateConsumerOffsetRequestHeader.setConsumerGroup("groupB");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, updateConsumerOffsetRequestHeader);
-        aclClient.doBeforeRequest("", remotingCommand);
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "123.4.5.6");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test(expected = AclException.class)
-    public void validateNullAccessKeyTest() {
-        SessionCredentials sessionCredentials = new SessionCredentials();
-        sessionCredentials.setAccessKey("RocketMQ1");
-        sessionCredentials.setSecretKey("1234");
-        AclClientRPCHook aclClientRPCHook = new AclClientRPCHook(sessionCredentials);
-        SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
-        messageRequestHeader.setTopic("topicB");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
-        aclClientRPCHook.doBeforeRequest("", remotingCommand);
-
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.1.1");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test(expected = AclException.class)
-    public void validateErrorSecretKeyTest() {
-        SessionCredentials sessionCredentials = new SessionCredentials();
-        sessionCredentials.setAccessKey("RocketMQ");
-        sessionCredentials.setSecretKey("1234");
-        AclClientRPCHook aclClientRPCHook = new AclClientRPCHook(sessionCredentials);
-        SendMessageRequestHeader messageRequestHeader = new SendMessageRequestHeader();
-        messageRequestHeader.setTopic("topicB");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, messageRequestHeader);
-        aclClientRPCHook.doBeforeRequest("", remotingCommand);
-
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "192.168.1.1");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void validateGetAllTopicConfigTest() {
-        String whiteRemoteAddress = "192.168.0.1";
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, null);
-
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), whiteRemoteAddress);
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-
-            Assert.fail("Should not throw IOException");
-        }
-    }
-
-    @Test
-    public void addAccessAclYamlConfigTest() throws InterruptedException {
-        String backupFileName = System.getProperty("rocketmq.home.dir")
-            + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
-        String targetFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
-        PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-
-        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-        plainAccessConfig.setAccessKey("rocketmq3");
-        plainAccessConfig.setSecretKey("1234567890");
-        plainAccessConfig.setWhiteRemoteAddress("192.168.0.*");
-        plainAccessConfig.setDefaultGroupPerm("PUB");
-        plainAccessConfig.setDefaultTopicPerm("SUB");
-        List<String> topicPerms = new ArrayList<>();
-        topicPerms.add("topicC=PUB|SUB");
-        topicPerms.add("topicB=PUB");
-        plainAccessConfig.setTopicPerms(topicPerms);
-        List<String> groupPerms = new ArrayList<>();
-        groupPerms.add("groupB=PUB|SUB");
-        groupPerms.add("groupC=DENY");
-        plainAccessConfig.setGroupPerms(groupPerms);
-
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        plainAccessValidator.updateAccessConfig(plainAccessConfig);
-        Thread.sleep(10000);
-
-        Map<String, Object> verifyMap = new HashMap<>();
-        AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
-        List<PlainAccessConfig> plainAccessConfigs = aclConfig.getPlainAccessConfigs();
-        for (PlainAccessConfig plainAccessConfig1 : plainAccessConfigs) {
-            if (plainAccessConfig1.getAccessKey().equals(plainAccessConfig.getAccessKey())) {
-                verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig1.getSecretKey());
-                verifyMap.put(AclConstants.CONFIG_DEFAULT_TOPIC_PERM, plainAccessConfig1.getDefaultTopicPerm());
-                verifyMap.put(AclConstants.CONFIG_DEFAULT_GROUP_PERM, plainAccessConfig1.getDefaultGroupPerm());
-                verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig1.isAdmin());
-                verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig1.getWhiteRemoteAddress());
-                verifyMap.put(AclConstants.CONFIG_TOPIC_PERMS, plainAccessConfig1.getTopicPerms());
-                verifyMap.put(AclConstants.CONFIG_GROUP_PERMS, plainAccessConfig1.getGroupPerms());
-            }
-        }
-
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_SECRET_KEY), "1234567890");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_DEFAULT_TOPIC_PERM), "SUB");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_DEFAULT_GROUP_PERM), "PUB");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_ADMIN_ROLE), false);
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_WHITE_ADDR), "192.168.0.*");
-        Assert.assertEquals(((List) verifyMap.get(AclConstants.CONFIG_TOPIC_PERMS)).size(), 2);
-        Assert.assertEquals(((List) verifyMap.get(AclConstants.CONFIG_GROUP_PERMS)).size(), 2);
-
-        String aclFileName = System.getProperty("rocketmq.home.dir") + File.separator + "conf/plain_acl.yml";
-        PlainAccessData readableMap = AclUtils.getYamlDataObject(aclFileName, PlainAccessData.class);
-        List<PlainAccessData.DataVersion> dataVersions = readableMap.getDataVersion();
-        Assert.assertEquals(1L, dataVersions.get(0).getCounter());
-
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-    }
-
-    @Test
-    public void getAccessAclYamlConfigTest() {
-        String accessKey = "rocketmq2";
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
-        List<PlainAccessConfig> plainAccessConfigs = aclConfig.getPlainAccessConfigs();
-        Map<String, Object> verifyMap = new HashMap<>();
-        for (PlainAccessConfig plainAccessConfig : plainAccessConfigs) {
-            if (plainAccessConfig.getAccessKey().equals(accessKey)) {
-                verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig.getSecretKey());
-                verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig.isAdmin());
-                verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig.getWhiteRemoteAddress());
-            }
-        }
-
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_SECRET_KEY), "12345678");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_ADMIN_ROLE), true);
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_WHITE_ADDR), "192.168.1.*");
-
-        String aclFileName = System.getProperty("rocketmq.home.dir")
-            + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
-        Map<String, DataVersion> dataVersionMap = plainAccessValidator.getAllAclConfigVersion();
-        DataVersion dataVersion = dataVersionMap.get(aclFileName);
-        Assert.assertEquals(0, dataVersion.getCounter().get());
-    }
-
-    @Test
-    public void updateAccessAclYamlConfigTest() throws InterruptedException {
-        String backupFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
-        String targetFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
-        PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-
-        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-        plainAccessConfig.setAccessKey("rocketmq3");
-        plainAccessConfig.setSecretKey("1234567890");
-        plainAccessConfig.setWhiteRemoteAddress("192.168.0.*");
-        plainAccessConfig.setDefaultGroupPerm("PUB");
-        plainAccessConfig.setDefaultTopicPerm("SUB");
-        List<String> topicPerms = new ArrayList<>();
-        topicPerms.add("topicC=PUB|SUB");
-        topicPerms.add("topicB=PUB");
-        plainAccessConfig.setTopicPerms(topicPerms);
-        List<String> groupPerms = new ArrayList<>();
-        groupPerms.add("groupB=PUB|SUB");
-        groupPerms.add("groupC=DENY");
-        plainAccessConfig.setGroupPerms(groupPerms);
-
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        plainAccessValidator.updateAccessConfig(plainAccessConfig);
-
-        Thread.sleep(10000);
-
-        PlainAccessConfig plainAccessConfig1 = new PlainAccessConfig();
-        plainAccessConfig1.setAccessKey("rocketmq3");
-        plainAccessConfig1.setSecretKey("1234567891");
-        plainAccessConfig1.setWhiteRemoteAddress("192.168.0.*");
-        plainAccessConfig1.setDefaultGroupPerm("PUB");
-        plainAccessConfig1.setDefaultTopicPerm("SUB");
-        List<String> topicPerms1 = new ArrayList<>();
-        topicPerms1.add("topicC=PUB|SUB");
-        topicPerms1.add("topicB=PUB");
-        plainAccessConfig1.setTopicPerms(topicPerms1);
-        List<String> groupPerms1 = new ArrayList<>();
-        groupPerms1.add("groupB=PUB|SUB");
-        groupPerms1.add("groupC=DENY");
-        plainAccessConfig1.setGroupPerms(groupPerms1);
-
-        plainAccessValidator.updateAccessConfig(plainAccessConfig1);
-
-        Thread.sleep(10000);
-
-        Map<String, Object> verifyMap = new HashMap<>();
-        AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
-        List<PlainAccessConfig> plainAccessConfigs = aclConfig.getPlainAccessConfigs();
-        for (PlainAccessConfig plainAccessConfig2 : plainAccessConfigs) {
-            if (plainAccessConfig2.getAccessKey().equals(plainAccessConfig1.getAccessKey())) {
-                verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig2.getSecretKey());
-                verifyMap.put(AclConstants.CONFIG_DEFAULT_TOPIC_PERM, plainAccessConfig2.getDefaultTopicPerm());
-                verifyMap.put(AclConstants.CONFIG_DEFAULT_GROUP_PERM, plainAccessConfig2.getDefaultGroupPerm());
-                verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig2.isAdmin());
-                verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig2.getWhiteRemoteAddress());
-                verifyMap.put(AclConstants.CONFIG_TOPIC_PERMS, plainAccessConfig2.getTopicPerms());
-                verifyMap.put(AclConstants.CONFIG_GROUP_PERMS, plainAccessConfig2.getGroupPerms());
-            }
-        }
-
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_SECRET_KEY), "1234567891");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_DEFAULT_TOPIC_PERM), "SUB");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_DEFAULT_GROUP_PERM), "PUB");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_ADMIN_ROLE), false);
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_WHITE_ADDR), "192.168.0.*");
-        Assert.assertEquals(((List) verifyMap.get(AclConstants.CONFIG_TOPIC_PERMS)).size(), 2);
-        Assert.assertEquals(((List) verifyMap.get(AclConstants.CONFIG_GROUP_PERMS)).size(), 2);
-
-        String aclFileName = System.getProperty("rocketmq.home.dir")
-            + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
-        PlainAccessData readableMap = AclUtils.getYamlDataObject(aclFileName, PlainAccessData.class);
-        List<PlainAccessData.DataVersion> dataVersions = readableMap.getDataVersion();
-        Assert.assertEquals(2L, dataVersions.get(0).getCounter());
-
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-    }
-
-    @Test
-    public void deleteAccessAclYamlConfigTest() throws InterruptedException {
-        String backupFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
-        String targetFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
-        PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-
-        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-        plainAccessConfig.setAccessKey("rocketmq3");
-        plainAccessConfig.setSecretKey("1234567890");
-        plainAccessConfig.setWhiteRemoteAddress("192.168.0.*");
-        plainAccessConfig.setDefaultGroupPerm("PUB");
-        plainAccessConfig.setDefaultTopicPerm("SUB");
-        List<String> topicPerms = new ArrayList<>();
-        topicPerms.add("topicC=PUB|SUB");
-        topicPerms.add("topicB=PUB");
-        plainAccessConfig.setTopicPerms(topicPerms);
-        List<String> groupPerms = new ArrayList<>();
-        groupPerms.add("groupB=PUB|SUB");
-        groupPerms.add("groupC=DENY");
-        plainAccessConfig.setGroupPerms(groupPerms);
-
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        plainAccessValidator.updateAccessConfig(plainAccessConfig);
-
-        String accessKey = "rocketmq3";
-        plainAccessValidator.deleteAccessConfig(accessKey);
-        Thread.sleep(10000);
-
-        Map<String, Object> verifyMap = new HashMap<>();
-        AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
-        List<PlainAccessConfig> plainAccessConfigs = aclConfig.getPlainAccessConfigs();
-        for (PlainAccessConfig plainAccessConfig1 : plainAccessConfigs) {
-            if (plainAccessConfig1.getAccessKey().equals(accessKey)) {
-                verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig1.getSecretKey());
-                verifyMap.put(AclConstants.CONFIG_DEFAULT_TOPIC_PERM, plainAccessConfig1.getDefaultTopicPerm());
-                verifyMap.put(AclConstants.CONFIG_DEFAULT_GROUP_PERM, plainAccessConfig1.getDefaultGroupPerm());
-                verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig1.isAdmin());
-                verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig1.getWhiteRemoteAddress());
-                verifyMap.put(AclConstants.CONFIG_TOPIC_PERMS, plainAccessConfig1.getTopicPerms());
-                verifyMap.put(AclConstants.CONFIG_GROUP_PERMS, plainAccessConfig1.getGroupPerms());
-            }
-        }
-
-        Assert.assertEquals(verifyMap.size(), 0);
-
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-    }
-
-    @Test
-    public void updateGlobalWhiteRemoteAddressesTest() throws InterruptedException {
-        String backupFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
-        String targetFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
-        PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-
-        List<String> globalWhiteAddrsList = new ArrayList<>();
-        globalWhiteAddrsList.add("192.168.1.*");
-
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        Assert.assertEquals(plainAccessValidator.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList, null), true);
-
-        String aclFileName = System.getProperty("rocketmq.home.dir")
-            + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
-        PlainAccessData readableMap = AclUtils.getYamlDataObject(aclFileName, PlainAccessData.class);
-        List<PlainAccessData.DataVersion> dataVersions = readableMap.getDataVersion();
-        Assert.assertEquals(1L, dataVersions.get(0).getCounter());
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-    }
-
-    @Test
-    public void addYamlConfigTest() throws IOException, InterruptedException {
-        String fileName = System.getProperty("rocketmq.home.dir")
-            + File.separator + "conf/acl/plain_acl_test.yml".replace("/", File.separator);
-        File transport = new File(fileName);
-        transport.delete();
-        transport.createNewFile();
-        FileWriter writer = new FileWriter(transport);
-        writer.write("accounts:\r\n");
-        writer.write("- accessKey: watchrocketmqx\r\n");
-        writer.write("  secretKey: 12345678\r\n");
-        writer.write("  whiteRemoteAddress: 127.0.0.1\r\n");
-        writer.write("  admin: true\r\n");
-        writer.flush();
-        writer.close();
-
-        Thread.sleep(1000);
-
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
-        List<PlainAccessConfig> plainAccessConfigs = aclConfig.getPlainAccessConfigs();
-        Map<String, Object> verifyMap = new HashMap<>();
-        for (PlainAccessConfig plainAccessConfig : plainAccessConfigs) {
-            if (plainAccessConfig.getAccessKey().equals("watchrocketmqx")) {
-                verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig.getSecretKey());
-                verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig.getWhiteRemoteAddress());
-                verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig.isAdmin());
-            }
-        }
-
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_SECRET_KEY), "12345678");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_WHITE_ADDR), "127.0.0.1");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_ADMIN_ROLE), true);
-
-        Map<String, DataVersion> dataVersionMap = plainAccessValidator.getAllAclConfigVersion();
-        DataVersion dataVersion = dataVersionMap.get(fileName);
-        Assert.assertEquals(0, dataVersion.getCounter().get());
-
-        transport.delete();
-    }
-
-    @Test
-    public void updateAccessAnotherAclYamlConfigTest() throws IOException, InterruptedException {
-        String fileName = System.getProperty("rocketmq.home.dir")
-            + File.separator + "conf/acl/plain_acl_test.yml".replace("/", File.separator);
-        File transport = new File(fileName);
-        transport.delete();
-        transport.createNewFile();
-        FileWriter writer = new FileWriter(transport);
-        writer.write("accounts:\r\n");
-        writer.write("- accessKey: watchrocketmqy\r\n");
-        writer.write("  secretKey: 12345678\r\n");
-        writer.write("  whiteRemoteAddress: 127.0.0.1\r\n");
-        writer.write("  admin: true\r\n");
-        writer.write("- accessKey: watchrocketmqx\r\n");
-        writer.write("  secretKey: 123456781\r\n");
-        writer.write("  whiteRemoteAddress: 127.0.0.1\r\n");
-        writer.write("  admin: true\r\n");
-        writer.flush();
-        writer.close();
-
-        Thread.sleep(1000);
-
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-
-        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-        plainAccessConfig.setAccessKey("watchrocketmqy");
-        plainAccessConfig.setSecretKey("1234567890");
-        plainAccessConfig.setWhiteRemoteAddress("127.0.0.1");
-        plainAccessConfig.setAdmin(false);
-
-        plainAccessValidator.updateAccessConfig(plainAccessConfig);
-
-        Thread.sleep(1000);
-
-        AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
-        List<PlainAccessConfig> plainAccessConfigs = aclConfig.getPlainAccessConfigs();
-        Map<String, Object> verifyMap = new HashMap<>();
-        for (PlainAccessConfig plainAccessConfig1 : plainAccessConfigs) {
-            if (plainAccessConfig1.getAccessKey().equals("watchrocketmqy")) {
-                verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig1.getSecretKey());
-                verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig1.getWhiteRemoteAddress());
-                verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig1.isAdmin());
-            }
-        }
-
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_SECRET_KEY), "1234567890");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_WHITE_ADDR), "127.0.0.1");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_ADMIN_ROLE), false);
-
-        Map<String, DataVersion> dataVersionMap = plainAccessValidator.getAllAclConfigVersion();
-        DataVersion dataVersion = dataVersionMap.get(fileName);
-        Assert.assertEquals(1, dataVersion.getCounter().get());
-
-        transport.delete();
-
-    }
-
-    @Test(expected = AclException.class)
-    public void createAndUpdateAccessAclNullSkExceptionTest() {
-        String backupFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
-        String targetFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
-        PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-
-        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-        plainAccessConfig.setAccessKey("RocketMQ33");
-        // secret key is null
-
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        plainAccessValidator.updateAccessConfig(plainAccessConfig);
-
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-    }
-
-    @Test
-    public void addAccessDefaultAclYamlConfigTest() throws InterruptedException {
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        String backupFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
-        String targetFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
-        PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-
-        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-        plainAccessConfig.setAccessKey("watchrocketmqh");
-        plainAccessConfig.setSecretKey("1234567890");
-        plainAccessConfig.setWhiteRemoteAddress("127.0.0.1");
-        plainAccessConfig.setAdmin(false);
-
-        plainAccessValidator.updateAccessConfig(plainAccessConfig);
-
-        Thread.sleep(10000);
-
-        AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
-        List<PlainAccessConfig> plainAccessConfigs = aclConfig.getPlainAccessConfigs();
-        Map<String, Object> verifyMap = new HashMap<>();
-        for (PlainAccessConfig plainAccessConfig1 : plainAccessConfigs) {
-            if (plainAccessConfig1.getAccessKey().equals("watchrocketmqh")) {
-                verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig1.getSecretKey());
-                verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig1.getWhiteRemoteAddress());
-                verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig1.isAdmin());
-            }
-        }
-
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_SECRET_KEY), "1234567890");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_WHITE_ADDR), "127.0.0.1");
-        Assert.assertEquals(verifyMap.get(AclConstants.CONFIG_ADMIN_ROLE), false);
-
-        PlainAccessData readableMap = AclUtils.getYamlDataObject(targetFileName, PlainAccessData.class);
-        List<PlainAccessData.DataVersion> dataVersions = readableMap.getDataVersion();
-        Assert.assertEquals(1L, dataVersions.get(0).getCounter());
-
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-    }
-
-    @Test
-    public void deleteAccessAnotherAclYamlConfigTest() throws IOException, InterruptedException {
-        String fileName = System.getProperty("rocketmq.home.dir")
-            + File.separator + "conf/acl/plain_acl_test.yml".replace("/", File.separator);
-        File transport = new File(fileName);
-        transport.delete();
-        transport.createNewFile();
-        FileWriter writer = new FileWriter(transport);
-        writer.write("accounts:\r\n");
-        writer.write("- accessKey: watchrocketmqx\r\n");
-        writer.write("  secretKey: 12345678\r\n");
-        writer.write("  whiteRemoteAddress: 127.0.0.1\r\n");
-        writer.write("  admin: true\r\n");
-        writer.write("- accessKey: watchrocketmqy\r\n");
-        writer.write("  secretKey: 1234567890\r\n");
-        writer.write("  whiteRemoteAddress: 127.0.0.1\r\n");
-        writer.write("  admin: false\r\n");
-        writer.flush();
-        writer.close();
-
-        Thread.sleep(1000);
-
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        plainAccessValidator.deleteAccessConfig("watchrocketmqx");
-        Thread.sleep(10000);
-
-        Map<String, Object> verifyMap = new HashMap<>();
-        AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
-        List<PlainAccessConfig> plainAccessConfigs = aclConfig.getPlainAccessConfigs();
-        for (PlainAccessConfig plainAccessConfig : plainAccessConfigs) {
-            if (plainAccessConfig.getAccessKey().equals("watchrocketmqx")) {
-                verifyMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig.getSecretKey());
-                verifyMap.put(AclConstants.CONFIG_DEFAULT_TOPIC_PERM, plainAccessConfig.getDefaultTopicPerm());
-                verifyMap.put(AclConstants.CONFIG_DEFAULT_GROUP_PERM, plainAccessConfig.getDefaultGroupPerm());
-                verifyMap.put(AclConstants.CONFIG_ADMIN_ROLE, plainAccessConfig.isAdmin());
-                verifyMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig.getWhiteRemoteAddress());
-                verifyMap.put(AclConstants.CONFIG_TOPIC_PERMS, plainAccessConfig.getTopicPerms());
-                verifyMap.put(AclConstants.CONFIG_GROUP_PERMS, plainAccessConfig.getGroupPerms());
-            }
-        }
-
-        Assert.assertEquals(verifyMap.size(), 0);
-
-        transport.delete();
-    }
-
-    @Test
-    public void getAllAclConfigTest() {
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        AclConfig aclConfig = plainAccessValidator.getAllAclConfig();
-        Assert.assertEquals(aclConfig.getGlobalWhiteAddrs().size(), 4);
-        Assert.assertEquals(aclConfig.getPlainAccessConfigs().size(), 2);
-    }
-
-    @Test
-    public void updateAccessConfigEmptyPermListTest() {
-        String backupFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
-        String targetFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
-        PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-        String accessKey = "updateAccessConfigEmptyPerm";
-        plainAccessConfig.setAccessKey(accessKey);
-        plainAccessConfig.setSecretKey("123456789111");
-        plainAccessConfig.setTopicPerms(Collections.singletonList("topicB=PUB"));
-        plainAccessValidator.updateAccessConfig(plainAccessConfig);
-
-        plainAccessConfig.setTopicPerms(new ArrayList<>());
-        plainAccessValidator.updateAccessConfig(plainAccessConfig);
-
-        List<PlainAccessConfig> plainAccessConfigs = plainAccessValidator.getAllAclConfig().getPlainAccessConfigs();
-        for (int i = 0; i < plainAccessConfigs.size(); i++) {
-            PlainAccessConfig plainAccessConfig1 = plainAccessConfigs.get(i);
-            if (plainAccessConfig1.getAccessKey() == accessKey) {
-                Assert.assertEquals(0, plainAccessConfig1.getTopicPerms().size());
-            }
-        }
-
-        plainAccessValidator.deleteAccessConfig(accessKey);
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-    }
-
-    @Test
-    public void updateAccessConfigEmptyWhiteRemoteAddressTest() {
-        String backupFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl_bak.yml".replace("/", File.separator);
-        String targetFileName = System.getProperty("rocketmq.home.dir")
-                + File.separator + "conf/plain_acl.yml".replace("/", File.separator);
-        PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(backupFileName, PlainAccessData.class);
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-        String accessKey = "updateAccessConfigEmptyWhiteRemoteAddress";
-        plainAccessConfig.setAccessKey(accessKey);
-        plainAccessConfig.setSecretKey("123456789111");
-        plainAccessConfig.setWhiteRemoteAddress("127.0.0.1");
-        plainAccessValidator.updateAccessConfig(plainAccessConfig);
-
-        plainAccessConfig.setWhiteRemoteAddress("");
-        plainAccessValidator.updateAccessConfig(plainAccessConfig);
-
-        List<PlainAccessConfig> plainAccessConfigs = plainAccessValidator.getAllAclConfig().getPlainAccessConfigs();
-        for (int i = 0; i < plainAccessConfigs.size(); i++) {
-            PlainAccessConfig plainAccessConfig1 = plainAccessConfigs.get(i);
-            if (plainAccessConfig1.getAccessKey() == accessKey) {
-                Assert.assertEquals("", plainAccessConfig1.getWhiteRemoteAddress());
-            }
-        }
-
-        plainAccessValidator.deleteAccessConfig(accessKey);
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-    }
-
-    @Test
-    public void deleteAccessAclToEmptyTest() {
-        final String bakAclFileProp = System.getProperty("rocketmq.acl.plain.file");
-        System.setProperty("rocketmq.acl.plain.file", "conf/empty.yml".replace("/", File.separator));
-        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-        plainAccessConfig.setAccessKey("deleteAccessAclToEmpty");
-        plainAccessConfig.setSecretKey("12345678");
-
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        plainAccessValidator.updateAccessConfig(plainAccessConfig);
-        boolean success = plainAccessValidator.deleteAccessConfig("deleteAccessAclToEmpty");
-        if (null != bakAclFileProp) {
-            System.setProperty("rocketmq.acl.plain.file", bakAclFileProp);
-        } else {
-            System.clearProperty("rocketmq.acl.plain.file");
-        }
-        Assert.assertTrue(success);
-    }
-
-    @Test
-    public void testValidateAfterUpdateAccessConfig() throws NoSuchFieldException, IllegalAccessException {
-        String targetFileName = System.getProperty("rocketmq.home.dir")
-            + File.separator + "conf/update.yml".replace("/", File.separator);
-        System.setProperty("rocketmq.acl.plain.file", "conf/update.yml".replace("/", File.separator));
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-        String accessKey = "updateAccessConfig";
-        String secretKey = "123456789111";
-        plainAccessConfig.setAccessKey(accessKey);
-        plainAccessConfig.setSecretKey(secretKey);
-        plainAccessConfig.setAdmin(true);
-        // update
-        plainAccessValidator.updateAccessConfig(plainAccessConfig);
-        // call load
-        Class clazz = PlainAccessValidator.class;
-        Field f = clazz.getDeclaredField("aclPlugEngine");
-        f.setAccessible(true);
-        PlainPermissionManager aclPlugEngine = (PlainPermissionManager) f.get(plainAccessValidator);
-        aclPlugEngine.load(targetFileName);
-
-        // call validate
-        PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();
-        pullMessageRequestHeader.setTopic("topicC");
-        pullMessageRequestHeader.setConsumerGroup("consumerGroupA");
-        RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, pullMessageRequestHeader);
-
-        AclClientRPCHook aclClient = new AclClientRPCHook(new SessionCredentials(accessKey, secretKey));
-        aclClient.doBeforeRequest("", remotingCommand);
-        ByteBuffer buf = remotingCommand.encodeHeader();
-        buf.getInt();
-        buf = ByteBuffer.allocate(buf.limit() - buf.position()).put(buf);
-        buf.position(0);
-        try {
-            PlainAccessResource accessResource = (PlainAccessResource) plainAccessValidator.parse(RemotingCommand.decode(buf), "1.1.1.1:9876");
-            plainAccessValidator.validate(accessResource);
-        } catch (RemotingCommandException e) {
-            e.printStackTrace();
-            Assert.fail("Should not throw IOException");
-        } finally {
-            System.setProperty("rocketmq.acl.plain.file", "conf/plain_acl.yml".replace("/", File.separator));
-        }
-    }
-
-    /**
-     * Fixme: this test case is not thread safe. The design itself is buggy.
-     * @throws IOException
-     */
-    @Test
-    public void testUpdateSpecifiedAclFileGlobalWhiteAddrsConfig() throws IOException {
-        String folder = "update_global_white_addr";
-        File home = AclTestHelper.copyResources(folder);
-        System.setProperty("rocketmq.home.dir", home.getAbsolutePath());
-        System.setProperty("rocketmq.acl.plain.file", "/conf/plain_acl.yml".replace("/", File.separator));
-
-        String targetFileName = Joiner.on(File.separator).join(new String[]{home.getAbsolutePath(), "conf", "plain_acl.yml"});
-        PlainAccessData backUpAclConfigMap = AclUtils.getYamlDataObject(targetFileName, PlainAccessData.class);
-
-        String targetFileName1 = Joiner.on(File.separator).join(new String[]{home.getAbsolutePath(), "conf", "acl", "plain_acl.yml"});
-        PlainAccessData backUpAclConfigMap1 = AclUtils.getYamlDataObject(targetFileName1, PlainAccessData.class);
-
-        String targetFileName2 = Joiner.on(File.separator).join(new String[]{home.getAbsolutePath(), "conf", "acl", "empty.yml"});
-        PlainAccessData backUpAclConfigMap2 = AclUtils.getYamlDataObject(targetFileName2, PlainAccessData.class);
-
-        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
-        List<String> globalWhiteAddrsList1 = new ArrayList<>();
-        globalWhiteAddrsList1.add("10.10.154.1");
-        List<String> globalWhiteAddrsList2 = new ArrayList<>();
-        globalWhiteAddrsList2.add("10.10.154.2");
-        List<String> globalWhiteAddrsList3 = new ArrayList<>();
-        globalWhiteAddrsList3.add("10.10.154.3");
-
-        //Test parameter p is null
-        plainAccessValidator.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList1, null);
-        String defaultAclFile = targetFileName;
-        PlainAccessData defaultAclFileMap = AclUtils.getYamlDataObject(defaultAclFile, PlainAccessData.class);
-        List<String> defaultAclFileGlobalWhiteAddrList = defaultAclFileMap.getGlobalWhiteRemoteAddresses();
-        Assert.assertTrue(defaultAclFileGlobalWhiteAddrList.contains("10.10.154.1"));
-        //Test parameter p is not null
-        plainAccessValidator.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList2, targetFileName1);
-        PlainAccessData aclFileMap1 =  AclUtils.getYamlDataObject(targetFileName1, PlainAccessData.class);
-        List<String> aclFileGlobalWhiteAddrList1 = aclFileMap1.getGlobalWhiteRemoteAddresses();
-        Assert.assertTrue(aclFileGlobalWhiteAddrList1.contains("10.10.154.2"));
-        //Test parameter p is not null, but the file does not have globalWhiteRemoteAddresses
-        plainAccessValidator.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList3, targetFileName2);
-        PlainAccessData aclFileMap2 =  AclUtils.getYamlDataObject(targetFileName2, PlainAccessData.class);
-        List<String> aclFileGlobalWhiteAddrList2 = aclFileMap2.getGlobalWhiteRemoteAddresses();
-        Assert.assertTrue(aclFileGlobalWhiteAddrList2.contains("10.10.154.3"));
-
-        AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
-        AclUtils.writeDataObject(targetFileName1, backUpAclConfigMap1);
-        AclUtils.writeDataObject(targetFileName2, backUpAclConfigMap2);
-
-        AclTestHelper.recursiveDelete(home);
-    }
-
-
-}
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java
deleted file mode 100644
index 941d8c77923..00000000000
--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.acl.plain;
-
-import com.google.common.base.Joiner;
-import org.apache.commons.lang3.reflect.FieldUtils;
-import org.apache.rocketmq.acl.common.AclConstants;
-import org.apache.rocketmq.acl.common.AclException;
-import org.apache.rocketmq.acl.common.AclUtils;
-import org.apache.rocketmq.acl.common.Permission;
-import org.apache.rocketmq.common.AclConfig;
-import org.apache.rocketmq.common.PlainAccessConfig;
-import org.apache.rocketmq.remoting.protocol.DataVersion;
-import org.assertj.core.api.Assertions;
-import org.assertj.core.util.Lists;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class PlainPermissionManagerTest {
-
-    PlainPermissionManager plainPermissionManager;
-    PlainAccessResource pubPlainAccessResource;
-    PlainAccessResource subPlainAccessResource;
-    PlainAccessResource anyPlainAccessResource;
-    PlainAccessResource denyPlainAccessResource;
-    PlainAccessResource plainAccessResource = new PlainAccessResource();
-    PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
-    Set<Integer> adminCode = new HashSet<>();
-
-    private static final String DEFAULT_TOPIC = "topic-acl";
-
-    private File confHome;
-
-    @Before
-    public void init() throws NoSuchFieldException, SecurityException, IOException {
-        // UPDATE_AND_CREATE_TOPIC
-        adminCode.add(17);
-        // UPDATE_BROKER_CONFIG
-        adminCode.add(25);
-        // DELETE_TOPIC_IN_BROKER
-        adminCode.add(215);
-        // UPDATE_AND_CREATE_SUBSCRIPTIONGROUP
-        adminCode.add(200);
-        // DELETE_SUBSCRIPTIONGROUP
-        adminCode.add(207);
-
-        pubPlainAccessResource = clonePlainAccessResource(Permission.PUB);
-        subPlainAccessResource = clonePlainAccessResource(Permission.SUB);
-        anyPlainAccessResource = clonePlainAccessResource(Permission.ANY);
-        denyPlainAccessResource = clonePlainAccessResource(Permission.DENY);
-
-        String folder = "conf";
-        confHome = AclTestHelper.copyResources(folder, true);
-        System.setProperty("rocketmq.home.dir", confHome.getAbsolutePath());
-        plainPermissionManager = new PlainPermissionManager();
-    }
-
-    public PlainAccessResource clonePlainAccessResource(byte perm) {
-        PlainAccessResource painAccessResource = new PlainAccessResource();
-        painAccessResource.setAccessKey("RocketMQ");
-        painAccessResource.setSecretKey("12345678");
-        painAccessResource.setWhiteRemoteAddress("127.0." + perm + ".*");
-        painAccessResource.setDefaultGroupPerm(perm);
-        painAccessResource.setDefaultTopicPerm(perm);
-        painAccessResource.addResourceAndPerm(PlainAccessResource.getRetryTopic("groupA"), Permission.PUB);
-        painAccessResource.addResourceAndPerm(PlainAccessResource.getRetryTopic("groupB"), Permission.SUB);
-        painAccessResource.addResourceAndPerm(PlainAccessResource.getRetryTopic("groupC"), Permission.ANY);
-        painAccessResource.addResourceAndPerm(PlainAccessResource.getRetryTopic("groupD"), Permission.DENY);
-
-        painAccessResource.addResourceAndPerm("topicA", Permission.PUB);
-        painAccessResource.addResourceAndPerm("topicB", Permission.SUB);
-        painAccessResource.addResourceAndPerm("topicC", Permission.ANY);
-        painAccessResource.addResourceAndPerm("topicD", Permission.DENY);
-        return painAccessResource;
-    }
-
-    @Test
-    public void buildPlainAccessResourceTest() {
-        PlainAccessResource plainAccessResource = null;
-        PlainAccessConfig plainAccess = new PlainAccessConfig();
-
-        plainAccess.setAccessKey("RocketMQ");
-        plainAccess.setSecretKey("12345678");
-        plainAccessResource = plainPermissionManager.buildPlainAccessResource(plainAccess);
-        Assert.assertEquals(plainAccessResource.getAccessKey(), "RocketMQ");
-        Assert.assertEquals(plainAccessResource.getSecretKey(), "12345678");
-
-        plainAccess.setWhiteRemoteAddress("127.0.0.1");
-        plainAccessResource = plainPermissionManager.buildPlainAccessResource(plainAccess);
-        Assert.assertEquals(plainAccessResource.getWhiteRemoteAddress(), "127.0.0.1");
-
-        plainAccess.setAdmin(true);
-        plainAccessResource = plainPermissionManager.buildPlainAccessResource(plainAccess);
-        Assert.assertEquals(plainAccessResource.isAdmin(), true);
-
-        List<String> groups = new ArrayList<>();
-        groups.add("groupA=DENY");
-        groups.add("groupB=PUB|SUB");
-        groups.add("groupC=PUB");
-        plainAccess.setGroupPerms(groups);
-        plainAccessResource = plainPermissionManager.buildPlainAccessResource(plainAccess);
-        Map<String, Byte> resourcePermMap = plainAccessResource.getResourcePermMap();
-        Assert.assertEquals(resourcePermMap.size(), 3);
-
-        Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupA")).byteValue(), Permission.DENY);
-        Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupB")).byteValue(), Permission.PUB | Permission.SUB);
-        Assert.assertEquals(resourcePermMap.get(PlainAccessResource.getRetryTopic("groupC")).byteValue(), Permission.PUB);
-
-        List<String> topics = new ArrayList<>();
-        topics.add("topicA=DENY");
-        topics.add("topicB=PUB|SUB");
-        topics.add("topicC=PUB");
-        plainAccess.setTopicPerms(topics);
-        plainAccessResource = plainPermissionManager.buildPlainAccessResource(plainAccess);
-        resourcePermMap = plainAccessResource.getResourcePermMap();
-        Assert.assertEquals(resourcePermMap.size(), 6);
-
-        Assert.assertEquals(resourcePermMap.get("topicA").byteValue(), Permission.DENY);
-        Assert.assertEquals(resourcePermMap.get("topicB").byteValue(), Permission.PUB | Permission.SUB);
-        Assert.assertEquals(resourcePermMap.get("topicC").byteValue(), Permission.PUB);
-    }
-
-    @Test(expected = AclException.class)
-    public void checkPermAdmin() {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setRequestCode(17);
-        plainPermissionManager.checkPerm(plainAccessResource, pubPlainAccessResource);
-    }
-
-    @Test
-    public void checkPerm() {
-
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        plainAccessResource.addResourceAndPerm("topicA", Permission.PUB);
-        plainPermissionManager.checkPerm(plainAccessResource, pubPlainAccessResource);
-        plainAccessResource.addResourceAndPerm("topicB", Permission.SUB);
-        plainPermissionManager.checkPerm(plainAccessResource, anyPlainAccessResource);
-
-        plainAccessResource = new PlainAccessResource();
-        plainAccessResource.addResourceAndPerm("topicB", Permission.SUB);
-        plainPermissionManager.checkPerm(plainAccessResource, subPlainAccessResource);
-        plainAccessResource.addResourceAndPerm("topicA", Permission.PUB);
-        plainPermissionManager.checkPerm(plainAccessResource, anyPlainAccessResource);
-
-    }
-
-    @Test(expected = AclException.class)
-    public void checkErrorPermDefaultValueNotMatch() {
-
-        plainAccessResource = new PlainAccessResource();
-        plainAccessResource.addResourceAndPerm("topicF", Permission.PUB);
-        plainPermissionManager.checkPerm(plainAccessResource, subPlainAccessResource);
-    }
-
-    @Test(expected = AclException.class)
-    public void accountNullTest() {
-        plainAccessConfig.setAccessKey(null);
-        plainPermissionManager.buildPlainAccessResource(plainAccessConfig);
-    }
-
-    @Test(expected = AclException.class)
-    public void accountThanTest() {
-        plainAccessConfig.setAccessKey("123");
-        plainPermissionManager.buildPlainAccessResource(plainAccessConfig);
-    }
-
-    @Test(expected = AclException.class)
-    public void passWordtNullTest() {
-        plainAccessConfig.setAccessKey(null);
-        plainPermissionManager.buildPlainAccessResource(plainAccessConfig);
-    }
-
-    @Test(expected = AclException.class)
-    public void passWordThanTest() {
-        plainAccessConfig.setSecretKey("123");
-        plainPermissionManager.buildPlainAccessResource(plainAccessConfig);
-    }
-
-    @SuppressWarnings("unchecked")
-    @Test
-    public void cleanAuthenticationInfoTest() throws IllegalAccessException {
-        // PlainPermissionManager.addPlainAccessResource(plainAccessResource);
-        Map<String, Map<String, PlainAccessResource>> plainAccessResourceMap = (Map<String, Map<String, PlainAccessResource>>) FieldUtils.readDeclaredField(plainPermissionManager, "aclPlainAccessResourceMap", true);
-        Assert.assertFalse(plainAccessResourceMap.isEmpty());
-
-        plainPermissionManager.clearPermissionInfo();
-        plainAccessResourceMap = (Map<String, Map<String, PlainAccessResource>>) FieldUtils.readDeclaredField(plainPermissionManager, "aclPlainAccessResourceMap", true);
-        Assert.assertTrue(plainAccessResourceMap.isEmpty());
-    }
-
-    @Test
-    public void isWatchStartTest() {
-
-        PlainPermissionManager plainPermissionManager = new PlainPermissionManager();
-        Assert.assertTrue(plainPermissionManager.isWatchStart());
-    }
-
-    @Test
-    public void testWatch() throws IOException, IllegalAccessException, InterruptedException {
-        String fileName =  Joiner.on(File.separator).join(new String[]{System.getProperty("rocketmq.home.dir"), "conf", "acl", "plain_acl_test.yml"});
-        File transport = new File(fileName);
-        transport.delete();
-        transport.createNewFile();
-        FileWriter writer = new FileWriter(transport);
-        writer.write("accounts:\r\n");
-        writer.write("- accessKey: watchrocketmqx\r\n");
-        writer.write("  secretKey: 12345678\r\n");
-        writer.write("  whiteRemoteAddress: 127.0.0.1\r\n");
-        writer.write("  admin: true\r\n");
-        writer.flush();
-        writer.close();
-
-        Thread.sleep(1000);
-
-        PlainPermissionManager plainPermissionManager = new PlainPermissionManager();
-        Assert.assertTrue(plainPermissionManager.isWatchStart());
-
-        Map<String, String> accessKeyTable = (Map<String, String>) FieldUtils.readDeclaredField(plainPermissionManager, "accessKeyTable", true);
-        String aclFileName = accessKeyTable.get("watchrocketmqx");
-        {
-            Map<String, Map<String, PlainAccessResource>> plainAccessResourceMap = (Map<String, Map<String, PlainAccessResource>>) FieldUtils.readDeclaredField(plainPermissionManager, "aclPlainAccessResourceMap", true);
-            PlainAccessResource accessResource = plainAccessResourceMap.get(aclFileName).get("watchrocketmqx");
-            Assert.assertNotNull(accessResource);
-            Assert.assertEquals(accessResource.getSecretKey(), "12345678");
-            Assert.assertTrue(accessResource.isAdmin());
-
-        }
-
-        PlainAccessData updatedMap = AclUtils.getYamlDataObject(fileName, PlainAccessData.class);
-        List<PlainAccessConfig> accounts = updatedMap.getAccounts();
-        accounts.get(0).setAccessKey("watchrocketmq1y");
-        accounts.get(0).setSecretKey("88888888");
-        accounts.get(0).setAdmin(false);
-        // Update file and flush to yaml file
-        AclUtils.writeDataObject(fileName, updatedMap);
-
-        Thread.sleep(10000);
-        {
-            Map<String, Map<String, PlainAccessResource>> plainAccessResourceMap = (Map<String, Map<String, PlainAccessResource>>) FieldUtils.readDeclaredField(plainPermissionManager, "aclPlainAccessResourceMap", true);
-            PlainAccessResource accessResource = plainAccessResourceMap.get(aclFileName).get("watchrocketmq1y");
-            Assert.assertNotNull(accessResource);
-            Assert.assertEquals(accessResource.getSecretKey(), "88888888");
-            Assert.assertFalse(accessResource.isAdmin());
-
-        }
-        transport.delete();
-    }
-
-    @Test
-    public void updateAccessConfigTest() {
-        Assert.assertThrows(AclException.class, () -> plainPermissionManager.updateAccessConfig(null));
-
-        plainAccessConfig.setAccessKey("admin_test");
-        // Invalid parameter
-        plainAccessConfig.setSecretKey("123456");
-        plainAccessConfig.setAdmin(true);
-        Assert.assertThrows(AclException.class, () -> plainPermissionManager.updateAccessConfig(plainAccessConfig));
-
-        plainAccessConfig.setSecretKey("12345678");
-        // Invalid parameter
-        plainAccessConfig.setGroupPerms(Lists.newArrayList("groupA!SUB"));
-        Assert.assertThrows(AclException.class, () -> plainPermissionManager.updateAccessConfig(plainAccessConfig));
-
-        // first update
-        plainAccessConfig.setGroupPerms(Lists.newArrayList("groupA=SUB"));
-        plainPermissionManager.updateAccessConfig(plainAccessConfig);
-
-        // second update
-        plainAccessConfig.setTopicPerms(Lists.newArrayList("topicA=SUB"));
-        plainPermissionManager.updateAccessConfig(plainAccessConfig);
-    }
-
-    @Test
-    public void getAllAclFilesTest() {
-        final List<String> notExistList = plainPermissionManager.getAllAclFiles("aa/bb");
-        Assertions.assertThat(notExistList).isEmpty();
-        final List<String> files = plainPermissionManager.getAllAclFiles(confHome.getAbsolutePath());
-        Assertions.assertThat(files).isNotEmpty();
-    }
-
-    @Test
-    public void loadTest() {
-        plainPermissionManager.load();
-        final Map<String, DataVersion> map = plainPermissionManager.getDataVersionMap();
-        Assertions.assertThat(map).isNotEmpty();
-    }
-
-    @Test
-    public void updateAclConfigFileVersionTest() {
-        String aclFileName = "test_plain_acl";
-        PlainAccessData updateAclConfigMap = new PlainAccessData();
-        List<PlainAccessData.DataVersion> versionElement = new ArrayList<>();
-        PlainAccessData.DataVersion accountsMap = new PlainAccessData.DataVersion();
-        accountsMap.setCounter(1);
-        accountsMap.setTimestamp(System.currentTimeMillis());
-        versionElement.add(accountsMap);
-
-        updateAclConfigMap.setDataVersion(versionElement);
-        final PlainAccessData map = plainPermissionManager.updateAclConfigFileVersion(aclFileName, updateAclConfigMap);
-        final List<PlainAccessData.DataVersion> version = map.getDataVersion();
-        Assert.assertEquals(2L, version.get(0).getCounter());
-    }
-
-    @Test
-    public void createAclAccessConfigMapTest() {
-        PlainAccessConfig existedAccountMap = new PlainAccessConfig();
-        plainAccessConfig.setAccessKey("admin123");
-        plainAccessConfig.setSecretKey("12345678");
-        plainAccessConfig.setWhiteRemoteAddress("192.168.1.1");
-        plainAccessConfig.setAdmin(false);
-        plainAccessConfig.setDefaultGroupPerm(AclConstants.SUB_PUB);
-        plainAccessConfig.setTopicPerms(Arrays.asList(DEFAULT_TOPIC + "=" + AclConstants.PUB));
-        plainAccessConfig.setGroupPerms(Lists.newArrayList("groupA=SUB"));
-
-        final PlainAccessConfig map = plainPermissionManager.createAclAccessConfigMap(existedAccountMap, plainAccessConfig);
-        Assert.assertEquals(AclConstants.SUB_PUB, map.getDefaultGroupPerm());
-        Assert.assertEquals("groupA=SUB", map.getGroupPerms().get(0));
-        Assert.assertEquals("12345678", map.getSecretKey());
-        Assert.assertEquals("admin123", map.getAccessKey());
-        Assert.assertEquals("192.168.1.1", map.getWhiteRemoteAddress());
-        Assert.assertEquals("topic-acl=PUB", map.getTopicPerms().get(0));
-        Assert.assertEquals(false, map.isAdmin());
-    }
-
-    @Test
-    public void deleteAccessConfigTest() throws InterruptedException {
-        // delete not exist accessConfig
-        final boolean flag1 = plainPermissionManager.deleteAccessConfig("test_delete");
-        assert !flag1;
-
-        plainAccessConfig.setAccessKey("test_delete");
-        plainAccessConfig.setSecretKey("12345678");
-        plainAccessConfig.setWhiteRemoteAddress("192.168.1.1");
-        plainAccessConfig.setAdmin(false);
-        plainAccessConfig.setDefaultGroupPerm(AclConstants.SUB_PUB);
-        plainAccessConfig.setTopicPerms(Arrays.asList(DEFAULT_TOPIC + "=" + AclConstants.PUB));
-        plainAccessConfig.setGroupPerms(Lists.newArrayList("groupA=SUB"));
-        plainPermissionManager.updateAccessConfig(plainAccessConfig);
-
-        //delete existed accessConfig
-        final boolean flag2 = plainPermissionManager.deleteAccessConfig("test_delete");
-        assert flag2;
-
-    }
-
-    @Test
-    public void updateGlobalWhiteAddrsConfigTest() {
-        final boolean flag = plainPermissionManager.updateGlobalWhiteAddrsConfig(Lists.newArrayList("192.168.1.2"));
-        assert flag;
-        final AclConfig config = plainPermissionManager.getAllAclConfig();
-        Assert.assertEquals(true, config.getGlobalWhiteAddrs().contains("192.168.1.2"));
-    }
-
-}
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java
deleted file mode 100644
index df7dd0c5460..00000000000
--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.acl.plain;
-
-import org.apache.rocketmq.acl.common.AclException;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class RemoteAddressStrategyTest {
-
-    RemoteAddressStrategyFactory remoteAddressStrategyFactory = new RemoteAddressStrategyFactory();
-
-    @Test
-    public void netAddressStrategyFactoryExceptionTest() {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource).getClass(),
-            RemoteAddressStrategyFactory.BlankRemoteAddressStrategy.class);
-    }
-
-    @Test
-    public void netAddressStrategyFactoryTest() {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-
-        plainAccessResource.setWhiteRemoteAddress("*");
-        RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy, RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY);
-
-        plainAccessResource.setWhiteRemoteAddress("*.*.*.*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy, RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.1");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.OneRemoteAddressStrategy.class);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.1,127.0.0.2,127.0.0.3");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.MultipleRemoteAddressStrategy.class);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.{1,2,3}");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.MultipleRemoteAddressStrategy.class);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.1-200");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.1-20.*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class);
-
-        plainAccessResource.setWhiteRemoteAddress("");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.BlankRemoteAddressStrategy.class);
-
-//        IPv6 test
-        plainAccessResource.setWhiteRemoteAddress("*:*:*:*:*:*:*:*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy, RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY);
-
-        plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:326b");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.OneRemoteAddressStrategy.class);
-
-        plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:3261,1050::0005:0600:300c:3262,1050::0005:0600:300c:3263");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.MultipleRemoteAddressStrategy.class);
-
-        plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:3261:{1,2,3}");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.MultipleRemoteAddressStrategy.class);
-
-        plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:3261:1-200");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class);
-
-        plainAccessResource.setWhiteRemoteAddress("1050:0005:0600:300c:3261:*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class);
-
-        plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:3261:1-20:*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class);
-    }
-
-    @Test(expected = AclException.class)
-    public void verifyTest() {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.1");
-        remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        plainAccessResource.setWhiteRemoteAddress("256.0.0.1");
-        remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        plainAccessResource.setWhiteRemoteAddress("::1ggg");
-        remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-    }
-
-    @Test
-    public void nullNetAddressStrategyTest() {
-        boolean isMatch = RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY.match(new PlainAccessResource());
-        Assert.assertTrue(isMatch);
-    }
-
-    @Test
-    public void blankNetAddressStrategyTest() {
-        boolean isMatch = RemoteAddressStrategyFactory.BLANK_NET_ADDRESS_STRATEGY.match(new PlainAccessResource());
-        Assert.assertFalse(isMatch);
-    }
-
-    @Test
-    public void oneNetAddressStrategyTest() {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.1");
-        RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        plainAccessResource.setWhiteRemoteAddress("");
-        boolean match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertFalse(match);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.2");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertFalse(match);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.1");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertTrue(match);
-
-//        Ipv6 test
-        plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setWhiteRemoteAddress("::1");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        plainAccessResource.setWhiteRemoteAddress("");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertFalse(match);
-
-        plainAccessResource.setWhiteRemoteAddress("::2");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertFalse(match);
-
-        plainAccessResource.setWhiteRemoteAddress("::1");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertTrue(match);
-
-        plainAccessResource.setWhiteRemoteAddress("0000:0000:0000:0000:0000:0000:0000:0001");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertTrue(match);
-    }
-
-    @Test
-    public void multipleNetAddressStrategyTest() {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.1,127.0.0.2,127.0.0.3");
-        RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        multipleNetAddressStrategyTest(remoteAddressStrategy);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.{1,2,3}");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        multipleNetAddressStrategyTest(remoteAddressStrategy);
-
-        plainAccessResource.setWhiteRemoteAddress("192.100-150.*.*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        plainAccessResource.setWhiteRemoteAddress("192.130.0.2");
-        boolean match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertTrue(match);
-
-        plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:1,1050::0005:0600:300c:2,1050::0005:0600:300c:3");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        multipleIPv6NetAddressStrategyTest(remoteAddressStrategy);
-
-        plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:{1,2,3}");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        multipleIPv6NetAddressStrategyTest(remoteAddressStrategy);
-
-    }
-
-    @Test(expected = AclException.class)
-    public void multipleNetAddressStrategyExceptionTest() {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.1,2,3}");
-        remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        plainAccessResource.setWhiteRemoteAddress("::1,2,3}");
-        remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        plainAccessResource.setWhiteRemoteAddress("192.168.1.{1}");
-        remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        plainAccessResource.setWhiteRemoteAddress("192.168.1.{1,2}");
-        remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        plainAccessResource.setWhiteRemoteAddress("192.168.{1}");
-        remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        plainAccessResource.setWhiteRemoteAddress("{192.168.1}");
-        remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        plainAccessResource.setWhiteRemoteAddress("{192.168.1.1}");
-        remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-    }
-
-    private void multipleNetAddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy) {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.1");
-        boolean match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertTrue(match);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.2");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertTrue(match);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.3");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertTrue(match);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.4");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertFalse(match);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.0");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertFalse(match);
-
-    }
-
-    private void multipleIPv6NetAddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy) {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:1");
-        boolean match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertTrue(match);
-
-        plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:2");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertTrue(match);
-
-        plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:3");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertTrue(match);
-
-        plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:4");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertFalse(match);
-
-        plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:0");
-        match = remoteAddressStrategy.match(plainAccessResource);
-        Assert.assertFalse(match);
-
-    }
-
-    @Test
-    public void rangeNetAddressStrategyTest() {
-        String head = "127.0.0.";
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.1-200");
-        RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        rangeNetAddressStrategyTest(remoteAddressStrategy, head, 1, 200, true);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.0.*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        rangeNetAddressStrategyTest(remoteAddressStrategy, head, 0, 255, true);
-
-        plainAccessResource.setWhiteRemoteAddress("127.0.1-200.*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        rangeNetAddressStrategyThirdlyTest(remoteAddressStrategy, head, 1, 200);
-
-        plainAccessResource.setWhiteRemoteAddress("127.*.*.*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        rangeNetAddressStrategyTest(remoteAddressStrategy, head, 0, 255, true);
-
-        plainAccessResource.setWhiteRemoteAddress("127.1-150.*.*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        rangeNetAddressStrategyThirdlyTest(remoteAddressStrategy, head, 1, 200);
-
-//        IPv6 test
-        head = "1050::0005:0600:300c:";
-        plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:1-200");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        rangeIPv6NetAddressStrategyTest(remoteAddressStrategy, head, "1", "200", true);
-
-        plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        rangeIPv6NetAddressStrategyTest(remoteAddressStrategy, head, "0", "ffff", true);
-
-        plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:3001:*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        rangeIPv6NetAddressStrategyTest(remoteAddressStrategy, head, "0", "ffff", false);
-
-        head = "1050::0005:0600:300c:1:";
-        plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:1-200:*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        rangeIPv6NetAddressStrategyTest(remoteAddressStrategy, head, "0", "ffff", true);
-
-        head = "1050::0005:0600:300c:201:";
-        plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:1-200:*");
-        remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-        rangeIPv6NetAddressStrategyTest(remoteAddressStrategy, head, "0", "ffff", false);
-
-    }
-
-    private void rangeNetAddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy, String head, int start,
-        int end,
-        boolean isFalse) {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        for (int i = -10; i < 300; i++) {
-            plainAccessResource.setWhiteRemoteAddress(head + i);
-            boolean match = remoteAddressStrategy.match(plainAccessResource);
-            if (isFalse && i >= start && i <= end) {
-                Assert.assertTrue(match);
-                continue;
-            }
-            Assert.assertFalse(match);
-
-        }
-    }
-
-    private void rangeNetAddressStrategyThirdlyTest(RemoteAddressStrategy remoteAddressStrategy, String head, int start,
-        int end) {
-        String newHead;
-        for (int i = -10; i < 300; i++) {
-            newHead = head + i;
-            if (i >= start && i <= end) {
-                rangeNetAddressStrategyTest(remoteAddressStrategy, newHead, 0, 255, false);
-            }
-        }
-    }
-
-    private void rangeIPv6NetAddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy, String head, String start,
-        String end,
-        boolean isFalse) {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        for (int i = -10; i < 65536 + 100; i++) {
-            String hex = Integer.toHexString(i);
-            plainAccessResource.setWhiteRemoteAddress(head + hex);
-            boolean match = remoteAddressStrategy.match(plainAccessResource);
-            int startNum = Integer.parseInt(start, 16);
-            int endNum = Integer.parseInt(end, 16);
-            if (isFalse && i >= startNum && i <= endNum) {
-                Assert.assertTrue(match);
-                continue;
-            }
-            Assert.assertFalse(match);
-
-        }
-    }
-
-    @Test(expected = AclException.class)
-    public void rangeNetAddressStrategyExceptionStartGreaterEndTest() {
-        rangeNetAddressStrategyExceptionTest("127.0.0.2-1");
-    }
-
-    @Test(expected = AclException.class)
-    public void rangeNetAddressStrategyExceptionScopeTest() {
-        rangeNetAddressStrategyExceptionTest("127.0.0.-1-200");
-    }
-
-    @Test(expected = AclException.class)
-    public void rangeNetAddressStrategyExceptionScopeTwoTest() {
-        rangeNetAddressStrategyExceptionTest("127.0.0.0-256");
-    }
-
-    private void rangeNetAddressStrategyExceptionTest(String netAddress) {
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setWhiteRemoteAddress(netAddress);
-        remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource);
-    }
-
-}
diff --git a/acl/src/test/resources/access_acl_conf/acl/plain_acl.yml b/acl/src/test/resources/access_acl_conf/acl/plain_acl.yml
deleted file mode 100644
index 28a8c488805..00000000000
--- a/acl/src/test/resources/access_acl_conf/acl/plain_acl.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-accounts:
-  - accessKey: rocketmq3
-    secretKey: 12345678
-    admin: false
-    defaultTopicPerm: DENY
-    defaultGroupPerm: DENY
-    topicPerms:
-      - topicA=PUB
-      - topicB=SUB
-      - topicC=PUB|SUB
-      - topicD=DENY
-    groupPerms:
-      - groupB=SUB
-      - groupC=PUB|SUB
-      - groupD=DENY
-
diff --git a/acl/src/test/resources/both_acl_file_folder_conf/conf/acl/plain_acl.yml b/acl/src/test/resources/both_acl_file_folder_conf/conf/acl/plain_acl.yml
deleted file mode 100644
index cf4ea7f4a5b..00000000000
--- a/acl/src/test/resources/both_acl_file_folder_conf/conf/acl/plain_acl.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## no global white addresses in this file, define them in ../plain_acl.yml
-accounts:
-  - accessKey: RocketMQ
-    secretKey: 12345678
-    whiteRemoteAddress: 192.168.0.*
-    admin: false
-    defaultTopicPerm: DENY
-    defaultGroupPerm: SUB
-    topicPerms:
-      - topicA=DENY
-      - topicB=PUB|SUB
-      - topicC=SUB
-    groupPerms:
-      # the group should convert to retry topic
-      - groupA=DENY
-      - groupB=SUB
-      - groupC=SUB
-
-  - accessKey: rocketmq2
-    secretKey: 12345678
-    whiteRemoteAddress: 192.168.1.*
-    # if it is admin, it could access all resources
-    admin: true
-
diff --git a/acl/src/test/resources/conf/acl/plain_acl.yml b/acl/src/test/resources/conf/acl/plain_acl.yml
deleted file mode 100644
index 34e46696d8e..00000000000
--- a/acl/src/test/resources/conf/acl/plain_acl.yml
+++ /dev/null
@@ -1,43 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## suggested format
-
-globalWhiteRemoteAddresses:
-  - 10.10.103.*
-  - 192.168.0.*
-
-accounts:
-  - accessKey: RocketMQ
-    secretKey: 12345678
-    whiteRemoteAddress: 192.168.0.*
-    admin: false
-    defaultTopicPerm: DENY
-    defaultGroupPerm: SUB
-    topicPerms:
-      - topicA=DENY
-      - topicB=PUB|SUB
-      - topicC=SUB
-    groupPerms:
-      # the group should convert to retry topic
-      - groupA=DENY
-      - groupB=SUB
-      - groupC=SUB
-
-  - accessKey: rocketmq2
-    secretKey: 12345678
-    whiteRemoteAddress: 192.168.1.*
-    # if it is admin, it could access all resources
-    admin: true
diff --git a/acl/src/test/resources/conf/plain_acl.yml b/acl/src/test/resources/conf/plain_acl.yml
deleted file mode 100644
index 59bd6d4ff29..00000000000
--- a/acl/src/test/resources/conf/plain_acl.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## suggested format
-
-globalWhiteRemoteAddresses:
-- 10.10.103.*
-- 192.168.0.*
-accounts:
-- accessKey: RocketMQ
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.0.*
-  admin: false
-  defaultTopicPerm: DENY
-  defaultGroupPerm: SUB
-  topicPerms:
-  - topicA=DENY
-  - topicB=PUB|SUB
-  - topicC=SUB
-  groupPerms:
-  - groupA=DENY
-  - groupB=SUB
-  - groupC=SUB
-- accessKey: rocketmq2
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.1.*
-  admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_bak.yml b/acl/src/test/resources/conf/plain_acl_bak.yml
deleted file mode 100644
index 59bd6d4ff29..00000000000
--- a/acl/src/test/resources/conf/plain_acl_bak.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## suggested format
-
-globalWhiteRemoteAddresses:
-- 10.10.103.*
-- 192.168.0.*
-accounts:
-- accessKey: RocketMQ
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.0.*
-  admin: false
-  defaultTopicPerm: DENY
-  defaultGroupPerm: SUB
-  topicPerms:
-  - topicA=DENY
-  - topicB=PUB|SUB
-  - topicC=SUB
-  groupPerms:
-  - groupA=DENY
-  - groupB=SUB
-  - groupC=SUB
-- accessKey: rocketmq2
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.1.*
-  admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_correct.yml b/acl/src/test/resources/conf/plain_acl_correct.yml
deleted file mode 100644
index 59bd6d4ff29..00000000000
--- a/acl/src/test/resources/conf/plain_acl_correct.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## suggested format
-
-globalWhiteRemoteAddresses:
-- 10.10.103.*
-- 192.168.0.*
-accounts:
-- accessKey: RocketMQ
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.0.*
-  admin: false
-  defaultTopicPerm: DENY
-  defaultGroupPerm: SUB
-  topicPerms:
-  - topicA=DENY
-  - topicB=PUB|SUB
-  - topicC=SUB
-  groupPerms:
-  - groupA=DENY
-  - groupB=SUB
-  - groupC=SUB
-- accessKey: rocketmq2
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.1.*
-  admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_delete.yml b/acl/src/test/resources/conf/plain_acl_delete.yml
deleted file mode 100644
index 59bd6d4ff29..00000000000
--- a/acl/src/test/resources/conf/plain_acl_delete.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## suggested format
-
-globalWhiteRemoteAddresses:
-- 10.10.103.*
-- 192.168.0.*
-accounts:
-- accessKey: RocketMQ
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.0.*
-  admin: false
-  defaultTopicPerm: DENY
-  defaultGroupPerm: SUB
-  topicPerms:
-  - topicA=DENY
-  - topicB=PUB|SUB
-  - topicC=SUB
-  groupPerms:
-  - groupA=DENY
-  - groupB=SUB
-  - groupC=SUB
-- accessKey: rocketmq2
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.1.*
-  admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_global_white_addrs.yml b/acl/src/test/resources/conf/plain_acl_global_white_addrs.yml
deleted file mode 100644
index 59bd6d4ff29..00000000000
--- a/acl/src/test/resources/conf/plain_acl_global_white_addrs.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## suggested format
-
-globalWhiteRemoteAddresses:
-- 10.10.103.*
-- 192.168.0.*
-accounts:
-- accessKey: RocketMQ
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.0.*
-  admin: false
-  defaultTopicPerm: DENY
-  defaultGroupPerm: SUB
-  topicPerms:
-  - topicA=DENY
-  - topicB=PUB|SUB
-  - topicC=SUB
-  groupPerms:
-  - groupA=DENY
-  - groupB=SUB
-  - groupC=SUB
-- accessKey: rocketmq2
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.1.*
-  admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_update_create.yml b/acl/src/test/resources/conf/plain_acl_update_create.yml
deleted file mode 100644
index 59bd6d4ff29..00000000000
--- a/acl/src/test/resources/conf/plain_acl_update_create.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## suggested format
-
-globalWhiteRemoteAddresses:
-- 10.10.103.*
-- 192.168.0.*
-accounts:
-- accessKey: RocketMQ
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.0.*
-  admin: false
-  defaultTopicPerm: DENY
-  defaultGroupPerm: SUB
-  topicPerms:
-  - topicA=DENY
-  - topicB=PUB|SUB
-  - topicC=SUB
-  groupPerms:
-  - groupA=DENY
-  - groupB=SUB
-  - groupC=SUB
-- accessKey: rocketmq2
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.1.*
-  admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_with_no_accouts.yml b/acl/src/test/resources/conf/plain_acl_with_no_accouts.yml
deleted file mode 100644
index 939f7c98ca6..00000000000
--- a/acl/src/test/resources/conf/plain_acl_with_no_accouts.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## suggested format
-
-globalWhiteRemoteAddresses:
-- 10.10.103.*
-- 192.168.0.*
\ No newline at end of file
diff --git a/acl/src/test/resources/conf/watch/plain_acl_watch.yml b/acl/src/test/resources/conf/watch/plain_acl_watch.yml
deleted file mode 100644
index 9d2c3954941..00000000000
--- a/acl/src/test/resources/conf/watch/plain_acl_watch.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## suggested format
-accounts:
-- accessKey: watchrocketmq
-  secretKey: 12345678
-  whiteRemoteAddress: 127.0.0.1
-  admin: true
-- accessKey: watchrocketmq1
-  secretKey: 88888888
-  whiteRemoteAddress: 127.0.0.1
-  admin: false
diff --git a/acl/src/test/resources/empty_acl_folder_conf/conf/plain_acl.yml b/acl/src/test/resources/empty_acl_folder_conf/conf/plain_acl.yml
deleted file mode 100644
index 6ade46723b7..00000000000
--- a/acl/src/test/resources/empty_acl_folder_conf/conf/plain_acl.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-
-globalWhiteRemoteAddresses:
-  - 10.10.103.*
-  - 192.168.0.*
diff --git a/acl/src/test/resources/only_acl_folder_conf/conf/acl/plain_acl.yml b/acl/src/test/resources/only_acl_folder_conf/conf/acl/plain_acl.yml
deleted file mode 100644
index cf4ea7f4a5b..00000000000
--- a/acl/src/test/resources/only_acl_folder_conf/conf/acl/plain_acl.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## no global white addresses in this file, define them in ../plain_acl.yml
-accounts:
-  - accessKey: RocketMQ
-    secretKey: 12345678
-    whiteRemoteAddress: 192.168.0.*
-    admin: false
-    defaultTopicPerm: DENY
-    defaultGroupPerm: SUB
-    topicPerms:
-      - topicA=DENY
-      - topicB=PUB|SUB
-      - topicC=SUB
-    groupPerms:
-      # the group should convert to retry topic
-      - groupA=DENY
-      - groupB=SUB
-      - groupC=SUB
-
-  - accessKey: rocketmq2
-    secretKey: 12345678
-    whiteRemoteAddress: 192.168.1.*
-    # if it is admin, it could access all resources
-    admin: true
-
diff --git a/acl/src/test/resources/rmq.logback-test.xml b/acl/src/test/resources/rmq.logback-test.xml
deleted file mode 100644
index 8695d52d57c..00000000000
--- a/acl/src/test/resources/rmq.logback-test.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-  -->
-<configuration>
-
-    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
-        <layout class="ch.qos.logback.classic.PatternLayout">
-            <Pattern>
-                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n
-            </Pattern>
-        </layout>
-    </appender>
-
-    <logger name="org.apache.rocketmq" level="error" additivity="false">
-        <appender-ref ref="CONSOLE"/>
-    </logger>
-
-    <root level="error">
-        <appender-ref ref="CONSOLE"/>
-    </root>
-
-</configuration>
\ No newline at end of file
diff --git a/acl/src/test/resources/update_global_white_addr/conf/acl/empty.yml b/acl/src/test/resources/update_global_white_addr/conf/acl/empty.yml
deleted file mode 100644
index 52ff50c2bef..00000000000
--- a/acl/src/test/resources/update_global_white_addr/conf/acl/empty.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## suggested format
-
-accounts: []
diff --git a/acl/src/test/resources/update_global_white_addr/conf/acl/plain_acl.yml b/acl/src/test/resources/update_global_white_addr/conf/acl/plain_acl.yml
deleted file mode 100644
index 59bd6d4ff29..00000000000
--- a/acl/src/test/resources/update_global_white_addr/conf/acl/plain_acl.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## suggested format
-
-globalWhiteRemoteAddresses:
-- 10.10.103.*
-- 192.168.0.*
-accounts:
-- accessKey: RocketMQ
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.0.*
-  admin: false
-  defaultTopicPerm: DENY
-  defaultGroupPerm: SUB
-  topicPerms:
-  - topicA=DENY
-  - topicB=PUB|SUB
-  - topicC=SUB
-  groupPerms:
-  - groupA=DENY
-  - groupB=SUB
-  - groupC=SUB
-- accessKey: rocketmq2
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.1.*
-  admin: true
diff --git a/acl/src/test/resources/update_global_white_addr/conf/plain_acl.yml b/acl/src/test/resources/update_global_white_addr/conf/plain_acl.yml
deleted file mode 100644
index 64c6e34169c..00000000000
--- a/acl/src/test/resources/update_global_white_addr/conf/plain_acl.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-## suggested format
-
-accounts:
-- accessKey: RocketMQ
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.0.*
-  admin: false
-  defaultTopicPerm: DENY
-  defaultGroupPerm: SUB
-  topicPerms:
-  - topicA=DENY
-  - topicB=PUB|SUB
-  - topicC=SUB
-  groupPerms:
-  - groupA=DENY
-  - groupB=SUB
-  - groupC=SUB
-- accessKey: rocketmq2
-  secretKey: 12345678
-  whiteRemoteAddress: 192.168.1.*
-  admin: true
diff --git a/acl/BUILD.bazel b/auth/BUILD.bazel
similarity index 67%
rename from acl/BUILD.bazel
rename to auth/BUILD.bazel
index ac6ac65c779..bc15ca3c9e0 100644
--- a/acl/BUILD.bazel
+++ b/auth/BUILD.bazel
@@ -17,26 +17,27 @@
 load("//bazel:GenTestRules.bzl", "GenTestRules")
 
 java_library(
-    name = "acl",
+    name = "auth",
     srcs = glob(["src/main/java/**/*.java"]),
     visibility = ["//visibility:public"],
     deps = [
         "//common",
         "//remoting",
         "//srvutil",
-        "@maven//:com_alibaba_fastjson",
-        "@maven//:com_github_luben_zstd_jni",
-        "@maven//:com_google_guava_guava",
-        "@maven//:com_google_protobuf_protobuf_java",
+        "//client",
         "@maven//:commons_codec_commons_codec",
-        "@maven//:commons_validator_commons_validator",
-        "@maven//:io_netty_netty_all",
         "@maven//:org_apache_commons_commons_lang3",
+        "@maven//:commons_collections_commons_collections",
+        "@maven//:com_alibaba_fastjson2_fastjson2",
         "@maven//:org_apache_rocketmq_rocketmq_proto",
-        "@maven//:org_lz4_lz4_java",
-        "@maven//:org_yaml_snakeyaml",
-        "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
-        "@maven//:io_github_aliyunmq_rocketmq_logback_classic",
+        "@maven//:org_slf4j_slf4j_api",
+        "@maven//:com_github_ben_manes_caffeine_caffeine",
+        "@maven//:io_grpc_grpc_api",
+        "@maven//:com_google_protobuf_protobuf_java",
+        "@maven//:com_google_protobuf_protobuf_java_util",
+        "@maven//:io_netty_netty_all",
+        "@maven//:com_google_guava_guava",
+        "@maven//:org_apache_rocketmq_rocketmq_rocksdb",
     ],
 )
 
@@ -46,28 +47,29 @@ java_library(
     resources = glob(["src/test/resources/**/*.yml"]),
     visibility = ["//visibility:public"],
     deps = [
-        ":acl",
+        ":auth",
         "//:test_deps",
         "//common",
         "//remoting",
-        "@maven//:com_alibaba_fastjson",
-        "@maven//:com_google_guava_guava",
+        "//client",
         "@maven//:commons_codec_commons_codec",
-        "@maven//:io_netty_netty_all",
         "@maven//:org_apache_commons_commons_lang3",
-        "@maven//:org_springframework_spring_core",
-        "@maven//:org_yaml_snakeyaml",
+        "@maven//:commons_collections_commons_collections",
+        "@maven//:com_alibaba_fastjson2_fastjson2",
+        "@maven//:org_apache_rocketmq_rocketmq_proto",
+        "@maven//:org_slf4j_slf4j_api",
+        "@maven//:com_github_ben_manes_caffeine_caffeine",
+        "@maven//:io_grpc_grpc_api",
+        "@maven//:com_google_protobuf_protobuf_java",
+        "@maven//:com_google_protobuf_protobuf_java_util",
+        "@maven//:io_netty_netty_all",
+        "@maven//:com_google_guava_guava",
+        "@maven//:org_apache_rocketmq_rocketmq_rocksdb",
     ],
 )
 
 GenTestRules(
     name = "GeneratedTestRules",
-    # The following tests are not hermetic. Fix them later.
-    exclude_tests = [
-    ],
-    medium_tests = [
-        "src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest",
-    ],
     test_files = glob(["src/test/java/**/*Test.java"]),
     deps = [
         ":tests",
diff --git a/acl/pom.xml b/auth/pom.xml
similarity index 65%
rename from acl/pom.xml
rename to auth/pom.xml
index 8a296e5ae9e..73b4d87dc6f 100644
--- a/acl/pom.xml
+++ b/auth/pom.xml
@@ -1,22 +1,22 @@
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
-	license agreements. See the NOTICE file distributed with this work for additional 
-	information regarding copyright ownership. The ASF licenses this file to 
-	You under the Apache License, Version 2.0 (the "License"); you may not use 
-	this file except in compliance with the License. You may obtain a copy of 
-	the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
-	by applicable law or agreed to in writing, software distributed under the 
-	License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
-	OF ANY KIND, either express or implied. See the License for the specific 
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+	license agreements. See the NOTICE file distributed with this work for additional
+	information regarding copyright ownership. The ASF licenses this file to
+	You under the Apache License, Version 2.0 (the "License"); you may not use
+	this file except in compliance with the License. You may obtain a copy of
+	the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+	by applicable law or agreed to in writing, software distributed under the
+	License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+	OF ANY KIND, either express or implied. See the License for the specific
 	language governing permissions and limitations under the License. -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
-    <artifactId>rocketmq-acl</artifactId>
-    <name>rocketmq-acl ${project.version}</name>
+    <artifactId>rocketmq-auth</artifactId>
+    <name>rocketmq-auth ${project.version}</name>
 
     <properties>
         <project.root>${basedir}/..</project.root>
@@ -29,27 +29,7 @@
         </dependency>
         <dependency>
             <groupId>${project.groupId}</groupId>
-            <artifactId>rocketmq-remoting</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>${project.groupId}</groupId>
-            <artifactId>rocketmq-common</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>${project.groupId}</groupId>
-            <artifactId>rocketmq-srvutil</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.github.aliyunmq</groupId>
-            <artifactId>rocketmq-slf4j-api</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.github.aliyunmq</groupId>
-            <artifactId>rocketmq-logback-classic</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.yaml</groupId>
-            <artifactId>snakeyaml</artifactId>
+            <artifactId>rocketmq-client</artifactId>
         </dependency>
         <dependency>
             <groupId>commons-codec</groupId>
@@ -59,19 +39,27 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
         </dependency>
-        <dependency>
-            <groupId>commons-validator</groupId>
-            <artifactId>commons-validator</artifactId>
-        </dependency>
         <dependency>
             <groupId>com.google.protobuf</groupId>
             <artifactId>protobuf-java-util</artifactId>
         </dependency>
-
         <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-core</artifactId>
-            <scope>test</scope>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.github.ben-manes.caffeine</groupId>
+            <artifactId>caffeine</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.checkerframework</groupId>
+                    <artifactId>checker-qual</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
         </dependency>
     </dependencies>
 
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/AuthenticationEvaluator.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/AuthenticationEvaluator.java
new file mode 100644
index 00000000000..3b7d45d18e8
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/AuthenticationEvaluator.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication;
+
+import java.util.function.Supplier;
+import org.apache.rocketmq.auth.authentication.context.AuthenticationContext;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.authentication.strategy.AuthenticationStrategy;
+import org.apache.rocketmq.auth.config.AuthConfig;
+
+public class AuthenticationEvaluator {
+
+    private final AuthenticationStrategy authenticationStrategy;
+
+    public AuthenticationEvaluator(AuthConfig authConfig) {
+        this(authConfig, null);
+    }
+
+    public AuthenticationEvaluator(AuthConfig authConfig, Supplier<?> metadataService) {
+        this.authenticationStrategy = AuthenticationFactory.getStrategy(authConfig, metadataService);
+    }
+
+    public void evaluate(AuthenticationContext context) {
+        if (context == null) {
+            return;
+        }
+        this.authenticationStrategy.evaluate(context);
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/builder/AuthenticationContextBuilder.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/builder/AuthenticationContextBuilder.java
new file mode 100644
index 00000000000..0176f01a34c
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/builder/AuthenticationContextBuilder.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.builder;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public interface AuthenticationContextBuilder<AuthenticationContext> {
+
+    AuthenticationContext build(Metadata metadata, GeneratedMessageV3 request);
+
+    AuthenticationContext build(ChannelHandlerContext context, RemotingCommand request);
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/builder/DefaultAuthenticationContextBuilder.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/builder/DefaultAuthenticationContextBuilder.java
new file mode 100644
index 00000000000..c1e970fa6eb
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/builder/DefaultAuthenticationContextBuilder.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.builder;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import io.netty.channel.ChannelHandlerContext;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.acl.common.AclUtils;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.common.MQVersion;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.constant.CommonConstants;
+import org.apache.rocketmq.common.constant.GrpcConstants;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class DefaultAuthenticationContextBuilder implements AuthenticationContextBuilder<DefaultAuthenticationContext> {
+
+    private static final String CREDENTIAL = "Credential";
+    private static final String SIGNATURE = "Signature";
+
+    @Override
+    public DefaultAuthenticationContext build(Metadata metadata, GeneratedMessageV3 request) {
+        try {
+            DefaultAuthenticationContext context = new DefaultAuthenticationContext();
+            context.setChannelId(metadata.get(GrpcConstants.CHANNEL_ID));
+            context.setRpcCode(request.getDescriptorForType().getFullName());
+            String authorization = metadata.get(GrpcConstants.AUTHORIZATION);
+            if (StringUtils.isEmpty(authorization)) {
+                return context;
+            }
+            String datetime = metadata.get(GrpcConstants.DATE_TIME);
+            if (StringUtils.isEmpty(datetime)) {
+                throw new AuthenticationException("datetime is null.");
+            }
+
+            String[] result = authorization.split(CommonConstants.SPACE, 2);
+            if (result.length != 2) {
+                throw new AuthenticationException("authentication header is incorrect.");
+            }
+            String[] keyValues = result[1].split(CommonConstants.COMMA);
+            for (String keyValue : keyValues) {
+                String[] kv = keyValue.trim().split(CommonConstants.EQUAL, 2);
+                int kvLength = kv.length;
+                if (kv.length != 2) {
+                    throw new AuthenticationException("authentication keyValues length is incorrect, actual length={}.", kvLength);
+                }
+                String authItem = kv[0];
+                if (CREDENTIAL.equals(authItem)) {
+                    String[] credential = kv[1].split(CommonConstants.SLASH);
+                    int credentialActualLength = credential.length;
+                    if (credentialActualLength == 0) {
+                        throw new AuthenticationException("authentication credential length is incorrect, actual length={}.", credentialActualLength);
+                    }
+                    context.setUsername(credential[0]);
+                    continue;
+                }
+                if (SIGNATURE.equals(authItem)) {
+                    context.setSignature(this.hexToBase64(kv[1]));
+                }
+            }
+
+            context.setContent(datetime.getBytes(StandardCharsets.UTF_8));
+
+            return context;
+        } catch (AuthenticationException e) {
+            throw e;
+        } catch (Throwable e) {
+            throw new AuthenticationException("create authentication context error.", e);
+        }
+    }
+
+    @Override
+    public DefaultAuthenticationContext build(ChannelHandlerContext context, RemotingCommand request) {
+        HashMap<String, String> fields = request.getExtFields();
+        DefaultAuthenticationContext result = new DefaultAuthenticationContext();
+        result.setChannelId(context.channel().id().asLongText());
+        result.setRpcCode(String.valueOf(request.getCode()));
+        if (MapUtils.isEmpty(fields)) {
+            return result;
+        }
+        if (!fields.containsKey(SessionCredentials.ACCESS_KEY)) {
+            return result;
+        }
+        result.setUsername(fields.get(SessionCredentials.ACCESS_KEY));
+        result.setSignature(fields.get(SessionCredentials.SIGNATURE));
+        // Content
+        SortedMap<String, String> map = new TreeMap<>();
+        for (Map.Entry<String, String> entry : fields.entrySet()) {
+            if (request.getVersion() <= MQVersion.Version.V4_9_3.ordinal() &&
+                MixAll.UNIQUE_MSG_QUERY_FLAG.equals(entry.getKey())) {
+                continue;
+            }
+            if (!SessionCredentials.SIGNATURE.equals(entry.getKey())) {
+                map.put(entry.getKey(), entry.getValue());
+            }
+        }
+        result.setContent(AclUtils.combineRequestContent(request, map));
+        return result;
+    }
+
+    public String hexToBase64(String input) throws DecoderException {
+        byte[] bytes = Hex.decodeHex(input);
+        return Base64.encodeBase64String(bytes);
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/chain/DefaultAuthenticationHandler.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/chain/DefaultAuthenticationHandler.java
new file mode 100644
index 00000000000..4b50de756ab
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/chain/DefaultAuthenticationHandler.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.chain;
+
+import java.security.MessageDigest;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.acl.common.AclSigner;
+import org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;
+import org.apache.rocketmq.auth.authentication.enums.UserStatus;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authentication.provider.AuthenticationMetadataProvider;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.chain.Handler;
+import org.apache.rocketmq.common.chain.HandlerChain;
+
+public class DefaultAuthenticationHandler implements Handler<DefaultAuthenticationContext, CompletableFuture<Void>> {
+
+    private final AuthenticationMetadataProvider authenticationMetadataProvider;
+
+    public DefaultAuthenticationHandler(AuthConfig config, Supplier<?> metadataService) {
+        this.authenticationMetadataProvider = AuthenticationFactory.getMetadataProvider(config, metadataService);
+    }
+
+    @Override
+    public CompletableFuture<Void> handle(DefaultAuthenticationContext context,
+        HandlerChain<DefaultAuthenticationContext, CompletableFuture<Void>> chain) {
+        return getUser(context).thenAccept(user -> doAuthenticate(context, user));
+    }
+
+    protected CompletableFuture<User> getUser(DefaultAuthenticationContext context) {
+        if (this.authenticationMetadataProvider == null) {
+            throw new AuthenticationException("The authenticationMetadataProvider is not configured");
+        }
+        if (StringUtils.isEmpty(context.getUsername())) {
+            throw new AuthenticationException("username cannot be null.");
+        }
+        return this.authenticationMetadataProvider.getUser(context.getUsername());
+    }
+
+    protected void doAuthenticate(DefaultAuthenticationContext context, User user) {
+        if (user == null) {
+            throw new AuthenticationException("User:{} is not found.", context.getUsername());
+        }
+        if (user.getUserStatus() == UserStatus.DISABLE) {
+            throw new AuthenticationException("User:{} is disabled.", context.getUsername());
+        }
+        String signature = AclSigner.calSignature(context.getContent(), user.getPassword());
+        if (context.getSignature() == null
+            || !MessageDigest.isEqual(signature.getBytes(AclSigner.DEFAULT_CHARSET), context.getSignature().getBytes(AclSigner.DEFAULT_CHARSET))) {
+            throw new AuthenticationException("check signature failed.");
+        }
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/context/AuthenticationContext.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/context/AuthenticationContext.java
new file mode 100644
index 00000000000..7c10f044c76
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/context/AuthenticationContext.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.context;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+
+public abstract class AuthenticationContext {
+
+    private String channelId;
+
+    private String rpcCode;
+
+    private Map<String, Object> extInfo;
+
+    public String getChannelId() {
+        return channelId;
+    }
+
+    public void setChannelId(String channelId) {
+        this.channelId = channelId;
+    }
+
+    public String getRpcCode() {
+        return rpcCode;
+    }
+
+    public void setRpcCode(String rpcCode) {
+        this.rpcCode = rpcCode;
+    }
+
+    @SuppressWarnings("unchecked")
+    public <T> T getExtInfo(String key) {
+        if (StringUtils.isBlank(key)) {
+            return null;
+        }
+        if (this.extInfo == null) {
+            return null;
+        }
+        Object value = this.extInfo.get(key);
+        if (value == null) {
+            return null;
+        }
+        return (T) value;
+    }
+
+    public void setExtInfo(String key, Object value) {
+        if (StringUtils.isBlank(key) || value == null) {
+            return;
+        }
+        if (this.extInfo == null) {
+            this.extInfo = new HashMap<>();
+        }
+        this.extInfo.put(key, value);
+    }
+
+    public boolean hasExtInfo(String key) {
+        Object value = getExtInfo(key);
+        return value != null;
+    }
+
+    public Map<String, Object> getExtInfo() {
+        return extInfo;
+    }
+
+    public void setExtInfo(Map<String, Object> extInfo) {
+        this.extInfo = extInfo;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/context/DefaultAuthenticationContext.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/context/DefaultAuthenticationContext.java
new file mode 100644
index 00000000000..a6fff86602c
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/context/DefaultAuthenticationContext.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.context;
+
+public class DefaultAuthenticationContext extends AuthenticationContext {
+
+    private String username;
+
+    private byte[] content;
+
+    private String signature;
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public byte[] getContent() {
+        return content;
+    }
+
+    public void setContent(byte[] content) {
+        this.content = content;
+    }
+
+    public String getSignature() {
+        return signature;
+    }
+
+    public void setSignature(String signature) {
+        this.signature = signature;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/enums/SubjectType.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/enums/SubjectType.java
new file mode 100644
index 00000000000..64a40f96ac2
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/enums/SubjectType.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.enums;
+
+import com.alibaba.fastjson2.annotation.JSONField;
+import org.apache.commons.lang3.StringUtils;
+
+public enum SubjectType {
+
+    USER((byte) 1, "User");
+
+    @JSONField(value = true)
+    private final byte code;
+    private final String name;
+
+    SubjectType(byte code, String name) {
+        this.code = code;
+        this.name = name;
+    }
+
+    public static SubjectType getByName(String name) {
+        for (SubjectType subjectType : SubjectType.values()) {
+            if (StringUtils.equalsIgnoreCase(subjectType.getName(), name)) {
+                return subjectType;
+            }
+        }
+        return null;
+    }
+
+    public byte getCode() {
+        return code;
+    }
+
+    public String getName() {
+        return name;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/enums/UserStatus.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/enums/UserStatus.java
new file mode 100644
index 00000000000..9bb25a2d559
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/enums/UserStatus.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.enums;
+
+import com.alibaba.fastjson2.annotation.JSONField;
+import org.apache.commons.lang3.StringUtils;
+
+public enum UserStatus {
+
+    ENABLE((byte) 1, "enable"),
+
+    DISABLE((byte) 2, "disable");
+
+    @JSONField(value = true)
+    private final byte code;
+
+    private final String name;
+
+    UserStatus(byte code, String name) {
+        this.code = code;
+        this.name = name;
+    }
+
+    public static UserStatus getByName(String name) {
+        for (UserStatus subjectType : UserStatus.values()) {
+            if (StringUtils.equalsIgnoreCase(subjectType.getName(), name)) {
+                return subjectType;
+            }
+        }
+        return null;
+    }
+
+    public byte getCode() {
+        return code;
+    }
+
+    public String getName() {
+        return name;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/enums/UserType.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/enums/UserType.java
new file mode 100644
index 00000000000..6dc786f0f60
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/enums/UserType.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.enums;
+
+import com.alibaba.fastjson2.annotation.JSONField;
+import org.apache.commons.lang3.StringUtils;
+
+public enum UserType {
+
+    SUPER((byte) 1, "Super"),
+
+    NORMAL((byte) 2, "Normal");
+
+    @JSONField(value = true)
+    private final byte code;
+
+    private final String name;
+
+    UserType(byte code, String name) {
+        this.code = code;
+        this.name = name;
+    }
+
+    public static UserType getByName(String name) {
+        for (UserType subjectType : UserType.values()) {
+            if (StringUtils.equalsIgnoreCase(subjectType.getName(), name)) {
+                return subjectType;
+            }
+        }
+        return null;
+    }
+
+    public byte getCode() {
+        return code;
+    }
+
+    public String getName() {
+        return name;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/exception/AuthenticationException.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/exception/AuthenticationException.java
new file mode 100644
index 00000000000..9b66c81bb17
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/exception/AuthenticationException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.exception;
+
+import org.slf4j.helpers.MessageFormatter;
+
+public class AuthenticationException extends RuntimeException {
+
+    public AuthenticationException(String message) {
+        super(message);
+    }
+
+    public AuthenticationException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public AuthenticationException(String messagePattern, Object... argArray) {
+        super(MessageFormatter.arrayFormat(messagePattern, argArray).getMessage());
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/factory/AuthenticationFactory.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/factory/AuthenticationFactory.java
new file mode 100644
index 00000000000..3ba82add5ab
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/factory/AuthenticationFactory.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.factory;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import io.netty.channel.ChannelHandlerContext;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authentication.AuthenticationEvaluator;
+import org.apache.rocketmq.auth.authentication.context.AuthenticationContext;
+import org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;
+import org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManagerImpl;
+import org.apache.rocketmq.auth.authentication.provider.AuthenticationMetadataProvider;
+import org.apache.rocketmq.auth.authentication.provider.AuthenticationProvider;
+import org.apache.rocketmq.auth.authentication.provider.DefaultAuthenticationProvider;
+import org.apache.rocketmq.auth.authentication.strategy.AuthenticationStrategy;
+import org.apache.rocketmq.auth.authentication.strategy.StatelessAuthenticationStrategy;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class AuthenticationFactory {
+
+    private static final Map<String, Object> INSTANCE_MAP = new HashMap<>();
+    private static final String PROVIDER_PREFIX = "PROVIDER_";
+    private static final String METADATA_PROVIDER_PREFIX = "METADATA_PROVIDER_";
+    private static final String EVALUATOR_PREFIX = "EVALUATOR_";
+
+    @SuppressWarnings("unchecked")
+    public static AuthenticationProvider<AuthenticationContext> getProvider(AuthConfig config) {
+        if (config == null) {
+            return null;
+        }
+        return computeIfAbsent(PROVIDER_PREFIX + config.getConfigName(), key -> {
+            try {
+                Class<? extends AuthenticationProvider<? extends AuthenticationContext>> clazz =
+                    DefaultAuthenticationProvider.class;
+                if (StringUtils.isNotBlank(config.getAuthenticationProvider())) {
+                    clazz = (Class<? extends AuthenticationProvider<? extends AuthenticationContext>>) Class.forName(config.getAuthenticationProvider());
+                }
+                return (AuthenticationProvider<AuthenticationContext>) clazz.getDeclaredConstructor().newInstance();
+            } catch (Exception e) {
+                throw new RuntimeException("Failed to load the authentication provider.", e);
+            }
+        });
+    }
+
+    public static AuthenticationMetadataProvider getMetadataProvider(AuthConfig config) {
+        return getMetadataProvider(config, null);
+    }
+
+    public static AuthenticationMetadataManager getMetadataManager(AuthConfig config) {
+        return new AuthenticationMetadataManagerImpl(config);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static AuthenticationMetadataProvider getMetadataProvider(AuthConfig config, Supplier<?> metadataService) {
+        if (config == null) {
+            return null;
+        }
+        return computeIfAbsent(METADATA_PROVIDER_PREFIX + config.getConfigName(), key -> {
+            try {
+                if (StringUtils.isBlank(config.getAuthenticationMetadataProvider())) {
+                    return null;
+                }
+                Class<? extends AuthenticationMetadataProvider> clazz = (Class<? extends AuthenticationMetadataProvider>)
+                    Class.forName(config.getAuthenticationMetadataProvider());
+                AuthenticationMetadataProvider result = clazz.getDeclaredConstructor().newInstance();
+                result.initialize(config, metadataService);
+                return result;
+            } catch (Exception e) {
+                throw new RuntimeException("Failed to load the authentication metadata provider", e);
+            }
+        });
+    }
+
+    public static AuthenticationEvaluator getEvaluator(AuthConfig config) {
+        return computeIfAbsent(EVALUATOR_PREFIX + config.getConfigName(), key -> new AuthenticationEvaluator(config));
+    }
+
+    public static AuthenticationEvaluator getEvaluator(AuthConfig config, Supplier<?> metadataService) {
+        return computeIfAbsent(EVALUATOR_PREFIX + config.getConfigName(), key -> new AuthenticationEvaluator(config, metadataService));
+    }
+
+    @SuppressWarnings("unchecked")
+    public static AuthenticationStrategy getStrategy(AuthConfig config, Supplier<?> metadataService) {
+        try {
+            Class<? extends AuthenticationStrategy> clazz = StatelessAuthenticationStrategy.class;
+            if (StringUtils.isNotBlank(config.getAuthenticationStrategy())) {
+                clazz = (Class<? extends AuthenticationStrategy>) Class.forName(config.getAuthenticationStrategy());
+            }
+            return clazz.getDeclaredConstructor(AuthConfig.class, Supplier.class).newInstance(config, metadataService);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static AuthenticationContext newContext(AuthConfig config, Metadata metadata, GeneratedMessageV3 request) {
+        AuthenticationProvider<AuthenticationContext> authenticationProvider = getProvider(config);
+        if (authenticationProvider == null) {
+            return null;
+        }
+        return authenticationProvider.newContext(metadata, request);
+    }
+
+    public static AuthenticationContext newContext(AuthConfig config, ChannelHandlerContext context,
+        RemotingCommand command) {
+        AuthenticationProvider<AuthenticationContext> authenticationProvider = getProvider(config);
+        if (authenticationProvider == null) {
+            return null;
+        }
+        return authenticationProvider.newContext(context, command);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <V> V computeIfAbsent(String key, Function<String, ? extends V> function) {
+        Object result = null;
+        if (INSTANCE_MAP.containsKey(key)) {
+            result = INSTANCE_MAP.get(key);
+        }
+        if (result == null) {
+            synchronized (INSTANCE_MAP) {
+                if (INSTANCE_MAP.containsKey(key)) {
+                    result = INSTANCE_MAP.get(key);
+                }
+                if (result == null) {
+                    result = function.apply(key);
+                    if (result != null) {
+                        INSTANCE_MAP.put(key, result);
+                    }
+                }
+            }
+        }
+        return result != null ? (V) result : null;
+    }
+}
\ No newline at end of file
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/manager/AuthenticationMetadataManager.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/manager/AuthenticationMetadataManager.java
new file mode 100644
index 00000000000..b3906437dc7
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/manager/AuthenticationMetadataManager.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.manager;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.config.AuthConfig;
+
+public interface AuthenticationMetadataManager {
+
+    void shutdown();
+
+    void initUser(AuthConfig authConfig);
+
+    CompletableFuture<Void> createUser(User user);
+
+    CompletableFuture<Void> updateUser(User user);
+
+    CompletableFuture<Void> deleteUser(String username);
+
+    CompletableFuture<User> getUser(String username);
+
+    CompletableFuture<List<User>> listUser(String filter);
+
+    CompletableFuture<Boolean> isSuperUser(String username);
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/manager/AuthenticationMetadataManagerImpl.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/manager/AuthenticationMetadataManagerImpl.java
new file mode 100644
index 00000000000..39620ca8d25
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/manager/AuthenticationMetadataManagerImpl.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.manager;
+
+import com.alibaba.fastjson2.JSON;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.auth.authentication.enums.UserStatus;
+import org.apache.rocketmq.auth.authentication.enums.UserType;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authentication.provider.AuthenticationMetadataProvider;
+import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;
+import org.apache.rocketmq.auth.authorization.provider.AuthorizationMetadataProvider;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.utils.ExceptionUtils;
+
+public class AuthenticationMetadataManagerImpl implements AuthenticationMetadataManager {
+
+    private final AuthenticationMetadataProvider authenticationMetadataProvider;
+
+    private final AuthorizationMetadataProvider authorizationMetadataProvider;
+
+    public AuthenticationMetadataManagerImpl(AuthConfig authConfig) {
+        this.authenticationMetadataProvider = AuthenticationFactory.getMetadataProvider(authConfig);
+        this.authorizationMetadataProvider = AuthorizationFactory.getMetadataProvider(authConfig);
+        this.initUser(authConfig);
+    }
+
+    @Override
+    public void shutdown() {
+        if (this.authenticationMetadataProvider != null) {
+            this.authenticationMetadataProvider.shutdown();
+        }
+        if (this.authorizationMetadataProvider != null) {
+            this.authorizationMetadataProvider.shutdown();
+        }
+    }
+
+    @Override
+    public void initUser(AuthConfig authConfig) {
+        if (authConfig == null) {
+            return;
+        }
+        if (StringUtils.isNotBlank(authConfig.getInitAuthenticationUser())) {
+            try {
+                User initUser = JSON.parseObject(authConfig.getInitAuthenticationUser(), User.class);
+                initUser.setUserType(UserType.SUPER);
+                this.getUser(initUser.getUsername()).thenCompose(user -> {
+                    if (user != null) {
+                        return CompletableFuture.completedFuture(null);
+                    }
+                    return this.createUser(initUser);
+                }).join();
+            } catch (Exception e) {
+                throw new AuthenticationException("Init authentication user error.", e);
+            }
+        }
+        if (StringUtils.isNotBlank(authConfig.getInnerClientAuthenticationCredentials())) {
+            try {
+                SessionCredentials credentials = JSON.parseObject(authConfig.getInnerClientAuthenticationCredentials(), SessionCredentials.class);
+                User innerUser = User.of(credentials.getAccessKey(), credentials.getSecretKey(), UserType.SUPER);
+                this.getUser(innerUser.getUsername()).thenCompose(user -> {
+                    if (user != null) {
+                        return CompletableFuture.completedFuture(null);
+                    }
+                    return this.createUser(innerUser);
+                }).join();
+            } catch (Exception e) {
+                throw new AuthenticationException("Init inner client authentication credentials error", e);
+            }
+        }
+    }
+
+    @Override
+    public CompletableFuture<Void> createUser(User user) {
+        CompletableFuture<Void> result = new CompletableFuture<>();
+        try {
+            this.validate(user, true);
+            if (user.getUserType() == null) {
+                user.setUserType(UserType.NORMAL);
+            }
+            if (user.getUserStatus() == null) {
+                user.setUserStatus(UserStatus.ENABLE);
+            }
+            result = this.getAuthenticationMetadataProvider().getUser(user.getUsername()).thenCompose(old -> {
+                if (old != null) {
+                    throw new AuthenticationException("The user is existed");
+                }
+                return this.getAuthenticationMetadataProvider().createUser(user);
+            });
+        } catch (Exception e) {
+            this.handleException(e, result);
+        }
+        return result;
+    }
+
+    @Override
+    public CompletableFuture<Void> updateUser(User user) {
+        CompletableFuture<Void> result = new CompletableFuture<>();
+        try {
+            this.validate(user, false);
+            result = this.getAuthenticationMetadataProvider().getUser(user.getUsername()).thenCompose(old -> {
+                if (old == null) {
+                    throw new AuthenticationException("The user is not exist");
+                }
+                if (StringUtils.isNotBlank(user.getPassword())) {
+                    old.setPassword(user.getPassword());
+                }
+                if (user.getUserType() != null) {
+                    old.setUserType(user.getUserType());
+                }
+                if (user.getUserStatus() != null) {
+                    old.setUserStatus(user.getUserStatus());
+                }
+                return this.getAuthenticationMetadataProvider().updateUser(old);
+            });
+        } catch (Exception e) {
+            this.handleException(e, result);
+        }
+        return result;
+    }
+
+    @Override
+    public CompletableFuture<Void> deleteUser(String username) {
+        CompletableFuture<Void> result = new CompletableFuture<>();
+        try {
+            if (StringUtils.isBlank(username)) {
+                throw new AuthenticationException("username can not be blank");
+            }
+            CompletableFuture<Void> deleteUser = this.getAuthenticationMetadataProvider().deleteUser(username);
+            CompletableFuture<Void> deleteAcl = this.getAuthorizationMetadataProvider().deleteAcl(User.of(username));
+            return CompletableFuture.allOf(deleteUser, deleteAcl);
+        } catch (Exception e) {
+            this.handleException(e, result);
+        }
+        return result;
+    }
+
+    @Override
+    public CompletableFuture<User> getUser(String username) {
+        CompletableFuture<User> result = new CompletableFuture<>();
+        try {
+            if (StringUtils.isBlank(username)) {
+                throw new AuthenticationException("username can not be blank");
+            }
+            result = this.getAuthenticationMetadataProvider().getUser(username);
+        } catch (Exception e) {
+            this.handleException(e, result);
+        }
+        return result;
+    }
+
+    @Override
+    public CompletableFuture<List<User>> listUser(String filter) {
+        CompletableFuture<List<User>> result = new CompletableFuture<>();
+        try {
+            result = this.getAuthenticationMetadataProvider().listUser(filter);
+        } catch (Exception e) {
+            this.handleException(e, result);
+        }
+        return result;
+    }
+
+    @Override
+    public CompletableFuture<Boolean> isSuperUser(String username) {
+        return this.getUser(username).thenApply(user -> {
+            if (user == null) {
+                throw new AuthenticationException("User:{} is not found", username);
+            }
+            return user.getUserType() == UserType.SUPER;
+        });
+    }
+
+    private void validate(User user, boolean isCreate) {
+        if (user == null) {
+            throw new AuthenticationException("user can not be null");
+        }
+        if (StringUtils.isBlank(user.getUsername())) {
+            throw new AuthenticationException("username can not be blank");
+        }
+        if (isCreate && StringUtils.isBlank(user.getPassword())) {
+            throw new AuthenticationException("password can not be blank");
+        }
+    }
+
+    private void handleException(Exception e, CompletableFuture<?> result) {
+        Throwable throwable = ExceptionUtils.getRealException(e);
+        result.completeExceptionally(throwable);
+    }
+
+    private AuthenticationMetadataProvider getAuthenticationMetadataProvider() {
+        if (authenticationMetadataProvider == null) {
+            throw new IllegalStateException("The authenticationMetadataProvider is not configured");
+        }
+        return authenticationMetadataProvider;
+    }
+
+    private AuthorizationMetadataProvider getAuthorizationMetadataProvider() {
+        if (authorizationMetadataProvider == null) {
+            throw new IllegalStateException("The authorizationMetadataProvider is not configured");
+        }
+        return authorizationMetadataProvider;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/model/Subject.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/model/Subject.java
new file mode 100644
index 00000000000..25b5dcb3162
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/model/Subject.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.model;
+
+import com.alibaba.fastjson2.annotation.JSONField;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authentication.enums.SubjectType;
+import org.apache.rocketmq.common.constant.CommonConstants;
+
+public interface Subject {
+
+    @JSONField(serialize = false)
+    String getSubjectKey();
+
+    SubjectType getSubjectType();
+
+    default boolean isSubject(SubjectType subjectType) {
+        return subjectType == this.getSubjectType();
+    }
+
+    @SuppressWarnings("unchecked")
+    static <T extends Subject> T of(String subjectKey) {
+        String type = StringUtils.substringBefore(subjectKey, CommonConstants.COLON);
+        SubjectType subjectType = SubjectType.getByName(type);
+        if (subjectType == null) {
+            return null;
+        }
+        if (subjectType == SubjectType.USER) {
+            return (T) User.of(StringUtils.substringAfter(subjectKey, CommonConstants.COLON));
+        }
+        return null;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/model/User.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/model/User.java
new file mode 100644
index 00000000000..4b009ca6163
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/model/User.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.model;
+
+import org.apache.rocketmq.auth.authentication.enums.SubjectType;
+import org.apache.rocketmq.auth.authentication.enums.UserStatus;
+import org.apache.rocketmq.auth.authentication.enums.UserType;
+import org.apache.rocketmq.common.constant.CommonConstants;
+
+public class User implements Subject {
+
+    private String username;
+
+    private String password;
+
+    private UserType userType;
+
+    private UserStatus userStatus;
+
+    public static User of(String username) {
+        User user = new User();
+        user.setUsername(username);
+        return user;
+    }
+
+    public static User of(String username, String password) {
+        User user = new User();
+        user.setUsername(username);
+        user.setPassword(password);
+        return user;
+    }
+
+    public static User of(String username, String password, UserType userType) {
+        User user = new User();
+        user.setUsername(username);
+        user.setPassword(password);
+        user.setUserType(userType);
+        return user;
+    }
+
+    @Override
+    public String getSubjectKey() {
+        return this.getSubjectType().getName() + CommonConstants.COLON + this.username;
+    }
+
+    @Override
+    public SubjectType getSubjectType() {
+        return SubjectType.USER;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public UserType getUserType() {
+        return userType;
+    }
+
+    public void setUserType(UserType userType) {
+        this.userType = userType;
+    }
+
+    public UserStatus getUserStatus() {
+        return userStatus;
+    }
+
+    public void setUserStatus(UserStatus userStatus) {
+        this.userStatus = userStatus;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/AuthenticationMetadataProvider.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/AuthenticationMetadataProvider.java
new file mode 100644
index 00000000000..59c3ec16022
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/AuthenticationMetadataProvider.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.provider;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.config.AuthConfig;
+
+public interface AuthenticationMetadataProvider {
+
+    void initialize(AuthConfig authConfig, Supplier<?> metadataService);
+
+    void shutdown();
+
+    CompletableFuture<Void> createUser(User user);
+
+    CompletableFuture<Void> deleteUser(String username);
+
+    CompletableFuture<Void> updateUser(User user);
+
+    CompletableFuture<User> getUser(String username);
+
+    CompletableFuture<List<User>> listUser(String filter);
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/AuthenticationProvider.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/AuthenticationProvider.java
new file mode 100644
index 00000000000..61660b5a18e
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/AuthenticationProvider.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.provider;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import io.netty.channel.ChannelHandlerContext;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public interface AuthenticationProvider<AuthenticationContext> {
+
+    void initialize(AuthConfig config, Supplier<?> metadataService);
+
+    CompletableFuture<Void> authenticate(AuthenticationContext context);
+
+    AuthenticationContext newContext(Metadata metadata, GeneratedMessageV3 request);
+
+    AuthenticationContext newContext(ChannelHandlerContext context, RemotingCommand command);
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/DefaultAuthenticationProvider.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/DefaultAuthenticationProvider.java
new file mode 100644
index 00000000000..98e7ede7ee3
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/DefaultAuthenticationProvider.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.provider;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import io.netty.channel.ChannelHandlerContext;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authentication.builder.AuthenticationContextBuilder;
+import org.apache.rocketmq.auth.authentication.builder.DefaultAuthenticationContextBuilder;
+import org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;
+import org.apache.rocketmq.auth.authentication.chain.DefaultAuthenticationHandler;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.chain.HandlerChain;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DefaultAuthenticationProvider implements AuthenticationProvider<DefaultAuthenticationContext> {
+
+    protected final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_AUTH_AUDIT_LOGGER_NAME);
+    protected AuthConfig authConfig;
+    protected Supplier<?> metadataService;
+    protected AuthenticationContextBuilder<DefaultAuthenticationContext> authenticationContextBuilder;
+
+    @Override
+    public void initialize(AuthConfig config, Supplier<?> metadataService) {
+        this.authConfig = config;
+        this.metadataService = metadataService;
+        this.authenticationContextBuilder = new DefaultAuthenticationContextBuilder();
+    }
+
+    @Override
+    public CompletableFuture<Void> authenticate(DefaultAuthenticationContext context) {
+        return this.newHandlerChain().handle(context)
+            .whenComplete((nil, ex) -> doAuditLog(context, ex));
+    }
+
+    @Override
+    public DefaultAuthenticationContext newContext(Metadata metadata, GeneratedMessageV3 request) {
+        return this.authenticationContextBuilder.build(metadata, request);
+    }
+
+    @Override
+    public DefaultAuthenticationContext newContext(ChannelHandlerContext context, RemotingCommand command) {
+        return this.authenticationContextBuilder.build(context, command);
+    }
+
+    protected HandlerChain<DefaultAuthenticationContext, CompletableFuture<Void>> newHandlerChain() {
+        return HandlerChain.<DefaultAuthenticationContext, CompletableFuture<Void>>create()
+            .addNext(new DefaultAuthenticationHandler(this.authConfig, metadataService));
+    }
+
+    protected void doAuditLog(DefaultAuthenticationContext context, Throwable ex) {
+        if (StringUtils.isBlank(context.getUsername())) {
+            return;
+        }
+        if (ex != null) {
+            log.info("[AUTHENTICATION] User:{} is authenticated failed with Signature = {}.", context.getUsername(), context.getSignature());
+        } else {
+            log.debug("[AUTHENTICATION] User:{} is authenticated success with Signature = {}.", context.getUsername(), context.getSignature());
+        }
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/LocalAuthenticationMetadataProvider.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/LocalAuthenticationMetadataProvider.java
new file mode 100644
index 00000000000..dcf90618229
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/provider/LocalAuthenticationMetadataProvider.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.provider;
+
+import com.alibaba.fastjson2.JSON;
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.config.ConfigRocksDBStorage;
+import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
+import org.rocksdb.RocksIterator;
+
+public class LocalAuthenticationMetadataProvider implements AuthenticationMetadataProvider {
+
+    private ConfigRocksDBStorage storage;
+
+    private LoadingCache<String, User> userCache;
+
+    @Override
+    public void initialize(AuthConfig authConfig, Supplier<?> metadataService) {
+        this.storage = new ConfigRocksDBStorage(authConfig.getAuthConfigPath() + File.separator + "users");
+        if (!this.storage.start()) {
+            throw new RuntimeException("Failed to load rocksdb for auth_user, please check whether it is occupied");
+        }
+
+        ThreadPoolExecutor cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor(
+            1,
+            1,
+            1000 * 60,
+            TimeUnit.MILLISECONDS,
+            "UserCacheRefresh",
+            100000
+        );
+
+        this.userCache = Caffeine.newBuilder()
+            .maximumSize(authConfig.getUserCacheMaxNum())
+            .expireAfterAccess(authConfig.getUserCacheExpiredSecond(), TimeUnit.SECONDS)
+            .refreshAfterWrite(authConfig.getUserCacheRefreshSecond(), TimeUnit.SECONDS)
+            .executor(cacheRefreshExecutor)
+            .build(new UserCacheLoader(this.storage));
+    }
+
+    @Override
+    public CompletableFuture<Void> createUser(User user) {
+        try {
+            byte[] keyBytes = user.getUsername().getBytes(StandardCharsets.UTF_8);
+            byte[] valueBytes = JSON.toJSONBytes(user);
+            this.storage.put(keyBytes, keyBytes.length, valueBytes);
+            this.storage.flushWAL();
+            this.userCache.invalidate(user.getUsername());
+        } catch (Exception e) {
+            throw new AuthenticationException("create user to RocksDB failed", e);
+        }
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public CompletableFuture<Void> deleteUser(String username) {
+        try {
+            this.storage.delete(username.getBytes(StandardCharsets.UTF_8));
+            this.storage.flushWAL();
+            this.userCache.invalidate(username);
+        } catch (Exception e) {
+            throw new AuthenticationException("delete user from RocksDB failed", e);
+        }
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public CompletableFuture<Void> updateUser(User user) {
+        try {
+            byte[] keyBytes = user.getUsername().getBytes(StandardCharsets.UTF_8);
+            byte[] valueBytes = JSON.toJSONBytes(user);
+            this.storage.put(keyBytes, keyBytes.length, valueBytes);
+            this.storage.flushWAL();
+            this.userCache.invalidate(user.getUsername());
+        } catch (Exception e) {
+            throw new AuthenticationException("update user to RocksDB failed", e);
+        }
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public CompletableFuture<User> getUser(String username) {
+        User user = this.userCache.get(username);
+        if (user == UserCacheLoader.EMPTY_USER) {
+            return CompletableFuture.completedFuture(null);
+        }
+        return CompletableFuture.completedFuture(user);
+    }
+
+    @Override
+    public CompletableFuture<List<User>> listUser(String filter) {
+        List<User> result = new ArrayList<>();
+        try (RocksIterator iterator = this.storage.iterator()) {
+            iterator.seekToFirst();
+            while (iterator.isValid()) {
+                String username = new String(iterator.key(), StandardCharsets.UTF_8);
+                if (StringUtils.isNotBlank(filter) && !username.contains(filter)) {
+                    iterator.next();
+                    continue;
+                }
+                User user = JSON.parseObject(new String(iterator.value(), StandardCharsets.UTF_8), User.class);
+                result.add(user);
+                iterator.next();
+            }
+        }
+        return CompletableFuture.completedFuture(result);
+    }
+
+    @Override
+    public void shutdown() {
+        if (this.storage != null) {
+            this.storage.shutdown();
+        }
+    }
+
+    private static class UserCacheLoader implements CacheLoader<String, User> {
+        private final ConfigRocksDBStorage storage;
+        public static final User EMPTY_USER = new User();
+
+        public UserCacheLoader(ConfigRocksDBStorage storage) {
+            this.storage = storage;
+        }
+
+        @Override
+        public User load(String username) {
+            try {
+                byte[] keyBytes = username.getBytes(StandardCharsets.UTF_8);
+                byte[] valueBytes = storage.get(keyBytes);
+                if (ArrayUtils.isEmpty(valueBytes)) {
+                    return EMPTY_USER;
+                }
+                return JSON.parseObject(new String(valueBytes, StandardCharsets.UTF_8), User.class);
+            } catch (Exception e) {
+                throw new AuthenticationException("Get user from RocksDB failed.", e);
+            }
+        }
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/AbstractAuthenticationStrategy.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/AbstractAuthenticationStrategy.java
new file mode 100644
index 00000000000..b27b6e33ec4
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/AbstractAuthenticationStrategy.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.strategy;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authentication.context.AuthenticationContext;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.authentication.provider.AuthenticationProvider;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.utils.ExceptionUtils;
+
+public abstract class AbstractAuthenticationStrategy implements AuthenticationStrategy {
+
+    protected final AuthConfig authConfig;
+    protected final List<String> authenticationWhitelist = new ArrayList<>();
+    protected final AuthenticationProvider<AuthenticationContext> authenticationProvider;
+
+    public AbstractAuthenticationStrategy(AuthConfig authConfig, Supplier<?> metadataService) {
+        this.authConfig = authConfig;
+        this.authenticationProvider = AuthenticationFactory.getProvider(authConfig);
+        if (this.authenticationProvider != null) {
+            this.authenticationProvider.initialize(authConfig, metadataService);
+        }
+        if (StringUtils.isNotBlank(authConfig.getAuthenticationWhitelist())) {
+            String[] whitelist = StringUtils.split(authConfig.getAuthenticationWhitelist(), ",");
+            for (String rpcCode : whitelist) {
+                this.authenticationWhitelist.add(StringUtils.trim(rpcCode));
+            }
+        }
+    }
+
+    protected void doEvaluate(AuthenticationContext context) {
+        if (context == null) {
+            return;
+        }
+        if (!authConfig.isAuthenticationEnabled()) {
+            return;
+        }
+        if (this.authenticationProvider == null) {
+            return;
+        }
+        if (this.authenticationWhitelist.contains(context.getRpcCode())) {
+            return;
+        }
+        try {
+            this.authenticationProvider.authenticate(context).join();
+        } catch (AuthenticationException ex) {
+            throw ex;
+        } catch (Throwable ex) {
+            Throwable exception = ExceptionUtils.getRealException(ex);
+            if (exception instanceof AuthenticationException) {
+                throw (AuthenticationException) exception;
+            }
+            throw new AuthenticationException("Authentication failed. Please verify the credentials and try again.", exception);
+        }
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/AuthenticationStrategy.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/AuthenticationStrategy.java
new file mode 100644
index 00000000000..22eee6f41be
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/AuthenticationStrategy.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.strategy;
+
+import org.apache.rocketmq.auth.authentication.context.AuthenticationContext;
+
+public interface AuthenticationStrategy {
+
+    void evaluate(AuthenticationContext context);
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/StatefulAuthenticationStrategy.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/StatefulAuthenticationStrategy.java
new file mode 100644
index 00000000000..914d99ac2e8
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/StatefulAuthenticationStrategy.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.strategy;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authentication.context.AuthenticationContext;
+import org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.Pair;
+import org.apache.rocketmq.common.constant.CommonConstants;
+
+public class StatefulAuthenticationStrategy extends AbstractAuthenticationStrategy {
+
+    protected Cache<String, Pair<Boolean, AuthenticationException>> authCache;
+
+    public StatefulAuthenticationStrategy(AuthConfig authConfig, Supplier<?> metadataService) {
+        super(authConfig, metadataService);
+        this.authCache = Caffeine.newBuilder()
+            .expireAfterWrite(authConfig.getStatefulAuthenticationCacheExpiredSecond(), TimeUnit.SECONDS)
+            .maximumSize(authConfig.getStatefulAuthenticationCacheMaxNum())
+            .build();
+    }
+
+    @Override
+    public void evaluate(AuthenticationContext context) {
+        if (StringUtils.isBlank(context.getChannelId())) {
+            this.doEvaluate(context);
+            return;
+        }
+        Pair<Boolean, AuthenticationException> result = this.authCache.get(buildKey(context), key -> {
+            try {
+                this.doEvaluate(context);
+                return Pair.of(true, null);
+            } catch (AuthenticationException ex) {
+                return Pair.of(false, ex);
+            }
+        });
+        if (result != null && result.getObject1() == Boolean.FALSE) {
+            throw result.getObject2();
+        }
+    }
+
+    private String buildKey(AuthenticationContext context) {
+        if (context instanceof DefaultAuthenticationContext) {
+            DefaultAuthenticationContext ctx = (DefaultAuthenticationContext) context;
+            if (StringUtils.isBlank(ctx.getUsername())) {
+                return ctx.getChannelId();
+            }
+            return ctx.getChannelId() + CommonConstants.POUND + ctx.getUsername();
+        }
+        throw new AuthenticationException("The request of {} is not support.", context.getClass().getSimpleName());
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/StatelessAuthenticationStrategy.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/StatelessAuthenticationStrategy.java
new file mode 100644
index 00000000000..05649824175
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/strategy/StatelessAuthenticationStrategy.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.strategy;
+
+import java.util.function.Supplier;
+import org.apache.rocketmq.auth.authentication.context.AuthenticationContext;
+import org.apache.rocketmq.auth.config.AuthConfig;
+
+public class StatelessAuthenticationStrategy extends AbstractAuthenticationStrategy {
+
+    public StatelessAuthenticationStrategy(AuthConfig authConfig, Supplier<?> metadataService) {
+        super(authConfig, metadataService);
+    }
+
+    @Override
+    public void evaluate(AuthenticationContext context) {
+        super.doEvaluate(context);
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/AuthorizationEvaluator.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/AuthorizationEvaluator.java
new file mode 100644
index 00000000000..f043810cc98
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/AuthorizationEvaluator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization;
+
+import java.util.List;
+import java.util.function.Supplier;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.auth.authorization.context.AuthorizationContext;
+import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;
+import org.apache.rocketmq.auth.authorization.strategy.AuthorizationStrategy;
+import org.apache.rocketmq.auth.config.AuthConfig;
+
+public class AuthorizationEvaluator {
+
+    private final AuthorizationStrategy authorizationStrategy;
+
+    public AuthorizationEvaluator(AuthConfig authConfig) {
+        this(authConfig, null);
+    }
+
+    public AuthorizationEvaluator(AuthConfig authConfig, Supplier<?> metadataService) {
+        this.authorizationStrategy = AuthorizationFactory.getStrategy(authConfig, metadataService);
+    }
+
+    public void evaluate(List<AuthorizationContext> contexts) {
+        if (CollectionUtils.isEmpty(contexts)) {
+            return;
+        }
+        contexts.forEach(this.authorizationStrategy::evaluate);
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/AuthorizationContextBuilder.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/AuthorizationContextBuilder.java
new file mode 100644
index 00000000000..e1e8dccfb81
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/AuthorizationContextBuilder.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.builder;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import io.netty.channel.ChannelHandlerContext;
+import java.util.List;
+import org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public interface AuthorizationContextBuilder {
+
+    List<DefaultAuthorizationContext> build(Metadata metadata, GeneratedMessageV3 message);
+
+    List<DefaultAuthorizationContext> build(ChannelHandlerContext context, RemotingCommand command);
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java
new file mode 100644
index 00000000000..fababc0ee71
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java
@@ -0,0 +1,504 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.builder;
+
+import apache.rocketmq.v2.AckMessageRequest;
+import apache.rocketmq.v2.ChangeInvisibleDurationRequest;
+import apache.rocketmq.v2.ClientType;
+import apache.rocketmq.v2.EndTransactionRequest;
+import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest;
+import apache.rocketmq.v2.HeartbeatRequest;
+import apache.rocketmq.v2.NotifyClientTerminationRequest;
+import apache.rocketmq.v2.QueryAssignmentRequest;
+import apache.rocketmq.v2.QueryRouteRequest;
+import apache.rocketmq.v2.RecallMessageRequest;
+import apache.rocketmq.v2.ReceiveMessageRequest;
+import apache.rocketmq.v2.SendMessageRequest;
+import apache.rocketmq.v2.Subscription;
+import apache.rocketmq.v2.SubscriptionEntry;
+import apache.rocketmq.v2.TelemetryCommand;
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import io.netty.channel.ChannelHandlerContext;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.acl.common.AclException;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.model.Resource;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.constant.CommonConstants;
+import org.apache.rocketmq.common.constant.GrpcConstants;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.resource.ResourcePattern;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.RequestHeaderRegistry;
+import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;
+import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+
+public class DefaultAuthorizationContextBuilder implements AuthorizationContextBuilder {
+
+    private static final String TOPIC = "topic";
+    private static final String GROUP = "group";
+    private static final String A = "a";
+    private static final String B = "b";
+    private static final String CONSUMER_GROUP = "consumerGroup";
+    private final AuthConfig authConfig;
+
+    private final RequestHeaderRegistry requestHeaderRegistry;
+
+    public DefaultAuthorizationContextBuilder(AuthConfig authConfig) {
+        this.authConfig = authConfig;
+        this.requestHeaderRegistry = RequestHeaderRegistry.getInstance();
+    }
+
+    @Override
+    public List<DefaultAuthorizationContext> build(Metadata metadata, GeneratedMessageV3 message) {
+        List<DefaultAuthorizationContext> result = null;
+        if (message instanceof SendMessageRequest) {
+            SendMessageRequest request = (SendMessageRequest) message;
+            if (request.getMessagesCount() <= 0) {
+                throw new AuthorizationException("message is null.");
+            }
+            result = newPubContext(metadata, request.getMessages(0).getTopic());
+        }
+        if (message instanceof RecallMessageRequest) {
+            RecallMessageRequest request = (RecallMessageRequest) message;
+            result = newPubContext(metadata, request.getTopic());
+        }
+        if (message instanceof EndTransactionRequest) {
+            EndTransactionRequest request = (EndTransactionRequest) message;
+            result = newPubContext(metadata, request.getTopic());
+        }
+        if (message instanceof HeartbeatRequest) {
+            HeartbeatRequest request = (HeartbeatRequest) message;
+            if (!isConsumerClientType(request.getClientType())) {
+                return null;
+            }
+            result = newGroupSubContexts(metadata, request.getGroup());
+        }
+        if (message instanceof ReceiveMessageRequest) {
+            ReceiveMessageRequest request = (ReceiveMessageRequest) message;
+            if (!request.hasMessageQueue()) {
+                throw new AuthorizationException("messageQueue is null.");
+            }
+            result = newSubContexts(metadata, request.getGroup(), request.getMessageQueue().getTopic());
+        }
+        if (message instanceof AckMessageRequest) {
+            AckMessageRequest request = (AckMessageRequest) message;
+            result = newSubContexts(metadata, request.getGroup(), request.getTopic());
+        }
+        if (message instanceof ForwardMessageToDeadLetterQueueRequest) {
+            ForwardMessageToDeadLetterQueueRequest request = (ForwardMessageToDeadLetterQueueRequest) message;
+            result = newSubContexts(metadata, request.getGroup(), request.getTopic());
+        }
+        if (message instanceof NotifyClientTerminationRequest) {
+            NotifyClientTerminationRequest request = (NotifyClientTerminationRequest) message;
+            if (StringUtils.isNotBlank(request.getGroup().getName())) {
+                result = newGroupSubContexts(metadata, request.getGroup());
+            }
+        }
+        if (message instanceof ChangeInvisibleDurationRequest) {
+            ChangeInvisibleDurationRequest request = (ChangeInvisibleDurationRequest) message;
+            result = newGroupSubContexts(metadata, request.getGroup());
+        }
+        if (message instanceof QueryRouteRequest) {
+            QueryRouteRequest request = (QueryRouteRequest) message;
+            result = newContext(metadata, request);
+        }
+        if (message instanceof QueryAssignmentRequest) {
+            QueryAssignmentRequest request = (QueryAssignmentRequest) message;
+            result = newSubContexts(metadata, request.getGroup(), request.getTopic());
+        }
+        if (message instanceof TelemetryCommand) {
+            TelemetryCommand request = (TelemetryCommand) message;
+            result = newContext(metadata, request);
+        }
+        if (CollectionUtils.isNotEmpty(result)) {
+            result.forEach(context -> {
+                context.setChannelId(metadata.get(GrpcConstants.CHANNEL_ID));
+                context.setRpcCode(message.getDescriptorForType().getFullName());
+            });
+        }
+        return result;
+    }
+
+    @Override
+    public List<DefaultAuthorizationContext> build(ChannelHandlerContext context, RemotingCommand command) {
+        List<DefaultAuthorizationContext> result = new ArrayList<>();
+        try {
+            HashMap<String, String> fields = command.getExtFields();
+            if (MapUtils.isEmpty(fields)) {
+                return result;
+            }
+            Subject subject = null;
+            if (fields.containsKey(SessionCredentials.ACCESS_KEY)) {
+                subject = User.of(fields.get(SessionCredentials.ACCESS_KEY));
+            }
+            String remoteAddr = RemotingHelper.parseChannelRemoteAddr(context.channel());
+            String sourceIp = StringUtils.substringBeforeLast(remoteAddr, CommonConstants.COLON);
+
+            Resource topic;
+            Resource group;
+            switch (command.getCode()) {
+                case RequestCode.GET_ROUTEINFO_BY_TOPIC:
+                    if (NamespaceUtil.isRetryTopic(fields.get(TOPIC))) {
+                        group = Resource.ofGroup(fields.get(TOPIC));
+                        result.add(DefaultAuthorizationContext.of(subject, group, Arrays.asList(Action.SUB, Action.GET), sourceIp));
+                    } else {
+                        topic = Resource.ofTopic(fields.get(TOPIC));
+                        result.add(DefaultAuthorizationContext.of(subject, topic, Arrays.asList(Action.PUB, Action.SUB, Action.GET), sourceIp));
+                    }
+                    break;
+                case RequestCode.SEND_MESSAGE:
+                    if (NamespaceUtil.isRetryTopic(fields.get(TOPIC))) {
+                        if (StringUtils.isNotBlank(fields.get(GROUP))) {
+                            group = Resource.ofGroup(fields.get(GROUP));
+                        } else {
+                            group = Resource.ofGroup(fields.get(TOPIC));
+                        }
+                        result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));
+                    } else {
+                        topic = Resource.ofTopic(fields.get(TOPIC));
+                        result.add(DefaultAuthorizationContext.of(subject, topic, Action.PUB, sourceIp));
+                    }
+                    break;
+                case RequestCode.SEND_MESSAGE_V2:
+                case RequestCode.SEND_BATCH_MESSAGE:
+                    if (NamespaceUtil.isRetryTopic(fields.get(B))) {
+                        if (StringUtils.isNotBlank(fields.get(A))) {
+                            group = Resource.ofGroup(fields.get(A));
+                        } else {
+                            group = Resource.ofGroup(fields.get(B));
+                        }
+                        result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));
+                    } else {
+                        topic = Resource.ofTopic(fields.get(B));
+                        result.add(DefaultAuthorizationContext.of(subject, topic, Action.PUB, sourceIp));
+                    }
+                    break;
+                case RequestCode.RECALL_MESSAGE:
+                    topic = Resource.ofTopic(fields.get(TOPIC));
+                    result.add(DefaultAuthorizationContext.of(subject, topic, Action.PUB, sourceIp));
+                    break;
+                case RequestCode.END_TRANSACTION:
+                    if (StringUtils.isNotBlank(fields.get(TOPIC))) {
+                        topic = Resource.ofTopic(fields.get(TOPIC));
+                        result.add(DefaultAuthorizationContext.of(subject, topic, Action.PUB, sourceIp));
+                    }
+                    break;
+                case RequestCode.CONSUMER_SEND_MSG_BACK:
+                    group = Resource.ofGroup(fields.get(GROUP));
+                    result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));
+                    break;
+                case RequestCode.PULL_MESSAGE:
+                    if (!NamespaceUtil.isRetryTopic(fields.get(TOPIC))) {
+                        topic = Resource.ofTopic(fields.get(TOPIC));
+                        result.add(DefaultAuthorizationContext.of(subject, topic, Action.SUB, sourceIp));
+                    }
+                    group = Resource.ofGroup(fields.get(CONSUMER_GROUP));
+                    result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));
+                    break;
+                case RequestCode.QUERY_MESSAGE:
+                    topic = Resource.ofTopic(fields.get(TOPIC));
+                    result.add(DefaultAuthorizationContext.of(subject, topic, Arrays.asList(Action.SUB, Action.GET), sourceIp));
+                    break;
+                case RequestCode.HEART_BEAT:
+                    HeartbeatData heartbeatData = HeartbeatData.decode(command.getBody(), HeartbeatData.class);
+                    for (ConsumerData data : heartbeatData.getConsumerDataSet()) {
+                        group = Resource.ofGroup(data.getGroupName());
+                        result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));
+                        for (SubscriptionData subscriptionData : data.getSubscriptionDataSet()) {
+                            if (NamespaceUtil.isRetryTopic(subscriptionData.getTopic())) {
+                                continue;
+                            }
+                            topic = Resource.ofTopic(subscriptionData.getTopic());
+                            result.add(DefaultAuthorizationContext.of(subject, topic, Action.SUB, sourceIp));
+                        }
+                    }
+                    break;
+                case RequestCode.UNREGISTER_CLIENT:
+                    final UnregisterClientRequestHeader unregisterClientRequestHeader =
+                        command.decodeCommandCustomHeader(UnregisterClientRequestHeader.class);
+                    if (StringUtils.isNotBlank(unregisterClientRequestHeader.getConsumerGroup())) {
+                        group = Resource.ofGroup(unregisterClientRequestHeader.getConsumerGroup());
+                        result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));
+                    }
+                    break;
+                case RequestCode.GET_CONSUMER_LIST_BY_GROUP:
+                    final GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader =
+                        command.decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class);
+                    group = Resource.ofGroup(getConsumerListByGroupRequestHeader.getConsumerGroup());
+                    result.add(DefaultAuthorizationContext.of(subject, group, Arrays.asList(Action.SUB, Action.GET), sourceIp));
+                    break;
+                case RequestCode.QUERY_CONSUMER_OFFSET:
+                    final QueryConsumerOffsetRequestHeader queryConsumerOffsetRequestHeader =
+                        command.decodeCommandCustomHeader(QueryConsumerOffsetRequestHeader.class);
+                    if (!NamespaceUtil.isRetryTopic(queryConsumerOffsetRequestHeader.getTopic())) {
+                        topic = Resource.ofTopic(queryConsumerOffsetRequestHeader.getTopic());
+                        result.add(DefaultAuthorizationContext.of(subject, topic, Arrays.asList(Action.SUB, Action.GET), sourceIp));
+                    }
+                    group = Resource.ofGroup(queryConsumerOffsetRequestHeader.getConsumerGroup());
+                    result.add(DefaultAuthorizationContext.of(subject, group, Arrays.asList(Action.SUB, Action.GET), sourceIp));
+                    break;
+                case RequestCode.UPDATE_CONSUMER_OFFSET:
+                    final UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader =
+                        command.decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class);
+                    if (!NamespaceUtil.isRetryTopic(updateConsumerOffsetRequestHeader.getTopic())) {
+                        topic = Resource.ofTopic(updateConsumerOffsetRequestHeader.getTopic());
+                        result.add(DefaultAuthorizationContext.of(subject, topic, Arrays.asList(Action.SUB, Action.UPDATE), sourceIp));
+                    }
+                    group = Resource.ofGroup(updateConsumerOffsetRequestHeader.getConsumerGroup());
+                    result.add(DefaultAuthorizationContext.of(subject, group, Arrays.asList(Action.SUB, Action.UPDATE), sourceIp));
+                    break;
+                case RequestCode.LOCK_BATCH_MQ:
+                    LockBatchRequestBody lockBatchRequestBody = LockBatchRequestBody.decode(command.getBody(), LockBatchRequestBody.class);
+                    group = Resource.ofGroup(lockBatchRequestBody.getConsumerGroup());
+                    result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));
+                    if (CollectionUtils.isNotEmpty(lockBatchRequestBody.getMqSet())) {
+                        for (MessageQueue messageQueue : lockBatchRequestBody.getMqSet()) {
+                            if (NamespaceUtil.isRetryTopic(messageQueue.getTopic())) {
+                                continue;
+                            }
+                            topic = Resource.ofTopic(messageQueue.getTopic());
+                            result.add(DefaultAuthorizationContext.of(subject, topic, Action.SUB, sourceIp));
+                        }
+                    }
+                    break;
+                case RequestCode.UNLOCK_BATCH_MQ:
+                    UnlockBatchRequestBody unlockBatchRequestBody = LockBatchRequestBody.decode(command.getBody(), UnlockBatchRequestBody.class);
+                    group = Resource.ofGroup(unlockBatchRequestBody.getConsumerGroup());
+                    result.add(DefaultAuthorizationContext.of(subject, group, Action.SUB, sourceIp));
+                    if (CollectionUtils.isNotEmpty(unlockBatchRequestBody.getMqSet())) {
+                        for (MessageQueue messageQueue : unlockBatchRequestBody.getMqSet()) {
+                            if (NamespaceUtil.isRetryTopic(messageQueue.getTopic())) {
+                                continue;
+                            }
+                            topic = Resource.ofTopic(messageQueue.getTopic());
+                            result.add(DefaultAuthorizationContext.of(subject, topic, Action.SUB, sourceIp));
+                        }
+                    }
+                    break;
+                default:
+                    result = buildContextByAnnotation(subject, command, sourceIp);
+                    break;
+            }
+            if (CollectionUtils.isNotEmpty(result)) {
+                result.forEach(r -> {
+                    r.setChannelId(context.channel().id().asLongText());
+                    r.setRpcCode(String.valueOf(command.getCode()));
+                });
+            }
+        } catch (AuthorizationException ex) {
+            throw ex;
+        } catch (Throwable t) {
+            throw new AuthorizationException("parse authorization context error.", t);
+        }
+        return result;
+    }
+
+    private List<DefaultAuthorizationContext> buildContextByAnnotation(Subject subject, RemotingCommand request,
+        String sourceIp) throws Exception {
+        List<DefaultAuthorizationContext> result = new ArrayList<>();
+
+        Class<? extends CommandCustomHeader> clazz = this.requestHeaderRegistry.getRequestHeader(request.getCode());
+        if (clazz == null) {
+            return result;
+        }
+        CommandCustomHeader header = request.decodeCommandCustomHeader(clazz);
+
+        RocketMQAction rocketMQAction = clazz.getAnnotation(RocketMQAction.class);
+        ResourceType resourceType = rocketMQAction.resource();
+        Action[] actions = rocketMQAction.action();
+        Resource resource = null;
+        if (resourceType == ResourceType.CLUSTER) {
+            resource = Resource.ofCluster(authConfig.getClusterName());
+        }
+
+        Field[] fields = clazz.getDeclaredFields();
+        if (ArrayUtils.isNotEmpty(fields)) {
+            for (Field field : fields) {
+                RocketMQResource rocketMQResource = field.getAnnotation(RocketMQResource.class);
+                if (rocketMQResource == null) {
+                    continue;
+                }
+                field.setAccessible(true);
+                try {
+                    resourceType = rocketMQResource.value();
+                    String splitter = rocketMQResource.splitter();
+                    Object value = field.get(header);
+                    if (value == null) {
+                        continue;
+                    }
+                    String[] resourceValues;
+                    if (StringUtils.isNotBlank(splitter)) {
+                        resourceValues = StringUtils.split(value.toString(), splitter);
+                    } else {
+                        resourceValues = new String[] {value.toString()};
+                    }
+                    for (String resourceValue : resourceValues) {
+                        if (resourceType == ResourceType.TOPIC && NamespaceUtil.isRetryTopic(resourceValue)) {
+                            resource = Resource.ofGroup(resourceValue);
+                            result.add(DefaultAuthorizationContext.of(subject, resource, Arrays.asList(actions), sourceIp));
+                        } else {
+                            resource = Resource.of(resourceType, resourceValue, ResourcePattern.LITERAL);
+                            result.add(DefaultAuthorizationContext.of(subject, resource, Arrays.asList(actions), sourceIp));
+                        }
+                    }
+                } finally {
+                    field.setAccessible(false);
+                }
+            }
+        }
+
+        if (CollectionUtils.isEmpty(result) && resource != null) {
+            result.add(DefaultAuthorizationContext.of(subject, resource, Arrays.asList(actions), sourceIp));
+        }
+
+        return result;
+    }
+
+    private List<DefaultAuthorizationContext> newContext(Metadata metadata, QueryRouteRequest request) {
+        apache.rocketmq.v2.Resource topic = request.getTopic();
+        if (StringUtils.isBlank(topic.getName())) {
+            throw new AuthorizationException("topic is null.");
+        }
+        Subject subject = null;
+        if (metadata.containsKey(GrpcConstants.AUTHORIZATION_AK)) {
+            subject = User.of(metadata.get(GrpcConstants.AUTHORIZATION_AK));
+        }
+        Resource resource = Resource.ofTopic(topic.getName());
+        String sourceIp = StringUtils.substringBeforeLast(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON);
+        DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, Arrays.asList(Action.PUB, Action.SUB), sourceIp);
+        return Collections.singletonList(context);
+    }
+
+    private static List<DefaultAuthorizationContext> newContext(Metadata metadata, TelemetryCommand request) {
+        if (request.getCommandCase() != TelemetryCommand.CommandCase.SETTINGS) {
+            return null;
+        }
+        if (!request.getSettings().hasPublishing() && !request.getSettings().hasSubscription()) {
+            throw new AclException("settings command doesn't have publishing or subscription.");
+        }
+        List<DefaultAuthorizationContext> result = new ArrayList<>();
+        if (request.getSettings().hasPublishing()) {
+            List<apache.rocketmq.v2.Resource> topicList = request.getSettings().getPublishing().getTopicsList();
+            for (apache.rocketmq.v2.Resource topic : topicList) {
+                result.addAll(newPubContext(metadata, topic));
+            }
+        }
+        if (request.getSettings().hasSubscription()) {
+            Subscription subscription = request.getSettings().getSubscription();
+            result.addAll(newSubContexts(metadata, ResourceType.GROUP, subscription.getGroup()));
+            for (SubscriptionEntry entry : subscription.getSubscriptionsList()) {
+                result.addAll(newSubContexts(metadata, ResourceType.TOPIC, entry.getTopic()));
+            }
+        }
+        return result;
+    }
+
+    private boolean isConsumerClientType(ClientType clientType) {
+        return Arrays.asList(ClientType.PUSH_CONSUMER, ClientType.SIMPLE_CONSUMER, ClientType.PULL_CONSUMER)
+            .contains(clientType);
+    }
+
+    private static List<DefaultAuthorizationContext> newPubContext(Metadata metadata, apache.rocketmq.v2.Resource topic) {
+        if (topic == null || StringUtils.isBlank(topic.getName())) {
+            throw new AuthorizationException("topic is null.");
+        }
+        Subject subject = null;
+        if (metadata.containsKey(GrpcConstants.AUTHORIZATION_AK)) {
+            subject = User.of(metadata.get(GrpcConstants.AUTHORIZATION_AK));
+        }
+        Resource resource = Resource.ofTopic(topic.getName());
+        String sourceIp = StringUtils.substringBeforeLast(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON);
+        DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, Action.PUB, sourceIp);
+        return Collections.singletonList(context);
+    }
+
+    private List<DefaultAuthorizationContext> newSubContexts(Metadata metadata, apache.rocketmq.v2.Resource group,
+        apache.rocketmq.v2.Resource topic) {
+        List<DefaultAuthorizationContext> result = new ArrayList<>();
+        result.addAll(newGroupSubContexts(metadata, group));
+        result.addAll(newTopicSubContexts(metadata, topic));
+        return result;
+    }
+
+    private static List<DefaultAuthorizationContext> newTopicSubContexts(Metadata metadata,
+        apache.rocketmq.v2.Resource resource) {
+        return newSubContexts(metadata, ResourceType.TOPIC, resource);
+    }
+
+    private static List<DefaultAuthorizationContext> newGroupSubContexts(Metadata metadata,
+        apache.rocketmq.v2.Resource resource) {
+        return newSubContexts(metadata, ResourceType.GROUP, resource);
+    }
+
+    private static List<DefaultAuthorizationContext> newSubContexts(Metadata metadata, ResourceType resourceType,
+        apache.rocketmq.v2.Resource resource) {
+        if (resourceType == ResourceType.GROUP) {
+            if (resource == null || StringUtils.isBlank(resource.getName())) {
+                throw new AuthorizationException("group is null.");
+            }
+            return newSubContexts(metadata, Resource.ofGroup(resource.getName()));
+        }
+        if (resourceType == ResourceType.TOPIC) {
+            if (resource == null || StringUtils.isBlank(resource.getName())) {
+                throw new AuthorizationException("topic is null.");
+            }
+            return newSubContexts(metadata, Resource.ofTopic(resource.getName()));
+        }
+        throw new AuthorizationException("unknown resource type.");
+    }
+
+    private static List<DefaultAuthorizationContext> newSubContexts(Metadata metadata, Resource resource) {
+        List<DefaultAuthorizationContext> result = new ArrayList<>();
+        Subject subject = null;
+        if (metadata.containsKey(GrpcConstants.AUTHORIZATION_AK)) {
+            subject = User.of(metadata.get(GrpcConstants.AUTHORIZATION_AK));
+        }
+        String sourceIp = StringUtils.substringBeforeLast(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON);
+        result.add(DefaultAuthorizationContext.of(subject, resource, Action.SUB, sourceIp));
+        return result;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/chain/AclAuthorizationHandler.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/chain/AclAuthorizationHandler.java
new file mode 100644
index 00000000000..06a130b2e0a
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/chain/AclAuthorizationHandler.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.chain;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;
+import org.apache.rocketmq.auth.authorization.enums.Decision;
+import org.apache.rocketmq.auth.authorization.enums.PolicyType;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.authorization.model.Environment;
+import org.apache.rocketmq.auth.authorization.model.Policy;
+import org.apache.rocketmq.auth.authorization.model.PolicyEntry;
+import org.apache.rocketmq.auth.authorization.model.Resource;
+import org.apache.rocketmq.auth.authorization.provider.AuthorizationMetadataProvider;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.chain.Handler;
+import org.apache.rocketmq.common.chain.HandlerChain;
+import org.apache.rocketmq.common.resource.ResourcePattern;
+import org.apache.rocketmq.common.resource.ResourceType;
+
+public class AclAuthorizationHandler implements Handler<DefaultAuthorizationContext, CompletableFuture<Void>> {
+
+    private final AuthorizationMetadataProvider authorizationMetadataProvider;
+
+    public AclAuthorizationHandler(AuthConfig config) {
+        this.authorizationMetadataProvider = AuthorizationFactory.getMetadataProvider(config);
+    }
+
+    public AclAuthorizationHandler(AuthConfig config, Supplier<?> metadataService) {
+        this.authorizationMetadataProvider = AuthorizationFactory.getMetadataProvider(config, metadataService);
+    }
+
+    @Override
+    public CompletableFuture<Void> handle(DefaultAuthorizationContext context,
+        HandlerChain<DefaultAuthorizationContext, CompletableFuture<Void>> chain) {
+        if (this.authorizationMetadataProvider == null) {
+            throw new AuthorizationException("The authorizationMetadataProvider is not configured");
+        }
+        return this.authorizationMetadataProvider.getAcl(context.getSubject()).thenAccept(acl -> {
+            if (acl == null) {
+                throwException(context, "no matched policies.");
+            }
+
+            // 1. get the defined acl entries which match the request.
+            PolicyEntry matchedEntry = matchPolicyEntries(context, acl);
+
+            // 2. if no matched acl entries, return deny
+            if (matchedEntry == null) {
+                throwException(context, "no matched policies.");
+            }
+
+            // 3. judge is the entries has denied decision.
+            if (matchedEntry.getDecision() == Decision.DENY) {
+                throwException(context, "the decision is deny.");
+            }
+        });
+    }
+
+    private PolicyEntry matchPolicyEntries(DefaultAuthorizationContext context, Acl acl) {
+        List<PolicyEntry> policyEntries = new ArrayList<>();
+
+        Policy policy = acl.getPolicy(PolicyType.CUSTOM);
+        if (policy != null) {
+            List<PolicyEntry> entries = matchPolicyEntries(context, policy.getEntries());
+            if (CollectionUtils.isNotEmpty(entries)) {
+                policyEntries.addAll(entries);
+            }
+        }
+
+        if (CollectionUtils.isEmpty(policyEntries)) {
+            policy = acl.getPolicy(PolicyType.DEFAULT);
+            if (policy != null) {
+                List<PolicyEntry> entries = matchPolicyEntries(context, policy.getEntries());
+                if (CollectionUtils.isNotEmpty(entries)) {
+                    policyEntries.addAll(entries);
+                }
+            }
+        }
+
+        if (CollectionUtils.isEmpty(policyEntries)) {
+            return null;
+        }
+
+        policyEntries.sort(this::comparePolicyEntries);
+
+        return policyEntries.get(0);
+    }
+
+    private List<PolicyEntry> matchPolicyEntries(DefaultAuthorizationContext context, List<PolicyEntry> entries) {
+        if (CollectionUtils.isEmpty(entries)) {
+            return null;
+        }
+        return entries.stream()
+            .filter(entry -> entry.isMatchResource(context.getResource()))
+            .filter(entry -> entry.isMatchAction(context.getActions()))
+            .filter(entry -> entry.isMatchEnvironment(Environment.of(context.getSourceIp())))
+            .collect(Collectors.toList());
+    }
+
+    private int comparePolicyEntries(PolicyEntry o1, PolicyEntry o2) {
+        int compare = 0;
+        Resource r1 = o1.getResource();
+        Resource r2 = o2.getResource();
+        if (r1.getResourceType() != r2.getResourceType()) {
+            if (r1.getResourceType() == ResourceType.ANY) {
+                compare = 1;
+            }
+            if (r2.getResourceType() == ResourceType.ANY) {
+                compare = -1;
+            }
+        } else if (r1.getResourcePattern() == r2.getResourcePattern()) {
+            if (r1.getResourcePattern() == ResourcePattern.PREFIXED) {
+                String n1 = r1.getResourceName();
+                String n2 = r2.getResourceName();
+                compare = Integer.compare(n1.length(), n2.length());
+            }
+        } else {
+            if (r1.getResourcePattern() == ResourcePattern.LITERAL) {
+                compare = 1;
+            }
+            if (r1.getResourcePattern() == ResourcePattern.LITERAL) {
+                compare = -1;
+            }
+            if (r1.getResourcePattern() == ResourcePattern.PREFIXED) {
+                compare = 1;
+            }
+            if (r1.getResourcePattern() == ResourcePattern.PREFIXED) {
+                compare = -1;
+            }
+        }
+
+        if (compare != 0) {
+            return compare;
+        }
+
+        // the decision deny has higher priority
+        Decision d1 = o1.getDecision();
+        Decision d2 = o2.getDecision();
+        return d1 == Decision.DENY ? 1 : d2 == Decision.DENY ? -1 : 0;
+    }
+
+    private static void throwException(DefaultAuthorizationContext context, String detail) {
+        throw new AuthorizationException("{} has no permission to access {} from {}, " + detail,
+            context.getSubject().getSubjectKey(), context.getResource().getResourceKey(), context.getSourceIp());
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/chain/UserAuthorizationHandler.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/chain/UserAuthorizationHandler.java
new file mode 100644
index 00000000000..1c391df54f5
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/chain/UserAuthorizationHandler.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.chain;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import org.apache.rocketmq.auth.authentication.enums.SubjectType;
+import org.apache.rocketmq.auth.authentication.enums.UserStatus;
+import org.apache.rocketmq.auth.authentication.enums.UserType;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authentication.provider.AuthenticationMetadataProvider;
+import org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.chain.Handler;
+import org.apache.rocketmq.common.chain.HandlerChain;
+
+public class UserAuthorizationHandler implements Handler<DefaultAuthorizationContext, CompletableFuture<Void>> {
+
+    private final AuthenticationMetadataProvider authenticationMetadataProvider;
+
+    public UserAuthorizationHandler(AuthConfig config, Supplier<?> metadataService) {
+        this.authenticationMetadataProvider = AuthenticationFactory.getMetadataProvider(config, metadataService);
+    }
+
+    @Override
+    public CompletableFuture<Void> handle(DefaultAuthorizationContext context, HandlerChain<DefaultAuthorizationContext, CompletableFuture<Void>> chain) {
+        if (!context.getSubject().isSubject(SubjectType.USER)) {
+            return chain.handle(context);
+        }
+        return this.getUser(context.getSubject()).thenCompose(user -> {
+            if (user.getUserType() == UserType.SUPER) {
+                return CompletableFuture.completedFuture(null);
+            }
+            return chain.handle(context);
+        });
+    }
+
+    private CompletableFuture<User> getUser(Subject subject) {
+        if (this.authenticationMetadataProvider == null) {
+            throw new AuthorizationException("The authenticationMetadataProvider is not configured");
+        }
+        User user = (User) subject;
+        return authenticationMetadataProvider.getUser(user.getUsername()).thenApply(result -> {
+            if (result == null) {
+                throw new AuthorizationException("User:{} not found.", user.getUsername());
+            }
+            if (user.getUserStatus() == UserStatus.DISABLE) {
+                throw new AuthenticationException("User:{} is disabled.", user.getUsername());
+            }
+            return result;
+        });
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/context/AuthorizationContext.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/context/AuthorizationContext.java
new file mode 100644
index 00000000000..d2286d787e4
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/context/AuthorizationContext.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.context;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+
+public abstract class AuthorizationContext {
+
+    private String channelId;
+
+    private String rpcCode;
+
+    private Map<String, Object> extInfo;
+
+    @SuppressWarnings("unchecked")
+    public <T> T getExtInfo(String key) {
+        if (StringUtils.isBlank(key)) {
+            return null;
+        }
+        if (this.extInfo == null) {
+            return null;
+        }
+        Object value = this.extInfo.get(key);
+        if (value == null) {
+            return null;
+        }
+        return (T) value;
+    }
+
+    public void setExtInfo(String key, Object value) {
+        if (StringUtils.isBlank(key) || value == null) {
+            return;
+        }
+        if (this.extInfo == null) {
+            this.extInfo = new HashMap<>();
+        }
+        this.extInfo.put(key, value);
+    }
+
+    public boolean hasExtInfo(String key) {
+        Object value = getExtInfo(key);
+        return value != null;
+    }
+
+    public String getChannelId() {
+        return channelId;
+    }
+
+    public void setChannelId(String channelId) {
+        this.channelId = channelId;
+    }
+
+    public String getRpcCode() {
+        return rpcCode;
+    }
+
+    public void setRpcCode(String rpcCode) {
+        this.rpcCode = rpcCode;
+    }
+
+    public Map<String, Object> getExtInfo() {
+        return extInfo;
+    }
+
+    public void setExtInfo(Map<String, Object> extInfo) {
+        this.extInfo = extInfo;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/context/DefaultAuthorizationContext.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/context/DefaultAuthorizationContext.java
new file mode 100644
index 00000000000..8e38ed20fae
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/context/DefaultAuthorizationContext.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.context;
+
+import java.util.Collections;
+import java.util.List;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authorization.model.Resource;
+import org.apache.rocketmq.common.action.Action;
+
+public class DefaultAuthorizationContext extends AuthorizationContext {
+
+    private Subject subject;
+
+    private Resource resource;
+
+    private List<Action> actions;
+
+    private String sourceIp;
+
+    public static DefaultAuthorizationContext of(Subject subject, Resource resource, Action action, String sourceIp) {
+        DefaultAuthorizationContext context = new DefaultAuthorizationContext();
+        context.setSubject(subject);
+        context.setResource(resource);
+        context.setActions(Collections.singletonList(action));
+        context.setSourceIp(sourceIp);
+        return context;
+    }
+
+    public static DefaultAuthorizationContext of(Subject subject, Resource resource, List<Action> actions, String sourceIp) {
+        DefaultAuthorizationContext context = new DefaultAuthorizationContext();
+        context.setSubject(subject);
+        context.setResource(resource);
+        context.setActions(actions);
+        context.setSourceIp(sourceIp);
+        return context;
+    }
+
+    public String getSubjectKey() {
+        return this.subject != null ? this.subject.getSubjectKey() : null;
+    }
+
+    public String getResourceKey() {
+        return this.resource != null ? this.resource.getResourceKey() : null;
+    }
+
+    public Subject getSubject() {
+        return subject;
+    }
+
+    public void setSubject(Subject subject) {
+        this.subject = subject;
+    }
+
+    public Resource getResource() {
+        return resource;
+    }
+
+    public void setResource(Resource resource) {
+        this.resource = resource;
+    }
+
+    public List<Action> getActions() {
+        return actions;
+    }
+
+    public void setActions(List<Action> actions) {
+        this.actions = actions;
+    }
+
+    public String getSourceIp() {
+        return sourceIp;
+    }
+
+    public void setSourceIp(String sourceIp) {
+        this.sourceIp = sourceIp;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/enums/Decision.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/enums/Decision.java
new file mode 100644
index 00000000000..b7e69b745b0
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/enums/Decision.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.enums;
+
+import com.alibaba.fastjson2.annotation.JSONField;
+import org.apache.commons.lang3.StringUtils;
+
+public enum Decision {
+
+    ALLOW((byte) 1, "Allow"),
+
+    DENY((byte) 2, "Deny");
+
+    @JSONField(value = true)
+    private final byte code;
+    private final String name;
+
+    Decision(byte code, String name) {
+        this.code = code;
+        this.name = name;
+    }
+
+    public static Decision getByName(String name) {
+        for (Decision decision : Decision.values()) {
+            if (StringUtils.equalsIgnoreCase(decision.getName(), name)) {
+                return decision;
+            }
+        }
+        return null;
+    }
+
+    public byte getCode() {
+        return code;
+    }
+
+    public String getName() {
+        return name;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/enums/PolicyType.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/enums/PolicyType.java
new file mode 100644
index 00000000000..fff71d60ead
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/enums/PolicyType.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.enums;
+
+import com.alibaba.fastjson2.annotation.JSONField;
+import org.apache.commons.lang3.StringUtils;
+
+public enum PolicyType {
+
+    CUSTOM((byte) 1, "Custom"),
+
+    DEFAULT((byte) 2, "Default");
+
+    @JSONField(value = true)
+    private final byte code;
+    private final String name;
+
+    PolicyType(byte code, String name) {
+        this.code = code;
+        this.name = name;
+    }
+
+    public static PolicyType getByName(String name) {
+        for (PolicyType policyType : PolicyType.values()) {
+            if (StringUtils.equalsIgnoreCase(policyType.getName(), name)) {
+                return policyType;
+            }
+        }
+        return null;
+    }
+
+    public byte getCode() {
+        return code;
+    }
+
+    public String getName() {
+        return name;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/exception/AuthorizationException.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/exception/AuthorizationException.java
new file mode 100644
index 00000000000..e2aadcbe6f1
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/exception/AuthorizationException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.exception;
+
+import org.slf4j.helpers.MessageFormatter;
+
+public class AuthorizationException extends RuntimeException {
+
+    public AuthorizationException(String message) {
+        super(message);
+    }
+
+    public AuthorizationException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public AuthorizationException(String messagePattern, Object... argArray) {
+        super(MessageFormatter.arrayFormat(messagePattern, argArray).getMessage());
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/factory/AuthorizationFactory.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/factory/AuthorizationFactory.java
new file mode 100644
index 00000000000..29748a9ed44
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/factory/AuthorizationFactory.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.factory;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import io.netty.channel.ChannelHandlerContext;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authorization.AuthorizationEvaluator;
+import org.apache.rocketmq.auth.authorization.context.AuthorizationContext;
+import org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;
+import org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManagerImpl;
+import org.apache.rocketmq.auth.authorization.provider.AuthorizationMetadataProvider;
+import org.apache.rocketmq.auth.authorization.provider.AuthorizationProvider;
+import org.apache.rocketmq.auth.authorization.provider.DefaultAuthorizationProvider;
+import org.apache.rocketmq.auth.authorization.strategy.AuthorizationStrategy;
+import org.apache.rocketmq.auth.authorization.strategy.StatelessAuthorizationStrategy;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class AuthorizationFactory {
+
+    private static final Map<String, Object> INSTANCE_MAP = new HashMap<>();
+    private static final String PROVIDER_PREFIX = "PROVIDER_";
+    private static final String METADATA_PROVIDER_PREFIX = "METADATA_PROVIDER_";
+    private static final String EVALUATOR_PREFIX = "EVALUATOR_";
+
+    @SuppressWarnings("unchecked")
+    public static AuthorizationProvider<AuthorizationContext> getProvider(AuthConfig config) {
+        if (config == null) {
+            return null;
+        }
+        return computeIfAbsent(PROVIDER_PREFIX + config.getConfigName(), key -> {
+            try {
+                Class<? extends AuthorizationProvider<? extends AuthorizationContext>> clazz =
+                    DefaultAuthorizationProvider.class;
+                if (StringUtils.isNotBlank(config.getAuthorizationProvider())) {
+                    clazz = (Class<? extends AuthorizationProvider<? extends AuthorizationContext>>) Class.forName(config.getAuthorizationProvider());
+                }
+                return (AuthorizationProvider<AuthorizationContext>) clazz
+                    .getDeclaredConstructor().newInstance();
+            } catch (Exception e) {
+                throw new RuntimeException("Failed to load the authorization provider.", e);
+            }
+        });
+    }
+
+    public static AuthorizationMetadataProvider getMetadataProvider(AuthConfig config) {
+        return getMetadataProvider(config, null);
+    }
+
+    public static AuthorizationMetadataManager getMetadataManager(AuthConfig config) {
+        return new AuthorizationMetadataManagerImpl(config);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static AuthorizationMetadataProvider getMetadataProvider(AuthConfig config, Supplier<?> metadataService) {
+        if (config == null) {
+            return null;
+        }
+        return computeIfAbsent(METADATA_PROVIDER_PREFIX + config.getConfigName(), key -> {
+            try {
+                if (StringUtils.isBlank(config.getAuthorizationMetadataProvider())) {
+                    return null;
+                }
+                Class<? extends AuthorizationMetadataProvider> clazz = (Class<? extends AuthorizationMetadataProvider>)
+                    Class.forName(config.getAuthorizationMetadataProvider());
+                AuthorizationMetadataProvider result = clazz.getDeclaredConstructor().newInstance();
+                result.initialize(config, metadataService);
+                return result;
+            } catch (Exception e) {
+                throw new RuntimeException("Failed to load the authorization metadata provider.", e);
+            }
+        });
+    }
+
+    public static AuthorizationEvaluator getEvaluator(AuthConfig config) {
+        return computeIfAbsent(EVALUATOR_PREFIX + config.getConfigName(), key -> new AuthorizationEvaluator(config));
+    }
+
+    public static AuthorizationEvaluator getEvaluator(AuthConfig config, Supplier<?> metadataService) {
+        return computeIfAbsent(EVALUATOR_PREFIX + config.getConfigName(), key -> new AuthorizationEvaluator(config, metadataService));
+    }
+
+    @SuppressWarnings("unchecked")
+    public static AuthorizationStrategy getStrategy(AuthConfig config, Supplier<?> metadataService) {
+        try {
+            Class<? extends AuthorizationStrategy> clazz = StatelessAuthorizationStrategy.class;
+            if (StringUtils.isNotBlank(config.getAuthorizationStrategy())) {
+                clazz = (Class<? extends AuthorizationStrategy>) Class.forName(config.getAuthorizationStrategy());
+            }
+            return clazz.getDeclaredConstructor(AuthConfig.class, Supplier.class).newInstance(config, metadataService);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static List<AuthorizationContext> newContexts(AuthConfig config, Metadata metadata,
+        GeneratedMessageV3 message) {
+        AuthorizationProvider<AuthorizationContext> authorizationProvider = getProvider(config);
+        if (authorizationProvider == null) {
+            return null;
+        }
+        return authorizationProvider.newContexts(metadata, message);
+    }
+
+    public static List<AuthorizationContext> newContexts(AuthConfig config, ChannelHandlerContext context,
+        RemotingCommand command) {
+        AuthorizationProvider<AuthorizationContext> authorizationProvider = getProvider(config);
+        if (authorizationProvider == null) {
+            return null;
+        }
+        return authorizationProvider.newContexts(context, command);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <V> V computeIfAbsent(String key, Function<String, ? extends V> function) {
+        Object result = null;
+        if (INSTANCE_MAP.containsKey(key)) {
+            result = INSTANCE_MAP.get(key);
+        }
+        if (result == null) {
+            synchronized (INSTANCE_MAP) {
+                if (INSTANCE_MAP.containsKey(key)) {
+                    result = INSTANCE_MAP.get(key);
+                }
+                if (result == null) {
+                    result = function.apply(key);
+                    if (result != null) {
+                        INSTANCE_MAP.put(key, result);
+                    }
+                }
+            }
+        }
+        return result != null ? (V) result : null;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/manager/AuthorizationMetadataManager.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/manager/AuthorizationMetadataManager.java
new file mode 100644
index 00000000000..ce96230606b
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/manager/AuthorizationMetadataManager.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.manager;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authorization.enums.PolicyType;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.authorization.model.Resource;
+
+public interface AuthorizationMetadataManager {
+
+    void shutdown();
+
+    CompletableFuture<Void> createAcl(Acl acl);
+
+    CompletableFuture<Void> updateAcl(Acl acl);
+
+    CompletableFuture<Void> deleteAcl(Subject subject);
+
+    CompletableFuture<Void> deleteAcl(Subject subject, PolicyType policyType, Resource resource);
+
+    CompletableFuture<Acl> getAcl(Subject subject);
+
+    CompletableFuture<List<Acl>> listAcl(String subjectFilter, String resourceFilter);
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/manager/AuthorizationMetadataManagerImpl.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/manager/AuthorizationMetadataManagerImpl.java
new file mode 100644
index 00000000000..52b62f72b3c
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/manager/AuthorizationMetadataManagerImpl.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.manager;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authentication.enums.SubjectType;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authentication.provider.AuthenticationMetadataProvider;
+import org.apache.rocketmq.auth.authorization.enums.PolicyType;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.authorization.model.Environment;
+import org.apache.rocketmq.auth.authorization.model.Policy;
+import org.apache.rocketmq.auth.authorization.model.PolicyEntry;
+import org.apache.rocketmq.auth.authorization.model.Resource;
+import org.apache.rocketmq.auth.authorization.provider.AuthorizationMetadataProvider;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.utils.ExceptionUtils;
+import org.apache.rocketmq.common.utils.IPAddressUtils;
+
+public class AuthorizationMetadataManagerImpl implements AuthorizationMetadataManager {
+
+    private final AuthorizationMetadataProvider authorizationMetadataProvider;
+
+    private final AuthenticationMetadataProvider authenticationMetadataProvider;
+
+    public AuthorizationMetadataManagerImpl(AuthConfig authConfig) {
+        this.authorizationMetadataProvider = AuthorizationFactory.getMetadataProvider(authConfig);
+        this.authenticationMetadataProvider = AuthenticationFactory.getMetadataProvider(authConfig);
+    }
+
+    @Override
+    public void shutdown() {
+        if (this.authenticationMetadataProvider != null) {
+            this.authenticationMetadataProvider.shutdown();
+        }
+        if (this.authorizationMetadataProvider != null) {
+            this.authorizationMetadataProvider.shutdown();
+        }
+    }
+
+    @Override
+    public CompletableFuture<Void> createAcl(Acl acl) {
+        try {
+            validate(acl);
+
+            initAcl(acl);
+
+            CompletableFuture<? extends Subject> subjectFuture;
+            if (acl.getSubject().isSubject(SubjectType.USER)) {
+                User user = (User) acl.getSubject();
+                subjectFuture = this.getAuthenticationMetadataProvider().getUser(user.getUsername());
+            } else {
+                subjectFuture = CompletableFuture.completedFuture(acl.getSubject());
+            }
+
+            return subjectFuture.thenCompose(subject -> {
+                if (subject == null) {
+                    throw new AuthorizationException("The subject of {} is not exist.", acl.getSubject().getSubjectKey());
+                }
+                return this.getAuthorizationMetadataProvider().getAcl(acl.getSubject());
+            }).thenCompose(oldAcl -> {
+                if (oldAcl == null) {
+                    return this.getAuthorizationMetadataProvider().createAcl(acl);
+                }
+                oldAcl.updatePolicy(acl.getPolicies());
+                return this.getAuthorizationMetadataProvider().updateAcl(oldAcl);
+            });
+
+        } catch (Exception e) {
+            return this.handleException(e);
+        }
+    }
+
+    @Override
+    public CompletableFuture<Void> updateAcl(Acl acl) {
+        try {
+            validate(acl);
+
+            initAcl(acl);
+
+            CompletableFuture<? extends Subject> subjectFuture;
+            if (acl.getSubject().isSubject(SubjectType.USER)) {
+                User user = (User) acl.getSubject();
+                subjectFuture = this.getAuthenticationMetadataProvider().getUser(user.getUsername());
+            } else {
+                subjectFuture = CompletableFuture.completedFuture(acl.getSubject());
+            }
+
+            return subjectFuture.thenCompose(subject -> {
+                if (subject == null) {
+                    throw new AuthorizationException("The subject of {} is not exist.", acl.getSubject().getSubjectKey());
+                }
+                return this.getAuthorizationMetadataProvider().getAcl(acl.getSubject());
+            }).thenCompose(oldAcl -> {
+                if (oldAcl == null) {
+                    return this.getAuthorizationMetadataProvider().createAcl(acl);
+                }
+                oldAcl.updatePolicy(acl.getPolicies());
+                return this.getAuthorizationMetadataProvider().updateAcl(oldAcl);
+            });
+
+        } catch (Exception e) {
+            return this.handleException(e);
+        }
+    }
+
+    @Override
+    public CompletableFuture<Void> deleteAcl(Subject subject) {
+        return this.deleteAcl(subject, null, null);
+    }
+
+    @Override
+    public CompletableFuture<Void> deleteAcl(Subject subject, PolicyType policyType, Resource resource) {
+        try {
+            if (subject == null) {
+                throw new AuthorizationException("The subject is null.");
+            }
+            if (policyType == null) {
+                policyType = PolicyType.CUSTOM;
+            }
+
+            CompletableFuture<? extends Subject> subjectFuture;
+            if (subject.isSubject(SubjectType.USER)) {
+                User user = (User) subject;
+                subjectFuture = this.getAuthenticationMetadataProvider().getUser(user.getUsername());
+            } else {
+                subjectFuture = CompletableFuture.completedFuture(subject);
+            }
+            CompletableFuture<Acl> aclFuture = this.getAuthorizationMetadataProvider().getAcl(subject);
+
+            PolicyType finalPolicyType = policyType;
+            return subjectFuture.thenCombine(aclFuture, (sub, oldAcl) -> {
+                if (sub == null) {
+                    throw new AuthorizationException("The subject is not exist.");
+                }
+                if (oldAcl == null) {
+                    throw new AuthorizationException("The acl is not exist.");
+                }
+                return oldAcl;
+            }).thenCompose(oldAcl -> {
+                if (resource != null) {
+                    oldAcl.deletePolicy(finalPolicyType, resource);
+                }
+                if (resource == null || CollectionUtils.isEmpty(oldAcl.getPolicies())) {
+                    return this.getAuthorizationMetadataProvider().deleteAcl(subject);
+                }
+                return this.getAuthorizationMetadataProvider().updateAcl(oldAcl);
+            });
+
+        } catch (Exception e) {
+            return this.handleException(e);
+        }
+    }
+
+    @Override
+    public CompletableFuture<Acl> getAcl(Subject subject) {
+        CompletableFuture<? extends Subject> subjectFuture;
+        if (subject.isSubject(SubjectType.USER)) {
+            User user = (User) subject;
+            subjectFuture = this.getAuthenticationMetadataProvider().getUser(user.getUsername());
+        } else {
+            subjectFuture = CompletableFuture.completedFuture(subject);
+        }
+        return subjectFuture.thenCompose(sub -> {
+            if (sub == null) {
+                throw new AuthorizationException("The subject is not exist.");
+            }
+            return this.getAuthorizationMetadataProvider().getAcl(subject);
+        });
+    }
+
+    @Override
+    public CompletableFuture<List<Acl>> listAcl(String subjectFilter, String resourceFilter) {
+        return this.getAuthorizationMetadataProvider().listAcl(subjectFilter, resourceFilter);
+    }
+
+    private static void initAcl(Acl acl) {
+        acl.getPolicies().forEach(policy -> {
+            if (policy.getPolicyType() == null) {
+                policy.setPolicyType(PolicyType.CUSTOM);
+            }
+        });
+    }
+
+    private void validate(Acl acl) {
+        Subject subject = acl.getSubject();
+        if (subject.getSubjectType() == null) {
+            throw new AuthorizationException("The subject type is null.");
+        }
+        List<Policy> policies = acl.getPolicies();
+        if (CollectionUtils.isEmpty(policies)) {
+            throw new AuthorizationException("The policies is empty.");
+        }
+        for (Policy policy : policies) {
+            this.validate(policy);
+        }
+    }
+
+    private void validate(Policy policy) {
+        List<PolicyEntry> policyEntries = policy.getEntries();
+        if (CollectionUtils.isEmpty(policyEntries)) {
+            throw new AuthorizationException("The policy entries is empty.");
+        }
+        for (PolicyEntry policyEntry : policyEntries) {
+            this.validate(policyEntry);
+        }
+    }
+
+    private void validate(PolicyEntry entry) {
+        Resource resource = entry.getResource();
+        if (resource == null) {
+            throw new AuthorizationException("The resource is null.");
+        }
+        if (resource.getResourceType() == null) {
+            throw new AuthorizationException("The resource type is null.");
+        }
+        if (resource.getResourcePattern() == null) {
+            throw new AuthorizationException("The resource pattern is null.");
+        }
+        if (CollectionUtils.isEmpty(entry.getActions())) {
+            throw new AuthorizationException("The actions is empty.");
+        }
+        if (entry.getActions().contains(Action.ANY)) {
+            throw new AuthorizationException("The actions can not be Any.");
+        }
+        Environment environment = entry.getEnvironment();
+        if (environment != null && CollectionUtils.isNotEmpty(environment.getSourceIps())) {
+            for (String sourceIp : environment.getSourceIps()) {
+                if (StringUtils.isBlank(sourceIp)) {
+                    throw new AuthorizationException("The source ip is empty.");
+                }
+                if (!IPAddressUtils.isValidIPOrCidr(sourceIp)) {
+                    throw new AuthorizationException("The source ip is invalid.");
+                }
+            }
+        }
+        if (entry.getDecision() == null) {
+            throw new AuthorizationException("The decision is null or illegal.");
+        }
+    }
+
+    private <T> CompletableFuture<T> handleException(Exception e) {
+        CompletableFuture<T> result = new CompletableFuture<>();
+        Throwable throwable = ExceptionUtils.getRealException(e);
+        result.completeExceptionally(throwable);
+        return result;
+    }
+
+    private AuthenticationMetadataProvider getAuthenticationMetadataProvider() {
+        if (authorizationMetadataProvider == null) {
+            throw new IllegalStateException("The authenticationMetadataProvider is not configured.");
+        }
+        return authenticationMetadataProvider;
+    }
+
+    private AuthorizationMetadataProvider getAuthorizationMetadataProvider() {
+        if (authenticationMetadataProvider == null) {
+            throw new IllegalStateException("The authenticationMetadataProvider is not configured.");
+        }
+        return authorizationMetadataProvider;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Acl.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Acl.java
new file mode 100644
index 00000000000..75d34b08145
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Acl.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.model;
+
+import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authorization.enums.Decision;
+import org.apache.rocketmq.auth.authorization.enums.PolicyType;
+import org.apache.rocketmq.common.action.Action;
+
+public class Acl {
+
+    private Subject subject;
+
+    private List<Policy> policies;
+
+    public static Acl of(Subject subject, Policy policy) {
+        return of(subject, Lists.newArrayList(policy));
+    }
+
+    public static Acl of(Subject subject, List<Policy> policies) {
+        Acl acl = new Acl();
+        acl.setSubject(subject);
+        acl.setPolicies(policies);
+        return acl;
+    }
+
+    public static Acl of(Subject subject, List<Resource> resources, List<Action> actions, Environment environment,
+        Decision decision) {
+        Acl acl = new Acl();
+        acl.setSubject(subject);
+        Policy policy = Policy.of(resources, actions, environment, decision);
+        acl.setPolicies(Lists.newArrayList(policy));
+        return acl;
+    }
+
+    public void updatePolicy(Policy policy) {
+        this.updatePolicy(Lists.newArrayList(policy));
+    }
+
+    public void updatePolicy(List<Policy> policies) {
+        if (this.policies == null) {
+            this.policies = new ArrayList<>();
+        }
+        policies.forEach(newPolicy -> {
+            Policy oldPolicy = this.getPolicy(newPolicy.getPolicyType());
+            if (oldPolicy == null) {
+                this.policies.add(newPolicy);
+            } else {
+                oldPolicy.updateEntry(newPolicy.getEntries());
+            }
+        });
+    }
+
+    public void deletePolicy(PolicyType policyType, Resource resource) {
+        Policy policy = getPolicy(policyType);
+        if (policy == null) {
+            return;
+        }
+        policy.deleteEntry(resource);
+        if (CollectionUtils.isEmpty(policy.getEntries())) {
+            this.policies.remove(policy);
+        }
+    }
+
+    public Policy getPolicy(PolicyType policyType) {
+        if (CollectionUtils.isEmpty(this.policies)) {
+            return null;
+        }
+        for (Policy policy : this.policies) {
+            if (policy.getPolicyType() == policyType) {
+                return policy;
+            }
+        }
+        return null;
+    }
+
+    public Subject getSubject() {
+        return subject;
+    }
+
+    public void setSubject(Subject subject) {
+        this.subject = subject;
+    }
+
+    public List<Policy> getPolicies() {
+        return policies;
+    }
+
+    public void setPolicies(List<Policy> policies) {
+        this.policies = policies;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Environment.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Environment.java
new file mode 100644
index 00000000000..f514e20750f
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Environment.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.model;
+
+import java.util.Collections;
+import java.util.List;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.utils.IPAddressUtils;
+
+public class Environment {
+
+    private List<String> sourceIps;
+
+    public static Environment of(String sourceIp) {
+        if (StringUtils.isEmpty(sourceIp)) {
+            return null;
+        }
+        return of(Collections.singletonList(sourceIp));
+    }
+
+    public static Environment of(List<String> sourceIps) {
+        if (CollectionUtils.isEmpty(sourceIps)) {
+            return null;
+        }
+        Environment environment = new Environment();
+        environment.setSourceIps(sourceIps);
+        return environment;
+    }
+
+    public boolean isMatch(Environment environment) {
+        if (CollectionUtils.isEmpty(this.sourceIps)) {
+            return true;
+        }
+        if (CollectionUtils.isEmpty(environment.getSourceIps())) {
+            return false;
+        }
+        String targetIp = environment.getSourceIps().get(0);
+        for (String sourceIp : this.sourceIps) {
+            if (IPAddressUtils.isIPInRange(targetIp, sourceIp)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public List<String> getSourceIps() {
+        return sourceIps;
+    }
+
+    public void setSourceIps(List<String> sourceIps) {
+        this.sourceIps = sourceIps;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Policy.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Policy.java
new file mode 100644
index 00000000000..fb4979d46ef
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Policy.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.auth.authorization.enums.Decision;
+import org.apache.rocketmq.auth.authorization.enums.PolicyType;
+
+public class Policy {
+
+    private PolicyType policyType;
+
+    private List<PolicyEntry> entries;
+
+    public static Policy of(List<Resource> resources, List<Action> actions, Environment environment,
+        Decision decision) {
+        return of(PolicyType.CUSTOM, resources, actions, environment, decision);
+    }
+
+    public static Policy of(PolicyType policyType, List<Resource> resources, List<Action> actions,
+        Environment environment,
+        Decision decision) {
+        Policy policy = new Policy();
+        policy.setPolicyType(policyType);
+        List<PolicyEntry> entries = resources.stream()
+            .map(resource -> PolicyEntry.of(resource, actions, environment, decision))
+            .collect(Collectors.toList());
+        policy.setEntries(entries);
+        return policy;
+    }
+
+    public static Policy of(PolicyType type, List<PolicyEntry> entries) {
+        Policy policy = new Policy();
+        policy.setPolicyType(type);
+        policy.setEntries(entries);
+        return policy;
+    }
+
+    public void updateEntry(List<PolicyEntry> newEntries) {
+        if (this.entries == null) {
+            this.entries = new ArrayList<>();
+        }
+        newEntries.forEach(newEntry -> {
+            PolicyEntry entry = getEntry(newEntry.getResource());
+            if (entry == null) {
+                this.entries.add(newEntry);
+            } else {
+                entry.updateEntry(newEntry.getActions(), newEntry.getEnvironment(), newEntry.getDecision());
+            }
+        });
+    }
+
+    public void deleteEntry(Resource resources) {
+        PolicyEntry entry = getEntry(resources);
+        if (entry != null) {
+            this.entries.remove(entry);
+        }
+    }
+
+    private PolicyEntry getEntry(Resource resource) {
+        if (CollectionUtils.isEmpty(this.entries)) {
+            return null;
+        }
+        for (PolicyEntry entry : this.entries) {
+            if (Objects.equals(entry.getResource(), resource)) {
+                return entry;
+            }
+        }
+        return null;
+    }
+
+    public PolicyType getPolicyType() {
+        return policyType;
+    }
+
+    public void setPolicyType(PolicyType policyType) {
+        this.policyType = policyType;
+    }
+
+    public List<PolicyEntry> getEntries() {
+        return entries;
+    }
+
+    public void setEntries(List<PolicyEntry> entries) {
+        this.entries = entries;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/PolicyEntry.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/PolicyEntry.java
new file mode 100644
index 00000000000..f1199842c68
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/PolicyEntry.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.model;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.auth.authorization.enums.Decision;
+
+public class PolicyEntry {
+
+    private Resource resource;
+
+    private List<Action> actions;
+
+    private Environment environment;
+
+    private Decision decision;
+
+    public static PolicyEntry of(Resource resource, List<Action> actions, Environment environment, Decision decision) {
+        PolicyEntry policyEntry = new PolicyEntry();
+        policyEntry.setResource(resource);
+        policyEntry.setActions(actions);
+        policyEntry.setEnvironment(environment);
+        policyEntry.setDecision(decision);
+        return policyEntry;
+    }
+
+    public void updateEntry(List<Action> actions, Environment environment,
+        Decision decision) {
+        this.setActions(actions);
+        this.setEnvironment(environment);
+        this.setDecision(decision);
+    }
+
+    public boolean isMatchResource(Resource resource) {
+        return this.resource.isMatch(resource);
+    }
+
+    public boolean isMatchAction(List<Action> actions) {
+        if (CollectionUtils.isEmpty(this.actions)) {
+            return false;
+        }
+        if (actions.contains(Action.ANY)) {
+            return true;
+        }
+        return actions.stream()
+            .anyMatch(action -> this.actions.contains(action)
+                || this.actions.contains(Action.ALL));
+    }
+
+    public boolean isMatchEnvironment(Environment environment) {
+        if (this.environment == null) {
+            return true;
+        }
+        return this.environment.isMatch(environment);
+    }
+
+    public String toResourceStr() {
+        if (resource == null) {
+            return null;
+        }
+        return resource.getResourceKey();
+    }
+
+    public List<String> toActionsStr() {
+        if (CollectionUtils.isEmpty(actions)) {
+            return null;
+        }
+        return actions.stream().map(Action::getName)
+            .collect(Collectors.toList());
+    }
+
+    public Resource getResource() {
+        return resource;
+    }
+
+    public void setResource(Resource resource) {
+        this.resource = resource;
+    }
+
+    public List<Action> getActions() {
+        return actions;
+    }
+
+    public void setActions(List<Action> actions) {
+        this.actions = actions;
+    }
+
+    public Environment getEnvironment() {
+        return environment;
+    }
+
+    public void setEnvironment(Environment environment) {
+        this.environment = environment;
+    }
+
+    public Decision getDecision() {
+        return decision;
+    }
+
+    public void setDecision(Decision decision) {
+        this.decision = decision;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/RequestContext.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/RequestContext.java
new file mode 100644
index 00000000000..88b814de5f1
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/RequestContext.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.model;
+
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.common.action.Action;
+
+public class RequestContext {
+
+    private Subject subject;
+
+    private Resource resource;
+
+    private Action action;
+
+    private String sourceIp;
+
+    public Subject getSubject() {
+        return subject;
+    }
+
+    public void setSubject(Subject subject) {
+        this.subject = subject;
+    }
+
+    public Resource getResource() {
+        return resource;
+    }
+
+    public void setResource(Resource resource) {
+        this.resource = resource;
+    }
+
+    public Action getAction() {
+        return action;
+    }
+
+    public void setAction(Action action) {
+        this.action = action;
+    }
+
+    public String getSourceIp() {
+        return sourceIp;
+    }
+
+    public void setSourceIp(String sourceIp) {
+        this.sourceIp = sourceIp;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Resource.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Resource.java
new file mode 100644
index 00000000000..67a37578dec
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/model/Resource.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.model;
+
+import com.alibaba.fastjson2.annotation.JSONField;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.ResourcePattern;
+import org.apache.rocketmq.common.constant.CommonConstants;
+import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
+
+public class Resource {
+
+    private ResourceType resourceType;
+
+    private String resourceName;
+
+    private ResourcePattern resourcePattern;
+
+    public static Resource ofCluster(String clusterName) {
+        return of(ResourceType.CLUSTER, clusterName, ResourcePattern.LITERAL);
+    }
+
+    public static Resource ofTopic(String topicName) {
+        return of(ResourceType.TOPIC, topicName, ResourcePattern.LITERAL);
+    }
+
+    public static Resource ofGroup(String groupName) {
+        if (NamespaceUtil.isRetryTopic(groupName)) {
+            groupName = NamespaceUtil.withOutRetryAndDLQ(groupName);
+        }
+        return of(ResourceType.GROUP, groupName, ResourcePattern.LITERAL);
+    }
+
+    public static Resource of(ResourceType resourceType, String resourceName, ResourcePattern resourcePattern) {
+        Resource resource = new Resource();
+        resource.resourceType = resourceType;
+        resource.resourceName = resourceName;
+        resource.resourcePattern = resourcePattern;
+        return resource;
+    }
+
+    public static List<Resource> of(List<String> resourceKeys) {
+        if (CollectionUtils.isEmpty(resourceKeys)) {
+            return null;
+        }
+        return resourceKeys.stream().map(Resource::of).collect(Collectors.toList());
+    }
+
+    public static Resource of(String resourceKey) {
+        if (StringUtils.isBlank(resourceKey)) {
+            return null;
+        }
+        if (StringUtils.equals(resourceKey, CommonConstants.ASTERISK)) {
+            return of(ResourceType.ANY, null, ResourcePattern.ANY);
+        }
+        String type = StringUtils.substringBefore(resourceKey, CommonConstants.COLON);
+        ResourceType resourceType = ResourceType.getByName(type);
+        if (resourceType == null) {
+            return null;
+        }
+        String resourceName = StringUtils.substringAfter(resourceKey, CommonConstants.COLON);
+        ResourcePattern resourcePattern = ResourcePattern.LITERAL;
+        if (StringUtils.equals(resourceName, CommonConstants.ASTERISK)) {
+            resourceName = null;
+            resourcePattern = ResourcePattern.ANY;
+        } else if (StringUtils.endsWith(resourceName, CommonConstants.ASTERISK)) {
+            resourceName = StringUtils.substringBefore(resourceName, CommonConstants.ASTERISK);
+            resourcePattern = ResourcePattern.PREFIXED;
+        }
+        return of(resourceType, resourceName, resourcePattern);
+    }
+
+    @JSONField(serialize = false)
+    public String getResourceKey() {
+        if (resourceType == ResourceType.ANY) {
+            return CommonConstants.ASTERISK;
+        }
+        switch (resourcePattern) {
+            case ANY:
+                return resourceType.getName() + CommonConstants.COLON + CommonConstants.ASTERISK;
+            case LITERAL:
+                return resourceType.getName() + CommonConstants.COLON + resourceName;
+            case PREFIXED:
+                return resourceType.getName() + CommonConstants.COLON + resourceName + CommonConstants.ASTERISK;
+            default:
+                return null;
+        }
+    }
+
+    public boolean isMatch(Resource resource) {
+        if (this.resourceType == ResourceType.ANY) {
+            return true;
+        }
+        if (this.resourceType != resource.resourceType) {
+            return false;
+        }
+        switch (resourcePattern) {
+            case ANY:
+                return true;
+            case LITERAL:
+                return StringUtils.equals(resource.resourceName, this.resourceName);
+            case PREFIXED:
+                return StringUtils.startsWith(resource.resourceName, this.resourceName);
+            default:
+                return false;
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        Resource resource = (Resource) o;
+        return resourceType == resource.resourceType
+            && Objects.equals(resourceName, resource.resourceName)
+            && resourcePattern == resource.resourcePattern;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(resourceType, resourceName, resourcePattern);
+    }
+
+    public ResourceType getResourceType() {
+        return resourceType;
+    }
+
+    public void setResourceType(ResourceType resourceType) {
+        this.resourceType = resourceType;
+    }
+
+    public String getResourceName() {
+        return resourceName;
+    }
+
+    public void setResourceName(String resourceName) {
+        this.resourceName = resourceName;
+    }
+
+    public ResourcePattern getResourcePattern() {
+        return resourcePattern;
+    }
+
+    public void setResourcePattern(ResourcePattern resourcePattern) {
+        this.resourcePattern = resourcePattern;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/AuthorizationMetadataProvider.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/AuthorizationMetadataProvider.java
new file mode 100644
index 00000000000..02bae743d54
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/AuthorizationMetadataProvider.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.provider;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.config.AuthConfig;
+
+public interface AuthorizationMetadataProvider {
+
+    void initialize(AuthConfig authConfig, Supplier<?> metadataService);
+
+    void shutdown();
+
+    CompletableFuture<Void> createAcl(Acl acl);
+
+    CompletableFuture<Void> deleteAcl(Subject subject);
+
+    CompletableFuture<Void> updateAcl(Acl acl);
+
+    CompletableFuture<Acl> getAcl(Subject subject);
+
+    CompletableFuture<List<Acl>> listAcl(String subjectFilter, String resourceFilter);
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/AuthorizationProvider.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/AuthorizationProvider.java
new file mode 100644
index 00000000000..98bd39c27b0
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/AuthorizationProvider.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.provider;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import io.netty.channel.ChannelHandlerContext;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public interface AuthorizationProvider<AuthorizationContext> {
+
+    void initialize(AuthConfig config);
+
+    void initialize(AuthConfig config, Supplier<?> metadataService);
+
+    CompletableFuture<Void> authorize(AuthorizationContext context);
+
+    List<AuthorizationContext> newContexts(Metadata metadata, GeneratedMessageV3 message);
+
+    List<AuthorizationContext> newContexts(ChannelHandlerContext context, RemotingCommand command);
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/DefaultAuthorizationProvider.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/DefaultAuthorizationProvider.java
new file mode 100644
index 00000000000..75111030328
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/DefaultAuthorizationProvider.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.provider;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import io.netty.channel.ChannelHandlerContext;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import org.apache.rocketmq.auth.authorization.builder.AuthorizationContextBuilder;
+import org.apache.rocketmq.auth.authorization.builder.DefaultAuthorizationContextBuilder;
+import org.apache.rocketmq.auth.authorization.chain.AclAuthorizationHandler;
+import org.apache.rocketmq.auth.authorization.chain.UserAuthorizationHandler;
+import org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;
+import org.apache.rocketmq.auth.authorization.enums.Decision;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.chain.HandlerChain;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DefaultAuthorizationProvider implements AuthorizationProvider<DefaultAuthorizationContext> {
+
+    protected final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_AUTH_AUDIT_LOGGER_NAME);
+    protected AuthConfig authConfig;
+    protected Supplier<?> metadataService;
+    protected AuthorizationContextBuilder authorizationContextBuilder;
+
+    @Override
+    public void initialize(AuthConfig config) {
+        this.initialize(config, null);
+    }
+
+    @Override
+    public void initialize(AuthConfig config, Supplier<?> metadataService) {
+        this.authConfig = config;
+        this.metadataService = metadataService;
+        this.authorizationContextBuilder = new DefaultAuthorizationContextBuilder(config);
+    }
+
+    @Override
+    public CompletableFuture<Void> authorize(DefaultAuthorizationContext context) {
+        return this.newHandlerChain().handle(context)
+            .whenComplete((nil, ex) -> doAuditLog(context, ex));
+    }
+
+    @Override
+    public List<DefaultAuthorizationContext> newContexts(Metadata metadata, GeneratedMessageV3 message) {
+        return this.authorizationContextBuilder.build(metadata, message);
+    }
+
+    @Override
+    public List<DefaultAuthorizationContext> newContexts(ChannelHandlerContext context, RemotingCommand command) {
+        return this.authorizationContextBuilder.build(context, command);
+    }
+
+    protected HandlerChain<DefaultAuthorizationContext, CompletableFuture<Void>> newHandlerChain() {
+        return HandlerChain.<DefaultAuthorizationContext, CompletableFuture<Void>>create()
+            .addNext(new UserAuthorizationHandler(authConfig, metadataService))
+            .addNext(new AclAuthorizationHandler(authConfig, metadataService));
+    }
+
+    protected void doAuditLog(DefaultAuthorizationContext context, Throwable ex) {
+        if (context.getSubject() == null) {
+            return;
+        }
+        Decision decision = Decision.ALLOW;
+        if (ex != null) {
+            decision = Decision.DENY;
+        }
+        String subject = context.getSubject().getSubjectKey();
+        String actions = context.getActions().stream().map(Action::getName)
+            .collect(Collectors.joining(","));
+        String sourceIp = context.getSourceIp();
+        String resource = context.getResource().getResourceKey();
+        String request = context.getRpcCode();
+        String format = "[AUTHORIZATION] Subject = {} is {} Action = {} from sourceIp = {} on resource = {} for request = {}.";
+        if (decision == Decision.ALLOW) {
+            log.debug(format, subject, decision.getName(), actions, sourceIp, resource, request);
+        } else {
+            log.info(format, subject, decision.getName(), actions, sourceIp, resource, request);
+        }
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/LocalAuthorizationMetadataProvider.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/LocalAuthorizationMetadataProvider.java
new file mode 100644
index 00000000000..bc631781084
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/provider/LocalAuthorizationMetadataProvider.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.provider;
+
+import com.alibaba.fastjson2.JSON;
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.authorization.model.Policy;
+import org.apache.rocketmq.auth.authorization.model.PolicyEntry;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.config.ConfigRocksDBStorage;
+import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
+import org.rocksdb.RocksIterator;
+
+public class LocalAuthorizationMetadataProvider implements AuthorizationMetadataProvider {
+
+    private ConfigRocksDBStorage storage;
+
+    private LoadingCache<String, Acl> aclCache;
+
+    @Override
+    public void initialize(AuthConfig authConfig, Supplier<?> metadataService) {
+        this.storage = new ConfigRocksDBStorage(authConfig.getAuthConfigPath() + File.separator + "acls");
+        if (!this.storage.start()) {
+            throw new RuntimeException("Failed to load rocksdb for auth_acl, please check whether it is occupied.");
+        }
+        ThreadPoolExecutor cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor(
+            1,
+            1,
+            1000 * 60,
+            TimeUnit.MILLISECONDS,
+            "AclCacheRefresh",
+            100000
+        );
+
+        this.aclCache = Caffeine.newBuilder()
+            .maximumSize(authConfig.getAclCacheMaxNum())
+            .expireAfterAccess(authConfig.getAclCacheExpiredSecond(), TimeUnit.SECONDS)
+            .refreshAfterWrite(authConfig.getAclCacheRefreshSecond(), TimeUnit.SECONDS)
+            .executor(cacheRefreshExecutor)
+            .build(new AclCacheLoader(this.storage));
+    }
+
+    @Override
+    public CompletableFuture<Void> createAcl(Acl acl) {
+        try {
+            Subject subject = acl.getSubject();
+            byte[] keyBytes = subject.getSubjectKey().getBytes(StandardCharsets.UTF_8);
+            byte[] valueBytes = JSON.toJSONBytes(acl);
+            this.storage.put(keyBytes, keyBytes.length, valueBytes);
+            this.storage.flushWAL();
+            this.aclCache.invalidate(subject.getSubjectKey());
+        } catch (Exception e) {
+            throw new AuthorizationException("create Acl to RocksDB failed.", e);
+        }
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public CompletableFuture<Void> deleteAcl(Subject subject) {
+        try {
+            byte[] keyBytes = subject.getSubjectKey().getBytes(StandardCharsets.UTF_8);
+            this.storage.delete(keyBytes);
+            this.storage.flushWAL();
+            this.aclCache.invalidate(subject.getSubjectKey());
+        } catch (Exception e) {
+            throw new AuthorizationException("delete Acl from RocksDB failed.", e);
+        }
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public CompletableFuture<Void> updateAcl(Acl acl) {
+        try {
+            Subject subject = acl.getSubject();
+            byte[] keyBytes = subject.getSubjectKey().getBytes(StandardCharsets.UTF_8);
+            byte[] valueBytes = JSON.toJSONBytes(acl);
+            this.storage.put(keyBytes, keyBytes.length, valueBytes);
+            this.storage.flushWAL();
+            this.aclCache.invalidate(subject.getSubjectKey());
+        } catch (Exception e) {
+            throw new AuthorizationException("update Acl to RocksDB failed.", e);
+        }
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public CompletableFuture<Acl> getAcl(Subject subject) {
+        Acl acl = aclCache.get(subject.getSubjectKey());
+        if (acl == AclCacheLoader.EMPTY_ACL) {
+            return CompletableFuture.completedFuture(null);
+        }
+        return CompletableFuture.completedFuture(acl);
+    }
+
+    @Override
+    public CompletableFuture<List<Acl>> listAcl(String subjectFilter, String resourceFilter) {
+        List<Acl> result = new ArrayList<>();
+        try (RocksIterator iterator = this.storage.iterator()) {
+            iterator.seekToFirst();
+            while (iterator.isValid()) {
+                String subjectKey = new String(iterator.key(), StandardCharsets.UTF_8);
+                if (StringUtils.isNotBlank(subjectFilter) && !subjectKey.contains(subjectFilter)) {
+                    iterator.next();
+                    continue;
+                }
+                Subject subject = Subject.of(subjectKey);
+                Acl acl = JSON.parseObject(new String(iterator.value(), StandardCharsets.UTF_8), Acl.class);
+                List<Policy> policies = acl.getPolicies();
+                if (!CollectionUtils.isNotEmpty(policies)) {
+                    iterator.next();
+                    continue;
+                }
+                Iterator<Policy> policyIterator = policies.iterator();
+                while (policyIterator.hasNext()) {
+                    Policy policy = policyIterator.next();
+                    List<PolicyEntry> entries = policy.getEntries();
+                    if (CollectionUtils.isEmpty(entries)) {
+                        continue;
+                    }
+                    if (StringUtils.isNotBlank(resourceFilter) && !subjectKey.contains(resourceFilter)) {
+                        entries.removeIf(entry -> !entry.toResourceStr().contains(resourceFilter));
+                    }
+                    if (CollectionUtils.isEmpty(entries)) {
+                        policyIterator.remove();
+                    }
+                }
+                if (CollectionUtils.isNotEmpty(policies)) {
+                    result.add(Acl.of(subject, policies));
+                }
+                iterator.next();
+            }
+        }
+        return CompletableFuture.completedFuture(result);
+    }
+
+    @Override
+    public void shutdown() {
+        if (this.storage != null) {
+            this.storage.shutdown();
+        }
+    }
+
+    private static class AclCacheLoader implements CacheLoader<String, Acl> {
+        private final ConfigRocksDBStorage storage;
+        public static final Acl EMPTY_ACL = new Acl();
+
+        public AclCacheLoader(ConfigRocksDBStorage storage) {
+            this.storage = storage;
+        }
+
+        @Override
+        public Acl load(String subjectKey) {
+            try {
+                byte[] keyBytes = subjectKey.getBytes(StandardCharsets.UTF_8);
+                Subject subject = Subject.of(subjectKey);
+
+                byte[] valueBytes = this.storage.get(keyBytes);
+                if (ArrayUtils.isEmpty(valueBytes)) {
+                    return EMPTY_ACL;
+                }
+                Acl acl = JSON.parseObject(valueBytes, Acl.class);
+                return Acl.of(subject, acl.getPolicies());
+            } catch (Exception e) {
+                throw new AuthorizationException("get Acl from RocksDB failed.", e);
+            }
+        }
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/AbstractAuthorizationStrategy.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/AbstractAuthorizationStrategy.java
new file mode 100644
index 00000000000..849c3082d31
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/AbstractAuthorizationStrategy.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.strategy;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authorization.context.AuthorizationContext;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;
+import org.apache.rocketmq.auth.authorization.provider.AuthorizationProvider;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.utils.ExceptionUtils;
+
+public abstract class AbstractAuthorizationStrategy implements AuthorizationStrategy {
+
+    protected final AuthConfig authConfig;
+    protected final List<String> authorizationWhitelist = new ArrayList<>();
+    protected final AuthorizationProvider<AuthorizationContext> authorizationProvider;
+
+    public AbstractAuthorizationStrategy(AuthConfig authConfig, Supplier<?> metadataService) {
+        this.authConfig = authConfig;
+        this.authorizationProvider = AuthorizationFactory.getProvider(authConfig);
+        if (this.authorizationProvider != null) {
+            this.authorizationProvider.initialize(authConfig, metadataService);
+        }
+        if (StringUtils.isNotBlank(authConfig.getAuthorizationWhitelist())) {
+            String[] whitelist = StringUtils.split(authConfig.getAuthorizationWhitelist(), ",");
+            for (String rpcCode : whitelist) {
+                this.authorizationWhitelist.add(StringUtils.trim(rpcCode));
+            }
+        }
+    }
+
+    public void doEvaluate(AuthorizationContext context) {
+        if (context == null) {
+            return;
+        }
+        if (!this.authConfig.isAuthorizationEnabled()) {
+            return;
+        }
+        if (this.authorizationProvider == null) {
+            return;
+        }
+        if (this.authorizationWhitelist.contains(context.getRpcCode())) {
+            return;
+        }
+        try {
+            this.authorizationProvider.authorize(context).join();
+        } catch (AuthorizationException ex) {
+            throw ex;
+        } catch (Throwable ex) {
+            Throwable exception = ExceptionUtils.getRealException(ex);
+            if (exception instanceof AuthorizationException) {
+                throw (AuthorizationException) exception;
+            }
+            throw new AuthorizationException("Authorization failed. Please verify your access rights and try again.", exception);
+        }
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/AuthorizationStrategy.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/AuthorizationStrategy.java
new file mode 100644
index 00000000000..d65ca82e10a
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/AuthorizationStrategy.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.strategy;
+
+import org.apache.rocketmq.auth.authorization.context.AuthorizationContext;
+
+public interface AuthorizationStrategy {
+
+    void evaluate(AuthorizationContext context);
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/StatefulAuthorizationStrategy.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/StatefulAuthorizationStrategy.java
new file mode 100644
index 00000000000..d5b252ee721
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/StatefulAuthorizationStrategy.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.strategy;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authorization.context.AuthorizationContext;
+import org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.Pair;
+import org.apache.rocketmq.common.constant.CommonConstants;
+
+public class StatefulAuthorizationStrategy extends AbstractAuthorizationStrategy {
+
+    protected Cache<String, Pair<Boolean, AuthorizationException>> authCache;
+
+    public StatefulAuthorizationStrategy(AuthConfig authConfig, Supplier<?> metadataService) {
+        super(authConfig, metadataService);
+        this.authCache = Caffeine.newBuilder()
+            .expireAfterWrite(authConfig.getStatefulAuthorizationCacheExpiredSecond(), TimeUnit.SECONDS)
+            .maximumSize(authConfig.getStatefulAuthorizationCacheMaxNum())
+            .build();
+    }
+
+    @Override
+    public void evaluate(AuthorizationContext context) {
+        if (StringUtils.isBlank(context.getChannelId())) {
+            this.doEvaluate(context);
+            return;
+        }
+        Pair<Boolean, AuthorizationException> result = this.authCache.get(buildKey(context), key -> {
+            try {
+                this.doEvaluate(context);
+                return Pair.of(true, null);
+            } catch (AuthorizationException ex) {
+                return Pair.of(false, ex);
+            }
+        });
+        if (result != null && result.getObject1() == Boolean.FALSE) {
+            throw result.getObject2();
+        }
+    }
+
+    private String buildKey(AuthorizationContext context) {
+        if (context instanceof DefaultAuthorizationContext) {
+            DefaultAuthorizationContext ctx = (DefaultAuthorizationContext) context;
+            return ctx.getChannelId()
+                + (ctx.getSubject() != null ? CommonConstants.POUND + ctx.getSubjectKey() : "")
+                + CommonConstants.POUND + ctx.getResourceKey()
+                + CommonConstants.POUND + StringUtils.join(ctx.getActions(), CommonConstants.COMMA)
+                + CommonConstants.POUND + ctx.getSourceIp();
+        }
+        throw new AuthorizationException("The request of {} is not support.", context.getClass().getSimpleName());
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/StatelessAuthorizationStrategy.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/StatelessAuthorizationStrategy.java
new file mode 100644
index 00000000000..e5d5e53f3ee
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/strategy/StatelessAuthorizationStrategy.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.strategy;
+
+import java.util.function.Supplier;
+import org.apache.rocketmq.auth.authorization.context.AuthorizationContext;
+import org.apache.rocketmq.auth.config.AuthConfig;
+
+public class StatelessAuthorizationStrategy extends AbstractAuthorizationStrategy {
+
+    public StatelessAuthorizationStrategy(AuthConfig authConfig, Supplier<?> metadataService) {
+        super(authConfig, metadataService);
+    }
+
+    @Override
+    public void evaluate(AuthorizationContext context) {
+        super.doEvaluate(context);
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/config/AuthConfig.java b/auth/src/main/java/org/apache/rocketmq/auth/config/AuthConfig.java
new file mode 100644
index 00000000000..ed294c8ecbf
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/config/AuthConfig.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.config;
+
+public class AuthConfig implements Cloneable {
+
+    private String configName;
+
+    private String clusterName;
+
+    private String authConfigPath;
+
+    private boolean authenticationEnabled = false;
+
+    private String authenticationProvider;
+
+    private String authenticationMetadataProvider;
+
+    private String authenticationStrategy;
+
+    private String authenticationWhitelist;
+
+    private String initAuthenticationUser;
+
+    private String innerClientAuthenticationCredentials;
+
+    private boolean authorizationEnabled = false;
+
+    private String authorizationProvider;
+
+    private String authorizationMetadataProvider;
+
+    private String authorizationStrategy;
+
+    private String authorizationWhitelist;
+
+    private boolean migrateAuthFromV1Enabled = false;
+
+    private int userCacheMaxNum = 1000;
+
+    private int userCacheExpiredSecond = 600;
+
+    private int userCacheRefreshSecond = 60;
+
+    private int aclCacheMaxNum = 1000;
+
+    private int aclCacheExpiredSecond = 600;
+
+    private int aclCacheRefreshSecond = 60;
+
+    private int statefulAuthenticationCacheMaxNum = 10000;
+
+    private int statefulAuthenticationCacheExpiredSecond = 60;
+
+    private int statefulAuthorizationCacheMaxNum = 10000;
+
+    private int statefulAuthorizationCacheExpiredSecond = 60;
+
+    @Override
+    public AuthConfig clone() {
+        try {
+            return (AuthConfig) super.clone();
+        } catch (CloneNotSupportedException e) {
+            throw new AssertionError();
+        }
+    }
+
+    public String getConfigName() {
+        return configName;
+    }
+
+    public void setConfigName(String configName) {
+        this.configName = configName;
+    }
+
+    public String getClusterName() {
+        return clusterName;
+    }
+
+    public void setClusterName(String clusterName) {
+        this.clusterName = clusterName;
+    }
+
+    public String getAuthConfigPath() {
+        return authConfigPath;
+    }
+
+    public void setAuthConfigPath(String authConfigPath) {
+        this.authConfigPath = authConfigPath;
+    }
+
+    public boolean isAuthenticationEnabled() {
+        return authenticationEnabled;
+    }
+
+    public void setAuthenticationEnabled(boolean authenticationEnabled) {
+        this.authenticationEnabled = authenticationEnabled;
+    }
+
+    public String getAuthenticationProvider() {
+        return authenticationProvider;
+    }
+
+    public void setAuthenticationProvider(String authenticationProvider) {
+        this.authenticationProvider = authenticationProvider;
+    }
+
+    public String getAuthenticationMetadataProvider() {
+        return authenticationMetadataProvider;
+    }
+
+    public void setAuthenticationMetadataProvider(String authenticationMetadataProvider) {
+        this.authenticationMetadataProvider = authenticationMetadataProvider;
+    }
+
+    public String getAuthenticationStrategy() {
+        return authenticationStrategy;
+    }
+
+    public void setAuthenticationStrategy(String authenticationStrategy) {
+        this.authenticationStrategy = authenticationStrategy;
+    }
+
+    public String getAuthenticationWhitelist() {
+        return authenticationWhitelist;
+    }
+
+    public void setAuthenticationWhitelist(String authenticationWhitelist) {
+        this.authenticationWhitelist = authenticationWhitelist;
+    }
+
+    public String getInitAuthenticationUser() {
+        return initAuthenticationUser;
+    }
+
+    public void setInitAuthenticationUser(String initAuthenticationUser) {
+        this.initAuthenticationUser = initAuthenticationUser;
+    }
+
+    public String getInnerClientAuthenticationCredentials() {
+        return innerClientAuthenticationCredentials;
+    }
+
+    public void setInnerClientAuthenticationCredentials(String innerClientAuthenticationCredentials) {
+        this.innerClientAuthenticationCredentials = innerClientAuthenticationCredentials;
+    }
+
+    public boolean isAuthorizationEnabled() {
+        return authorizationEnabled;
+    }
+
+    public void setAuthorizationEnabled(boolean authorizationEnabled) {
+        this.authorizationEnabled = authorizationEnabled;
+    }
+
+    public String getAuthorizationProvider() {
+        return authorizationProvider;
+    }
+
+    public void setAuthorizationProvider(String authorizationProvider) {
+        this.authorizationProvider = authorizationProvider;
+    }
+
+    public String getAuthorizationMetadataProvider() {
+        return authorizationMetadataProvider;
+    }
+
+    public void setAuthorizationMetadataProvider(String authorizationMetadataProvider) {
+        this.authorizationMetadataProvider = authorizationMetadataProvider;
+    }
+
+    public String getAuthorizationStrategy() {
+        return authorizationStrategy;
+    }
+
+    public void setAuthorizationStrategy(String authorizationStrategy) {
+        this.authorizationStrategy = authorizationStrategy;
+    }
+
+    public String getAuthorizationWhitelist() {
+        return authorizationWhitelist;
+    }
+
+    public void setAuthorizationWhitelist(String authorizationWhitelist) {
+        this.authorizationWhitelist = authorizationWhitelist;
+    }
+
+    public boolean isMigrateAuthFromV1Enabled() {
+        return migrateAuthFromV1Enabled;
+    }
+
+    public void setMigrateAuthFromV1Enabled(boolean migrateAuthFromV1Enabled) {
+        this.migrateAuthFromV1Enabled = migrateAuthFromV1Enabled;
+    }
+
+    public int getUserCacheMaxNum() {
+        return userCacheMaxNum;
+    }
+
+    public void setUserCacheMaxNum(int userCacheMaxNum) {
+        this.userCacheMaxNum = userCacheMaxNum;
+    }
+
+    public int getUserCacheExpiredSecond() {
+        return userCacheExpiredSecond;
+    }
+
+    public void setUserCacheExpiredSecond(int userCacheExpiredSecond) {
+        this.userCacheExpiredSecond = userCacheExpiredSecond;
+    }
+
+    public int getUserCacheRefreshSecond() {
+        return userCacheRefreshSecond;
+    }
+
+    public void setUserCacheRefreshSecond(int userCacheRefreshSecond) {
+        this.userCacheRefreshSecond = userCacheRefreshSecond;
+    }
+
+    public int getAclCacheMaxNum() {
+        return aclCacheMaxNum;
+    }
+
+    public void setAclCacheMaxNum(int aclCacheMaxNum) {
+        this.aclCacheMaxNum = aclCacheMaxNum;
+    }
+
+    public int getAclCacheExpiredSecond() {
+        return aclCacheExpiredSecond;
+    }
+
+    public void setAclCacheExpiredSecond(int aclCacheExpiredSecond) {
+        this.aclCacheExpiredSecond = aclCacheExpiredSecond;
+    }
+
+    public int getAclCacheRefreshSecond() {
+        return aclCacheRefreshSecond;
+    }
+
+    public void setAclCacheRefreshSecond(int aclCacheRefreshSecond) {
+        this.aclCacheRefreshSecond = aclCacheRefreshSecond;
+    }
+
+    public int getStatefulAuthenticationCacheMaxNum() {
+        return statefulAuthenticationCacheMaxNum;
+    }
+
+    public void setStatefulAuthenticationCacheMaxNum(int statefulAuthenticationCacheMaxNum) {
+        this.statefulAuthenticationCacheMaxNum = statefulAuthenticationCacheMaxNum;
+    }
+
+    public int getStatefulAuthenticationCacheExpiredSecond() {
+        return statefulAuthenticationCacheExpiredSecond;
+    }
+
+    public void setStatefulAuthenticationCacheExpiredSecond(int statefulAuthenticationCacheExpiredSecond) {
+        this.statefulAuthenticationCacheExpiredSecond = statefulAuthenticationCacheExpiredSecond;
+    }
+
+    public int getStatefulAuthorizationCacheMaxNum() {
+        return statefulAuthorizationCacheMaxNum;
+    }
+
+    public void setStatefulAuthorizationCacheMaxNum(int statefulAuthorizationCacheMaxNum) {
+        this.statefulAuthorizationCacheMaxNum = statefulAuthorizationCacheMaxNum;
+    }
+
+    public int getStatefulAuthorizationCacheExpiredSecond() {
+        return statefulAuthorizationCacheExpiredSecond;
+    }
+
+    public void setStatefulAuthorizationCacheExpiredSecond(int statefulAuthorizationCacheExpiredSecond) {
+        this.statefulAuthorizationCacheExpiredSecond = statefulAuthorizationCacheExpiredSecond;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/migration/AuthMigrator.java b/auth/src/main/java/org/apache/rocketmq/auth/migration/AuthMigrator.java
new file mode 100644
index 00000000000..d2ab4dda88f
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/migration/AuthMigrator.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.migration;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.acl.common.AclConstants;
+import org.apache.rocketmq.auth.authentication.enums.UserType;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authorization.enums.Decision;
+import org.apache.rocketmq.auth.authorization.enums.PolicyType;
+import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;
+import org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.authorization.model.Policy;
+import org.apache.rocketmq.auth.authorization.model.PolicyEntry;
+import org.apache.rocketmq.auth.authorization.model.Resource;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.auth.migration.v1.PlainPermissionManager;
+import org.apache.rocketmq.auth.migration.v1.AclConfig;
+import org.apache.rocketmq.auth.migration.v1.PlainAccessConfig;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.constant.CommonConstants;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.resource.ResourcePattern;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+
+public class AuthMigrator {
+
+    protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+
+    private final AuthConfig authConfig;
+
+    private final PlainPermissionManager plainPermissionManager;
+
+    private final AuthenticationMetadataManager authenticationMetadataManager;
+
+    private final AuthorizationMetadataManager authorizationMetadataManager;
+
+    public AuthMigrator(AuthConfig authConfig) {
+        this.authConfig = authConfig;
+        this.plainPermissionManager = new PlainPermissionManager();
+        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(authConfig);
+        this.authorizationMetadataManager = AuthorizationFactory.getMetadataManager(authConfig);
+    }
+
+    public void migrate() {
+        if (!authConfig.isMigrateAuthFromV1Enabled()) {
+            return;
+        }
+
+        AclConfig aclConfig = this.plainPermissionManager.getAllAclConfig();
+        List<PlainAccessConfig> accessConfigs = aclConfig.getPlainAccessConfigs();
+        if (CollectionUtils.isEmpty(accessConfigs)) {
+            return;
+        }
+
+        for (PlainAccessConfig accessConfig : accessConfigs) {
+            doMigrate(accessConfig);
+        }
+    }
+
+    private void doMigrate(PlainAccessConfig accessConfig) {
+        this.isUserExisted(accessConfig.getAccessKey()).thenCompose(existed -> {
+            if (existed) {
+                return CompletableFuture.completedFuture(null);
+            }
+            return createUserAndAcl(accessConfig);
+        }).exceptionally(ex -> {
+            LOG.error("[ACL MIGRATE] An error occurred while migrating ACL configurations for AccessKey:{}.", accessConfig.getAccessKey(), ex);
+            return null;
+        }).join();
+    }
+
+    private CompletableFuture<Void> createUserAndAcl(PlainAccessConfig accessConfig) {
+        return createUser(accessConfig).thenCompose(nil -> createAcl(accessConfig));
+    }
+
+    private CompletableFuture<Void> createUser(PlainAccessConfig accessConfig) {
+        User user = new User();
+        user.setUsername(accessConfig.getAccessKey());
+        user.setPassword(accessConfig.getSecretKey());
+        if (accessConfig.isAdmin()) {
+            user.setUserType(UserType.SUPER);
+        } else {
+            user.setUserType(UserType.NORMAL);
+        }
+        return this.authenticationMetadataManager.createUser(user);
+    }
+
+    private CompletableFuture<Void> createAcl(PlainAccessConfig config) {
+        Subject subject = User.of(config.getAccessKey());
+        List<Policy> policies = new ArrayList<>();
+
+        Policy customPolicy = null;
+        if (CollectionUtils.isNotEmpty(config.getTopicPerms())) {
+            for (String topicPerm : config.getTopicPerms()) {
+                String[] temp = StringUtils.split(topicPerm, CommonConstants.EQUAL);
+                if (temp.length != 2) {
+                    continue;
+                }
+                String topicName = StringUtils.trim(temp[0]);
+                String perm = StringUtils.trim(temp[1]);
+                Resource resource = Resource.ofTopic(topicName);
+                List<Action> actions = parseActions(perm);
+                Decision decision = parseDecision(perm);
+                PolicyEntry policyEntry = PolicyEntry.of(resource, actions, null, decision);
+                if (customPolicy == null) {
+                    customPolicy = Policy.of(PolicyType.CUSTOM, new ArrayList<>());
+                }
+                customPolicy.getEntries().add(policyEntry);
+            }
+        }
+        if (CollectionUtils.isNotEmpty(config.getGroupPerms())) {
+            for (String groupPerm : config.getGroupPerms()) {
+                String[] temp = StringUtils.split(groupPerm, CommonConstants.EQUAL);
+                if (temp.length != 2) {
+                    continue;
+                }
+                String groupName = StringUtils.trim(temp[0]);
+                String perm = StringUtils.trim(temp[1]);
+                Resource resource = Resource.ofGroup(groupName);
+                List<Action> actions = parseActions(perm);
+                Decision decision = parseDecision(perm);
+                PolicyEntry policyEntry = PolicyEntry.of(resource, actions, null, decision);
+                if (customPolicy == null) {
+                    customPolicy = Policy.of(PolicyType.CUSTOM, new ArrayList<>());
+                }
+                customPolicy.getEntries().add(policyEntry);
+            }
+        }
+        if (customPolicy != null) {
+            policies.add(customPolicy);
+        }
+
+        Policy defaultPolicy = null;
+        if (StringUtils.isNotBlank(config.getDefaultTopicPerm())) {
+            String topicPerm = StringUtils.trim(config.getDefaultTopicPerm());
+            Resource resource = Resource.of(ResourceType.TOPIC, null, ResourcePattern.ANY);
+            List<Action> actions = parseActions(topicPerm);
+            Decision decision = parseDecision(topicPerm);
+            PolicyEntry policyEntry = PolicyEntry.of(resource, actions, null, decision);
+            defaultPolicy = Policy.of(PolicyType.DEFAULT, new ArrayList<>());
+            defaultPolicy.getEntries().add(policyEntry);
+        }
+        if (StringUtils.isNotBlank(config.getDefaultGroupPerm())) {
+            String groupPerm = StringUtils.trim(config.getDefaultGroupPerm());
+            Resource resource = Resource.of(ResourceType.GROUP, null, ResourcePattern.ANY);
+            List<Action> actions = parseActions(groupPerm);
+            Decision decision = parseDecision(groupPerm);
+            PolicyEntry policyEntry = PolicyEntry.of(resource, actions, null, decision);
+            if (defaultPolicy == null) {
+                defaultPolicy = Policy.of(PolicyType.DEFAULT, new ArrayList<>());
+            }
+            defaultPolicy.getEntries().add(policyEntry);
+        }
+        if (defaultPolicy != null) {
+            policies.add(defaultPolicy);
+        }
+
+        if (CollectionUtils.isEmpty(policies)) {
+            return CompletableFuture.completedFuture(null);
+        }
+
+        Acl acl = Acl.of(subject, policies);
+        return this.authorizationMetadataManager.createAcl(acl);
+    }
+
+    private Decision parseDecision(String str) {
+        if (StringUtils.isBlank(str)) {
+            return Decision.DENY;
+        }
+        return StringUtils.equals(str, AclConstants.DENY) ? Decision.DENY : Decision.ALLOW;
+    }
+
+    private List<Action> parseActions(String str) {
+        List<Action> result = new ArrayList<>();
+        if (StringUtils.isBlank(str)) {
+            result.add(Action.ALL);
+        }
+        switch (StringUtils.trim(str)) {
+            case AclConstants.PUB:
+                result.add(Action.PUB);
+                break;
+            case AclConstants.SUB:
+                result.add(Action.SUB);
+                break;
+            case AclConstants.PUB_SUB:
+            case AclConstants.SUB_PUB:
+                result.add(Action.PUB);
+                result.add(Action.SUB);
+                break;
+            case AclConstants.DENY:
+                result.add(Action.ALL);
+                break;
+            default:
+                result.add(Action.ALL);
+                break;
+        }
+        return result;
+    }
+
+    private CompletableFuture<Boolean> isUserExisted(String username) {
+        return this.authenticationMetadataManager.getUser(username).thenApply(Objects::nonNull);
+    }
+}
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/AccessResource.java b/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/AccessResource.java
similarity index 94%
rename from acl/src/main/java/org/apache/rocketmq/acl/AccessResource.java
rename to auth/src/main/java/org/apache/rocketmq/auth/migration/v1/AccessResource.java
index e30febc5719..0a706b7b97e 100644
--- a/acl/src/main/java/org/apache/rocketmq/acl/AccessResource.java
+++ b/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/AccessResource.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.acl;
+package org.apache.rocketmq.auth.migration.v1;
 
 public interface AccessResource {
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/AclConfig.java b/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/AclConfig.java
similarity index 97%
rename from common/src/main/java/org/apache/rocketmq/common/AclConfig.java
rename to auth/src/main/java/org/apache/rocketmq/auth/migration/v1/AclConfig.java
index 49b9e05e2e1..dbea87717b2 100644
--- a/common/src/main/java/org/apache/rocketmq/common/AclConfig.java
+++ b/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/AclConfig.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.common;
+package org.apache.rocketmq.auth.migration.v1;
 
 import java.util.List;
 
diff --git a/common/src/main/java/org/apache/rocketmq/common/PlainAccessConfig.java b/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainAccessConfig.java
similarity index 98%
rename from common/src/main/java/org/apache/rocketmq/common/PlainAccessConfig.java
rename to auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainAccessConfig.java
index 24596daa529..f4368d6faa5 100644
--- a/common/src/main/java/org/apache/rocketmq/common/PlainAccessConfig.java
+++ b/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainAccessConfig.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.common;
+package org.apache.rocketmq.auth.migration.v1;
 
 import java.io.Serializable;
 import java.util.List;
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessData.java b/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainAccessData.java
similarity index 94%
rename from acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessData.java
rename to auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainAccessData.java
index 83c8cc40c49..d0270d28701 100644
--- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessData.java
+++ b/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainAccessData.java
@@ -14,9 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.acl.plain;
-
-import org.apache.rocketmq.common.PlainAccessConfig;
+package org.apache.rocketmq.auth.migration.v1;
 
 import java.io.Serializable;
 import java.util.ArrayList;
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainAccessResource.java b/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainAccessResource.java
new file mode 100644
index 00000000000..edeb8e5a4b5
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainAccessResource.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.migration.v1;
+
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.MixAll;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class PlainAccessResource implements AccessResource {
+
+    // Identify the user
+    private String accessKey;
+
+    private String secretKey;
+
+    private String whiteRemoteAddress;
+
+    private boolean admin;
+
+    private byte defaultTopicPerm = 1;
+
+    private byte defaultGroupPerm = 1;
+
+    private Map<String, Byte> resourcePermMap;
+
+    private int requestCode;
+
+    // The content to calculate the content
+    private byte[] content;
+
+    private String signature;
+
+    private String secretToken;
+
+    private String recognition;
+
+    public PlainAccessResource() {
+    }
+
+    public static String getGroupFromRetryTopic(String retryTopic) {
+        if (retryTopic == null) {
+            return null;
+        }
+        return KeyBuilder.parseGroup(retryTopic);
+    }
+
+    public static String getRetryTopic(String group) {
+        if (group == null) {
+            return null;
+        }
+        return MixAll.getRetryTopic(group);
+    }
+
+    public void addResourceAndPerm(String resource, byte perm) {
+        if (resource == null) {
+            return;
+        }
+        if (resourcePermMap == null) {
+            resourcePermMap = new HashMap<>();
+        }
+        resourcePermMap.put(resource, perm);
+    }
+
+    public String getAccessKey() {
+        return accessKey;
+    }
+
+    public void setAccessKey(String accessKey) {
+        this.accessKey = accessKey;
+    }
+
+    public String getSecretKey() {
+        return secretKey;
+    }
+
+    public void setSecretKey(String secretKey) {
+        this.secretKey = secretKey;
+    }
+
+    public String getWhiteRemoteAddress() {
+        return whiteRemoteAddress;
+    }
+
+    public void setWhiteRemoteAddress(String whiteRemoteAddress) {
+        this.whiteRemoteAddress = whiteRemoteAddress;
+    }
+
+    public boolean isAdmin() {
+        return admin;
+    }
+
+    public void setAdmin(boolean admin) {
+        this.admin = admin;
+    }
+
+    public byte getDefaultTopicPerm() {
+        return defaultTopicPerm;
+    }
+
+    public void setDefaultTopicPerm(byte defaultTopicPerm) {
+        this.defaultTopicPerm = defaultTopicPerm;
+    }
+
+    public byte getDefaultGroupPerm() {
+        return defaultGroupPerm;
+    }
+
+    public void setDefaultGroupPerm(byte defaultGroupPerm) {
+        this.defaultGroupPerm = defaultGroupPerm;
+    }
+
+    public Map<String, Byte> getResourcePermMap() {
+        return resourcePermMap;
+    }
+
+    public String getRecognition() {
+        return recognition;
+    }
+
+    public void setRecognition(String recognition) {
+        this.recognition = recognition;
+    }
+
+    public int getRequestCode() {
+        return requestCode;
+    }
+
+    public void setRequestCode(int requestCode) {
+        this.requestCode = requestCode;
+    }
+
+    public String getSecretToken() {
+        return secretToken;
+    }
+
+    public void setSecretToken(String secretToken) {
+        this.secretToken = secretToken;
+    }
+
+    public String getSignature() {
+        return signature;
+    }
+
+    public void setSignature(String signature) {
+        this.signature = signature;
+    }
+
+    @Override
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this);
+    }
+
+    public byte[] getContent() {
+        return content;
+    }
+
+    public void setContent(byte[] content) {
+        this.content = content;
+    }
+}
diff --git a/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainPermissionManager.java b/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainPermissionManager.java
new file mode 100644
index 00000000000..78828592611
--- /dev/null
+++ b/auth/src/main/java/org/apache/rocketmq/auth/migration/v1/PlainPermissionManager.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.migration.v1;
+
+import org.apache.rocketmq.acl.common.AclException;
+import org.apache.rocketmq.acl.common.AclUtils;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class PlainPermissionManager {
+
+    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
+
+    private String fileHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY,
+        System.getenv(MixAll.ROCKETMQ_HOME_ENV));
+
+    private String defaultAclDir;
+
+    private String defaultAclFile;
+
+    private List<String> fileList = new ArrayList<>();
+
+
+    public PlainPermissionManager() {
+        this.defaultAclDir = MixAll.dealFilePath(fileHome + File.separator + "conf" + File.separator + "acl");
+        this.defaultAclFile = MixAll.dealFilePath(fileHome + File.separator + System.getProperty("rocketmq.acl.plain.file", "conf" + File.separator + "plain_acl.yml"));
+        load();
+    }
+
+    public List<String> getAllAclFiles(String path) {
+        if (!new File(path).exists()) {
+            log.info("The default acl dir {} is not exist", path);
+            return new ArrayList<>();
+        }
+        List<String> allAclFileFullPath = new ArrayList<>();
+        File file = new File(path);
+        File[] files = file.listFiles();
+        for (int i = 0; files != null && i < files.length; i++) {
+            String fileName = files[i].getAbsolutePath();
+            File f = new File(fileName);
+            if (fileName.equals(fileHome + MixAll.ACL_CONF_TOOLS_FILE)) {
+                continue;
+            } else if (fileName.endsWith(".yml") || fileName.endsWith(".yaml")) {
+                allAclFileFullPath.add(fileName);
+            } else if (f.isDirectory()) {
+                allAclFileFullPath.addAll(getAllAclFiles(fileName));
+            }
+        }
+        return allAclFileFullPath;
+    }
+
+    public void load() {
+        if (fileHome == null || fileHome.isEmpty()) {
+            return;
+        }
+
+        assureAclConfigFilesExist();
+
+        fileList = getAllAclFiles(defaultAclDir);
+        if (new File(defaultAclFile).exists() && !fileList.contains(defaultAclFile)) {
+            fileList.add(defaultAclFile);
+        }
+    }
+
+    /**
+     * Currently GlobalWhiteAddress is defined in {@link #defaultAclFile}, so make sure it exists.
+     */
+    private void assureAclConfigFilesExist() {
+        final Path defaultAclFilePath = Paths.get(this.defaultAclFile);
+        if (!Files.exists(defaultAclFilePath)) {
+            try {
+                Files.createFile(defaultAclFilePath);
+            } catch (FileAlreadyExistsException e) {
+                // Maybe created by other threads
+            } catch (IOException e) {
+                log.error("Error in creating " + this.defaultAclFile, e);
+                throw new AclException(e.getMessage());
+            }
+        }
+    }
+
+    public AclConfig getAllAclConfig() {
+        AclConfig aclConfig = new AclConfig();
+        List<PlainAccessConfig> configs = new ArrayList<>();
+        List<String> whiteAddrs = new ArrayList<>();
+        Set<String> accessKeySets = new HashSet<>();
+
+        for (String path : fileList) {
+            PlainAccessData plainAclConfData = AclUtils.getYamlDataObject(path, PlainAccessData.class);
+            if (plainAclConfData == null) {
+                continue;
+            }
+            List<String> globalWhiteAddrs = plainAclConfData.getGlobalWhiteRemoteAddresses();
+            if (globalWhiteAddrs != null && !globalWhiteAddrs.isEmpty()) {
+                whiteAddrs.addAll(globalWhiteAddrs);
+            }
+
+            List<PlainAccessConfig> plainAccessConfigs = plainAclConfData.getAccounts();
+            if (plainAccessConfigs != null && !plainAccessConfigs.isEmpty()) {
+                for (PlainAccessConfig accessConfig : plainAccessConfigs) {
+                    if (!accessKeySets.contains(accessConfig.getAccessKey())) {
+                        accessKeySets.add(accessConfig.getAccessKey());
+                        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+                        plainAccessConfig.setGroupPerms(accessConfig.getGroupPerms());
+                        plainAccessConfig.setDefaultTopicPerm(accessConfig.getDefaultTopicPerm());
+                        plainAccessConfig.setDefaultGroupPerm(accessConfig.getDefaultGroupPerm());
+                        plainAccessConfig.setAccessKey(accessConfig.getAccessKey());
+                        plainAccessConfig.setSecretKey(accessConfig.getSecretKey());
+                        plainAccessConfig.setAdmin(accessConfig.isAdmin());
+                        plainAccessConfig.setTopicPerms(accessConfig.getTopicPerms());
+                        plainAccessConfig.setWhiteRemoteAddress(accessConfig.getWhiteRemoteAddress());
+                        configs.add(plainAccessConfig);
+                    }
+                }
+            }
+        }
+        aclConfig.setPlainAccessConfigs(configs);
+        aclConfig.setGlobalWhiteAddrs(whiteAddrs);
+        return aclConfig;
+    }
+}
diff --git a/auth/src/test/java/org/apache/rocketmq/auth/authentication/AuthenticationEvaluatorTest.java b/auth/src/test/java/org/apache/rocketmq/auth/authentication/AuthenticationEvaluatorTest.java
new file mode 100644
index 00000000000..dc20a0bb6d4
--- /dev/null
+++ b/auth/src/test/java/org/apache/rocketmq/auth/authentication/AuthenticationEvaluatorTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.auth.helper.AuthTestHelper;
+import org.apache.rocketmq.common.MixAll;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class AuthenticationEvaluatorTest {
+
+    private AuthConfig authConfig;
+    private AuthenticationEvaluator evaluator;
+    private AuthenticationMetadataManager authenticationMetadataManager;
+
+    @Before
+    public void setUp() throws Exception {
+        if (MixAll.isMac()) {
+            return;
+        }
+        this.authConfig = AuthTestHelper.createDefaultConfig();
+        this.evaluator = new AuthenticationEvaluator(authConfig);
+        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(authConfig);
+        this.clearAllUsers();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        if (MixAll.isMac()) {
+            return;
+        }
+        this.clearAllUsers();
+        this.authenticationMetadataManager.shutdown();
+    }
+
+    @Test
+    public void evaluate1() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user);
+
+        DefaultAuthenticationContext context = new DefaultAuthenticationContext();
+        context.setRpcCode("11");
+        context.setUsername("test");
+        context.setContent("test".getBytes(StandardCharsets.UTF_8));
+        context.setSignature("DJRRXBXlCVuKh6ULoN87847QX+Y=");
+        this.evaluator.evaluate(context);
+    }
+
+    @Test
+    public void evaluate2() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        DefaultAuthenticationContext context = new DefaultAuthenticationContext();
+        context.setRpcCode("11");
+        context.setUsername("test");
+        context.setContent("test".getBytes(StandardCharsets.UTF_8));
+        context.setSignature("DJRRXBXlCVuKh6ULoN87847QX+Y=");
+        Assert.assertThrows(AuthenticationException.class, () -> this.evaluator.evaluate(context));
+    }
+
+    @Test
+    public void evaluate3() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user);
+
+        DefaultAuthenticationContext context = new DefaultAuthenticationContext();
+        context.setRpcCode("11");
+        context.setUsername("test");
+        context.setContent("test".getBytes(StandardCharsets.UTF_8));
+        context.setSignature("test");
+        Assert.assertThrows(AuthenticationException.class, () -> this.evaluator.evaluate(context));
+    }
+
+    @Test
+    public void evaluate4() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        this.authConfig.setAuthenticationWhitelist("11");
+        this.evaluator = new AuthenticationEvaluator(authConfig);
+
+        DefaultAuthenticationContext context = new DefaultAuthenticationContext();
+        context.setRpcCode("11");
+        context.setUsername("test");
+        context.setContent("test".getBytes(StandardCharsets.UTF_8));
+        context.setSignature("test");
+        this.evaluator.evaluate(context);
+    }
+
+    @Test
+    public void evaluate5() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        this.authConfig.setAuthenticationEnabled(false);
+        this.evaluator = new AuthenticationEvaluator(authConfig);
+
+        DefaultAuthenticationContext context = new DefaultAuthenticationContext();
+        context.setRpcCode("11");
+        context.setUsername("test");
+        context.setContent("test".getBytes(StandardCharsets.UTF_8));
+        context.setSignature("test");
+        this.evaluator.evaluate(context);
+    }
+
+    private void clearAllUsers() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        List<User> users = this.authenticationMetadataManager.listUser(null).join();
+        if (CollectionUtils.isEmpty(users)) {
+            return;
+        }
+        users.forEach(user -> this.authenticationMetadataManager.deleteUser(user.getUsername()).join());
+    }
+}
\ No newline at end of file
diff --git a/auth/src/test/java/org/apache/rocketmq/auth/authentication/builder/DefaultAuthenticationContextBuilderTest.java b/auth/src/test/java/org/apache/rocketmq/auth/authentication/builder/DefaultAuthenticationContextBuilderTest.java
new file mode 100644
index 00000000000..e8e0144fcb6
--- /dev/null
+++ b/auth/src/test/java/org/apache/rocketmq/auth/authentication/builder/DefaultAuthenticationContextBuilderTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.builder;
+
+import apache.rocketmq.v2.Message;
+import apache.rocketmq.v2.Resource;
+import apache.rocketmq.v2.SendMessageRequest;
+import com.google.protobuf.ByteString;
+import io.grpc.Metadata;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelId;
+import java.nio.charset.StandardCharsets;
+import org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;
+import org.apache.rocketmq.common.constant.GrpcConstants;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.Silent.class)
+public class DefaultAuthenticationContextBuilderTest {
+
+    private DefaultAuthenticationContextBuilder builder;
+
+    @Mock
+    private ChannelHandlerContext channelHandlerContext;
+
+    @Mock
+    private Channel channel;
+
+    @Before
+    public void setUp() throws Exception {
+        builder = new DefaultAuthenticationContextBuilder();
+    }
+
+    @Test
+    public void build1() {
+        Resource topic = Resource.newBuilder().setName("topic-test").build();
+        {
+            SendMessageRequest request = SendMessageRequest.newBuilder()
+                .addMessages(Message.newBuilder().setTopic(topic)
+                    .setBody(ByteString.copyFromUtf8("message-body"))
+                    .build())
+                .build();
+            Metadata metadata = new Metadata();
+            metadata.put(GrpcConstants.AUTHORIZATION, "MQv2-HMAC-SHA1 Credential=abc, SignedHeaders=x-mq-date-time, Signature=D18A9CBCDDBA9041D6693268FEF15A989E64430B");
+            metadata.put(GrpcConstants.DATE_TIME, "20231227T194619Z");
+            DefaultAuthenticationContext context = builder.build(metadata, request);
+            Assert.assertNotNull(context);
+            Assert.assertEquals("abc", context.getUsername());
+            Assert.assertEquals("0YqcvN26kEHWaTJo/vFamJ5kQws=", context.getSignature());
+            Assert.assertEquals("20231227T194619Z", new String(context.getContent(), StandardCharsets.UTF_8));
+        }
+    }
+
+    @Test
+    public void build2() {
+        when(channel.id()).thenReturn(mockChannelId("channel-id"));
+        when(channelHandlerContext.channel()).thenReturn(channel);
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        requestHeader.setTopic("topic-test");
+        requestHeader.setQueueId(0);
+        requestHeader.setBornTimestamp(117036786441330L);
+        requestHeader.setBname("brokerName-1");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "abc");
+        request.addExtField("Signature", "ZG26exJ5u9q1fwZlO4DCmz2Rs88=");
+        request.makeCustomHeaderToNet();
+        DefaultAuthenticationContext context = builder.build(channelHandlerContext, request);
+        Assert.assertNotNull(context);
+        Assert.assertEquals("abc", context.getUsername());
+        Assert.assertEquals("ZG26exJ5u9q1fwZlO4DCmz2Rs88=", context.getSignature());
+        Assert.assertEquals("abcbrokerName-11170367864413300topic-test", new String(context.getContent(), StandardCharsets.UTF_8));
+    }
+
+    private ChannelId mockChannelId(String channelId) {
+        return new ChannelId() {
+            @Override
+            public String asShortText() {
+                return channelId;
+            }
+
+            @Override
+            public String asLongText() {
+                return channelId;
+            }
+
+            @Override
+            public int compareTo(ChannelId o) {
+                return 0;
+            }
+        };
+    }
+}
\ No newline at end of file
diff --git a/auth/src/test/java/org/apache/rocketmq/auth/authentication/manager/AuthenticationMetadataManagerTest.java b/auth/src/test/java/org/apache/rocketmq/auth/authentication/manager/AuthenticationMetadataManagerTest.java
new file mode 100644
index 00000000000..844deb37568
--- /dev/null
+++ b/auth/src/test/java/org/apache/rocketmq/auth/authentication/manager/AuthenticationMetadataManagerTest.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authentication.manager;
+
+import java.util.List;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.auth.authentication.enums.UserType;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.auth.helper.AuthTestHelper;
+import org.apache.rocketmq.common.MixAll;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class AuthenticationMetadataManagerTest {
+
+    private AuthConfig authConfig;
+    private AuthenticationMetadataManager authenticationMetadataManager;
+
+    @Before
+    public void setUp() throws Exception {
+        if (MixAll.isMac()) {
+            return;
+        }
+        this.authConfig = AuthTestHelper.createDefaultConfig();
+        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(this.authConfig);
+        this.clearAllUsers();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        if (MixAll.isMac()) {
+            return;
+        }
+        this.clearAllUsers();
+        this.authenticationMetadataManager.shutdown();
+    }
+
+    @Test
+    public void createUser() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+        user = this.authenticationMetadataManager.getUser("test").join();
+        Assert.assertNotNull(user);
+        Assert.assertEquals(user.getUsername(), "test");
+        Assert.assertEquals(user.getPassword(), "test");
+        Assert.assertEquals(user.getUserType(), UserType.NORMAL);
+
+        user = User.of("super", "super", UserType.SUPER);
+        this.authenticationMetadataManager.createUser(user).join();
+        user = this.authenticationMetadataManager.getUser("super").join();
+        Assert.assertNotNull(user);
+        Assert.assertEquals(user.getUsername(), "super");
+        Assert.assertEquals(user.getPassword(), "super");
+        Assert.assertEquals(user.getUserType(), UserType.SUPER);
+
+        Assert.assertThrows(AuthenticationException.class, () -> {
+            try {
+                User user2 = User.of("test", "test");
+                this.authenticationMetadataManager.createUser(user2).join();
+            } catch (Exception e) {
+                AuthTestHelper.handleException(e);
+            }
+        });
+    }
+
+    @Test
+    public void updateUser() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+        user = this.authenticationMetadataManager.getUser("test").join();
+        Assert.assertNotNull(user);
+        Assert.assertEquals(user.getUsername(), "test");
+        Assert.assertEquals(user.getPassword(), "test");
+        Assert.assertEquals(user.getUserType(), UserType.NORMAL);
+
+        user.setPassword("123");
+        this.authenticationMetadataManager.updateUser(user).join();
+        user = this.authenticationMetadataManager.getUser("test").join();
+        Assert.assertNotNull(user);
+        Assert.assertEquals(user.getUsername(), "test");
+        Assert.assertEquals(user.getPassword(), "123");
+        Assert.assertEquals(user.getUserType(), UserType.NORMAL);
+
+        user.setUserType(UserType.SUPER);
+        this.authenticationMetadataManager.updateUser(user).join();
+        user = this.authenticationMetadataManager.getUser("test").join();
+        Assert.assertNotNull(user);
+        Assert.assertEquals(user.getUsername(), "test");
+        Assert.assertEquals(user.getPassword(), "123");
+        Assert.assertEquals(user.getUserType(), UserType.SUPER);
+
+        Assert.assertThrows(AuthenticationException.class, () -> {
+            try {
+                User user2 = User.of("no_user", "no_user");
+                this.authenticationMetadataManager.updateUser(user2).join();
+            } catch (Exception e) {
+                AuthTestHelper.handleException(e);
+            }
+        });
+    }
+
+    @Test
+    public void deleteUser() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+        user = this.authenticationMetadataManager.getUser("test").join();
+        Assert.assertNotNull(user);
+        this.authenticationMetadataManager.deleteUser("test").join();
+        user = this.authenticationMetadataManager.getUser("test").join();
+        Assert.assertNull(user);
+
+        this.authenticationMetadataManager.deleteUser("no_user").join();
+    }
+
+    @Test
+    public void getUser() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+        user = this.authenticationMetadataManager.getUser("test").join();
+        Assert.assertNotNull(user);
+        Assert.assertEquals(user.getUsername(), "test");
+        Assert.assertEquals(user.getPassword(), "test");
+        Assert.assertEquals(user.getUserType(), UserType.NORMAL);
+
+        user = this.authenticationMetadataManager.getUser("no_user").join();
+        Assert.assertNull(user);
+    }
+
+    @Test
+    public void listUser() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        List<User> users = this.authenticationMetadataManager.listUser(null).join();
+        Assert.assertTrue(CollectionUtils.isEmpty(users));
+
+        User user = User.of("test-1", "test-1");
+        this.authenticationMetadataManager.createUser(user).join();
+        users = this.authenticationMetadataManager.listUser(null).join();
+        Assert.assertEquals(users.size(), 1);
+
+        user = User.of("test-2", "test-2");
+        this.authenticationMetadataManager.createUser(user).join();
+        users = this.authenticationMetadataManager.listUser("test").join();
+        Assert.assertEquals(users.size(), 2);
+    }
+
+    private void clearAllUsers() {
+        List<User> users = this.authenticationMetadataManager.listUser(null).join();
+        if (CollectionUtils.isEmpty(users)) {
+            return;
+        }
+        users.forEach(user -> this.authenticationMetadataManager.deleteUser(user.getUsername()).join());
+    }
+}
\ No newline at end of file
diff --git a/auth/src/test/java/org/apache/rocketmq/auth/authorization/AuthorizationEvaluatorTest.java b/auth/src/test/java/org/apache/rocketmq/auth/authorization/AuthorizationEvaluatorTest.java
new file mode 100644
index 00000000000..d8b839d7fb9
--- /dev/null
+++ b/auth/src/test/java/org/apache/rocketmq/auth/authorization/AuthorizationEvaluatorTest.java
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authorization.context.AuthorizationContext;
+import org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;
+import org.apache.rocketmq.auth.authorization.enums.Decision;
+import org.apache.rocketmq.auth.authorization.enums.PolicyType;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;
+import org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.authorization.model.Resource;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.auth.helper.AuthTestHelper;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.action.Action;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class AuthorizationEvaluatorTest {
+
+    private AuthConfig authConfig;
+    private AuthorizationEvaluator evaluator;
+    private AuthenticationMetadataManager authenticationMetadataManager;
+    private AuthorizationMetadataManager authorizationMetadataManager;
+
+    @Before
+    public void setUp() throws Exception {
+        if (MixAll.isMac()) {
+            return;
+        }
+        this.authConfig = AuthTestHelper.createDefaultConfig();
+        this.evaluator = new AuthorizationEvaluator(authConfig);
+        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(authConfig);
+        this.authorizationMetadataManager = AuthorizationFactory.getMetadataManager(authConfig);
+        this.clearAllAcls();
+        this.clearAllUsers();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        if (MixAll.isMac()) {
+            return;
+        }
+        this.clearAllAcls();
+        this.clearAllUsers();
+        this.authenticationMetadataManager.shutdown();
+    }
+
+    @Test
+    public void evaluate1() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+
+        Acl acl = AuthTestHelper.buildAcl("User:test", "Topic:test*", "Pub", "192.168.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.createAcl(acl).join();
+
+        Subject subject = Subject.of("User:test");
+        Resource resource = Resource.ofTopic("test");
+        Action action = Action.PUB;
+        String sourceIp = "192.168.0.1";
+        DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+        context.setRpcCode("10");
+        this.evaluator.evaluate(Collections.singletonList(context));
+
+        // acl sourceIp is null
+        acl = AuthTestHelper.buildAcl("User:test", "Topic:test*", "Pub", null, Decision.ALLOW);
+        this.authorizationMetadataManager.updateAcl(acl).join();
+
+        subject = Subject.of("User:test");
+        resource = Resource.ofTopic("test");
+        action = Action.PUB;
+        sourceIp = "192.168.0.1";
+        context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+        context.setRpcCode("10");
+        this.evaluator.evaluate(Collections.singletonList(context));
+    }
+
+    @Test
+    public void evaluate2() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+
+        Acl acl = AuthTestHelper.buildAcl("User:test", "Topic:test*,Group:test*", "Sub", "192.168.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.createAcl(acl).join();
+
+        List<AuthorizationContext> contexts = new ArrayList<>();
+
+        Subject subject = Subject.of("User:test");
+        Resource resource = Resource.ofTopic("test");
+        Action action = Action.SUB;
+        String sourceIp = "192.168.0.1";
+        DefaultAuthorizationContext context1 = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+        context1.setRpcCode("11");
+        contexts.add(context1);
+
+        subject = Subject.of("User:test");
+        resource = Resource.ofGroup("test");
+        action = Action.SUB;
+        sourceIp = "192.168.0.1";
+        DefaultAuthorizationContext context2 = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+        context2.setRpcCode("11");
+        contexts.add(context2);
+
+        this.evaluator.evaluate(contexts);
+    }
+
+    @Test
+    public void evaluate4() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+
+        Acl acl = AuthTestHelper.buildAcl("User:test", "Topic:test*", "Pub", "192.168.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.createAcl(acl).join();
+
+        // user not exist
+        Assert.assertThrows(AuthorizationException.class, () -> {
+            Subject subject = Subject.of("User:abc");
+            Resource resource = Resource.ofTopic("test");
+            Action action = Action.PUB;
+            String sourceIp = "192.168.0.1";
+            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+            context.setRpcCode("10");
+            this.evaluator.evaluate(Collections.singletonList(context));
+        });
+
+        // resource not match
+        Assert.assertThrows(AuthorizationException.class, () -> {
+            Subject subject = Subject.of("User:test");
+            Resource resource = Resource.ofTopic("abc");
+            Action action = Action.PUB;
+            String sourceIp = "192.168.0.1";
+            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+            context.setRpcCode("10");
+            this.evaluator.evaluate(Collections.singletonList(context));
+        });
+
+        // action not match
+        Assert.assertThrows(AuthorizationException.class, () -> {
+            Subject subject = Subject.of("User:test");
+            Resource resource = Resource.ofTopic("test");
+            Action action = Action.SUB;
+            String sourceIp = "192.168.0.1";
+            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+            context.setRpcCode("10");
+            this.evaluator.evaluate(Collections.singletonList(context));
+        });
+
+        // sourceIp not match
+        Assert.assertThrows(AuthorizationException.class, () -> {
+            Subject subject = Subject.of("User:test");
+            Resource resource = Resource.ofTopic("test");
+            Action action = Action.PUB;
+            String sourceIp = "10.10.0.1";
+            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+            context.setRpcCode("10");
+            this.evaluator.evaluate(Collections.singletonList(context));
+        });
+
+        // decision is deny
+        acl = AuthTestHelper.buildAcl("User:test", "Topic:test*", "Pub", "192.168.0.0/24", Decision.DENY);
+        this.authorizationMetadataManager.updateAcl(acl).join();
+        Assert.assertThrows(AuthorizationException.class, () -> {
+            Subject subject = Subject.of("User:test");
+            Resource resource = Resource.ofTopic("test");
+            Action action = Action.PUB;
+            String sourceIp = "192.168.0.1";
+            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+            context.setRpcCode("10");
+            this.evaluator.evaluate(Collections.singletonList(context));
+        });
+    }
+
+    @Test
+    public void evaluate5() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+
+        Acl acl = AuthTestHelper.buildAcl("User:test", "*", "Pub,Sub", "192.168.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.createAcl(acl).join();
+
+        acl = AuthTestHelper.buildAcl("User:test", "Topic:*", "Pub,Sub", "192.168.0.0/24", Decision.DENY);
+        this.authorizationMetadataManager.updateAcl(acl).join();
+
+        acl = AuthTestHelper.buildAcl("User:test", "Topic:test*", "Pub,Sub", "192.168.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.updateAcl(acl).join();
+
+        acl = AuthTestHelper.buildAcl("User:test", "Topic:test-1", "Pub,Sub", "192.168.0.0/24", Decision.DENY);
+        this.authorizationMetadataManager.updateAcl(acl).join();
+
+        Assert.assertThrows(AuthorizationException.class, () -> {
+            Subject subject = Subject.of("User:test");
+            Resource resource = Resource.ofTopic("test-1");
+            Action action = Action.PUB;
+            String sourceIp = "192.168.0.1";
+            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+            context.setRpcCode("10");
+            this.evaluator.evaluate(Collections.singletonList(context));
+        });
+
+        {
+            Subject subject = Subject.of("User:test");
+            Resource resource = Resource.ofTopic("test-2");
+            Action action = Action.PUB;
+            String sourceIp = "192.168.0.1";
+            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+            context.setRpcCode("10");
+            this.evaluator.evaluate(Collections.singletonList(context));
+        }
+
+        Assert.assertThrows(AuthorizationException.class, () -> {
+            Subject subject = Subject.of("User:test");
+            Resource resource = Resource.ofTopic("abc");
+            Action action = Action.PUB;
+            String sourceIp = "192.168.0.1";
+            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+            context.setRpcCode("10");
+            this.evaluator.evaluate(Collections.singletonList(context));
+        });
+
+        {
+            Subject subject = Subject.of("User:test");
+            Resource resource = Resource.ofGroup("test-2");
+            Action action = Action.SUB;
+            String sourceIp = "192.168.0.1";
+            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+            context.setRpcCode("10");
+            this.evaluator.evaluate(Collections.singletonList(context));
+        }
+    }
+
+    @Test
+    public void evaluate6() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        this.authConfig.setAuthorizationWhitelist("10");
+        this.evaluator = new AuthorizationEvaluator(this.authConfig);
+
+        Subject subject = Subject.of("User:test");
+        Resource resource = Resource.ofTopic("test");
+        Action action = Action.PUB;
+        String sourceIp = "192.168.0.1";
+        DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+        context.setRpcCode("10");
+        this.evaluator.evaluate(Collections.singletonList(context));
+    }
+
+    @Test
+    public void evaluate7() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        this.authConfig.setAuthorizationEnabled(false);
+        this.evaluator = new AuthorizationEvaluator(this.authConfig);
+
+        Subject subject = Subject.of("User:test");
+        Resource resource = Resource.ofTopic("test");
+        Action action = Action.PUB;
+        String sourceIp = "192.168.0.1";
+        DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+        context.setRpcCode("10");
+        this.evaluator.evaluate(Collections.singletonList(context));
+    }
+
+    @Test
+    public void evaluate8() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+
+        Acl acl = AuthTestHelper.buildAcl("User:test", "Topic:test*", "Pub", "192.168.0.0/24", Decision.DENY);
+        this.authorizationMetadataManager.createAcl(acl).join();
+
+
+        Assert.assertThrows(AuthorizationException.class, () -> {
+            Subject subject = Subject.of("User:test");
+            Resource resource = Resource.ofTopic("test");
+            Action action = Action.PUB;
+            String sourceIp = "192.168.0.1";
+            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+            context.setRpcCode("10");
+            this.evaluator.evaluate(Collections.singletonList(context));
+        });
+
+        Assert.assertThrows(AuthorizationException.class, () -> {
+            Subject subject = Subject.of("User:test");
+            Resource resource = Resource.ofTopic("abc");
+            Action action = Action.PUB;
+            String sourceIp = "192.168.0.1";
+            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+            context.setRpcCode("10");
+            this.evaluator.evaluate(Collections.singletonList(context));
+        });
+
+        acl = AuthTestHelper.buildAcl("User:test", PolicyType.DEFAULT, "Topic:*", "Pub", null, Decision.ALLOW);
+        this.authorizationMetadataManager.updateAcl(acl).join();
+        {
+            Subject subject = Subject.of("User:test");
+            Resource resource = Resource.ofTopic("abc");
+            Action action = Action.PUB;
+            String sourceIp = "192.168.0.1";
+            DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, action, sourceIp);
+            context.setRpcCode("10");
+            this.evaluator.evaluate(Collections.singletonList(context));
+        }
+    }
+
+    private void clearAllUsers() {
+        List<User> users = this.authenticationMetadataManager.listUser(null).join();
+        if (CollectionUtils.isEmpty(users)) {
+            return;
+        }
+        users.forEach(user -> this.authenticationMetadataManager.deleteUser(user.getUsername()).join());
+    }
+
+    private void clearAllAcls() {
+        List<Acl> acls = this.authorizationMetadataManager.listAcl(null, null).join();
+        if (CollectionUtils.isEmpty(acls)) {
+            return;
+        }
+        acls.forEach(acl -> this.authorizationMetadataManager.deleteAcl(acl.getSubject(), null, null).join());
+    }
+}
\ No newline at end of file
diff --git a/auth/src/test/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilderTest.java b/auth/src/test/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilderTest.java
new file mode 100644
index 00000000000..c73e07d7529
--- /dev/null
+++ b/auth/src/test/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilderTest.java
@@ -0,0 +1,580 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.builder;
+
+import apache.rocketmq.v2.AckMessageRequest;
+import apache.rocketmq.v2.ChangeInvisibleDurationRequest;
+import apache.rocketmq.v2.ClientType;
+import apache.rocketmq.v2.EndTransactionRequest;
+import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest;
+import apache.rocketmq.v2.HeartbeatRequest;
+import apache.rocketmq.v2.Message;
+import apache.rocketmq.v2.MessageQueue;
+import apache.rocketmq.v2.NotifyClientTerminationRequest;
+import apache.rocketmq.v2.Publishing;
+import apache.rocketmq.v2.QueryAssignmentRequest;
+import apache.rocketmq.v2.QueryRouteRequest;
+import apache.rocketmq.v2.RecallMessageRequest;
+import apache.rocketmq.v2.ReceiveMessageRequest;
+import apache.rocketmq.v2.Resource;
+import apache.rocketmq.v2.SendMessageRequest;
+import apache.rocketmq.v2.Settings;
+import apache.rocketmq.v2.Subscription;
+import apache.rocketmq.v2.SubscriptionEntry;
+import apache.rocketmq.v2.TelemetryCommand;
+import com.alibaba.fastjson2.JSON;
+import com.google.common.collect.Sets;
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelId;
+import io.netty.util.Attribute;
+import io.netty.util.AttributeKey;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.TopicFilterType;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.constant.GrpcConstants;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.remoting.netty.AttributeKeys;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.RequestHeaderRegistry;
+import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.CreateUserRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.HeartbeatRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;
+import org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumerData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.Silent.class)
+public class DefaultAuthorizationContextBuilderTest {
+
+    private AuthorizationContextBuilder builder;
+
+    @Mock
+    private ChannelHandlerContext channelHandlerContext;
+
+    @Mock
+    private Channel channel;
+
+    @Before
+    public void setUp() throws Exception {
+        AuthConfig authConfig = new AuthConfig();
+        authConfig.setClusterName("DefaultCluster");
+        builder = new DefaultAuthorizationContextBuilder(authConfig);
+        RequestHeaderRegistry.getInstance().initialize();
+    }
+
+    @Test
+    public void buildGrpc() {
+        Metadata metadata = new Metadata();
+        metadata.put(GrpcConstants.AUTHORIZATION_AK, "rocketmq");
+        metadata.put(GrpcConstants.REMOTE_ADDRESS, "192.168.0.1");
+        metadata.put(GrpcConstants.CHANNEL_ID, "channel-id");
+
+        GeneratedMessageV3 request = SendMessageRequest.newBuilder()
+            .addMessages(Message.newBuilder()
+                .setTopic(Resource.newBuilder().setName("topic").build())
+                .build())
+            .build();
+        List<DefaultAuthorizationContext> result = builder.build(metadata, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(result.get(0).getResource().getResourceKey(), "Topic:topic");
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));
+        Assert.assertEquals(result.get(0).getSourceIp(), "192.168.0.1");
+        Assert.assertEquals(result.get(0).getChannelId(), "channel-id");
+        Assert.assertEquals(result.get(0).getRpcCode(), SendMessageRequest.getDescriptor().getFullName());
+
+        request = RecallMessageRequest.newBuilder()
+            .setTopic(Resource.newBuilder().setName("topic").build())
+            .setRecallHandle("handle")
+            .build();
+        result = builder.build(metadata, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(result.get(0).getResource().getResourceKey(), "Topic:topic");
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));
+        Assert.assertEquals(result.get(0).getSourceIp(), "192.168.0.1");
+        Assert.assertEquals(result.get(0).getChannelId(), "channel-id");
+        Assert.assertEquals(result.get(0).getRpcCode(), RecallMessageRequest.getDescriptor().getFullName());
+
+        request = EndTransactionRequest.newBuilder()
+            .setTopic(Resource.newBuilder().setName("topic").build())
+            .build();
+        result = builder.build(metadata, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(result.get(0).getResource().getResourceKey(), "Topic:topic");
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));
+
+        request = HeartbeatRequest.newBuilder()
+            .setClientType(ClientType.PUSH_CONSUMER)
+            .setGroup(Resource.newBuilder().setName("group").build())
+            .build();
+        result = builder.build(metadata, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(result.get(0).getResource().getResourceKey(), "Group:group");
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        request = ReceiveMessageRequest.newBuilder()
+            .setMessageQueue(MessageQueue.newBuilder()
+                .setTopic(Resource.newBuilder().setName("topic").build())
+                .build())
+            .setGroup(Resource.newBuilder().setName("group").build())
+            .build();
+        result = builder.build(metadata, request);
+        Assert.assertEquals(2, result.size());
+        Assert.assertEquals(getContext(result, ResourceType.GROUP).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(getContext(result, ResourceType.GROUP).getResource().getResourceKey(), "Group:group");
+        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));
+        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getResource().getResourceKey(), "Topic:topic");
+        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        request = AckMessageRequest.newBuilder()
+            .setTopic(Resource.newBuilder().setName("topic").build())
+            .setGroup(Resource.newBuilder().setName("group").build())
+            .build();
+        result = builder.build(metadata, request);
+        Assert.assertEquals(2, result.size());
+        Assert.assertEquals(getContext(result, ResourceType.GROUP).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(getContext(result, ResourceType.GROUP).getResource().getResourceKey(), "Group:group");
+        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));
+        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getResource().getResourceKey(), "Topic:topic");
+        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        request = ForwardMessageToDeadLetterQueueRequest.newBuilder()
+            .setTopic(Resource.newBuilder().setName("topic").build())
+            .setGroup(Resource.newBuilder().setName("group").build())
+            .build();
+        result = builder.build(metadata, request);
+        Assert.assertEquals(2, result.size());
+        Assert.assertEquals(getContext(result, ResourceType.GROUP).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(getContext(result, ResourceType.GROUP).getResource().getResourceKey(), "Group:group");
+        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));
+        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getResource().getResourceKey(), "Topic:topic");
+        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        request = NotifyClientTerminationRequest.newBuilder()
+            .setGroup(Resource.newBuilder().setName("group").build())
+            .build();
+        result = builder.build(metadata, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(result.get(0).getResource().getResourceKey(), "Group:group");
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        request = ChangeInvisibleDurationRequest.newBuilder()
+            .setGroup(Resource.newBuilder().setName("group").build())
+            .build();
+        result = builder.build(metadata, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(result.get(0).getResource().getResourceKey(), "Group:group");
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        request = QueryRouteRequest.newBuilder()
+            .setTopic(Resource.newBuilder().setName("topic").build())
+            .build();
+        result = builder.build(metadata, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(result.get(0).getResource().getResourceKey(), "Topic:topic");
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB, Action.SUB)));
+
+        request = QueryAssignmentRequest.newBuilder()
+            .setTopic(Resource.newBuilder().setName("topic").build())
+            .setGroup(Resource.newBuilder().setName("group").build())
+            .build();
+        result = builder.build(metadata, request);
+        Assert.assertEquals(2, result.size());
+        Assert.assertEquals(getContext(result, ResourceType.GROUP).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(getContext(result, ResourceType.GROUP).getResource().getResourceKey(), "Group:group");
+        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));
+        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getResource().getResourceKey(), "Topic:topic");
+        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        request = TelemetryCommand.newBuilder()
+            .setSettings(Settings.newBuilder()
+                .setPublishing(Publishing.newBuilder()
+                    .addTopics(Resource.newBuilder().setName("topic").build())
+                    .build())
+                .build())
+            .build();
+        result = builder.build(metadata, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getResource().getResourceKey(), "Topic:topic");
+        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.PUB)));
+
+        request = TelemetryCommand.newBuilder()
+            .setSettings(Settings.newBuilder()
+                .setSubscription(Subscription.newBuilder()
+                    .setGroup(Resource.newBuilder().setName("group").build())
+                    .addSubscriptions(SubscriptionEntry.newBuilder()
+                        .setTopic(Resource.newBuilder().setName("topic").build())
+                        .build())
+                    .build())
+                .build())
+            .build();
+        result = builder.build(metadata, request);
+        Assert.assertEquals(2, result.size());
+        Assert.assertEquals(getContext(result, ResourceType.GROUP).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(getContext(result, ResourceType.GROUP).getResource().getResourceKey(), "Group:group");
+        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));
+        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey(), "User:rocketmq");
+        Assert.assertEquals(getContext(result, ResourceType.TOPIC).getResource().getResourceKey(), "Topic:topic");
+        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));
+    }
+
+    @Test
+    public void buildRemoting() {
+        when(channel.id()).thenReturn(mockChannelId("channel-id"));
+        when(channel.hasAttr(eq(AttributeKeys.PROXY_PROTOCOL_ADDR))).thenReturn(true);
+        when(channel.attr(eq(AttributeKeys.PROXY_PROTOCOL_ADDR))).thenReturn(mockAttribute("192.168.0.1"));
+        when(channel.hasAttr(eq(AttributeKeys.PROXY_PROTOCOL_PORT))).thenReturn(true);
+        when(channel.attr(eq(AttributeKeys.PROXY_PROTOCOL_PORT))).thenReturn(mockAttribute("1234"));
+        when(channelHandlerContext.channel()).thenReturn(channel);
+
+        SendMessageRequestHeader sendMessageRequestHeader = new SendMessageRequestHeader();
+        sendMessageRequestHeader.setTopic("topic");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, sendMessageRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        List<DefaultAuthorizationContext> result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey());
+        Assert.assertEquals("Topic:topic", result.get(0).getResource().getResourceKey());
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));
+        Assert.assertEquals("192.168.0.1", result.get(0).getSourceIp());
+        Assert.assertEquals("channel-id", result.get(0).getChannelId());
+        Assert.assertEquals(RequestCode.SEND_MESSAGE + "", result.get(0).getRpcCode());
+
+        sendMessageRequestHeader = new SendMessageRequestHeader();
+        sendMessageRequestHeader.setTopic("%RETRY%group");
+        request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, sendMessageRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey());
+        Assert.assertEquals("Group:group", result.get(0).getResource().getResourceKey());
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        SendMessageRequestHeaderV2 sendMessageRequestHeaderV2 = new SendMessageRequestHeaderV2();
+        sendMessageRequestHeaderV2.setTopic("topic");
+        request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, sendMessageRequestHeaderV2);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey());
+        Assert.assertEquals("Topic:topic", result.get(0).getResource().getResourceKey());
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));
+
+        sendMessageRequestHeaderV2 = new SendMessageRequestHeaderV2();
+        sendMessageRequestHeaderV2.setTopic("%RETRY%group");
+        request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, sendMessageRequestHeaderV2);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey());
+        Assert.assertEquals("Group:group", result.get(0).getResource().getResourceKey());
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        RecallMessageRequestHeader recallMessageRequestHeader = new RecallMessageRequestHeader();
+        recallMessageRequestHeader.setTopic("topic");
+        recallMessageRequestHeader.setRecallHandle("handle");
+        request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, recallMessageRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey());
+        Assert.assertEquals("Topic:topic", result.get(0).getResource().getResourceKey());
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));
+        Assert.assertEquals("192.168.0.1", result.get(0).getSourceIp());
+        Assert.assertEquals("channel-id", result.get(0).getChannelId());
+        Assert.assertEquals(RequestCode.RECALL_MESSAGE + "", result.get(0).getRpcCode());
+
+        EndTransactionRequestHeader endTransactionRequestHeader = new EndTransactionRequestHeader();
+        endTransactionRequestHeader.setTopic("topic");
+        request = RemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, endTransactionRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey());
+        Assert.assertEquals("Topic:topic", result.get(0).getResource().getResourceKey());
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB)));
+
+        endTransactionRequestHeader = new EndTransactionRequestHeader();
+        request = RemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, endTransactionRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(0, result.size());
+
+        ConsumerSendMsgBackRequestHeader consumerSendMsgBackRequestHeader = new ConsumerSendMsgBackRequestHeader();
+        consumerSendMsgBackRequestHeader.setGroup("group");
+        request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, consumerSendMsgBackRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey());
+        Assert.assertEquals("Group:group", result.get(0).getResource().getResourceKey());
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        PullMessageRequestHeader pullMessageRequestHeader = new PullMessageRequestHeader();
+        pullMessageRequestHeader.setTopic("topic");
+        pullMessageRequestHeader.setConsumerGroup("group");
+        request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, pullMessageRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(2, result.size());
+        Assert.assertEquals("User:rocketmq", getContext(result, ResourceType.GROUP).getSubject().getSubjectKey());
+        Assert.assertEquals("Group:group", getContext(result, ResourceType.GROUP).getResource().getResourceKey());
+        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));
+        Assert.assertEquals("User:rocketmq", getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey());
+        Assert.assertEquals("Topic:topic", getContext(result, ResourceType.TOPIC).getResource().getResourceKey());
+        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        QueryMessageRequestHeader queryMessageRequestHeader = new QueryMessageRequestHeader();
+        queryMessageRequestHeader.setTopic("topic");
+        request = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, queryMessageRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey());
+        Assert.assertEquals("Topic:topic", result.get(0).getResource().getResourceKey());
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB, Action.GET)));
+
+        HeartbeatRequestHeader heartbeatRequestHeader = new HeartbeatRequestHeader();
+        request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, heartbeatRequestHeader);
+        HeartbeatData heartbeatData = new HeartbeatData();
+        ConsumerData consumerData = new ConsumerData();
+        consumerData.setGroupName("group");
+        SubscriptionData subscriptionData = new SubscriptionData();
+        subscriptionData.setTopic("topic");
+        consumerData.setSubscriptionDataSet(Sets.newHashSet(subscriptionData));
+        heartbeatData.setConsumerDataSet(Sets.newHashSet(consumerData));
+        request.setBody(JSON.toJSONBytes(heartbeatData));
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(2, result.size());
+        Assert.assertEquals("User:rocketmq", getContext(result, ResourceType.GROUP).getSubject().getSubjectKey());
+        Assert.assertEquals("Group:group", getContext(result, ResourceType.GROUP).getResource().getResourceKey());
+        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));
+        Assert.assertEquals("User:rocketmq", getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey());
+        Assert.assertEquals("Topic:topic", getContext(result, ResourceType.TOPIC).getResource().getResourceKey());
+        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        UnregisterClientRequestHeader unregisterClientRequestHeader = new UnregisterClientRequestHeader();
+        unregisterClientRequestHeader.setConsumerGroup("group");
+        request = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT, unregisterClientRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey());
+        Assert.assertEquals("Group:group", result.get(0).getResource().getResourceKey());
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader = new GetConsumerListByGroupRequestHeader();
+        getConsumerListByGroupRequestHeader.setConsumerGroup("group");
+        request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, getConsumerListByGroupRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey());
+        Assert.assertEquals("Group:group", result.get(0).getResource().getResourceKey());
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB, Action.GET)));
+
+        QueryConsumerOffsetRequestHeader queryConsumerOffsetRequestHeader = new QueryConsumerOffsetRequestHeader();
+        queryConsumerOffsetRequestHeader.setTopic("topic");
+        queryConsumerOffsetRequestHeader.setConsumerGroup("group");
+        request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, queryConsumerOffsetRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(2, result.size());
+        Assert.assertEquals("User:rocketmq", getContext(result, ResourceType.GROUP).getSubject().getSubjectKey());
+        Assert.assertEquals("Group:group", getContext(result, ResourceType.GROUP).getResource().getResourceKey());
+        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB)));
+        Assert.assertEquals("User:rocketmq", getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey());
+        Assert.assertEquals("Topic:topic", getContext(result, ResourceType.TOPIC).getResource().getResourceKey());
+        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB)));
+
+        UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader = new UpdateConsumerOffsetRequestHeader();
+        updateConsumerOffsetRequestHeader.setTopic("topic");
+        updateConsumerOffsetRequestHeader.setConsumerGroup("group");
+        request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, updateConsumerOffsetRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(2, result.size());
+        Assert.assertEquals("User:rocketmq", getContext(result, ResourceType.GROUP).getSubject().getSubjectKey());
+        Assert.assertEquals("Group:group", getContext(result, ResourceType.GROUP).getResource().getResourceKey());
+        Assert.assertTrue(getContext(result, ResourceType.GROUP).getActions().containsAll(Arrays.asList(Action.SUB, Action.UPDATE)));
+        Assert.assertEquals("User:rocketmq", getContext(result, ResourceType.TOPIC).getSubject().getSubjectKey());
+        Assert.assertEquals("Topic:topic", getContext(result, ResourceType.TOPIC).getResource().getResourceKey());
+        Assert.assertTrue(getContext(result, ResourceType.TOPIC).getActions().containsAll(Arrays.asList(Action.SUB, Action.UPDATE)));
+
+        CreateTopicRequestHeader createTopicRequestHeader = new CreateTopicRequestHeader();
+        createTopicRequestHeader.setTopic("topic");
+        createTopicRequestHeader.setTopicFilterType(TopicFilterType.SINGLE_TAG.name());
+        request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, createTopicRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey());
+        Assert.assertEquals("Topic:topic", result.get(0).getResource().getResourceKey());
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.CREATE)));
+
+        CreateUserRequestHeader createUserRequestHeader = new CreateUserRequestHeader();
+        createUserRequestHeader.setUsername("abc");
+        request = RemotingCommand.createRequestCommand(RequestCode.AUTH_CREATE_USER, createUserRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        result = builder.build(channelHandlerContext, request);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey());
+        Assert.assertEquals("Cluster:DefaultCluster", result.get(0).getResource().getResourceKey());
+        Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.UPDATE)));
+    }
+
+    private DefaultAuthorizationContext getContext(List<DefaultAuthorizationContext> contexts,
+        ResourceType resourceType) {
+        return contexts.stream().filter(context -> context.getResource().getResourceType() == resourceType)
+            .findFirst().orElse(null);
+    }
+
+    private ChannelId mockChannelId(String channelId) {
+        return new ChannelId() {
+            @Override
+            public String asShortText() {
+                return channelId;
+            }
+
+            @Override
+            public String asLongText() {
+                return channelId;
+            }
+
+            @Override
+            public int compareTo(ChannelId o) {
+                return 0;
+            }
+        };
+    }
+
+    private Attribute<String> mockAttribute(String value) {
+        return new Attribute<String>() {
+            @Override
+            public AttributeKey<String> key() {
+                return null;
+            }
+
+            @Override
+            public String get() {
+                return value;
+            }
+
+            @Override
+            public void set(String value) {
+            }
+
+            @Override
+            public String getAndSet(String value) {
+                return null;
+            }
+
+            @Override
+            public String setIfAbsent(String value) {
+                return null;
+            }
+
+            @Override
+            public String getAndRemove() {
+                return null;
+            }
+
+            @Override
+            public boolean compareAndSet(String oldValue, String newValue) {
+                return false;
+            }
+
+            @Override
+            public void remove() {
+
+            }
+        };
+    }
+}
\ No newline at end of file
diff --git a/auth/src/test/java/org/apache/rocketmq/auth/authorization/manager/AuthorizationMetadataManagerTest.java b/auth/src/test/java/org/apache/rocketmq/auth/authorization/manager/AuthorizationMetadataManagerTest.java
new file mode 100644
index 00000000000..21ae30aca94
--- /dev/null
+++ b/auth/src/test/java/org/apache/rocketmq/auth/authorization/manager/AuthorizationMetadataManagerTest.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.manager;
+
+import java.util.List;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authorization.enums.Decision;
+import org.apache.rocketmq.auth.authorization.enums.PolicyType;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.authorization.model.Policy;
+import org.apache.rocketmq.auth.authorization.model.Resource;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.auth.helper.AuthTestHelper;
+import org.apache.rocketmq.common.MixAll;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class AuthorizationMetadataManagerTest {
+
+    private AuthConfig authConfig;
+
+    private AuthenticationMetadataManager authenticationMetadataManager;
+
+    private AuthorizationMetadataManager authorizationMetadataManager;
+
+    @Before
+    public void setUp() throws Exception {
+        if (MixAll.isMac()) {
+            return;
+        }
+        this.authConfig = AuthTestHelper.createDefaultConfig();
+        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(this.authConfig);
+        this.authorizationMetadataManager = AuthorizationFactory.getMetadataManager(this.authConfig);
+        this.clearAllAcls();
+        this.clearAllUsers();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        if (MixAll.isMac()) {
+            return;
+        }
+        this.clearAllAcls();
+        this.clearAllUsers();
+        this.authenticationMetadataManager.shutdown();
+        this.authorizationMetadataManager.shutdown();
+    }
+
+    @Test
+    public void createAcl() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+
+        Acl acl1 = AuthTestHelper.buildAcl("User:test", "Topic:test,Group:test", "PUB,SUB",
+            "192.168.0.0/24,10.10.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.createAcl(acl1).join();
+        Acl acl2 = this.authorizationMetadataManager.getAcl(Subject.of("User:test")).join();
+        Assert.assertTrue(AuthTestHelper.isEquals(acl1, acl2));
+
+        user = User.of("abc", "abc");
+        this.authenticationMetadataManager.createUser(user).join();
+
+        acl1 = AuthTestHelper.buildAcl("User:abc", PolicyType.DEFAULT, "Topic:*,Group:*", "PUB,SUB",
+            null, Decision.DENY);
+        this.authorizationMetadataManager.createAcl(acl1).join();
+        acl2 = this.authorizationMetadataManager.getAcl(Subject.of("User:abc")).join();
+        Assert.assertTrue(AuthTestHelper.isEquals(acl1, acl2));
+
+        Acl acl3 = AuthTestHelper.buildAcl("User:test", "Topic:test,Group:test", "PUB,SUB",
+            "192.168.0.0/24,10.10.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.createAcl(acl3).join();
+        Acl acl4 = this.authorizationMetadataManager.getAcl(Subject.of("User:test")).join();
+        Assert.assertTrue(AuthTestHelper.isEquals(acl3, acl4));
+
+        Assert.assertThrows(AuthorizationException.class, () -> {
+            try {
+                Acl acl5 = AuthTestHelper.buildAcl("User:ddd", "Topic:test,Group:test", "PUB,SUB",
+                    "192.168.0.0/24,10.10.0.0/24", Decision.ALLOW);
+                this.authorizationMetadataManager.createAcl(acl5).join();
+            } catch (Exception e) {
+                AuthTestHelper.handleException(e);
+            }
+        });
+    }
+
+    @Test
+    public void updateAcl() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+
+        Acl acl1 = AuthTestHelper.buildAcl("User:test", "Topic:test,Group:test", "PUB,SUB",
+            "192.168.0.0/24,10.10.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.createAcl(acl1).join();
+
+        Acl acl2 = AuthTestHelper.buildAcl("User:test", "Topic:abc,Group:abc", "PUB,SUB",
+            "192.168.0.0/24,10.10.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.updateAcl(acl2).join();
+
+        Acl acl3 = AuthTestHelper.buildAcl("User:test", "Topic:test,Group:test,Topic:abc,Group:abc", "PUB,SUB",
+            "192.168.0.0/24,10.10.0.0/24", Decision.ALLOW);
+        Acl acl4 = this.authorizationMetadataManager.getAcl(Subject.of("User:test")).join();
+        Assert.assertTrue(AuthTestHelper.isEquals(acl3, acl4));
+
+        Policy policy = AuthTestHelper.buildPolicy("Topic:test,Group:test", "PUB,SUB,Create", "192.168.0.0/24", Decision.DENY);
+        acl4.updatePolicy(policy);
+        this.authorizationMetadataManager.updateAcl(acl4);
+        Acl acl5 = this.authorizationMetadataManager.getAcl(Subject.of("User:test")).join();
+        Assert.assertTrue(AuthTestHelper.isEquals(acl4, acl5));
+
+        User user2 = User.of("abc", "abc");
+        this.authenticationMetadataManager.createUser(user2).join();
+        Acl acl6 = AuthTestHelper.buildAcl("User:abc", "Topic:test,Group:test", "PUB,SUB",
+            "192.168.0.0/24,10.10.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.updateAcl(acl6).join();
+        Acl acl7 = this.authorizationMetadataManager.getAcl(Subject.of("User:abc")).join();
+        Assert.assertTrue(AuthTestHelper.isEquals(acl6, acl7));
+    }
+
+    @Test
+    public void deleteAcl() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+
+        Acl acl1 = AuthTestHelper.buildAcl("User:test", "Topic:test,Group:test", "PUB,SUB",
+            "192.168.0.0/24,10.10.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.createAcl(acl1).join();
+
+        this.authorizationMetadataManager.deleteAcl(Subject.of("User:test"), PolicyType.CUSTOM, Resource.ofTopic("abc")).join();
+        Acl acl2 = this.authorizationMetadataManager.getAcl(Subject.of("User:test")).join();
+        Assert.assertTrue(AuthTestHelper.isEquals(acl1, acl2));
+
+        this.authorizationMetadataManager.deleteAcl(Subject.of("User:test"), PolicyType.CUSTOM, Resource.ofTopic("test")).join();
+        Acl acl3 = AuthTestHelper.buildAcl("User:test", "Group:test", "PUB,SUB",
+            "192.168.0.0/24,10.10.0.0/24", Decision.ALLOW);
+        Acl acl4 = this.authorizationMetadataManager.getAcl(Subject.of("User:test")).join();
+        Assert.assertTrue(AuthTestHelper.isEquals(acl3, acl4));
+
+        this.authorizationMetadataManager.deleteAcl(Subject.of("User:test"));
+        Acl acl5 = this.authorizationMetadataManager.getAcl(Subject.of("User:test")).join();
+        Assert.assertNull(acl5);
+
+        Assert.assertThrows(AuthorizationException.class, () -> {
+            try {
+                this.authorizationMetadataManager.deleteAcl(Subject.of("User:abc")).join();
+            } catch (Exception e) {
+                AuthTestHelper.handleException(e);
+            }
+        });
+    }
+
+    @Test
+    public void getAcl() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user = User.of("test", "test");
+        this.authenticationMetadataManager.createUser(user).join();
+
+        Acl acl1 = AuthTestHelper.buildAcl("User:test", "Topic:test,Group:test", "PUB,SUB",
+            "192.168.0.0/24,10.10.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.createAcl(acl1).join();
+        Acl acl2 = this.authorizationMetadataManager.getAcl(Subject.of("User:test")).join();
+        Assert.assertTrue(AuthTestHelper.isEquals(acl1, acl2));
+
+        Assert.assertThrows(AuthorizationException.class, () -> {
+            try {
+                this.authorizationMetadataManager.getAcl(Subject.of("User:abc")).join();
+            } catch (Exception e) {
+                AuthTestHelper.handleException(e);
+            }
+        });
+    }
+
+    @Test
+    public void listAcl() {
+        if (MixAll.isMac()) {
+            return;
+        }
+        User user1 = User.of("test-1", "test-1");
+        this.authenticationMetadataManager.createUser(user1).join();
+        User user2 = User.of("test-2", "test-2");
+        this.authenticationMetadataManager.createUser(user2).join();
+
+        Acl acl1 = AuthTestHelper.buildAcl("User:test-1", "Topic:test-1,Group:test-1", "PUB,SUB",
+            "192.168.0.0/24,10.10.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.createAcl(acl1).join();
+
+        Acl acl2 = AuthTestHelper.buildAcl("User:test-2", "Topic:test-2,Group:test-2", "PUB,SUB",
+            "192.168.0.0/24,10.10.0.0/24", Decision.ALLOW);
+        this.authorizationMetadataManager.createAcl(acl2).join();
+
+        List<Acl> acls1 = this.authorizationMetadataManager.listAcl(null, null).join();
+        Assert.assertEquals(acls1.size(), 2);
+
+        List<Acl> acls2 = this.authorizationMetadataManager.listAcl("User:test-1", null).join();
+        Assert.assertEquals(acls2.size(), 1);
+
+        List<Acl> acls3 = this.authorizationMetadataManager.listAcl("test", null).join();
+        Assert.assertEquals(acls3.size(), 2);
+
+        List<Acl> acls4 = this.authorizationMetadataManager.listAcl(null, "Topic:test-1").join();
+        Assert.assertEquals(acls4.size(), 1);
+        Assert.assertEquals(acls4.get(0).getPolicy(PolicyType.CUSTOM).getEntries().size(), 1);
+
+        List<Acl> acls5 = this.authorizationMetadataManager.listAcl(null, "test-1").join();
+        Assert.assertEquals(acls5.size(), 1);
+        Assert.assertEquals(acls4.get(0).getPolicy(PolicyType.CUSTOM).getEntries().size(), 1);
+
+        List<Acl> acls6 = this.authorizationMetadataManager.listAcl("User:abc", null).join();
+        Assert.assertTrue(CollectionUtils.isEmpty(acls6));
+
+        List<Acl> acls7 = this.authorizationMetadataManager.listAcl(null, "Topic:abc").join();
+        Assert.assertTrue(CollectionUtils.isEmpty(acls7));
+    }
+
+    private void clearAllUsers() {
+        List<User> users = this.authenticationMetadataManager.listUser(null).join();
+        if (CollectionUtils.isEmpty(users)) {
+            return;
+        }
+        users.forEach(user -> this.authenticationMetadataManager.deleteUser(user.getUsername()).join());
+    }
+
+    private void clearAllAcls() {
+        List<Acl> acls = this.authorizationMetadataManager.listAcl(null, null).join();
+        if (CollectionUtils.isEmpty(acls)) {
+            return;
+        }
+        acls.forEach(acl -> this.authorizationMetadataManager.deleteAcl(acl.getSubject(), null, null).join());
+    }
+}
\ No newline at end of file
diff --git a/auth/src/test/java/org/apache/rocketmq/auth/authorization/model/ResourceTest.java b/auth/src/test/java/org/apache/rocketmq/auth/authorization/model/ResourceTest.java
new file mode 100644
index 00000000000..a17a4ab6bec
--- /dev/null
+++ b/auth/src/test/java/org/apache/rocketmq/auth/authorization/model/ResourceTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.model;
+
+import org.apache.rocketmq.common.resource.ResourcePattern;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ResourceTest {
+
+    @Test
+    public void parseResource() {
+        Resource resource = Resource.of("*");
+        Assert.assertEquals(resource.getResourceType(), ResourceType.ANY);
+        Assert.assertNull(resource.getResourceName());
+        Assert.assertEquals(resource.getResourcePattern(), ResourcePattern.ANY);
+
+        resource = Resource.of("Topic:*");
+        Assert.assertEquals(resource.getResourceType(), ResourceType.TOPIC);
+        Assert.assertNull(resource.getResourceName());
+        Assert.assertEquals(resource.getResourcePattern(), ResourcePattern.ANY);
+
+        resource = Resource.of("Topic:test-*");
+        Assert.assertEquals(resource.getResourceType(), ResourceType.TOPIC);
+        Assert.assertEquals(resource.getResourceName(), "test-");
+        Assert.assertEquals(resource.getResourcePattern(), ResourcePattern.PREFIXED);
+
+        resource = Resource.of("Topic:test-1");
+        Assert.assertEquals(resource.getResourceType(), ResourceType.TOPIC);
+        Assert.assertEquals(resource.getResourceName(), "test-1");
+        Assert.assertEquals(resource.getResourcePattern(), ResourcePattern.LITERAL);
+    }
+
+    @Test
+    public void isMatch() {
+
+    }
+}
\ No newline at end of file
diff --git a/auth/src/test/java/org/apache/rocketmq/auth/authorization/strategy/StatefulAuthorizationStrategyTest.java b/auth/src/test/java/org/apache/rocketmq/auth/authorization/strategy/StatefulAuthorizationStrategyTest.java
new file mode 100644
index 00000000000..80e1f0b49e7
--- /dev/null
+++ b/auth/src/test/java/org/apache/rocketmq/auth/authorization/strategy/StatefulAuthorizationStrategyTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.authorization.strategy;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import org.apache.commons.lang3.reflect.MethodUtils;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authorization.context.AuthorizationContext;
+import org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.model.Resource;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.Pair;
+import org.apache.rocketmq.common.action.Action;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class StatefulAuthorizationStrategyTest {
+
+    @Mock
+    private AuthConfig authConfig;
+
+    private StatefulAuthorizationStrategy statefulAuthorizationStrategy;
+
+    @Before
+    public void setUp() {
+        when(authConfig.getStatefulAuthorizationCacheExpiredSecond()).thenReturn(60);
+        when(authConfig.getStatefulAuthorizationCacheMaxNum()).thenReturn(100);
+        Supplier<?> metadataService = mock(Supplier.class);
+        statefulAuthorizationStrategy = spy(new StatefulAuthorizationStrategy(authConfig, metadataService));
+    }
+
+    @Test
+    public void testEvaluateChannelIdBlankDoesNotUseCache() {
+        AuthorizationContext context = mock(AuthorizationContext.class);
+        when(context.getChannelId()).thenReturn(null);
+        statefulAuthorizationStrategy.evaluate(context);
+        verify(statefulAuthorizationStrategy, times(1)).doEvaluate(context);
+    }
+
+    @Test
+    public void testEvaluateChannelIdNotNullCacheHit() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {
+        DefaultAuthorizationContext context = new DefaultAuthorizationContext();
+        context.setChannelId("channelId");
+        context.setSubject(Subject.of("User"));
+        context.setResource(Resource.of("Cluster"));
+        context.setActions(new ArrayList<>());
+        context.setSourceIp("sourceIp");
+        Pair<Boolean, AuthorizationException> pair = Pair.of(true, null);
+        Cache<String, Pair<Boolean, AuthorizationException>> authCache = Caffeine.newBuilder()
+                .expireAfterWrite(60, TimeUnit.SECONDS)
+                .maximumSize(100)
+                .build();
+        authCache.put(buildKey(context), pair);
+        statefulAuthorizationStrategy.authCache = authCache;
+        statefulAuthorizationStrategy.evaluate(context);
+        verify(statefulAuthorizationStrategy, never()).doEvaluate(context);
+    }
+
+    @Test
+    public void testEvaluateChannelIdNotNullCacheMiss() {
+        DefaultAuthorizationContext context = new DefaultAuthorizationContext();
+        context.setChannelId("channelId");
+        context.setSubject(Subject.of("User"));
+        context.setResource(Resource.of("Cluster"));
+        context.setActions(Collections.singletonList(Action.PUB));
+        context.setSourceIp("sourceIp");
+        statefulAuthorizationStrategy.authCache = Caffeine.newBuilder()
+                .expireAfterWrite(60, TimeUnit.SECONDS)
+                .maximumSize(100)
+                .build();
+        statefulAuthorizationStrategy.evaluate(context);
+        verify(statefulAuthorizationStrategy, times(1)).doEvaluate(context);
+    }
+
+    @Test
+    public void testEvaluateChannelIdNotNullCacheException() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {
+        DefaultAuthorizationContext context = new DefaultAuthorizationContext();
+        context.setChannelId("channelId");
+        context.setSubject(Subject.of("subjectKey"));
+        context.setResource(Resource.of("resourceKey"));
+        context.setActions(Collections.singletonList(Action.PUB));
+        context.setSourceIp("sourceIp");
+        AuthorizationException exception = new AuthorizationException("test");
+        Pair<Boolean, AuthorizationException> pair = Pair.of(false, exception);
+        Cache<String, Pair<Boolean, AuthorizationException>> authCache = Caffeine.newBuilder()
+                .expireAfterWrite(60, TimeUnit.SECONDS)
+                .maximumSize(100)
+                .build();
+        authCache.put(buildKey(context), pair);
+        statefulAuthorizationStrategy.authCache = authCache;
+        try {
+            statefulAuthorizationStrategy.evaluate(context);
+            fail("Expected AuthorizationException to be thrown");
+        } catch (final AuthorizationException ex) {
+            assertEquals(exception, ex);
+        }
+        verify(statefulAuthorizationStrategy, never()).doEvaluate(context);
+    }
+
+    private String buildKey(AuthorizationContext context) throws InvocationTargetException, NoSuchMethodException, IllegalAccessException {
+        return (String) MethodUtils.invokeMethod(statefulAuthorizationStrategy, true, "buildKey", context);
+    }
+}
diff --git a/auth/src/test/java/org/apache/rocketmq/auth/helper/AuthTestHelper.java b/auth/src/test/java/org/apache/rocketmq/auth/helper/AuthTestHelper.java
new file mode 100644
index 00000000000..e31732a26de
--- /dev/null
+++ b/auth/src/test/java/org/apache/rocketmq/auth/helper/AuthTestHelper.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.helper;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authentication.provider.DefaultAuthenticationProvider;
+import org.apache.rocketmq.auth.authentication.provider.LocalAuthenticationMetadataProvider;
+import org.apache.rocketmq.auth.authorization.enums.Decision;
+import org.apache.rocketmq.auth.authorization.enums.PolicyType;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.authorization.model.Environment;
+import org.apache.rocketmq.auth.authorization.model.Policy;
+import org.apache.rocketmq.auth.authorization.model.PolicyEntry;
+import org.apache.rocketmq.auth.authorization.model.Resource;
+import org.apache.rocketmq.auth.authorization.provider.DefaultAuthorizationProvider;
+import org.apache.rocketmq.auth.authorization.provider.LocalAuthorizationMetadataProvider;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.utils.ExceptionUtils;
+
+public class AuthTestHelper {
+
+    public static AuthConfig createDefaultConfig() {
+        AuthConfig authConfig = new AuthConfig();
+        authConfig.setConfigName("test-" + System.nanoTime());
+        authConfig.setAuthConfigPath("~/config");
+        authConfig.setAuthenticationEnabled(true);
+        authConfig.setAuthenticationProvider(DefaultAuthenticationProvider.class.getName());
+        authConfig.setAuthenticationMetadataProvider(LocalAuthenticationMetadataProvider.class.getName());
+        authConfig.setAuthorizationEnabled(true);
+        authConfig.setAuthorizationProvider(DefaultAuthorizationProvider.class.getName());
+        authConfig.setAuthorizationMetadataProvider(LocalAuthorizationMetadataProvider.class.getName());
+        return authConfig;
+    }
+
+    public static Acl buildAcl(String subjectKey, String resources, String actions, String sourceIps,
+        Decision decision) {
+        return buildAcl(subjectKey, null, resources, actions, sourceIps, decision);
+    }
+
+    public static Acl buildAcl(String subjectKey, PolicyType policyType, String resources, String actions,
+        String sourceIps, Decision decision) {
+        Subject subject = Subject.of(subjectKey);
+        Policy policy = buildPolicy(policyType, resources, actions, sourceIps, decision);
+        return Acl.of(subject, policy);
+    }
+
+    public static Policy buildPolicy(String resources, String actions, String sourceIps,
+        Decision decision) {
+        return buildPolicy(null, resources, actions, sourceIps, decision);
+    }
+
+    public static Policy buildPolicy(PolicyType policyType, String resources, String actions, String sourceIps,
+        Decision decision) {
+        List<Resource> resourceList = Arrays.stream(StringUtils.split(resources, ","))
+            .map(Resource::of).collect(Collectors.toList());
+        List<Action> actionList = Arrays.stream(StringUtils.split(actions, ","))
+            .map(Action::getByName).collect(Collectors.toList());
+        Environment environment = null;
+        if (StringUtils.isNotBlank(sourceIps)) {
+            environment = Environment.of(Arrays.stream(StringUtils.split(sourceIps, ","))
+                .collect(Collectors.toList()));
+        }
+        return Policy.of(policyType, resourceList, actionList, environment, decision);
+    }
+
+    public static boolean isEquals(Acl acl1, Acl acl2) {
+        if (acl1 == null && acl2 == null) {
+            return true;
+        }
+        if (acl1 == null || acl2 == null) {
+            return false;
+        }
+        Subject subject1 = acl1.getSubject();
+        Subject subject2 = acl2.getSubject();
+        if (!isEquals(subject1, subject2)) {
+            return false;
+        }
+        Map<PolicyType, Policy> policyMap1 = new HashMap<>();
+        Map<PolicyType, Policy> policyMap2 = new HashMap<>();
+        if (CollectionUtils.isNotEmpty(acl1.getPolicies())) {
+            acl1.getPolicies().forEach(policy -> {
+                if (policy.getPolicyType() == null) {
+                    policy.setPolicyType(PolicyType.CUSTOM);
+                }
+                policyMap1.put(policy.getPolicyType(), policy);
+            });
+        }
+        if (CollectionUtils.isNotEmpty(acl2.getPolicies())) {
+            acl2.getPolicies().forEach(policy -> {
+                if (policy.getPolicyType() == null) {
+                    policy.setPolicyType(PolicyType.CUSTOM);
+                }
+                policyMap2.put(policy.getPolicyType(), policy);
+            });
+        }
+        if (policyMap1.size() != policyMap2.size()) {
+            return false;
+        }
+        Policy customPolicy1 = policyMap1.get(PolicyType.CUSTOM);
+        Policy customPolicy2 = policyMap2.get(PolicyType.CUSTOM);
+        if (!isEquals(customPolicy1, customPolicy2)) {
+            return false;
+        }
+
+        Policy defaultPolicy1 = policyMap1.get(PolicyType.DEFAULT);
+        Policy defaultPolicy2 = policyMap2.get(PolicyType.DEFAULT);
+        if (!isEquals(defaultPolicy1, defaultPolicy2)) {
+            return false;
+        }
+
+        return true;
+    }
+
+    private static boolean isEquals(Policy policy1, Policy policy2) {
+        if (policy1 == null && policy2 == null) {
+            return true;
+        }
+        if (policy1 == null || policy2 == null) {
+            return false;
+        }
+        if (policy1.getPolicyType() != policy2.getPolicyType()) {
+            return false;
+        }
+        Map<String, PolicyEntry> policyEntryMap1 = new HashMap<>();
+        Map<String, PolicyEntry> policyEntryMap2 = new HashMap<>();
+        if (CollectionUtils.isNotEmpty(policy1.getEntries())) {
+            policy1.getEntries().forEach(policyEntry -> {
+                policyEntryMap1.put(policyEntry.getResource().getResourceKey(), policyEntry);
+            });
+        }
+        if (CollectionUtils.isNotEmpty(policy2.getEntries())) {
+            policy2.getEntries().forEach(policyEntry -> {
+                policyEntryMap2.put(policyEntry.getResource().getResourceKey(), policyEntry);
+            });
+        }
+        if (policyEntryMap1.size() != policyEntryMap2.size()) {
+            return false;
+        }
+
+        for (String resourceKey : policyEntryMap1.keySet()) {
+            if (!isEquals(policyEntryMap1.get(resourceKey), policyEntryMap2.get(resourceKey))) {
+                return false;
+            }
+        }
+
+        for (String resourceKey : policyEntryMap2.keySet()) {
+            if (!isEquals(policyEntryMap1.get(resourceKey), policyEntryMap2.get(resourceKey))) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    private static boolean isEquals(PolicyEntry entry1, PolicyEntry entry2) {
+        if (entry1 == null && entry2 == null) {
+            return true;
+        }
+        if (entry1 == null || entry2 == null) {
+            return false;
+        }
+        Resource resource1 = entry1.getResource();
+        Resource resource2 = entry2.getResource();
+        if (!isEquals(resource1, resource2)) {
+            return false;
+        }
+        List<Action> actions1 = entry1.getActions();
+        List<Action> actions2 = entry2.getActions();
+        if (CollectionUtils.isEmpty(actions1) && CollectionUtils.isNotEmpty(actions2)) {
+            return false;
+        }
+        if (CollectionUtils.isNotEmpty(actions1) && CollectionUtils.isEmpty(actions2)) {
+            return false;
+        }
+        if (CollectionUtils.isNotEmpty(actions1) && CollectionUtils.isNotEmpty(actions2)
+            && !CollectionUtils.isEqualCollection(actions1, actions2)) {
+            return false;
+        }
+        Environment environment1 = entry1.getEnvironment();
+        Environment environment2 = entry2.getEnvironment();
+        if (!isEquals(environment1, environment2)) {
+            return false;
+        }
+        return entry1.getDecision() == entry2.getDecision();
+    }
+
+    private static boolean isEquals(Resource resource1, Resource resource2) {
+        if (resource1 == null && resource2 == null) {
+            return true;
+        }
+        if (resource1 == null || resource2 == null) {
+            return false;
+        }
+        return Objects.equals(resource1, resource2);
+    }
+
+    private static boolean isEquals(Environment environment1, Environment environment2) {
+        if (environment1 == null && environment2 == null) {
+            return true;
+        }
+        if (environment1 == null || environment2 == null) {
+            return false;
+        }
+        List<String> sourceIp1 = environment1.getSourceIps();
+        List<String> sourceIp2 = environment2.getSourceIps();
+        if (CollectionUtils.isEmpty(sourceIp1) && CollectionUtils.isEmpty(sourceIp2)) {
+            return true;
+        }
+        if (CollectionUtils.isEmpty(sourceIp1) || CollectionUtils.isEmpty(sourceIp2)) {
+            return false;
+        }
+        return CollectionUtils.isEqualCollection(sourceIp1, sourceIp2);
+    }
+
+    private static boolean isEquals(Subject subject1, Subject subject2) {
+        if (subject1 == null && subject2 == null) {
+            return true;
+        }
+        if (subject1 == null || subject2 == null) {
+            return false;
+        }
+        return subject1.getSubjectType() == subject2.getSubjectType()
+            && StringUtils.equals(subject1.getSubjectKey(), subject2.getSubjectKey());
+    }
+
+    public static void handleException(Throwable e) {
+        Throwable throwable = ExceptionUtils.getRealException(e);
+        if (throwable instanceof AuthenticationException) {
+            throw (AuthenticationException) throwable;
+        }
+        if (throwable instanceof AuthorizationException) {
+            throw (AuthorizationException) throwable;
+        }
+        throw new RuntimeException(e);
+    }
+}
diff --git a/auth/src/test/java/org/apache/rocketmq/auth/migration/AuthMigratorTest.java b/auth/src/test/java/org/apache/rocketmq/auth/migration/AuthMigratorTest.java
new file mode 100644
index 00000000000..1b95051d32a
--- /dev/null
+++ b/auth/src/test/java/org/apache/rocketmq/auth/migration/AuthMigratorTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.auth.migration;
+
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.auth.migration.v1.PlainPermissionManager;
+import org.apache.rocketmq.auth.migration.v1.AclConfig;
+import org.apache.rocketmq.auth.migration.v1.PlainAccessConfig;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class AuthMigratorTest {
+
+    @Mock
+    private AuthenticationMetadataManager authenticationMetadataManager;
+
+    @Mock
+    private AuthorizationMetadataManager authorizationMetadataManager;
+
+    @Mock
+    private PlainPermissionManager plainPermissionManager;
+
+    @Mock
+    private AuthConfig authConfig;
+
+    private AuthMigrator authMigrator;
+
+    @Before
+    public void setUp() throws IllegalAccessException {
+        when(authConfig.isMigrateAuthFromV1Enabled()).thenReturn(true);
+        authMigrator = new AuthMigrator(authConfig);
+        FieldUtils.writeDeclaredField(authMigrator, "authenticationMetadataManager", authenticationMetadataManager, true);
+        FieldUtils.writeDeclaredField(authMigrator, "authorizationMetadataManager", authorizationMetadataManager, true);
+        FieldUtils.writeDeclaredField(authMigrator, "plainPermissionManager", plainPermissionManager, true);
+    }
+
+    @Test
+    public void testMigrateNoAclConfigDoesNothing() {
+        AclConfig aclConfig = mock(AclConfig.class);
+        when(aclConfig.getPlainAccessConfigs()).thenReturn(new ArrayList<>());
+        when(plainPermissionManager.getAllAclConfig()).thenReturn(aclConfig);
+        authMigrator.migrate();
+        verify(authConfig, times(1)).isMigrateAuthFromV1Enabled();
+        verify(plainPermissionManager, times(1)).getAllAclConfig();
+        verify(authenticationMetadataManager, never()).createUser(any());
+        verify(authorizationMetadataManager, never()).createAcl(any());
+    }
+
+    @Test
+    public void testMigrateWithAclConfigCreatesUserAndAcl() {
+        AclConfig aclConfig = mock(AclConfig.class);
+        List<PlainAccessConfig> accessConfigs = new ArrayList<>();
+        accessConfigs.add(createPlainAccessConfig());
+        when(aclConfig.getPlainAccessConfigs()).thenReturn(accessConfigs);
+        when(plainPermissionManager.getAllAclConfig()).thenReturn(aclConfig);
+        when(authenticationMetadataManager.getUser(anyString()))
+                .thenReturn(CompletableFuture.completedFuture(null));
+        when(authenticationMetadataManager.createUser(any()))
+                .thenReturn(CompletableFuture.completedFuture(null));
+        authMigrator.migrate();
+        verify(authConfig, times(1)).isMigrateAuthFromV1Enabled();
+        verify(plainPermissionManager, times(1)).getAllAclConfig();
+        verify(authenticationMetadataManager, times(1)).createUser(any());
+        verify(authorizationMetadataManager, times(1)).createAcl(any());
+    }
+
+    @Test
+    public void testMigrateExceptionInMigrateLogsError() {
+        PlainAccessConfig accessConfig = mock(PlainAccessConfig.class);
+        when(accessConfig.getAccessKey()).thenReturn("testAk");
+        when(authenticationMetadataManager.createUser(any(User.class)))
+                .thenThrow(new RuntimeException("Test Exception"));
+        AclConfig aclConfig = mock(AclConfig.class);
+        List<PlainAccessConfig> accessConfigs = new ArrayList<>();
+        accessConfigs.add(accessConfig);
+        when(aclConfig.getPlainAccessConfigs()).thenReturn(accessConfigs);
+        when(plainPermissionManager.getAllAclConfig()).thenReturn(aclConfig);
+        when(authenticationMetadataManager.getUser(anyString()))
+                .thenReturn(CompletableFuture.completedFuture(null));
+        try {
+            authMigrator.migrate();
+            verify(authConfig, times(1)).isMigrateAuthFromV1Enabled();
+            verify(plainPermissionManager, times(1)).getAllAclConfig();
+            verify(authenticationMetadataManager, times(1)).createUser(any());
+            verify(authorizationMetadataManager, never()).createAcl(any());
+        } catch (final RuntimeException ex) {
+            assertEquals("Test Exception", ex.getMessage());
+        }
+    }
+
+    private PlainAccessConfig createPlainAccessConfig() {
+        PlainAccessConfig result = mock(PlainAccessConfig.class);
+        when(result.getAccessKey()).thenReturn("testAk");
+        when(result.getSecretKey()).thenReturn("testSk");
+        when(result.isAdmin()).thenReturn(false);
+        when(result.getTopicPerms()).thenReturn(new ArrayList<>());
+        when(result.getGroupPerms()).thenReturn(new ArrayList<>());
+        when(result.getDefaultTopicPerm()).thenReturn("PUB");
+        when(result.getDefaultGroupPerm()).thenReturn(null);
+        return result;
+    }
+}
diff --git a/broker/BUILD.bazel b/broker/BUILD.bazel
index ab413d3d060..9d61c0a1ff0 100644
--- a/broker/BUILD.bazel
+++ b/broker/BUILD.bazel
@@ -21,15 +21,18 @@ java_library(
     srcs = glob(["src/main/java/**/*.java"]),
     visibility = ["//visibility:public"],
     deps = [
-        "//acl",
+        "//auth",
         "//client",
         "//common",
         "//filter",
         "//remoting",
         "//srvutil",
         "//store",
+        "//tieredstore",
+        "@maven//:org_slf4j_slf4j_api",
         "@maven//:ch_qos_logback_logback_classic",
         "@maven//:com_alibaba_fastjson",
+        "@maven//:com_alibaba_fastjson2_fastjson2",
         "@maven//:com_github_luben_zstd_jni",
         "@maven//:com_google_guava_guava",
         "@maven//:com_googlecode_concurrentlinkedhashmap_concurrentlinkedhashmap_lru",
@@ -49,6 +52,7 @@ java_library(
         "@maven//:io_opentelemetry_opentelemetry_sdk_common",
         "@maven//:io_opentelemetry_opentelemetry_sdk_metrics",
         "@maven//:org_apache_commons_commons_lang3",
+        "@maven//:commons_codec_commons_codec",
         "@maven//:org_lz4_lz4_java",
         "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
         "@maven//:io_github_aliyunmq_rocketmq_logback_classic",
@@ -63,7 +67,6 @@ java_library(
     name = "tests",
     srcs = glob(["src/test/java/**/*.java"]),
     resources = [
-        "src/test/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator",
         "src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener",
         "src/test/resources/META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService",
         "src/test/resources/rmq.logback-test.xml",
@@ -72,25 +75,39 @@ java_library(
     deps = [
         ":broker",
         "//:test_deps",
-        "//acl",
+        "//auth",
         "//client",
         "//common",
         "//filter",
         "//remoting",
         "//store",
+        "//tieredstore",
         "@maven//:com_alibaba_fastjson",
+        "@maven//:com_alibaba_fastjson2_fastjson2",
+        "@maven//:org_slf4j_slf4j_api",
         "@maven//:com_google_guava_guava",
         "@maven//:io_netty_netty_all",
         "@maven//:org_apache_commons_commons_lang3",
+        "@maven//:commons_codec_commons_codec",
+        "@maven//:commons_io_commons_io",
         "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
         "@maven//:org_powermock_powermock_core",
         "@maven//:io_opentelemetry_opentelemetry_api",
+        "@maven//:com_googlecode_concurrentlinkedhashmap_concurrentlinkedhashmap_lru",
+        "@maven//:org_apache_rocketmq_rocketmq_rocksdb",
+        "@maven//:commons_collections_commons_collections",
+        "@maven//:org_junit_jupiter_junit_jupiter_api",
     ],
 )
 
 GenTestRules(
     name = "GeneratedTestRules",
     test_files = glob(["src/test/java/**/*Test.java"]),
+    exclude_tests = [
+            # These tests are extremely slow and flaky, exclude them before they are properly fixed.
+            "src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest",
+            "src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest",
+        ],
     deps = [
         ":tests",
     ],
diff --git a/broker/pom.xml b/broker/pom.xml
index add83045dcb..ce0fc016557 100644
--- a/broker/pom.xml
+++ b/broker/pom.xml
@@ -13,7 +13,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -59,8 +59,8 @@
             <artifactId>rocketmq-filter</artifactId>
         </dependency>
         <dependency>
-            <groupId>${project.groupId}</groupId>
-            <artifactId>rocketmq-acl</artifactId>
+            <groupId>org.apache.rocketmq</groupId>
+            <artifactId>rocketmq-auth</artifactId>
         </dependency>
         <dependency>
             <groupId>commons-io</groupId>
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
index af90e5f87eb..5a9fa263e52 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
@@ -16,35 +16,16 @@
  */
 package org.apache.rocketmq.broker;
 
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.AbstractMap;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
-
-import org.apache.rocketmq.acl.AccessValidator;
-import org.apache.rocketmq.acl.plain.PlainAccessValidator;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;
+import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;
+import org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.auth.migration.AuthMigrator;
+import org.apache.rocketmq.broker.auth.pipeline.AuthenticationPipeline;
+import org.apache.rocketmq.broker.auth.pipeline.AuthorizationPipeline;
 import org.apache.rocketmq.broker.client.ClientHousekeepingService;
 import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
 import org.apache.rocketmq.broker.client.ConsumerManager;
@@ -54,6 +35,15 @@
 import org.apache.rocketmq.broker.client.rebalance.RebalanceLockManager;
 import org.apache.rocketmq.broker.coldctr.ColdDataCgCtrService;
 import org.apache.rocketmq.broker.coldctr.ColdDataPullRequestHoldService;
+import org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager;
+import org.apache.rocketmq.broker.config.v1.RocksDBLmqSubscriptionGroupManager;
+import org.apache.rocketmq.broker.config.v1.RocksDBLmqTopicConfigManager;
+import org.apache.rocketmq.broker.config.v1.RocksDBSubscriptionGroupManager;
+import org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager;
+import org.apache.rocketmq.broker.config.v2.ConfigStorage;
+import org.apache.rocketmq.broker.config.v2.ConsumerOffsetManagerV2;
+import org.apache.rocketmq.broker.config.v2.SubscriptionGroupManagerV2;
+import org.apache.rocketmq.broker.config.v2.TopicConfigManagerV2;
 import org.apache.rocketmq.broker.controller.ReplicasManager;
 import org.apache.rocketmq.broker.dledger.DLedgerRoleChangeHandler;
 import org.apache.rocketmq.broker.failover.EscapeBridge;
@@ -70,10 +60,9 @@
 import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
 import org.apache.rocketmq.broker.offset.ConsumerOrderInfoManager;
 import org.apache.rocketmq.broker.offset.LmqConsumerOffsetManager;
-import org.apache.rocketmq.broker.offset.RocksDBConsumerOffsetManager;
-import org.apache.rocketmq.broker.offset.RocksDBLmqConsumerOffsetManager;
 import org.apache.rocketmq.broker.out.BrokerOuterAPI;
 import org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin;
+import org.apache.rocketmq.broker.pop.PopConsumerService;
 import org.apache.rocketmq.broker.processor.AckMessageProcessor;
 import org.apache.rocketmq.broker.processor.AdminBrokerProcessor;
 import org.apache.rocketmq.broker.processor.ChangeInvisibleTimeProcessor;
@@ -88,17 +77,14 @@
 import org.apache.rocketmq.broker.processor.PullMessageProcessor;
 import org.apache.rocketmq.broker.processor.QueryAssignmentProcessor;
 import org.apache.rocketmq.broker.processor.QueryMessageProcessor;
+import org.apache.rocketmq.broker.processor.RecallMessageProcessor;
 import org.apache.rocketmq.broker.processor.ReplyMessageProcessor;
 import org.apache.rocketmq.broker.processor.SendMessageProcessor;
 import org.apache.rocketmq.broker.schedule.ScheduleMessageService;
 import org.apache.rocketmq.broker.slave.SlaveSynchronize;
 import org.apache.rocketmq.broker.subscription.LmqSubscriptionGroupManager;
-import org.apache.rocketmq.broker.subscription.RocksDBLmqSubscriptionGroupManager;
-import org.apache.rocketmq.broker.subscription.RocksDBSubscriptionGroupManager;
 import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
 import org.apache.rocketmq.broker.topic.LmqTopicConfigManager;
-import org.apache.rocketmq.broker.topic.RocksDBLmqTopicConfigManager;
-import org.apache.rocketmq.broker.topic.RocksDBTopicConfigManager;
 import org.apache.rocketmq.broker.topic.TopicConfigManager;
 import org.apache.rocketmq.broker.topic.TopicQueueMappingCleanService;
 import org.apache.rocketmq.broker.topic.TopicQueueMappingManager;
@@ -118,6 +104,7 @@
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.config.ConfigManagerVersion;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.constant.PermName;
 import org.apache.rocketmq.common.message.MessageExt;
@@ -137,11 +124,12 @@
 import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.remoting.netty.RequestTask;
 import org.apache.rocketmq.remoting.netty.TlsSystemConfig;
+import org.apache.rocketmq.remoting.pipeline.RequestPipeline;
 import org.apache.rocketmq.remoting.protocol.BrokerSyncInfo;
 import org.apache.rocketmq.remoting.protocol.DataVersion;
 import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
-import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.RequestHeaderRegistry;
 import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;
 import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;
 import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
@@ -168,6 +156,30 @@
 import org.apache.rocketmq.store.timer.TimerMessageStore;
 import org.apache.rocketmq.store.timer.TimerMetrics;
 
+import java.net.InetSocketAddress;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
 public class BrokerController {
     protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private static final Logger LOG_PROTECTION = LoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME);
@@ -178,12 +190,14 @@ public class BrokerController {
     private final NettyServerConfig nettyServerConfig;
     private final NettyClientConfig nettyClientConfig;
     protected final MessageStoreConfig messageStoreConfig;
-    protected final ConsumerOffsetManager consumerOffsetManager;
+    private final AuthConfig authConfig;
+    protected ConsumerOffsetManager consumerOffsetManager;
     protected final BroadcastOffsetManager broadcastOffsetManager;
     protected final ConsumerManager consumerManager;
     protected final ConsumerFilterManager consumerFilterManager;
     protected final ConsumerOrderInfoManager consumerOrderInfoManager;
     protected final PopInflightMessageCounter popInflightMessageCounter;
+    protected final PopConsumerService popConsumerService;
     protected final ProducerManager producerManager;
     protected final ScheduleMessageService scheduleMessageService;
     protected final ClientHousekeepingService clientHousekeepingService;
@@ -197,6 +211,7 @@ public class BrokerController {
     protected final QueryAssignmentProcessor queryAssignmentProcessor;
     protected final ClientManageProcessor clientManageProcessor;
     protected final SendMessageProcessor sendMessageProcessor;
+    protected final RecallMessageProcessor recallMessageProcessor;
     protected final ReplyMessageProcessor replyMessageProcessor;
     protected final PullRequestHoldService pullRequestHoldService;
     protected final MessageArrivingListener messageArrivingListener;
@@ -223,13 +238,18 @@ public class BrokerController {
     protected final BlockingQueue<Runnable> endTransactionThreadPoolQueue;
     protected final BlockingQueue<Runnable> adminBrokerThreadPoolQueue;
     protected final BlockingQueue<Runnable> loadBalanceThreadPoolQueue;
-    protected final BrokerStatsManager brokerStatsManager;
+    protected BrokerStatsManager brokerStatsManager;
     protected final List<SendMessageHook> sendMessageHookList = new ArrayList<>();
     protected final List<ConsumeMessageHook> consumeMessageHookList = new ArrayList<>();
     protected MessageStore messageStore;
-    protected RemotingServer remotingServer;
+    protected static final String TCP_REMOTING_SERVER = "TCP_REMOTING_SERVER";
+    protected static final String FAST_REMOTING_SERVER = "FAST_REMOTING_SERVER";
+    protected final Map<String, RemotingServer> remotingServerMap = new ConcurrentHashMap<>();
     protected CountDownLatch remotingServerStartLatch;
-    protected RemotingServer fastRemotingServer;
+    /**
+     * If {Topic, SubscriptionGroup, Offset}ManagerV2 are used, config entries are stored in RocksDB.
+     */
+    protected ConfigStorage configStorage;
     protected TopicConfigManager topicConfigManager;
     protected SubscriptionGroupManager subscriptionGroupManager;
     protected TopicQueueMappingManager topicQueueMappingManager;
@@ -258,7 +278,6 @@ public class BrokerController {
     protected TransactionalMessageCheckService transactionalMessageCheckService;
     protected TransactionalMessageService transactionalMessageService;
     protected AbstractTransactionalMessageCheckListener transactionalMessageCheckListener;
-    protected Map<Class, AccessValidator> accessValidatorMap = new HashMap<>();
     protected volatile boolean shutdown = false;
     protected ShutdownHook shutdownHook;
     private volatile boolean isScheduleServiceStart = false;
@@ -279,15 +298,18 @@ public class BrokerController {
     private ColdDataPullRequestHoldService coldDataPullRequestHoldService;
     private ColdDataCgCtrService coldDataCgCtrService;
     private TransactionMetricsFlushService transactionMetricsFlushService;
+    private AuthenticationMetadataManager authenticationMetadataManager;
+    private AuthorizationMetadataManager authorizationMetadataManager;
 
     public BrokerController(
         final BrokerConfig brokerConfig,
         final NettyServerConfig nettyServerConfig,
         final NettyClientConfig nettyClientConfig,
         final MessageStoreConfig messageStoreConfig,
+        final AuthConfig authConfig,
         final ShutdownHook shutdownHook
     ) {
-        this(brokerConfig, nettyServerConfig, nettyClientConfig, messageStoreConfig);
+        this(brokerConfig, nettyServerConfig, nettyClientConfig, messageStoreConfig, authConfig);
         this.shutdownHook = shutdownHook;
     }
 
@@ -295,7 +317,7 @@ public BrokerController(
         final BrokerConfig brokerConfig,
         final MessageStoreConfig messageStoreConfig
     ) {
-        this(brokerConfig, null, null, messageStoreConfig);
+        this(brokerConfig, null, null, messageStoreConfig, null);
     }
 
     public BrokerController(
@@ -303,24 +325,42 @@ public BrokerController(
         final NettyServerConfig nettyServerConfig,
         final NettyClientConfig nettyClientConfig,
         final MessageStoreConfig messageStoreConfig
+    ) {
+        this(brokerConfig, nettyServerConfig, nettyClientConfig, messageStoreConfig, null);
+    }
+
+    public BrokerController(
+        final BrokerConfig brokerConfig,
+        final NettyServerConfig nettyServerConfig,
+        final NettyClientConfig nettyClientConfig,
+        final MessageStoreConfig messageStoreConfig,
+        final AuthConfig authConfig
     ) {
         this.brokerConfig = brokerConfig;
         this.nettyServerConfig = nettyServerConfig;
         this.nettyClientConfig = nettyClientConfig;
         this.messageStoreConfig = messageStoreConfig;
+        this.authConfig = authConfig;
         this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), getListenPort()));
-        this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat());
+        this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat());
         this.broadcastOffsetManager = new BroadcastOffsetManager(this);
-        if (this.messageStoreConfig.isEnableRocksDBStore()) {
+        if (ConfigManagerVersion.V2.getVersion().equals(brokerConfig.getConfigManagerVersion())) {
+            this.configStorage = new ConfigStorage(messageStoreConfig);
+            this.topicConfigManager = new TopicConfigManagerV2(this, configStorage);
+            this.subscriptionGroupManager = new SubscriptionGroupManagerV2(this, configStorage);
+            this.consumerOffsetManager = new ConsumerOffsetManagerV2(this, configStorage);
+        } else if (this.messageStoreConfig.isEnableRocksDBStore()) {
             this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqTopicConfigManager(this) : new RocksDBTopicConfigManager(this);
             this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqSubscriptionGroupManager(this) : new RocksDBSubscriptionGroupManager(this);
-            this.consumerOffsetManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqConsumerOffsetManager(this) : new RocksDBConsumerOffsetManager(this);
+            this.consumerOffsetManager = new RocksDBConsumerOffsetManager(this);
         } else {
             this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new LmqTopicConfigManager(this) : new TopicConfigManager(this);
             this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new LmqSubscriptionGroupManager(this) : new SubscriptionGroupManager(this);
             this.consumerOffsetManager = messageStoreConfig.isEnableLmq() ? new LmqConsumerOffsetManager(this) : new ConsumerOffsetManager(this);
         }
         this.topicQueueMappingManager = new TopicQueueMappingManager(this);
+        this.authenticationMetadataManager = AuthenticationFactory.getMetadataManager(this.authConfig);
+        this.authorizationMetadataManager = AuthorizationFactory.getMetadataManager(this.authConfig);
         this.pullMessageProcessor = new PullMessageProcessor(this);
         this.peekMessageProcessor = new PeekMessageProcessor(this);
         this.pullRequestHoldService = messageStoreConfig.isEnableLmq() ? new LmqPullRequestHoldService(this) : new PullRequestHoldService(this);
@@ -330,6 +370,7 @@ public BrokerController(
         this.ackMessageProcessor = new AckMessageProcessor(this);
         this.changeInvisibleTimeProcessor = new ChangeInvisibleTimeProcessor(this);
         this.sendMessageProcessor = new SendMessageProcessor(this);
+        this.recallMessageProcessor = new RecallMessageProcessor(this);
         this.replyMessageProcessor = new ReplyMessageProcessor(this);
         this.messageArrivingListener = new NotifyMessageArrivingListener(this.pullRequestHoldService, this.popMessageProcessor, this.notificationProcessor);
         this.consumerIdsChangeListener = new DefaultConsumerIdsChangeListener(this);
@@ -338,6 +379,7 @@ public BrokerController(
         this.consumerFilterManager = new ConsumerFilterManager(this);
         this.consumerOrderInfoManager = new ConsumerOrderInfoManager(this);
         this.popInflightMessageCounter = new PopInflightMessageCounter(this);
+        this.popConsumerService = brokerConfig.isPopConsumerKVServiceInit() ? new PopConsumerService(this) : null;
         this.clientHousekeepingService = new ClientHousekeepingService(this);
         this.broker2Client = new Broker2Client(this);
         this.scheduleMessageService = new ScheduleMessageService(this);
@@ -345,7 +387,7 @@ public BrokerController(
         this.coldDataCgCtrService = new ColdDataCgCtrService(this);
 
         if (nettyClientConfig != null) {
-            this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig);
+            this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig, authConfig);
         }
 
         this.queryAssignmentProcessor = new QueryAssignmentProcessor(this);
@@ -382,7 +424,7 @@ public BrokerController(
             this.brokerConfig, this.nettyServerConfig, this.nettyClientConfig, this.messageStoreConfig
         );
 
-        this.brokerStatsManager.setProduerStateGetter(new BrokerStatsManager.StateGetter() {
+        this.brokerStatsManager.setProducerStateGetter(new BrokerStatsManager.StateGetter() {
             @Override
             public boolean online(String instanceId, String group, String topic) {
                 if (getTopicConfigManager().getTopicConfigTable().containsKey(NamespaceUtil.wrapNamespace(instanceId, topic))) {
@@ -414,6 +456,14 @@ public boolean online(String instanceId, String group, String topic) {
         if (this.brokerConfig.isEnableSlaveActingMaster() && !this.brokerConfig.isSkipPreOnline()) {
             this.brokerPreOnlineService = new BrokerPreOnlineService(this);
         }
+
+        if (this.authConfig != null && this.authConfig.isMigrateAuthFromV1Enabled()) {
+            new AuthMigrator(this.authConfig).migrate();
+        }
+    }
+
+    public AuthConfig getAuthConfig() {
+        return authConfig;
     }
 
     public BrokerConfig getBrokerConfig() {
@@ -441,7 +491,7 @@ public BrokerMetricsManager getBrokerMetricsManager() {
     }
 
     protected void initializeRemotingServer() throws CloneNotSupportedException {
-        this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService);
+        RemotingServer tcpRemotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService);
         NettyServerConfig fastConfig = (NettyServerConfig) this.nettyServerConfig.clone();
 
         int listeningPort = nettyServerConfig.getListenPort() - 2;
@@ -450,7 +500,10 @@ protected void initializeRemotingServer() throws CloneNotSupportedException {
         }
         fastConfig.setListenPort(listeningPort);
 
-        this.fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService);
+        RemotingServer fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService);
+
+        remotingServerMap.put(TCP_REMOTING_SERVER, tcpRemotingServer);
+        remotingServerMap.put(FAST_REMOTING_SERVER, fastRemotingServer);
     }
 
     /**
@@ -490,7 +543,7 @@ protected void initializeResources() {
             1000 * 60,
             TimeUnit.MILLISECONDS,
             this.putThreadPoolQueue,
-            new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity()));
+            new ThreadFactoryImpl("PutMessageThread_", getBrokerIdentity()));
 
         this.ackMessageExecutor = ThreadUtils.newThreadPoolExecutor(
             this.brokerConfig.getAckMessageThreadPoolNums(),
@@ -634,6 +687,18 @@ public void run() {
             }
         }, 10, 1, TimeUnit.SECONDS);
 
+        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    BrokerController.this.messageStore.getTimerMessageStore().getTimerMetrics()
+                            .cleanMetrics(BrokerController.this.topicConfigManager.getTopicConfigTable().keySet());
+                } catch (Throwable e) {
+                    LOG.error("BrokerController: failed to clean unused timer metrics.", e);
+                }
+            }
+        }, 3, 3, TimeUnit.MINUTES);
+
         this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
 
             @Override
@@ -713,7 +778,7 @@ public void run() {
                         LOG.error("Failed to update nameServer address list", e);
                     }
                 }
-            }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS);
+            }, 1000 * 10, this.brokerConfig.getUpdateNameServerAddrPeriod(), TimeUnit.MILLISECONDS);
         } else if (this.brokerConfig.isFetchNamesrvAddrByAddressServer()) {
             this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
 
@@ -738,7 +803,11 @@ private void updateNamesrvAddr() {
     }
 
     public boolean initializeMetadata() {
-        boolean result = this.topicConfigManager.load();
+        boolean result = true;
+        if (null != configStorage) {
+            result = configStorage.start();
+        }
+        result = result && this.topicConfigManager.load();
         result = result && this.topicQueueMappingManager.load();
         result = result && this.consumerOffsetManager.load();
         result = result && this.subscriptionGroupManager.load();
@@ -755,6 +824,9 @@ public boolean initializeMessageStore() {
                 defaultMessageStore = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable());
             } else {
                 defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable());
+                if (messageStoreConfig.isRocksdbCQDoubleWriteEnable()) {
+                    defaultMessageStore.enableRocksdbCQWrite();
+                }
             }
 
             if (messageStoreConfig.isEnableDLegerCommitLog()) {
@@ -778,7 +850,7 @@ public boolean initializeMessageStore() {
                 this.timerMessageStore.registerEscapeBridgeHook(msg -> escapeBridge.putMessage(msg));
                 this.messageStore.setTimerMessageStore(this.timerMessageStore);
             }
-        } catch (IOException e) {
+        } catch (Exception e) {
             result = false;
             LOG.error("BrokerController#initialize: unexpected error occurs", e);
         }
@@ -841,10 +913,10 @@ public boolean recoverAndInitService() throws CloneNotSupportedException {
 
             initialTransaction();
 
-            initialAcl();
-
             initialRpcHooks();
 
+            initialRequestPipeline();
+
             if (TlsSystemConfig.tlsMode != TlsMode.DISABLED) {
                 // Register a listener to reload SslContext
                 try {
@@ -877,8 +949,12 @@ public void onChanged(String path) {
                             }
 
                             private void reloadServerSslContext() {
-                                ((NettyRemotingServer) remotingServer).loadSslContext();
-                                ((NettyRemotingServer) fastRemotingServer).loadSslContext();
+                                for (Map.Entry<String, RemotingServer> entry : remotingServerMap.entrySet()) {
+                                    RemotingServer remotingServer = entry.getValue();
+                                    if (remotingServer instanceof NettyRemotingServer) {
+                                        ((NettyRemotingServer) remotingServer).loadSslContext();
+                                    }
+                                }
                             }
                         });
                 } catch (Exception e) {
@@ -970,37 +1046,6 @@ private void initialTransaction() {
 
     }
 
-    private void initialAcl() {
-        if (!this.brokerConfig.isAclEnable()) {
-            LOG.info("The broker dose not enable acl");
-            return;
-        }
-
-        List<AccessValidator> accessValidators = ServiceProvider.load(AccessValidator.class);
-        if (accessValidators.isEmpty()) {
-            LOG.info("ServiceProvider loaded no AccessValidator, using default org.apache.rocketmq.acl.plain.PlainAccessValidator");
-            accessValidators.add(new PlainAccessValidator());
-        }
-
-        for (AccessValidator accessValidator : accessValidators) {
-            final AccessValidator validator = accessValidator;
-            accessValidatorMap.put(validator.getClass(), validator);
-            this.registerServerRPCHook(new RPCHook() {
-
-                @Override
-                public void doBeforeRequest(String remoteAddr, RemotingCommand request) {
-                    //Do not catch the exception
-                    validator.validate(validator.parse(request, remoteAddr));
-                }
-
-                @Override
-                public void doAfterResponse(String remoteAddr, RemotingCommand request, RemotingCommand response) {
-                }
-
-            });
-        }
-    }
-
     private void initialRpcHooks() {
 
         List<RPCHook> rpcHooks = ServiceProvider.load(RPCHook.class);
@@ -1012,58 +1057,80 @@ private void initialRpcHooks() {
         }
     }
 
+    private void initialRequestPipeline() {
+        if (this.authConfig == null) {
+            return;
+        }
+        RequestPipeline pipeline = (ctx, request) -> {
+        };
+        // add pipeline
+        // the last pipe add will execute at the first
+        try {
+            pipeline = pipeline.pipe(new AuthorizationPipeline(authConfig))
+                .pipe(new AuthenticationPipeline(authConfig));
+            this.setRequestPipeline(pipeline);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public void registerProcessor() {
+        RemotingServer remotingServer = remotingServerMap.get(TCP_REMOTING_SERVER);
+        RemotingServer fastRemotingServer = remotingServerMap.get(FAST_REMOTING_SERVER);
+
         /*
          * SendMessageProcessor
          */
         sendMessageProcessor.registerSendMessageHook(sendMessageHookList);
         sendMessageProcessor.registerConsumeMessageHook(consumeMessageHookList);
 
-        this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
-        this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageProcessor, this.sendMessageExecutor);
-        this.remotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
-        this.remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageProcessor, this.sendMessageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageProcessor, this.sendMessageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageProcessor, this.sendMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageProcessor, this.sendMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageProcessor, this.sendMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.RECALL_MESSAGE, recallMessageProcessor, this.sendMessageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageProcessor, this.sendMessageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageProcessor, this.sendMessageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.RECALL_MESSAGE, recallMessageProcessor, this.sendMessageExecutor);
         /**
          * PullMessageProcessor
          */
-        this.remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, this.pullMessageProcessor, this.pullMessageExecutor);
-        this.remotingServer.registerProcessor(RequestCode.LITE_PULL_MESSAGE, this.pullMessageProcessor, this.litePullMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, this.pullMessageProcessor, this.pullMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.LITE_PULL_MESSAGE, this.pullMessageProcessor, this.litePullMessageExecutor);
         this.pullMessageProcessor.registerConsumeMessageHook(consumeMessageHookList);
         /**
          * PeekMessageProcessor
          */
-        this.remotingServer.registerProcessor(RequestCode.PEEK_MESSAGE, this.peekMessageProcessor, this.pullMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.PEEK_MESSAGE, this.peekMessageProcessor, this.pullMessageExecutor);
         /**
          * PopMessageProcessor
          */
-        this.remotingServer.registerProcessor(RequestCode.POP_MESSAGE, this.popMessageProcessor, this.pullMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.POP_MESSAGE, this.popMessageProcessor, this.pullMessageExecutor);
 
         /**
          * AckMessageProcessor
          */
-        this.remotingServer.registerProcessor(RequestCode.ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);
 
-        this.remotingServer.registerProcessor(RequestCode.BATCH_ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.BATCH_ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.BATCH_ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.BATCH_ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor);
         /**
          * ChangeInvisibleTimeProcessor
          */
-        this.remotingServer.registerProcessor(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, this.changeInvisibleTimeProcessor, this.ackMessageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, this.changeInvisibleTimeProcessor, this.ackMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, this.changeInvisibleTimeProcessor, this.ackMessageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, this.changeInvisibleTimeProcessor, this.ackMessageExecutor);
         /**
          * notificationProcessor
          */
-        this.remotingServer.registerProcessor(RequestCode.NOTIFICATION, this.notificationProcessor, this.pullMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.NOTIFICATION, this.notificationProcessor, this.pullMessageExecutor);
 
         /**
          * pollingInfoProcessor
          */
-        this.remotingServer.registerProcessor(RequestCode.POLLING_INFO, this.pollingInfoProcessor, this.pullMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.POLLING_INFO, this.pollingInfoProcessor, this.pullMessageExecutor);
 
         /**
          * ReplyMessageProcessor
@@ -1071,64 +1138,69 @@ public void registerProcessor() {
 
         replyMessageProcessor.registerSendMessageHook(sendMessageHookList);
 
-        this.remotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor);
-        this.remotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor);
 
         /**
          * QueryMessageProcessor
          */
         NettyRequestProcessor queryProcessor = new QueryMessageProcessor(this);
-        this.remotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.queryMessageExecutor);
-        this.remotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.queryMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.queryMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.queryMessageExecutor);
 
-        this.fastRemotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.queryMessageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.queryMessageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.queryMessageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.queryMessageExecutor);
 
         /**
          * ClientManageProcessor
          */
-        this.remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManageProcessor, this.heartbeatExecutor);
-        this.remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManageProcessor, this.clientManageExecutor);
-        this.remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManageProcessor, this.clientManageExecutor);
+        remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManageProcessor, this.heartbeatExecutor);
+        remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManageProcessor, this.clientManageExecutor);
+        remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManageProcessor, this.clientManageExecutor);
 
-        this.fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManageProcessor, this.heartbeatExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManageProcessor, this.clientManageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManageProcessor, this.clientManageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManageProcessor, this.heartbeatExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManageProcessor, this.clientManageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManageProcessor, this.clientManageExecutor);
 
         /**
          * ConsumerManageProcessor
          */
         ConsumerManageProcessor consumerManageProcessor = new ConsumerManageProcessor(this);
-        this.remotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor);
-        this.remotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
-        this.remotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
+        remotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor);
+        remotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
+        remotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
 
-        this.fastRemotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
 
         /**
          * QueryAssignmentProcessor
          */
-        this.remotingServer.registerProcessor(RequestCode.QUERY_ASSIGNMENT, queryAssignmentProcessor, loadBalanceExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.QUERY_ASSIGNMENT, queryAssignmentProcessor, loadBalanceExecutor);
-        this.remotingServer.registerProcessor(RequestCode.SET_MESSAGE_REQUEST_MODE, queryAssignmentProcessor, loadBalanceExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.SET_MESSAGE_REQUEST_MODE, queryAssignmentProcessor, loadBalanceExecutor);
+        remotingServer.registerProcessor(RequestCode.QUERY_ASSIGNMENT, queryAssignmentProcessor, loadBalanceExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.QUERY_ASSIGNMENT, queryAssignmentProcessor, loadBalanceExecutor);
+        remotingServer.registerProcessor(RequestCode.SET_MESSAGE_REQUEST_MODE, queryAssignmentProcessor, loadBalanceExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.SET_MESSAGE_REQUEST_MODE, queryAssignmentProcessor, loadBalanceExecutor);
 
         /**
          * EndTransactionProcessor
          */
-        this.remotingServer.registerProcessor(RequestCode.END_TRANSACTION, endTransactionProcessor, this.endTransactionExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.END_TRANSACTION, endTransactionProcessor, this.endTransactionExecutor);
+        remotingServer.registerProcessor(RequestCode.END_TRANSACTION, endTransactionProcessor, this.endTransactionExecutor);
+        fastRemotingServer.registerProcessor(RequestCode.END_TRANSACTION, endTransactionProcessor, this.endTransactionExecutor);
 
         /*
          * Default
          */
         AdminBrokerProcessor adminProcessor = new AdminBrokerProcessor(this);
-        this.remotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor);
-        this.fastRemotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor);
+        remotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor);
+        fastRemotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor);
+
+        /*
+         * Initialize the mapping of request codes to request headers.
+         */
+        RequestHeaderRegistry.getInstance().initialize();
     }
 
     public BrokerStats getBrokerStats() {
@@ -1184,16 +1256,40 @@ public long headSlowTimeMills4QueryThreadPoolQueue() {
         return this.headSlowTimeMills(this.queryThreadPoolQueue);
     }
 
+    public long headSlowTimeMills4AckThreadPoolQueue() {
+        return this.headSlowTimeMills(this.ackThreadPoolQueue);
+    }
+
+    public long headSlowTimeMills4EndTransactionThreadPoolQueue() {
+        return this.headSlowTimeMills(this.endTransactionThreadPoolQueue);
+    }
+
+    public long headSlowTimeMills4ClientManagerThreadPoolQueue() {
+        return this.headSlowTimeMills(this.clientManagerThreadPoolQueue);
+    }
+
+    public long headSlowTimeMills4HeartbeatThreadPoolQueue() {
+        return this.headSlowTimeMills(this.heartbeatThreadPoolQueue);
+    }
+
+    public long headSlowTimeMills4AdminBrokerThreadPoolQueue() {
+        return this.headSlowTimeMills(this.adminBrokerThreadPoolQueue);
+    }
+
     public void printWaterMark() {
-        LOG_WATER_MARK.info("[WATERMARK] Send Queue Size: {} SlowTimeMills: {}", this.sendThreadPoolQueue.size(), headSlowTimeMills4SendThreadPoolQueue());
-        LOG_WATER_MARK.info("[WATERMARK] Pull Queue Size: {} SlowTimeMills: {}", this.pullThreadPoolQueue.size(), headSlowTimeMills4PullThreadPoolQueue());
-        LOG_WATER_MARK.info("[WATERMARK] Query Queue Size: {} SlowTimeMills: {}", this.queryThreadPoolQueue.size(), headSlowTimeMills4QueryThreadPoolQueue());
-        LOG_WATER_MARK.info("[WATERMARK] Lite Pull Queue Size: {} SlowTimeMills: {}", this.litePullThreadPoolQueue.size(), headSlowTimeMills4LitePullThreadPoolQueue());
-        LOG_WATER_MARK.info("[WATERMARK] Transaction Queue Size: {} SlowTimeMills: {}", this.endTransactionThreadPoolQueue.size(), headSlowTimeMills(this.endTransactionThreadPoolQueue));
-        LOG_WATER_MARK.info("[WATERMARK] ClientManager Queue Size: {} SlowTimeMills: {}", this.clientManagerThreadPoolQueue.size(), this.headSlowTimeMills(this.clientManagerThreadPoolQueue));
-        LOG_WATER_MARK.info("[WATERMARK] Heartbeat Queue Size: {} SlowTimeMills: {}", this.heartbeatThreadPoolQueue.size(), this.headSlowTimeMills(this.heartbeatThreadPoolQueue));
-        LOG_WATER_MARK.info("[WATERMARK] Ack Queue Size: {} SlowTimeMills: {}", this.ackThreadPoolQueue.size(), headSlowTimeMills(this.ackThreadPoolQueue));
-        LOG_WATER_MARK.info("[WATERMARK] Admin Queue Size: {} SlowTimeMills: {}", this.adminBrokerThreadPoolQueue.size(), headSlowTimeMills(this.adminBrokerThreadPoolQueue));
+        logWaterMarkQueueInfo("Send", this.sendThreadPoolQueue, this::headSlowTimeMills4SendThreadPoolQueue);
+        logWaterMarkQueueInfo("Pull", this.pullThreadPoolQueue, this::headSlowTimeMills4PullThreadPoolQueue);
+        logWaterMarkQueueInfo("Query", this.queryThreadPoolQueue, this::headSlowTimeMills4QueryThreadPoolQueue);
+        logWaterMarkQueueInfo("Lite Pull", this.litePullThreadPoolQueue, this::headSlowTimeMills4LitePullThreadPoolQueue);
+        logWaterMarkQueueInfo("Transaction", this.endTransactionThreadPoolQueue, this::headSlowTimeMills4EndTransactionThreadPoolQueue);
+        logWaterMarkQueueInfo("ClientManager", this.clientManagerThreadPoolQueue, this::headSlowTimeMills4ClientManagerThreadPoolQueue);
+        logWaterMarkQueueInfo("Heartbeat", this.heartbeatThreadPoolQueue, this::headSlowTimeMills4HeartbeatThreadPoolQueue);
+        logWaterMarkQueueInfo("Ack", this.ackThreadPoolQueue, this::headSlowTimeMills4AckThreadPoolQueue);
+        logWaterMarkQueueInfo("Admin", this.adminBrokerThreadPoolQueue, this::headSlowTimeMills4AdminBrokerThreadPoolQueue);
+    }
+
+    private void logWaterMarkQueueInfo(String queueName, BlockingQueue<?> queue, Supplier<Long> slowTimeSupplier) {
+        LOG_WATER_MARK.info("[WATERMARK] {} Queue Size: {} SlowTimeMills: {}", queueName, queue.size(), slowTimeSupplier.get());
     }
 
     public MessageStore getMessageStore() {
@@ -1231,10 +1327,19 @@ public PopInflightMessageCounter getPopInflightMessageCounter() {
         return popInflightMessageCounter;
     }
 
+    public PopConsumerService getPopConsumerService() {
+        return popConsumerService;
+    }
+
     public ConsumerOffsetManager getConsumerOffsetManager() {
         return consumerOffsetManager;
     }
 
+    public void setConsumerOffsetManager(ConsumerOffsetManager consumerOffsetManager) {
+        this.consumerOffsetManager = consumerOffsetManager;
+    }
+
+
     public BroadcastOffsetManager getBroadcastOffsetManager() {
         return broadcastOffsetManager;
     }
@@ -1247,14 +1352,6 @@ public ProducerManager getProducerManager() {
         return producerManager;
     }
 
-    public void setFastRemotingServer(RemotingServer fastRemotingServer) {
-        this.fastRemotingServer = fastRemotingServer;
-    }
-
-    public RemotingServer getFastRemotingServer() {
-        return fastRemotingServer;
-    }
-
     public PullMessageProcessor getPullMessageProcessor() {
         return pullMessageProcessor;
     }
@@ -1305,12 +1402,11 @@ protected void shutdownBasicService() {
             this.shutdownHook.beforeShutdown(this);
         }
 
-        if (this.remotingServer != null) {
-            this.remotingServer.shutdown();
-        }
-
-        if (this.fastRemotingServer != null) {
-            this.fastRemotingServer.shutdown();
+        for (Map.Entry<String, RemotingServer> entry : remotingServerMap.entrySet()) {
+            RemotingServer remotingServer = entry.getValue();
+            if (remotingServer != null) {
+                remotingServer.shutdown();
+            }
         }
 
         if (this.brokerMetricsManager != null) {
@@ -1329,13 +1425,23 @@ protected void shutdownBasicService() {
             this.pullRequestHoldService.shutdown();
         }
 
-        {
+        if (this.popConsumerService != null) {
+            this.popConsumerService.shutdown();
+        }
+
+        if (this.popMessageProcessor.getPopLongPollingService() != null) {
             this.popMessageProcessor.getPopLongPollingService().shutdown();
+        }
+
+        if (this.popMessageProcessor.getQueueLockManager() != null) {
             this.popMessageProcessor.getQueueLockManager().shutdown();
         }
 
-        {
+        if (this.popMessageProcessor.getPopBufferMergeService() != null) {
             this.popMessageProcessor.getPopBufferMergeService().shutdown();
+        }
+
+        if (this.ackMessageProcessor.getPopReviveServices() != null) {
             this.ackMessageProcessor.shutdownPopReviveService();
         }
 
@@ -1412,10 +1518,6 @@ protected void shutdownBasicService() {
             this.consumerFilterManager.persist();
         }
 
-        if (this.consumerOrderInfoManager != null) {
-            this.consumerOrderInfoManager.persist();
-        }
-
         if (this.scheduleMessageService != null) {
             this.scheduleMessageService.persist();
             this.scheduleMessageService.shutdown();
@@ -1450,7 +1552,7 @@ protected void shutdownBasicService() {
         }
 
         if (this.escapeBridge != null) {
-            escapeBridge.shutdown();
+            this.escapeBridge.shutdown();
         }
 
         if (this.topicRouteInfoManager != null) {
@@ -1487,6 +1589,23 @@ protected void shutdownBasicService() {
             this.consumerOffsetManager.stop();
         }
 
+        if (this.consumerOrderInfoManager != null) {
+            this.consumerOrderInfoManager.persist();
+            this.consumerOrderInfoManager.shutdown();
+        }
+
+        if (this.configStorage != null) {
+            this.configStorage.shutdown();
+        }
+
+        if (this.authenticationMetadataManager != null) {
+            this.authenticationMetadataManager.shutdown();
+        }
+
+        if (this.authorizationMetadataManager != null) {
+            this.authorizationMetadataManager.shutdown();
+        }
+
         for (BrokerAttachedPlugin brokerAttachedPlugin : brokerAttachedPlugins) {
             if (brokerAttachedPlugin != null) {
                 brokerAttachedPlugin.shutdown();
@@ -1550,19 +1669,20 @@ protected void startBasicService() throws Exception {
             remotingServerStartLatch.await();
         }
 
-        if (this.remotingServer != null) {
-            this.remotingServer.start();
+        for (Map.Entry<String, RemotingServer> entry : remotingServerMap.entrySet()) {
+            RemotingServer remotingServer = entry.getValue();
+            if (remotingServer != null) {
+                remotingServer.start();
 
-            // In test scenarios where it is up to OS to pick up an available port, set the listening port back to config
-            if (null != nettyServerConfig && 0 == nettyServerConfig.getListenPort()) {
-                nettyServerConfig.setListenPort(remotingServer.localListenPort());
+                if (TCP_REMOTING_SERVER.equals(entry.getKey())) {
+                    // In test scenarios where it is up to OS to pick up an available port, set the listening port back to config
+                    if (null != nettyServerConfig && 0 == nettyServerConfig.getListenPort()) {
+                        nettyServerConfig.setListenPort(remotingServer.localListenPort());
+                    }
+                }
             }
         }
 
-        if (this.fastRemotingServer != null) {
-            this.fastRemotingServer.start();
-        }
-
         this.storeHost = new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), this.getNettyServerConfig().getListenPort());
 
         for (BrokerAttachedPlugin brokerAttachedPlugin : brokerAttachedPlugins) {
@@ -1573,18 +1693,26 @@ protected void startBasicService() throws Exception {
 
         if (this.popMessageProcessor != null) {
             this.popMessageProcessor.getPopLongPollingService().start();
-            this.popMessageProcessor.getPopBufferMergeService().start();
+            if (brokerConfig.isPopConsumerFSServiceInit()) {
+                this.popMessageProcessor.getPopBufferMergeService().start();
+            }
             this.popMessageProcessor.getQueueLockManager().start();
         }
 
         if (this.ackMessageProcessor != null) {
-            this.ackMessageProcessor.startPopReviveService();
+            if (brokerConfig.isPopConsumerFSServiceInit()) {
+                this.ackMessageProcessor.startPopReviveService();
+            }
         }
 
         if (this.notificationProcessor != null) {
             this.notificationProcessor.getPopLongPollingService().start();
         }
 
+        if (this.popConsumerService != null) {
+            this.popConsumerService.start();
+        }
+
         if (this.topicQueueMappingCleanService != null) {
             this.topicQueueMappingCleanService.start();
         }
@@ -2154,6 +2282,26 @@ public TopicQueueMappingManager getTopicQueueMappingManager() {
         return topicQueueMappingManager;
     }
 
+    public AuthenticationMetadataManager getAuthenticationMetadataManager() {
+        return authenticationMetadataManager;
+    }
+
+    @VisibleForTesting
+    public void setAuthenticationMetadataManager(
+        AuthenticationMetadataManager authenticationMetadataManager) {
+        this.authenticationMetadataManager = authenticationMetadataManager;
+    }
+
+    public AuthorizationMetadataManager getAuthorizationMetadataManager() {
+        return authorizationMetadataManager;
+    }
+
+    @VisibleForTesting
+    public void setAuthorizationMetadataManager(
+        AuthorizationMetadataManager authorizationMetadataManager) {
+        this.authorizationMetadataManager = authorizationMetadataManager;
+    }
+
     public String getHAServerAddr() {
         return this.brokerConfig.getBrokerIP2() + ":" + this.messageStoreConfig.getHaListenPort();
     }
@@ -2194,6 +2342,10 @@ public BrokerStatsManager getBrokerStatsManager() {
         return brokerStatsManager;
     }
 
+    public void setBrokerStatsManager(BrokerStatsManager brokerStatsManager) {
+        this.brokerStatsManager = brokerStatsManager;
+    }
+
     public List<SendMessageHook> getSendMessageHookList() {
         return sendMessageHookList;
     }
@@ -2213,16 +2365,49 @@ public void registerConsumeMessageHook(final ConsumeMessageHook hook) {
     }
 
     public void registerServerRPCHook(RPCHook rpcHook) {
-        getRemotingServer().registerRPCHook(rpcHook);
-        this.fastRemotingServer.registerRPCHook(rpcHook);
+        for (Map.Entry<String, RemotingServer> entry : remotingServerMap.entrySet()) {
+            RemotingServer remotingServer = entry.getValue();
+            if (remotingServer != null) {
+                remotingServer.registerRPCHook(rpcHook);
+            }
+        }
+    }
+
+    public void setRequestPipeline(RequestPipeline pipeline) {
+        for (Map.Entry<String, RemotingServer> entry : remotingServerMap.entrySet()) {
+            RemotingServer remotingServer = entry.getValue();
+            if (remotingServer != null) {
+                remotingServer.setRequestPipeline(pipeline);
+            }
+        }
     }
 
     public RemotingServer getRemotingServer() {
-        return remotingServer;
+        return remotingServerMap.get(TCP_REMOTING_SERVER);
     }
 
     public void setRemotingServer(RemotingServer remotingServer) {
-        this.remotingServer = remotingServer;
+        remotingServerMap.put(TCP_REMOTING_SERVER, remotingServer);
+    }
+
+    public RemotingServer getFastRemotingServer() {
+        return remotingServerMap.get(FAST_REMOTING_SERVER);
+    }
+
+    public void setFastRemotingServer(RemotingServer fastRemotingServer) {
+        remotingServerMap.put(FAST_REMOTING_SERVER, fastRemotingServer);
+    }
+
+    public RemotingServer getRemotingServerByName(String name) {
+        return remotingServerMap.get(name);
+    }
+
+    public void setRemotingServerByName(String name, RemotingServer remotingServer) {
+        remotingServerMap.put(name, remotingServer);
+    }
+
+    public ClientHousekeepingService getClientHousekeepingService() {
+        return clientHousekeepingService;
     }
 
     public CountDownLatch getRemotingServerStartLatch() {
@@ -2288,10 +2473,6 @@ public BlockingQueue<Runnable> getEndTransactionThreadPoolQueue() {
 
     }
 
-    public Map<Class, AccessValidator> getAccessValidatorMap() {
-        return accessValidatorMap;
-    }
-
     public ExecutorService getSendMessageExecutor() {
         return sendMessageExecutor;
     }
@@ -2300,6 +2481,10 @@ public SendMessageProcessor getSendMessageProcessor() {
         return sendMessageProcessor;
     }
 
+    public RecallMessageProcessor getRecallMessageProcessor() {
+        return recallMessageProcessor;
+    }
+
     public QueryAssignmentProcessor getQueryAssignmentProcessor() {
         return queryAssignmentProcessor;
     }
@@ -2428,4 +2613,6 @@ public ColdDataCgCtrService getColdDataCgCtrService() {
     public void setColdDataCgCtrService(ColdDataCgCtrService coldDataCgCtrService) {
         this.coldDataCgCtrService = coldDataCgCtrService;
     }
+
+
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java
index 0b2f52f32eb..1c37774fe05 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java
@@ -32,43 +32,47 @@ public static void setBrokerConfigPath(String path) {
     }
 
     public static String getTopicConfigPath(final String rootDir) {
-        return rootDir + File.separator + "config" + File.separator + "topics.json";
+        return getConfigDir(rootDir) + "topics.json";
     }
 
     public static String getTopicQueueMappingPath(final String rootDir) {
-        return rootDir + File.separator + "config" + File.separator + "topicQueueMapping.json";
+        return getConfigDir(rootDir) + "topicQueueMapping.json";
     }
 
     public static String getConsumerOffsetPath(final String rootDir) {
-        return rootDir + File.separator + "config" + File.separator + "consumerOffset.json";
+        return getConfigDir(rootDir) + "consumerOffset.json";
     }
 
     public static String getLmqConsumerOffsetPath(final String rootDir) {
-        return rootDir + File.separator + "config" + File.separator + "lmqConsumerOffset.json";
+        return getConfigDir(rootDir) + "lmqConsumerOffset.json";
     }
 
     public static String getConsumerOrderInfoPath(final String rootDir) {
-        return rootDir + File.separator + "config" + File.separator + "consumerOrderInfo.json";
+        return getConfigDir(rootDir) + "consumerOrderInfo.json";
     }
 
     public static String getSubscriptionGroupPath(final String rootDir) {
-        return rootDir + File.separator + "config" + File.separator + "subscriptionGroup.json";
+        return getConfigDir(rootDir) + "subscriptionGroup.json";
     }
     public static String getTimerCheckPath(final String rootDir) {
-        return rootDir + File.separator + "config" + File.separator + "timercheck";
+        return getConfigDir(rootDir) + "timercheck";
     }
     public static String getTimerMetricsPath(final String rootDir) {
-        return rootDir + File.separator + "config" + File.separator + "timermetrics";
+        return getConfigDir(rootDir) + "timermetrics";
     }
     public static String getTransactionMetricsPath(final String rootDir) {
-        return rootDir + File.separator + "config" + File.separator + "transactionMetrics";
+        return getConfigDir(rootDir) + "transactionMetrics";
     }
 
     public static String getConsumerFilterPath(final String rootDir) {
-        return rootDir + File.separator + "config" + File.separator + "consumerFilter.json";
+        return getConfigDir(rootDir) + "consumerFilter.json";
     }
 
     public static String getMessageRequestModePath(final String rootDir) {
-        return rootDir + File.separator + "config" + File.separator + "messageRequestMode.json";
+        return getConfigDir(rootDir) + "messageRequestMode.json";
+    }
+
+    private static String getConfigDir(final String rootDir) {
+        return rootDir + File.separator + "config" + File.separator;
     }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java
index 3151683161d..90006b087e6 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java
@@ -17,6 +17,7 @@
 package org.apache.rocketmq.broker;
 
 import java.io.BufferedInputStream;
+import java.io.File;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -27,6 +28,7 @@
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.config.AuthConfig;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.MQVersion;
 import org.apache.rocketmq.common.MixAll;
@@ -86,6 +88,7 @@ public static BrokerController buildBrokerController(String[] args) throws Excep
         final NettyServerConfig nettyServerConfig = new NettyServerConfig();
         final NettyClientConfig nettyClientConfig = new NettyClientConfig();
         final MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+        final AuthConfig authConfig = new AuthConfig();
         nettyServerConfig.setListenPort(10911);
         messageStoreConfig.setHaListenPort(0);
 
@@ -112,6 +115,7 @@ public static BrokerController buildBrokerController(String[] args) throws Excep
             MixAll.properties2Object(properties, nettyServerConfig);
             MixAll.properties2Object(properties, nettyClientConfig);
             MixAll.properties2Object(properties, messageStoreConfig);
+            MixAll.properties2Object(properties, authConfig);
         }
 
         MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), brokerConfig);
@@ -204,8 +208,12 @@ public static BrokerController buildBrokerController(String[] args) throws Excep
         MixAll.printObjectProperties(log, nettyClientConfig);
         MixAll.printObjectProperties(log, messageStoreConfig);
 
+        authConfig.setConfigName(brokerConfig.getBrokerName());
+        authConfig.setClusterName(brokerConfig.getBrokerClusterName());
+        authConfig.setAuthConfigPath(messageStoreConfig.getStorePathRootDir() + File.separator + "config");
+
         final BrokerController controller = new BrokerController(
-            brokerConfig, nettyServerConfig, nettyClientConfig, messageStoreConfig);
+            brokerConfig, nettyServerConfig, nettyClientConfig, messageStoreConfig, authConfig);
 
         // Remember all configs to prevent discard
         controller.getConfiguration().registerConfig(properties);
diff --git a/common/src/main/java/org/apache/rocketmq/common/config/RocksDBConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/RocksDBConfigManager.java
similarity index 61%
rename from common/src/main/java/org/apache/rocketmq/common/config/RocksDBConfigManager.java
rename to broker/src/main/java/org/apache/rocketmq/broker/RocksDBConfigManager.java
index d1ec894685f..ee2d4e54a6a 100644
--- a/common/src/main/java/org/apache/rocketmq/common/config/RocksDBConfigManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/RocksDBConfigManager.java
@@ -14,46 +14,66 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.common.config;
+package org.apache.rocketmq.broker;
 
+import com.alibaba.fastjson.JSON;
+import java.nio.charset.StandardCharsets;
+import java.util.function.BiConsumer;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.config.ConfigRocksDBStorage;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.rocksdb.CompressionType;
 import org.rocksdb.FlushOptions;
 import org.rocksdb.RocksIterator;
 import org.rocksdb.Statistics;
 import org.rocksdb.WriteBatch;
 
-import java.util.function.BiConsumer;
-
 public class RocksDBConfigManager {
     protected static final Logger BROKER_LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
-
-    protected volatile boolean isStop = false;
-    protected ConfigRocksDBStorage configRocksDBStorage = null;
+    public volatile boolean isStop = false;
+    public ConfigRocksDBStorage configRocksDBStorage = null;
     private FlushOptions flushOptions = null;
     private volatile long lastFlushMemTableMicroSecond = 0;
+    private final String filePath;
     private final long memTableFlushInterval;
+    private final CompressionType compressionType;
+    private DataVersion kvDataVersion = new DataVersion();
 
-    public RocksDBConfigManager(long memTableFlushInterval) {
+    public RocksDBConfigManager(String filePath, long memTableFlushInterval, CompressionType compressionType) {
+        this.filePath = filePath;
         this.memTableFlushInterval = memTableFlushInterval;
+        this.compressionType = compressionType;
     }
 
-    public boolean load(String configFilePath, BiConsumer<byte[], byte[]> biConsumer) {
+    public boolean init() {
         this.isStop = false;
-        this.configRocksDBStorage = new ConfigRocksDBStorage(configFilePath);
-        if (!this.configRocksDBStorage.start()) {
-            return false;
-        }
-        RocksIterator iterator = this.configRocksDBStorage.iterator();
+        this.configRocksDBStorage = new ConfigRocksDBStorage(filePath, compressionType);
+        return this.configRocksDBStorage.start();
+    }
+    public boolean loadDataVersion() {
+        String currDataVersionString = null;
         try {
+            byte[] dataVersion = this.configRocksDBStorage.getKvDataVersion();
+            if (dataVersion != null && dataVersion.length > 0) {
+                currDataVersionString = new String(dataVersion, StandardCharsets.UTF_8);
+            }
+            kvDataVersion = StringUtils.isNotBlank(currDataVersionString) ? JSON.parseObject(currDataVersionString, DataVersion.class) : new DataVersion();
+            return true;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public boolean loadData(BiConsumer<byte[], byte[]> biConsumer) {
+        try (RocksIterator iterator = this.configRocksDBStorage.iterator()) {
             iterator.seekToFirst();
             while (iterator.isValid()) {
                 biConsumer.accept(iterator.key(), iterator.value());
                 iterator.next();
             }
-        } finally {
-            iterator.close();
         }
 
         this.flushOptions = new FlushOptions();
@@ -103,6 +123,20 @@ public void delete(final byte[] keyBytes) throws Exception {
         this.configRocksDBStorage.delete(keyBytes);
     }
 
+    public void updateKvDataVersion() throws Exception {
+        kvDataVersion.nextVersion();
+        this.configRocksDBStorage.updateKvDataVersion(JSON.toJSONString(kvDataVersion).getBytes(StandardCharsets.UTF_8));
+    }
+
+    public DataVersion getKvDataVersion() {
+        return kvDataVersion;
+    }
+
+    public void updateForbidden(String key, String value) throws Exception {
+        this.configRocksDBStorage.updateForbidden(key.getBytes(StandardCharsets.UTF_8), value.getBytes(StandardCharsets.UTF_8));
+    }
+
+
     public void batchPutWithWal(final WriteBatch batch) throws Exception {
         this.configRocksDBStorage.batchPutWithWal(batch);
     }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/auth/converter/AclConverter.java b/broker/src/main/java/org/apache/rocketmq/broker/auth/converter/AclConverter.java
new file mode 100644
index 00000000000..8cef5c6f61a
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/auth/converter/AclConverter.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.auth.converter;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authorization.enums.Decision;
+import org.apache.rocketmq.auth.authorization.enums.PolicyType;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.authorization.model.Environment;
+import org.apache.rocketmq.auth.authorization.model.Policy;
+import org.apache.rocketmq.auth.authorization.model.PolicyEntry;
+import org.apache.rocketmq.auth.authorization.model.Resource;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.remoting.protocol.body.AclInfo;
+
+public class AclConverter {
+
+    public static Acl convertAcl(AclInfo aclInfo) {
+        if (aclInfo == null) {
+            return null;
+        }
+        Subject subject = Subject.of(aclInfo.getSubject());
+        List<Policy> policies = new ArrayList<>();
+        for (AclInfo.PolicyInfo policy : aclInfo.getPolicies()) {
+            PolicyType policyType = PolicyType.getByName(policy.getPolicyType());
+
+            List<AclInfo.PolicyEntryInfo> entryInfos = policy.getEntries();
+            if (CollectionUtils.isEmpty(entryInfos)) {
+                continue;
+            }
+            List<PolicyEntry> entries = new ArrayList<>();
+            for (AclInfo.PolicyEntryInfo entryInfo : entryInfos) {
+                Resource resource = Resource.of(entryInfo.getResource());
+
+                List<Action> actions = new ArrayList<>();
+                for (String a : entryInfo.getActions()) {
+                    Action action = Action.getByName(a);
+                    if (action == null) {
+                        continue;
+                    }
+                    actions.add(action);
+                }
+
+                Environment environment = new Environment();
+                if (CollectionUtils.isNotEmpty(entryInfo.getSourceIps())) {
+                    environment.setSourceIps(entryInfo.getSourceIps());
+                }
+
+                Decision decision = Decision.getByName(entryInfo.getDecision());
+
+                entries.add(PolicyEntry.of(resource, actions, environment, decision));
+            }
+
+            policies.add(Policy.of(policyType, entries));
+        }
+
+        return Acl.of(subject, policies);
+    }
+
+    public static List<AclInfo> convertAcls(List<Acl> acls) {
+        if (CollectionUtils.isEmpty(acls)) {
+            return null;
+        }
+        return acls.stream().map(AclConverter::convertAcl)
+            .collect(Collectors.toList());
+    }
+
+    public static AclInfo convertAcl(Acl acl) {
+        if (acl == null) {
+            return null;
+        }
+        AclInfo aclInfo = new AclInfo();
+        aclInfo.setSubject(acl.getSubject().getSubjectKey());
+        if (CollectionUtils.isEmpty(acl.getPolicies())) {
+            return aclInfo;
+        }
+        List<AclInfo.PolicyInfo> policyInfos = acl.getPolicies().stream()
+            .map(AclConverter::convertPolicy)
+            .collect(Collectors.toList());
+        aclInfo.setPolicies(policyInfos);
+        return aclInfo;
+    }
+
+    private static AclInfo.PolicyInfo convertPolicy(Policy policy) {
+        AclInfo.PolicyInfo policyInfo = new AclInfo.PolicyInfo();
+        if (policy.getPolicyType() != null) {
+            policyInfo.setPolicyType(policy.getPolicyType().getName());
+        }
+        if (CollectionUtils.isEmpty(policy.getEntries())) {
+            return policyInfo;
+        }
+        List<AclInfo.PolicyEntryInfo> entryInfos = policy.getEntries().stream()
+            .map(AclConverter::convertPolicyEntry).collect(Collectors.toList());
+        policyInfo.setEntries(entryInfos);
+        return policyInfo;
+    }
+
+    private static AclInfo.PolicyEntryInfo convertPolicyEntry(PolicyEntry entry) {
+        AclInfo.PolicyEntryInfo entryInfo = new AclInfo.PolicyEntryInfo();
+        entryInfo.setResource(entry.toResourceStr());
+        entryInfo.setActions(entry.toActionsStr());
+        if (entry.getEnvironment() != null) {
+            entryInfo.setSourceIps(entry.getEnvironment().getSourceIps());
+        }
+        entryInfo.setDecision(entry.getDecision().getName());
+        return entryInfo;
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/auth/converter/UserConverter.java b/broker/src/main/java/org/apache/rocketmq/broker/auth/converter/UserConverter.java
new file mode 100644
index 00000000000..12756d8fd3c
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/auth/converter/UserConverter.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.auth.converter;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.rocketmq.auth.authentication.enums.UserStatus;
+import org.apache.rocketmq.auth.authentication.enums.UserType;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.remoting.protocol.body.UserInfo;
+
+public class UserConverter {
+
+    public static List<UserInfo> convertUsers(List<User> users) {
+        return users.stream().map(UserConverter::convertUser)
+            .collect(Collectors.toList());
+    }
+
+    public static UserInfo convertUser(User user) {
+        UserInfo result = new UserInfo();
+        result.setUsername(user.getUsername());
+        result.setPassword(user.getPassword());
+        if (user.getUserType() != null) {
+            result.setUserType(user.getUserType().getName());
+        }
+        if (user.getUserStatus() != null) {
+            result.setUserStatus(user.getUserStatus().getName());
+        }
+        return result;
+    }
+
+    public static User convertUser(UserInfo userInfo) {
+        User result = new User();
+        result.setUsername(userInfo.getUsername());
+        result.setPassword(userInfo.getPassword());
+        result.setUserType(UserType.getByName(userInfo.getUserType()));
+        result.setUserStatus(UserStatus.getByName(userInfo.getUserStatus()));
+        return result;
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/auth/pipeline/AuthenticationPipeline.java b/broker/src/main/java/org/apache/rocketmq/broker/auth/pipeline/AuthenticationPipeline.java
new file mode 100644
index 00000000000..c38e415d7aa
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/auth/pipeline/AuthenticationPipeline.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.auth.pipeline;
+
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.auth.authentication.AuthenticationEvaluator;
+import org.apache.rocketmq.auth.authentication.context.AuthenticationContext;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.AbortProcessException;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+
+public class AuthenticationPipeline implements RequestPipeline {
+    protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private final AuthConfig authConfig;
+    private final AuthenticationEvaluator evaluator;
+
+    public AuthenticationPipeline(AuthConfig authConfig) {
+        this.authConfig = authConfig;
+        this.evaluator = AuthenticationFactory.getEvaluator(authConfig);
+    }
+
+    @Override
+    public void execute(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {
+        if (!authConfig.isAuthenticationEnabled()) {
+            return;
+        }
+        try {
+            AuthenticationContext authenticationContext = newContext(ctx, request);
+            evaluator.evaluate(authenticationContext);
+        } catch (AuthenticationException ex) {
+            throw new AbortProcessException(ResponseCode.NO_PERMISSION, ex.getMessage());
+        } catch (Throwable ex) {
+            LOGGER.error("authenticate failed, request:{}", request, ex);
+            throw ex;
+        }
+    }
+
+    protected AuthenticationContext newContext(ChannelHandlerContext ctx, RemotingCommand request) {
+        return AuthenticationFactory.newContext(authConfig, ctx, request);
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/auth/pipeline/AuthorizationPipeline.java b/broker/src/main/java/org/apache/rocketmq/broker/auth/pipeline/AuthorizationPipeline.java
new file mode 100644
index 00000000000..c588dae4e83
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/auth/pipeline/AuthorizationPipeline.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.auth.pipeline;
+
+import io.netty.channel.ChannelHandlerContext;
+import java.util.List;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authorization.AuthorizationEvaluator;
+import org.apache.rocketmq.auth.authorization.context.AuthorizationContext;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.AbortProcessException;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+
+public class AuthorizationPipeline implements RequestPipeline {
+    protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private final AuthConfig authConfig;
+    private final AuthorizationEvaluator evaluator;
+
+    public AuthorizationPipeline(AuthConfig authConfig) {
+        this.authConfig = authConfig;
+        this.evaluator = AuthorizationFactory.getEvaluator(authConfig);
+    }
+
+    @Override
+    public void execute(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {
+        if (!authConfig.isAuthorizationEnabled()) {
+            return;
+        }
+        try {
+            List<AuthorizationContext> contexts = newContexts(ctx, request);
+            evaluator.evaluate(contexts);
+        } catch (AuthorizationException | AuthenticationException ex) {
+            throw new AbortProcessException(ResponseCode.NO_PERMISSION, ex.getMessage());
+        }  catch (Throwable ex) {
+            LOGGER.error("authorization failed, request:{}", request, ex);
+            throw ex;
+        }
+    }
+
+    protected List<AuthorizationContext> newContexts(ChannelHandlerContext ctx, RemotingCommand request) {
+        return AuthorizationFactory.newContexts(authConfig, ctx, request);
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientChannelAttributeHelper.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientChannelAttributeHelper.java
new file mode 100644
index 00000000000..29085398d08
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientChannelAttributeHelper.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.client;
+
+import io.netty.channel.Channel;
+import io.netty.util.AttributeKey;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class ClientChannelAttributeHelper {
+    private static final AttributeKey<String> ATTR_CG = AttributeKey.valueOf("CHANNEL_CONSUMER_GROUP");
+    private static final AttributeKey<String> ATTR_PG = AttributeKey.valueOf("CHANNEL_PRODUCER_GROUP");
+    private static final String SEPARATOR = "|";
+
+    public static void addProducerGroup(Channel channel, String group) {
+        addGroup(channel, group, ATTR_PG);
+    }
+
+    public static void addConsumerGroup(Channel channel, String group) {
+        addGroup(channel, group, ATTR_CG);
+    }
+
+    public static List<String> getProducerGroups(Channel channel) {
+        return getGroups(channel, ATTR_PG);
+    }
+
+    public static List<String> getConsumerGroups(Channel channel) {
+        return getGroups(channel, ATTR_CG);
+    }
+
+    private static void addGroup(Channel channel, String group, AttributeKey<String> key) {
+        if (null == channel || !channel.isActive()) {  // no side effect if check active status.
+            return;
+        }
+        if (null == group || group.length() == 0 || null == key) {
+            return;
+        }
+        String groups = channel.attr(key).get();
+        if (null == groups) {
+            channel.attr(key).set(group + SEPARATOR);
+        } else {
+            if (groups.contains(SEPARATOR + group + SEPARATOR)) {
+                return;
+            } else {
+                channel.attr(key).compareAndSet(groups, groups + group + SEPARATOR);
+            }
+        }
+    }
+
+    private static List<String> getGroups(Channel channel, AttributeKey<String> key) {
+        if (null == channel) {
+            return Collections.emptyList();
+        }
+        if (null == key) {
+            return Collections.emptyList();
+        }
+        String groups = channel.attr(key).get();
+        return null == groups ? Collections.<String>emptyList() : Arrays.asList(groups.split("\\|"));
+    }
+
+}
\ No newline at end of file
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
index 42e71e7e997..04238e2c300 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java
@@ -42,18 +42,22 @@ public class ConsumerManager {
     private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private final ConcurrentMap<String, ConsumerGroupInfo> consumerTable =
         new ConcurrentHashMap<>(1024);
+    private final ConcurrentMap<String, Set<String>> topicGroupTable =
+            new ConcurrentHashMap<>(1024);
     private final ConcurrentMap<String, ConsumerGroupInfo> consumerCompensationTable =
         new ConcurrentHashMap<>(1024);
     private final List<ConsumerIdsChangeListener> consumerIdsChangeListenerList = new CopyOnWriteArrayList<>();
     protected final BrokerStatsManager brokerStatsManager;
     private final long channelExpiredTimeout;
     private final long subscriptionExpiredTimeout;
+    private final BrokerConfig brokerConfig;
 
     public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener, long expiredTimeout) {
         this.consumerIdsChangeListenerList.add(consumerIdsChangeListener);
         this.brokerStatsManager = null;
         this.channelExpiredTimeout = expiredTimeout;
         this.subscriptionExpiredTimeout = expiredTimeout;
+        this.brokerConfig = null;
     }
 
     public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener,
@@ -62,6 +66,7 @@ public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener
         this.brokerStatsManager = brokerStatsManager;
         this.channelExpiredTimeout = brokerConfig.getChannelExpiredTimeout();
         this.subscriptionExpiredTimeout = brokerConfig.getSubscriptionExpiredTimeout();
+        this.brokerConfig = brokerConfig;
     }
 
     public ClientChannelInfo findChannel(final String group, final String clientId) {
@@ -130,12 +135,44 @@ public int findSubscriptionDataCount(final String group) {
 
     public boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) {
         boolean removed = false;
+        if (this.brokerConfig != null && this.brokerConfig.isEnableFastChannelEventProcess()) {
+            List<String> groups = ClientChannelAttributeHelper.getConsumerGroups(channel);
+            if (this.brokerConfig.isPrintChannelGroups() && groups.size() >= 5 && groups.size() >= this.brokerConfig.getPrintChannelGroupsMinNum()) {
+                LOGGER.warn("channel close event, too many consumer groups one channel, {}, {}, {}", groups.size(), remoteAddr, groups);
+            }
+            for (String group : groups) {
+                if (null == group || group.length() == 0) {
+                    continue;
+                }
+                ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
+                if (null == consumerGroupInfo) {
+                    continue;
+                }
+                ClientChannelInfo clientChannelInfo = consumerGroupInfo.doChannelCloseEvent(remoteAddr, channel);
+                if (clientChannelInfo != null) {
+                    removed = true;
+                    callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo, consumerGroupInfo.getSubscribeTopics());
+                    if (consumerGroupInfo.getChannelInfoTable().isEmpty()) {
+                        ConsumerGroupInfo remove = this.consumerTable.remove(group);
+                        if (remove != null) {
+                            LOGGER.info("unregister consumer ok, no any connection, and remove consumer group, {}",
+                                    group);
+                            callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, group);
+                            clearTopicGroupTable(remove);
+                        }
+                    }
+                    callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
+                }
+            }
+            return removed;
+        }
         Iterator<Entry<String, ConsumerGroupInfo>> it = this.consumerTable.entrySet().iterator();
         while (it.hasNext()) {
             Entry<String, ConsumerGroupInfo> next = it.next();
             ConsumerGroupInfo info = next.getValue();
             ClientChannelInfo clientChannelInfo = info.doChannelCloseEvent(remoteAddr, channel);
             if (clientChannelInfo != null) {
+                removed = true;
                 callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_UNREGISTER, next.getKey(), clientChannelInfo, info.getSubscribeTopics());
                 if (info.getChannelInfoTable().isEmpty()) {
                     ConsumerGroupInfo remove = this.consumerTable.remove(next.getKey());
@@ -143,15 +180,29 @@ public boolean doChannelCloseEvent(final String remoteAddr, final Channel channe
                         LOGGER.info("unregister consumer ok, no any connection, and remove consumer group, {}",
                             next.getKey());
                         callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, next.getKey());
+                        clearTopicGroupTable(remove);
                     }
                 }
-
-                callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel());
+                if (!isBroadcastMode(info.getMessageModel())) {
+                    callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel());
+                }
             }
         }
         return removed;
     }
 
+    private void clearTopicGroupTable(final ConsumerGroupInfo groupInfo) {
+        for (String subscribeTopic : groupInfo.getSubscribeTopics()) {
+            Set<String> groups = this.topicGroupTable.get(subscribeTopic);
+            if (groups != null) {
+                groups.remove(groupInfo.getGroupName());
+            }
+            if (groups != null && groups.isEmpty()) {
+                this.topicGroupTable.remove(subscribeTopic);
+            }
+        }
+    }
+
     // compensate consumer info for consumer without heartbeat
     public void compensateBasicConsumerInfo(String group, ConsumeType consumeType, MessageModel messageModel) {
         ConsumerGroupInfo consumerGroupInfo = consumerCompensationTable.computeIfAbsent(group, ConsumerGroupInfo::new);
@@ -178,26 +229,43 @@ public boolean registerConsumer(final String group, final ClientChannelInfo clie
         long start = System.currentTimeMillis();
         ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
         if (null == consumerGroupInfo) {
-            callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_REGISTER, group, clientChannelInfo,
-                subList.stream().map(SubscriptionData::getTopic).collect(Collectors.toSet()));
             ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere);
             ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp);
             consumerGroupInfo = prev != null ? prev : tmp;
         }
 
+        for (SubscriptionData subscriptionData : subList) {
+            Set<String> groups = this.topicGroupTable.get(subscriptionData.getTopic());
+            if (groups == null) {
+                Set<String> tmp = new HashSet<>();
+                Set<String> prev = this.topicGroupTable.putIfAbsent(subscriptionData.getTopic(), tmp);
+                groups = prev != null ? prev : tmp;
+            }
+            groups.add(subscriptionData.getTopic());
+        }
+
         boolean r1 =
             consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel,
                 consumeFromWhere);
+        if (r1) {
+            callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_REGISTER, group, clientChannelInfo,
+                subList.stream().map(SubscriptionData::getTopic).collect(Collectors.toSet()));
+        }
         boolean r2 = false;
         if (updateSubscription) {
             r2 = consumerGroupInfo.updateSubscription(subList);
         }
 
         if (r1 || r2) {
-            if (isNotifyConsumerIdsChangedEnable) {
+            if (isNotifyConsumerIdsChangedEnable && !isBroadcastMode(consumerGroupInfo.getMessageModel())) {
                 callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
             }
         }
+
+        if (this.brokerConfig != null && this.brokerConfig.isEnableFastChannelEventProcess() && r1) {
+            ClientChannelAttributeHelper.addConsumerGroup(clientChannelInfo.getChannel(), group);
+        }
+
         if (null != this.brokerStatsManager) {
             this.brokerStatsManager.incConsumerRegisterTime((int) (System.currentTimeMillis() - start));
         }
@@ -216,8 +284,19 @@ public boolean registerConsumerWithoutSub(final String group, final ClientChanne
             ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp);
             consumerGroupInfo = prev != null ? prev : tmp;
         }
+
+        for (SubscriptionData subscriptionData : consumerGroupInfo.getSubscriptionTable().values()) {
+            Set<String> groups = this.topicGroupTable.get(subscriptionData.getTopic());
+            if (groups == null) {
+                Set<String> tmp = new HashSet<>();
+                Set<String> prev = this.topicGroupTable.putIfAbsent(subscriptionData.getTopic(), tmp);
+                groups = prev != null ? prev : tmp;
+            }
+            groups.add(subscriptionData.getTopic());
+        }
+
         boolean updateChannelRst = consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel, consumeFromWhere);
-        if (updateChannelRst && isNotifyConsumerIdsChangedEnable) {
+        if (updateChannelRst && isNotifyConsumerIdsChangedEnable && !isBroadcastMode(consumerGroupInfo.getMessageModel())) {
             callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
         }
         if (null != this.brokerStatsManager) {
@@ -240,9 +319,10 @@ public void unregisterConsumer(final String group, final ClientChannelInfo clien
                     LOGGER.info("unregister consumer ok, no any connection, and remove consumer group, {}", group);
 
                     callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, group);
+                    clearTopicGroupTable(remove);
                 }
             }
-            if (isNotifyConsumerIdsChangedEnable) {
+            if (isNotifyConsumerIdsChangedEnable && !isBroadcastMode(consumerGroupInfo.getMessageModel())) {
                 callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
             }
         }
@@ -307,14 +387,8 @@ public void scanNotActiveChannel() {
 
     public HashSet<String> queryTopicConsumeByWho(final String topic) {
         HashSet<String> groups = new HashSet<>();
-        Iterator<Entry<String, ConsumerGroupInfo>> it = this.consumerTable.entrySet().iterator();
-        while (it.hasNext()) {
-            Entry<String, ConsumerGroupInfo> entry = it.next();
-            ConcurrentMap<String, SubscriptionData> subscriptionTable =
-                entry.getValue().getSubscriptionTable();
-            if (subscriptionTable.containsKey(topic)) {
-                groups.add(entry.getKey());
-            }
+        if (this.topicGroupTable.get(topic) != null) {
+            groups.addAll(this.topicGroupTable.get(topic));
         }
         return groups;
     }
@@ -332,4 +406,8 @@ protected void callConsumerIdsChangeListener(ConsumerGroupEvent event, String gr
             }
         }
     }
+
+    private boolean isBroadcastMode(final MessageModel messageModel) {
+        return MessageModel.BROADCASTING.equals(messageModel);
+    }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
index d17a2a5470c..e0461769568 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
@@ -23,6 +23,8 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.AbstractBrokerRunnable;
 import org.apache.rocketmq.common.constant.LoggerName;
@@ -41,6 +43,8 @@ public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListen
 
     private ConcurrentHashMap<String,List<Channel>> consumerChannelMap = new ConcurrentHashMap<>(cacheSize);
 
+    private final ConcurrentHashMap<String, NotifyTaskControl> activeGroupNotifyMap = new ConcurrentHashMap<>();
+
     public DefaultConsumerIdsChangeListener(BrokerController brokerController) {
         this.brokerController = brokerController;
 
@@ -70,9 +74,25 @@ public void handle(ConsumerGroupEvent event, String group, Object... args) {
                 List<Channel> channels = (List<Channel>) args[0];
                 if (channels != null && brokerController.getBrokerConfig().isNotifyConsumerIdsChangedEnable()) {
                     if (this.brokerController.getBrokerConfig().isRealTimeNotifyConsumerChange()) {
-                        for (Channel chl : channels) {
+                        NotifyTaskControl currentNotifyTaskControl = new NotifyTaskControl(channels);
+                        activeGroupNotifyMap.compute(group, (k, oldVal) -> {
+                            if (null != oldVal) {
+                                oldVal.interrupt();
+                            }
+                            return currentNotifyTaskControl;
+                        });
+
+                        boolean isNormalCompletion = true;
+                        for (Channel chl : currentNotifyTaskControl.getChannels()) {
+                            if (currentNotifyTaskControl.isInterrupted()) {
+                                isNormalCompletion = false;
+                                break;
+                            }
                             this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, group);
                         }
+                        if (isNormalCompletion) {
+                            activeGroupNotifyMap.computeIfPresent(group, (k, val) -> val == currentNotifyTaskControl ? null : val);
+                        }
                     } else {
                         consumerChannelMap.put(group, channels);
                     }
@@ -125,4 +145,27 @@ private void notifyConsumerChange() {
     public void shutdown() {
         this.scheduledExecutorService.shutdown();
     }
+
+    private static class NotifyTaskControl {
+
+        private final AtomicBoolean interrupted = new AtomicBoolean(false);
+
+        private final List<Channel> channels;
+
+        public NotifyTaskControl(List<Channel> channels) {
+            this.channels = channels;
+        }
+
+        public boolean isInterrupted() {
+            return interrupted.get();
+        }
+
+        public void interrupt() {
+            interrupted.set(true);
+        }
+
+        public List<Channel> getChannels() {
+            return channels;
+        }
+    }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java
index f9fe1193e22..bc8400c19a2 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java
@@ -25,8 +25,10 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import org.apache.rocketmq.broker.util.PositiveAtomicCounter;
+import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
@@ -39,19 +41,27 @@ public class ProducerManager {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120;
     private static final int GET_AVAILABLE_CHANNEL_RETRY_COUNT = 3;
-    private final ConcurrentHashMap<String /* group name */, ConcurrentHashMap<Channel, ClientChannelInfo>> groupChannelTable =
+    private final ConcurrentMap<String /* group name */, ConcurrentMap<Channel, ClientChannelInfo>> groupChannelTable =
         new ConcurrentHashMap<>();
-    private final ConcurrentHashMap<String, Channel> clientChannelTable = new ConcurrentHashMap<>();
+    private final ConcurrentMap<String, Channel> clientChannelTable = new ConcurrentHashMap<>();
     protected final BrokerStatsManager brokerStatsManager;
-    private PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter();
+    private final BrokerConfig brokerConfig;
+    private final PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter();
     private final List<ProducerChangeListener> producerChangeListenerList = new CopyOnWriteArrayList<>();
 
     public ProducerManager() {
         this.brokerStatsManager = null;
+        this.brokerConfig = null;
     }
 
     public ProducerManager(final BrokerStatsManager brokerStatsManager) {
         this.brokerStatsManager = brokerStatsManager;
+        this.brokerConfig = null;
+    }
+
+    public ProducerManager(final BrokerStatsManager brokerStatsManager, final BrokerConfig brokerConfig) {
+        this.brokerStatsManager = brokerStatsManager;
+        this.brokerConfig = brokerConfig;
     }
 
     public int groupSize() {
@@ -63,22 +73,22 @@ public boolean groupOnline(String group) {
         return channels != null && !channels.isEmpty();
     }
 
-    public ConcurrentHashMap<String, ConcurrentHashMap<Channel, ClientChannelInfo>> getGroupChannelTable() {
+    public ConcurrentMap<String, ConcurrentMap<Channel, ClientChannelInfo>> getGroupChannelTable() {
         return groupChannelTable;
     }
 
     public ProducerTableInfo getProducerTable() {
         Map<String, List<ProducerInfo>> map = new HashMap<>();
         for (String group : this.groupChannelTable.keySet()) {
-            for (Entry<Channel, ClientChannelInfo> entry: this.groupChannelTable.get(group).entrySet()) {
+            for (Entry<Channel, ClientChannelInfo> entry : this.groupChannelTable.get(group).entrySet()) {
                 ClientChannelInfo clientChannelInfo = entry.getValue();
                 if (map.containsKey(group)) {
                     map.get(group).add(new ProducerInfo(
-                            clientChannelInfo.getClientId(),
-                            clientChannelInfo.getChannel().remoteAddress().toString(),
-                            clientChannelInfo.getLanguage(),
-                            clientChannelInfo.getVersion(),
-                            clientChannelInfo.getLastUpdateTimestamp()
+                        clientChannelInfo.getClientId(),
+                        clientChannelInfo.getChannel().remoteAddress().toString(),
+                        clientChannelInfo.getLanguage(),
+                        clientChannelInfo.getVersion(),
+                        clientChannelInfo.getLastUpdateTimestamp()
                     ));
                 } else {
                     map.put(group, new ArrayList<>(Collections.singleton(new ProducerInfo(
@@ -95,13 +105,13 @@ public ProducerTableInfo getProducerTable() {
     }
 
     public void scanNotActiveChannel() {
-        Iterator<Map.Entry<String, ConcurrentHashMap<Channel, ClientChannelInfo>>> iterator = this.groupChannelTable.entrySet().iterator();
+        Iterator<Map.Entry<String, ConcurrentMap<Channel, ClientChannelInfo>>> iterator = this.groupChannelTable.entrySet().iterator();
 
         while (iterator.hasNext()) {
-            Map.Entry<String, ConcurrentHashMap<Channel, ClientChannelInfo>> entry = iterator.next();
+            Map.Entry<String, ConcurrentMap<Channel, ClientChannelInfo>> entry = iterator.next();
 
             final String group = entry.getKey();
-            final ConcurrentHashMap<Channel, ClientChannelInfo> chlMap = entry.getValue();
+            final ConcurrentMap<Channel, ClientChannelInfo> chlMap = entry.getValue();
 
             Iterator<Entry<Channel, ClientChannelInfo>> it = chlMap.entrySet().iterator();
             while (it.hasNext()) {
@@ -117,8 +127,8 @@ public void scanNotActiveChannel() {
                         clientChannelTable.remove(info.getClientId());
                     }
                     log.warn(
-                            "ProducerManager#scanNotActiveChannel: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}",
-                            RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group);
+                        "ProducerManager#scanNotActiveChannel: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}",
+                        RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group);
                     callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, info);
                     RemotingHelper.closeChannel(info.getChannel());
                 }
@@ -132,25 +142,55 @@ public void scanNotActiveChannel() {
         }
     }
 
-    public synchronized boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) {
+    public boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) {
         boolean removed = false;
         if (channel != null) {
-            for (final Map.Entry<String, ConcurrentHashMap<Channel, ClientChannelInfo>> entry : this.groupChannelTable
-                    .entrySet()) {
+            if (this.brokerConfig != null && this.brokerConfig.isEnableFastChannelEventProcess()) {
+                List<String> groups = ClientChannelAttributeHelper.getProducerGroups(channel);
+                if (this.brokerConfig.isPrintChannelGroups() && groups.size() >= 5 && groups.size() >= this.brokerConfig.getPrintChannelGroupsMinNum()) {
+                    log.warn("channel close event, too many producer groups one channel, {}, {}, {}", groups.size(), remoteAddr, groups);
+                }
+                for (String group : groups) {
+                    if (null == group || group.length() == 0) {
+                        continue;
+                    }
+                    ConcurrentMap<Channel, ClientChannelInfo> clientChannelInfoTable = this.groupChannelTable.get(group);
+                    if (null == clientChannelInfoTable) {
+                        continue;
+                    }
+                    final ClientChannelInfo clientChannelInfo =
+                            clientChannelInfoTable.remove(channel);
+                    if (clientChannelInfo != null) {
+                        clientChannelTable.remove(clientChannelInfo.getClientId());
+                        removed = true;
+                        log.info(
+                                "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}",
+                                clientChannelInfo.toString(), remoteAddr, group);
+                        callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo);
+                        if (clientChannelInfoTable.isEmpty()) {
+                            ConcurrentMap<Channel, ClientChannelInfo> oldGroupTable = this.groupChannelTable.remove(group);
+                            if (oldGroupTable != null) {
+                                log.info("unregister a producer group[{}] from groupChannelTable", group);
+                                callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null);
+                            }
+                        }
+                    }
+                }
+                return removed;  // must return here, degrade to scanNotActiveChannel at worst.
+            }
+            for (final Map.Entry<String, ConcurrentMap<Channel, ClientChannelInfo>> entry : this.groupChannelTable.entrySet()) {
                 final String group = entry.getKey();
-                final ConcurrentHashMap<Channel, ClientChannelInfo> clientChannelInfoTable =
-                        entry.getValue();
-                final ClientChannelInfo clientChannelInfo =
-                        clientChannelInfoTable.remove(channel);
+                final ConcurrentMap<Channel, ClientChannelInfo> clientChannelInfoTable = entry.getValue();
+                final ClientChannelInfo clientChannelInfo = clientChannelInfoTable.remove(channel);
                 if (clientChannelInfo != null) {
                     clientChannelTable.remove(clientChannelInfo.getClientId());
                     removed = true;
                     log.info(
-                            "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}",
-                            clientChannelInfo.toString(), remoteAddr, group);
+                        "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}",
+                        clientChannelInfo.toString(), remoteAddr, group);
                     callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo);
                     if (clientChannelInfoTable.isEmpty()) {
-                        ConcurrentHashMap<Channel, ClientChannelInfo> oldGroupTable = this.groupChannelTable.remove(group);
+                        ConcurrentMap<Channel, ClientChannelInfo> oldGroupTable = this.groupChannelTable.remove(group);
                         if (oldGroupTable != null) {
                             log.info("unregister a producer group[{}] from groupChannelTable", group);
                             callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null);
@@ -163,37 +203,68 @@ public synchronized boolean doChannelCloseEvent(final String remoteAddr, final C
         return removed;
     }
 
-    public synchronized void registerProducer(final String group, final ClientChannelInfo clientChannelInfo) {
-        ClientChannelInfo clientChannelInfoFound = null;
+    public void registerProducer(final String group, final ClientChannelInfo clientChannelInfo) {
+
+        long start = System.currentTimeMillis();
+        ClientChannelInfo clientChannelInfoFound;
+
+        ConcurrentMap<Channel, ClientChannelInfo> channelTable = this.groupChannelTable.get(group);
+        // note that we must take care of the exist groups and channels,
+        // only can return when groups or channels not exist.
+        if (this.brokerConfig != null
+                && !this.brokerConfig.isEnableRegisterProducer()
+                && this.brokerConfig.isRejectTransactionMessage()) {
+            boolean needRegister = true;
+            if (null == channelTable) {
+                needRegister = false;
+            } else {
+                clientChannelInfoFound = channelTable.get(clientChannelInfo.getChannel());
+                if (null == clientChannelInfoFound) {
+                    needRegister = false;
+                }
+            }
+            if (!needRegister) {
+                if (null != this.brokerStatsManager) {
+                    this.brokerStatsManager.incProducerRegisterTime((int) (System.currentTimeMillis() - start));
+                }
+                return;
+            }
+        }
 
-        ConcurrentHashMap<Channel, ClientChannelInfo> channelTable = this.groupChannelTable.get(group);
         if (null == channelTable) {
             channelTable = new ConcurrentHashMap<>();
-            this.groupChannelTable.put(group, channelTable);
+            ConcurrentMap<Channel, ClientChannelInfo> prev = this.groupChannelTable.putIfAbsent(group, channelTable);
+            channelTable = prev != null ? prev : channelTable;
         }
 
         clientChannelInfoFound = channelTable.get(clientChannelInfo.getChannel());
+        // Add client-channel info to existing producer group
         if (null == clientChannelInfoFound) {
             channelTable.put(clientChannelInfo.getChannel(), clientChannelInfo);
             clientChannelTable.put(clientChannelInfo.getClientId(), clientChannelInfo.getChannel());
-            log.info("new producer connected, group: {} channel: {}", group,
-                    clientChannelInfo.toString());
+            log.info("new producer connected, group: {} channel: {}", group, clientChannelInfo.toString());
+            if (this.brokerConfig != null && this.brokerConfig.isEnableFastChannelEventProcess()) {
+                ClientChannelAttributeHelper.addProducerGroup(clientChannelInfo.getChannel(), group);
+            }
         }
 
-
+        // Refresh existing client-channel-info update-timestamp
         if (clientChannelInfoFound != null) {
             clientChannelInfoFound.setLastUpdateTimestamp(System.currentTimeMillis());
         }
+
+        if (null != this.brokerStatsManager) {
+            this.brokerStatsManager.incProducerRegisterTime((int) (System.currentTimeMillis() - start));
+        }
     }
 
-    public synchronized void unregisterProducer(final String group, final ClientChannelInfo clientChannelInfo) {
-        ConcurrentHashMap<Channel, ClientChannelInfo> channelTable = this.groupChannelTable.get(group);
+    public void unregisterProducer(final String group, final ClientChannelInfo clientChannelInfo) {
+        ConcurrentMap<Channel, ClientChannelInfo> channelTable = this.groupChannelTable.get(group);
         if (null != channelTable && !channelTable.isEmpty()) {
             ClientChannelInfo old = channelTable.remove(clientChannelInfo.getChannel());
             clientChannelTable.remove(clientChannelInfo.getClientId());
             if (old != null) {
-                log.info("unregister a producer[{}] from groupChannelTable {}", group,
-                        clientChannelInfo.toString());
+                log.info("unregister a producer[{}] from groupChannelTable {}", group, clientChannelInfo.toString());
                 callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo);
             }
 
@@ -210,7 +281,7 @@ public Channel getAvailableChannel(String groupId) {
             return null;
         }
         List<Channel> channelList;
-        ConcurrentHashMap<Channel, ClientChannelInfo> channelClientChannelInfoHashMap = groupChannelTable.get(groupId);
+        ConcurrentMap<Channel, ClientChannelInfo> channelClientChannelInfoHashMap = groupChannelTable.get(groupId);
         if (channelClientChannelInfoHashMap != null) {
             channelList = new ArrayList<>(channelClientChannelInfoHashMap.keySet());
         } else {
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java
index 8d95d843dba..f8984963f94 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java
@@ -37,6 +37,7 @@
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
 import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
@@ -49,6 +50,7 @@
 import org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.NotifyConsumerIdsChangedRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 
 public class Broker2Client {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -100,19 +102,18 @@ public void notifyConsumerIdsChanged(
         }
     }
 
-
-    public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce) {
+    public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce) throws RemotingCommandException {
         return resetOffset(topic, group, timeStamp, isForce, false);
     }
 
     public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce,
-        boolean isC) {
+        boolean isC) throws RemotingCommandException {
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
 
         TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
         if (null == topicConfig) {
             log.error("[reset-offset] reset offset failed, no topic in this broker. topic={}", topic);
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.TOPIC_NOT_EXIST);
             response.setRemark("[reset-offset] reset offset failed, no topic in this broker. topic=" + topic);
             return response;
         }
@@ -135,8 +136,11 @@ public RemotingCommand resetOffset(String topic, String group, long timeStamp, b
 
             long timeStampOffset;
             if (timeStamp == -1) {
-
-                timeStampOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+                try {
+                    timeStampOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+                } catch (ConsumeQueueException e) {
+                    throw new RemotingCommandException("Failed to get max offset in queue", e);
+                }
             } else {
                 timeStampOffset = this.brokerController.getMessageStore().getOffsetInQueueByTime(topic, i, timeStamp);
             }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java
index bf7d0964bef..e00be4fcda8 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java
@@ -98,8 +98,6 @@ public boolean tryLock(final String group, final MessageQueue mq, final String c
                 log.error("RebalanceLockManager#tryLock: unexpected error, group={}, mq={}, clientId={}", group, mq,
                     clientId, e);
             }
-        } else {
-
         }
 
         return true;
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/coldctr/ColdDataCgCtrService.java b/broker/src/main/java/org/apache/rocketmq/broker/coldctr/ColdDataCgCtrService.java
index dd9278fb755..5b8b2fb9cec 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/coldctr/ColdDataCgCtrService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/coldctr/ColdDataCgCtrService.java
@@ -24,8 +24,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 
-import com.alibaba.fastjson.JSONObject;
-
+import com.alibaba.fastjson2.JSONObject;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.MixAll;
@@ -188,7 +187,7 @@ public boolean isCgNeedColdDataFlowCtr(String consumerGroup) {
         if (!this.messageStoreConfig.isColdDataFlowControlEnable()) {
             return false;
         }
-        if (MixAll.isSysConsumerGroupForNoColdReadLimit(consumerGroup)) {
+        if (MixAll.isSysConsumerGroupPullMessage(consumerGroup)) {
             return false;
         }
         AccAndTimeStamp accAndTimeStamp = cgColdThresholdMapRuntime.get(consumerGroup);
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBConsumerOffsetManager.java
similarity index 52%
rename from broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java
rename to broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBConsumerOffsetManager.java
index 05b53b0bcf2..963c5046f24 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBConsumerOffsetManager.java
@@ -14,33 +14,81 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.broker.offset;
+package org.apache.rocketmq.broker.config.v1;
 
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
 import java.io.File;
 import java.util.Iterator;
 import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentMap;
-
 import org.apache.rocketmq.broker.BrokerController;
-import org.apache.rocketmq.common.config.RocksDBConfigManager;
+import org.apache.rocketmq.broker.RocksDBConfigManager;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.utils.DataConverter;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.rocksdb.CompressionType;
 import org.rocksdb.WriteBatch;
 
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
-
 public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager {
 
+    protected static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+
+    protected transient RocksDBConfigManager rocksDBConfigManager;
+
     public RocksDBConsumerOffsetManager(BrokerController brokerController) {
         super(brokerController);
-        this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs());
+        this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs(),
+            CompressionType.getCompressionType(brokerController.getMessageStoreConfig().getRocksdbCompressionType()));
+
     }
 
     @Override
     public boolean load() {
-        return this.rocksDBConfigManager.load(configFilePath(), this::decode0);
+        if (!rocksDBConfigManager.init()) {
+            return false;
+        }
+        if (!loadDataVersion() || !loadConsumerOffset()) {
+            return false;
+        }
+
+        return true;
     }
 
+    public boolean loadConsumerOffset() {
+        return this.rocksDBConfigManager.loadData(this::decodeOffset) && merge();
+    }
+
+    private boolean merge() {
+        if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) {
+            log.info("consumerOffset json file does not exist, so skip merge");
+            return true;
+        }
+        if (!super.loadDataVersion()) {
+            log.error("load json consumerOffset dataVersion error, startup will exit");
+            return false;
+        }
+
+        final DataVersion dataVersion = super.getDataVersion();
+        final DataVersion kvDataVersion = this.getDataVersion();
+        if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) {
+            if (!super.load()) {
+                log.error("load json consumerOffset info failed, startup will exit");
+                return false;
+            }
+            this.persist();
+            this.getDataVersion().assignNewOne(dataVersion);
+            updateDataVersion();
+            log.info("update offset from json, dataVersion:{}, offsetTable: {} ", this.getDataVersion(), JSON.toJSONString(this.getOffsetTable()));
+        }
+        return true;
+    }
+
+
     @Override
     public boolean stop() {
         return this.rocksDBConfigManager.stop();
@@ -52,21 +100,19 @@ protected void removeConsumerOffset(String topicAtGroup) {
             byte[] keyBytes = topicAtGroup.getBytes(DataConverter.CHARSET_UTF8);
             this.rocksDBConfigManager.delete(keyBytes);
         } catch (Exception e) {
-            LOG.error("kv remove consumerOffset Failed, {}", topicAtGroup);
+            log.error("kv remove consumerOffset Failed, {}", topicAtGroup);
         }
     }
 
-    @Override
-    protected void decode0(final byte[] key, final byte[] body) {
+    protected void decodeOffset(final byte[] key, final byte[] body) {
         String topicAtGroup = new String(key, DataConverter.CHARSET_UTF8);
         RocksDBOffsetSerializeWrapper wrapper = JSON.parseObject(body, RocksDBOffsetSerializeWrapper.class);
 
         this.offsetTable.put(topicAtGroup, wrapper.getOffsetTable());
-        LOG.info("load exist local offset, {}, {}", topicAtGroup, wrapper.getOffsetTable());
+        log.info("load exist local offset, {}, {}", topicAtGroup, wrapper.getOffsetTable());
     }
 
-    @Override
-    public String configFilePath() {
+    public String rocksdbConfigFilePath() {
         return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "consumerOffsets" + File.separator;
     }
 
@@ -86,12 +132,17 @@ public synchronized void persist() {
             this.rocksDBConfigManager.batchPutWithWal(writeBatch);
             this.rocksDBConfigManager.flushWAL();
         } catch (Exception e) {
-            LOG.error("consumer offset persist Failed", e);
+            log.error("consumer offset persist Failed", e);
         } finally {
             writeBatch.close();
         }
     }
 
+    public synchronized void exportToJson() {
+        log.info("RocksDBConsumerOffsetManager export consumer offset to json file");
+        super.persist();
+    }
+
     private void putWriteBatch(final WriteBatch writeBatch, final String topicGroupName, final ConcurrentMap<Integer, Long> offsetMap) throws Exception {
         byte[] keyBytes = topicGroupName.getBytes(DataConverter.CHARSET_UTF8);
         RocksDBOffsetSerializeWrapper wrapper = new RocksDBOffsetSerializeWrapper();
@@ -99,4 +150,23 @@ private void putWriteBatch(final WriteBatch writeBatch, final String topicGroupN
         byte[] valueBytes = JSON.toJSONBytes(wrapper, SerializerFeature.BrowserCompatible);
         writeBatch.put(keyBytes, valueBytes);
     }
+
+    @Override
+    public boolean loadDataVersion() {
+        return this.rocksDBConfigManager.loadDataVersion();
+    }
+
+    @Override
+    public DataVersion getDataVersion() {
+        return rocksDBConfigManager.getKvDataVersion();
+    }
+
+    public void updateDataVersion() {
+        try {
+            rocksDBConfigManager.updateKvDataVersion();
+        } catch (Exception e) {
+            log.error("update consumer offset dataVersion error", e);
+            throw new RuntimeException(e);
+        }
+    }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqSubscriptionGroupManager.java
similarity index 87%
rename from broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java
rename to broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqSubscriptionGroupManager.java
index 8c05d0bd98f..05f3f7d2ec3 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqSubscriptionGroupManager.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.broker.subscription;
+package org.apache.rocketmq.broker.config.v1;
 
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.MixAll;
@@ -43,4 +43,13 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config)
         }
         super.updateSubscriptionGroupConfig(config);
     }
+
+    @Override
+    public boolean containsSubscriptionGroup(String group) {
+        if (MixAll.isLmq(group)) {
+            return true;
+        } else {
+            return super.containsSubscriptionGroup(group);
+        }
+    }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBLmqTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqTopicConfigManager.java
similarity index 97%
rename from broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBLmqTopicConfigManager.java
rename to broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqTopicConfigManager.java
index d049a8dbcde..7b278013965 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBLmqTopicConfigManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqTopicConfigManager.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.broker.topic;
+package org.apache.rocketmq.broker.config.v1;
 
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.MixAll;
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapper.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBOffsetSerializeWrapper.java
similarity index 96%
rename from broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapper.java
rename to broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBOffsetSerializeWrapper.java
index 7a90fd62fb8..4801cfc681c 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapper.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBOffsetSerializeWrapper.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.broker.offset;
+package org.apache.rocketmq.broker.config.v1;
 
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManager.java
new file mode 100644
index 00000000000..b208169e416
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManager.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v1;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import java.io.File;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.function.BiConsumer;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.RocksDBConfigManager;
+import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.utils.DataConverter;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.rocksdb.CompressionType;
+import org.rocksdb.RocksIterator;
+
+public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager {
+
+    protected transient RocksDBConfigManager rocksDBConfigManager;
+
+    public RocksDBSubscriptionGroupManager(BrokerController brokerController) {
+        super(brokerController, false);
+        this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs(),
+            CompressionType.getCompressionType(brokerController.getMessageStoreConfig().getRocksdbCompressionType()));
+    }
+
+    @Override
+    public boolean load() {
+        if (!rocksDBConfigManager.init()) {
+            return false;
+        }
+        if (!loadDataVersion() || !loadSubscriptionGroupAndForbidden()) {
+            return false;
+        }
+        this.init();
+        return true;
+    }
+
+    public boolean loadDataVersion() {
+        return this.rocksDBConfigManager.loadDataVersion();
+    }
+
+    public boolean loadSubscriptionGroupAndForbidden() {
+        return this.rocksDBConfigManager.loadData(this::decodeSubscriptionGroup)
+                && this.loadForbidden(this::decodeForbidden)
+                && merge();
+    }
+
+    public boolean loadForbidden(BiConsumer<byte[], byte[]> biConsumer) {
+        try (RocksIterator iterator = this.rocksDBConfigManager.configRocksDBStorage.forbiddenIterator()) {
+            iterator.seekToFirst();
+            while (iterator.isValid()) {
+                biConsumer.accept(iterator.key(), iterator.value());
+                iterator.next();
+            }
+        }
+        return true;
+    }
+
+    private boolean merge() {
+        if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) {
+            log.info("subGroup json file does not exist, so skip merge");
+            return true;
+        }
+        if (!super.loadDataVersion()) {
+            log.error("load json subGroup dataVersion error, startup will exit");
+            return false;
+        }
+        final DataVersion dataVersion = super.getDataVersion();
+        final DataVersion kvDataVersion = this.getDataVersion();
+        if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) {
+            if (!super.load()) {
+                log.error("load group and forbidden info from json file error, startup will exit");
+                return false;
+            }
+            final ConcurrentMap<String, SubscriptionGroupConfig> groupTable = this.getSubscriptionGroupTable();
+            for (Map.Entry<String, SubscriptionGroupConfig> entry : groupTable.entrySet()) {
+                putSubscriptionGroupConfig(entry.getValue());
+                log.info("import subscription config to rocksdb, group={}", entry.getValue());
+            }
+            final ConcurrentMap<String, ConcurrentMap<String, Integer>> forbiddenTable = this.getForbiddenTable();
+            for (Map.Entry<String, ConcurrentMap<String, Integer>> entry : forbiddenTable.entrySet()) {
+                try {
+                    this.rocksDBConfigManager.updateForbidden(entry.getKey(), JSON.toJSONString(entry.getValue()));
+                    log.info("import forbidden config to rocksdb, group={}", entry.getValue());
+                } catch (Exception e) {
+                    log.error("import forbidden config to rocksdb failed, group={}", entry.getValue());
+                    return false;
+                }
+            }
+            this.getDataVersion().assignNewOne(dataVersion);
+            updateDataVersion();
+        } else {
+            log.info("dataVersion is not greater than kvDataVersion, no need to merge group metaData, dataVersion={}, kvDataVersion={}", dataVersion, kvDataVersion);
+        }
+        log.info("finish marge subscription config from json file and merge to rocksdb");
+        this.persist();
+
+        return true;
+    }
+
+    @Override
+    public boolean stop() {
+        return this.rocksDBConfigManager.stop();
+    }
+
+    @Override
+    public SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) {
+        String groupName = subscriptionGroupConfig.getGroupName();
+        SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig);
+
+        try {
+            byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8);
+            byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible);
+            this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes);
+        } catch (Exception e) {
+            log.error("kv put sub Failed, {}", subscriptionGroupConfig.toString());
+        }
+        return oldConfig;
+    }
+
+    @Override
+    protected SubscriptionGroupConfig putSubscriptionGroupConfigIfAbsent(SubscriptionGroupConfig subscriptionGroupConfig) {
+        String groupName = subscriptionGroupConfig.getGroupName();
+        SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.putIfAbsent(groupName, subscriptionGroupConfig);
+        if (oldConfig == null) {
+            try {
+                byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8);
+                byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible);
+                this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes);
+            } catch (Exception e) {
+                log.error("kv put sub Failed, {}", subscriptionGroupConfig.toString());
+            }
+        }
+        return oldConfig;
+    }
+
+    @Override
+    protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) {
+        SubscriptionGroupConfig subscriptionGroupConfig = this.subscriptionGroupTable.remove(groupName);
+        try {
+            this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.CHARSET_UTF8));
+        } catch (Exception e) {
+            log.error("kv delete sub Failed, {}", subscriptionGroupConfig.toString());
+        }
+        return subscriptionGroupConfig;
+    }
+
+
+    protected void decodeSubscriptionGroup(byte[] key, byte[] body) {
+        String groupName = new String(key, DataConverter.CHARSET_UTF8);
+        SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(body, SubscriptionGroupConfig.class);
+
+        this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig);
+        log.info("load exist local sub, {}", subscriptionGroupConfig.toString());
+    }
+
+    @Override
+    public synchronized void persist() {
+        if (brokerController.getMessageStoreConfig().isRealTimePersistRocksDBConfig()) {
+            this.rocksDBConfigManager.flushWAL();
+        }
+    }
+
+    public synchronized void exportToJson() {
+        log.info("RocksDBSubscriptionGroupManager export subscription group to json file");
+        super.persist();
+    }
+
+    public String rocksdbConfigFilePath() {
+        return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "subscriptionGroups" + File.separator;
+    }
+
+    @Override
+    public DataVersion getDataVersion() {
+        return rocksDBConfigManager.getKvDataVersion();
+    }
+
+    @Override
+    public void updateDataVersion() {
+        try {
+            rocksDBConfigManager.updateKvDataVersion();
+        } catch (Exception e) {
+            log.error("update group config dataVersion error", e);
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected void decodeForbidden(byte[] key, byte[] body) {
+        String forbiddenGroupName = new String(key, DataConverter.CHARSET_UTF8);
+        JSONObject jsonObject = JSON.parseObject(new String(body, DataConverter.CHARSET_UTF8));
+        Set<Map.Entry<String, Object>> entries = jsonObject.entrySet();
+        ConcurrentMap<String, Integer> forbiddenGroup = new ConcurrentHashMap<>(entries.size());
+        for (Map.Entry<String, Object> entry : entries) {
+            forbiddenGroup.put(entry.getKey(), (Integer) entry.getValue());
+        }
+        this.getForbiddenTable().put(forbiddenGroupName, forbiddenGroup);
+        log.info("load forbidden,{} value {}", forbiddenGroupName, forbiddenGroup.toString());
+    }
+
+    @Override
+    public void updateForbidden(String group, String topic, int forbiddenIndex, boolean setOrClear) {
+        try {
+            super.updateForbidden(group, topic, forbiddenIndex, setOrClear);
+            this.rocksDBConfigManager.updateForbidden(group, JSON.toJSONString(this.getForbiddenTable().get(group)));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void setForbidden(String group, String topic, int forbiddenIndex) {
+        try {
+            super.setForbidden(group, topic, forbiddenIndex);
+            this.rocksDBConfigManager.updateForbidden(group, JSON.toJSONString(this.getForbiddenTable().get(group)));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void clearForbidden(String group, String topic, int forbiddenIndex) {
+        try {
+            super.clearForbidden(group, topic, forbiddenIndex);
+            this.rocksDBConfigManager.updateForbidden(group, JSON.toJSONString(this.getForbiddenTable().get(group)));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManager.java
new file mode 100644
index 00000000000..d64f808067c
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManager.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v1;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import java.io.File;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.RocksDBConfigManager;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.utils.DataConverter;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.rocksdb.CompressionType;
+
+public class RocksDBTopicConfigManager extends TopicConfigManager {
+
+    protected transient RocksDBConfigManager rocksDBConfigManager;
+
+    public RocksDBTopicConfigManager(BrokerController brokerController) {
+        super(brokerController, false);
+        this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs(),
+            CompressionType.getCompressionType(brokerController.getMessageStoreConfig().getRocksdbCompressionType()));
+    }
+
+    @Override
+    public boolean load() {
+        if (!rocksDBConfigManager.init()) {
+            return false;
+        }
+        if (!loadDataVersion() || !loadTopicConfig()) {
+            return false;
+        }
+        this.init();
+        return true;
+    }
+
+    public boolean loadTopicConfig() {
+        return this.rocksDBConfigManager.loadData(this::decodeTopicConfig) && merge();
+    }
+
+    public boolean loadDataVersion() {
+        return this.rocksDBConfigManager.loadDataVersion();
+    }
+
+    private boolean merge() {
+        if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) {
+            log.info("topic json file does not exist, so skip merge");
+            return true;
+        }
+
+        if (!super.loadDataVersion()) {
+            log.error("load json topic dataVersion error, startup will exit");
+            return false;
+        }
+
+        final DataVersion dataVersion = super.getDataVersion();
+        final DataVersion kvDataVersion = this.getDataVersion();
+        if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) {
+            if (!super.load()) {
+                log.error("load topic config from json file error, startup will exit");
+                return false;
+            }
+            final ConcurrentMap<String, TopicConfig> topicConfigTable = this.getTopicConfigTable();
+            for (Map.Entry<String, TopicConfig> entry : topicConfigTable.entrySet()) {
+                putTopicConfig(entry.getValue());
+                log.info("import topic config to rocksdb, topic={}", entry.getValue());
+            }
+            this.getDataVersion().assignNewOne(dataVersion);
+            updateDataVersion();
+        } else {
+            log.info("dataVersion is not greater than kvDataVersion, no need to merge topic metaData, dataVersion={}, kvDataVersion={}", dataVersion, kvDataVersion);
+        }
+        log.info("finish read topic config from json file and merge to rocksdb");
+        this.persist();
+        return true;
+    }
+
+
+    @Override
+    public boolean stop() {
+        return this.rocksDBConfigManager.stop();
+    }
+
+    protected void decodeTopicConfig(byte[] key, byte[] body) {
+        String topicName = new String(key, DataConverter.CHARSET_UTF8);
+        TopicConfig topicConfig = JSON.parseObject(body, TopicConfig.class);
+
+        this.topicConfigTable.put(topicName, topicConfig);
+        log.info("load exist local topic, {}", topicConfig.toString());
+    }
+
+    @Override
+    public TopicConfig putTopicConfig(TopicConfig topicConfig) {
+        String topicName = topicConfig.getTopicName();
+        TopicConfig oldTopicConfig = this.topicConfigTable.put(topicName, topicConfig);
+        try {
+            byte[] keyBytes = topicName.getBytes(DataConverter.CHARSET_UTF8);
+            byte[] valueBytes = JSON.toJSONBytes(topicConfig, SerializerFeature.BrowserCompatible);
+            this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes);
+        } catch (Exception e) {
+            log.error("kv put topic Failed, {}", topicConfig.toString(), e);
+        }
+        return oldTopicConfig;
+    }
+
+    @Override
+    protected TopicConfig removeTopicConfig(String topicName) {
+        TopicConfig topicConfig = this.topicConfigTable.remove(topicName);
+        try {
+            this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.CHARSET_UTF8));
+        } catch (Exception e) {
+            log.error("kv remove topic Failed, {}", topicConfig.toString());
+        }
+        return topicConfig;
+    }
+
+    @Override
+    public synchronized void persist() {
+        if (brokerController.getMessageStoreConfig().isRealTimePersistRocksDBConfig()) {
+            this.rocksDBConfigManager.flushWAL();
+        }
+    }
+
+    public synchronized void exportToJson() {
+        log.info("RocksDBTopicConfigManager export topic config to json file");
+        super.persist();
+    }
+
+    public String rocksdbConfigFilePath() {
+        return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "topics" + File.separator;
+    }
+
+
+    @Override
+    public DataVersion getDataVersion() {
+        return rocksDBConfigManager.getKvDataVersion();
+    }
+
+    @Override
+    public void updateDataVersion() {
+        try {
+            rocksDBConfigManager.updateKvDataVersion();
+        } catch (Exception e) {
+            log.error("update topic config dataVersion error", e);
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigHelper.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigHelper.java
new file mode 100644
index 00000000000..29a7c313bab
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigHelper.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+import com.alibaba.fastjson2.JSON;
+import com.google.common.base.Preconditions;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.rocketmq.common.config.AbstractRocksDBStorage;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.WriteBatch;
+
+public class ConfigHelper {
+
+    /**
+     * <p>
+     * Layout of data version key:
+     * [table-prefix, 1 byte][table-id, 2 byte][record-prefix, 1 byte][data-version-bytes]
+     * </p>
+     *
+     * <p>
+     * Layout of data version value:
+     * [state-machine-version, 8 bytes][timestamp, 8 bytes][sequence counter, 8 bytes]
+     * </p>
+     *
+     * @throws RocksDBException if RocksDB raises an error
+     */
+    public static Optional<ByteBuf> loadDataVersion(ConfigStorage configStorage, TableId tableId)
+        throws RocksDBException {
+        int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */
+            + ConfigStorage.DATA_VERSION_KEY_BYTES.length;
+        ByteBuf keyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        try {
+            keyBuf.writeByte(TablePrefix.TABLE.getValue());
+            keyBuf.writeShort(tableId.getValue());
+            keyBuf.writeByte(RecordPrefix.DATA_VERSION.getValue());
+            keyBuf.writeBytes(ConfigStorage.DATA_VERSION_KEY_BYTES);
+            byte[] valueByes = configStorage.get(keyBuf.nioBuffer());
+            if (null != valueByes) {
+                ByteBuf valueBuf = Unpooled.wrappedBuffer(valueByes);
+                return Optional.of(valueBuf);
+            }
+        } finally {
+            keyBuf.release();
+        }
+        return Optional.empty();
+    }
+
+    public static void stampDataVersion(WriteBatch writeBatch, TableId table, DataVersion dataVersion, long stateMachineVersion)
+        throws RocksDBException {
+        // Increase data version
+        dataVersion.nextVersion(stateMachineVersion);
+
+        int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */
+            + ConfigStorage.DATA_VERSION_KEY_BYTES.length;
+        ByteBuf keyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        ByteBuf valueBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(Long.BYTES * 3);
+        try {
+            keyBuf.writeByte(TablePrefix.TABLE.getValue());
+            keyBuf.writeShort(table.getValue());
+            keyBuf.writeByte(RecordPrefix.DATA_VERSION.getValue());
+            keyBuf.writeBytes(ConfigStorage.DATA_VERSION_KEY_BYTES);
+            valueBuf.writeLong(dataVersion.getStateVersion());
+            valueBuf.writeLong(dataVersion.getTimestamp());
+            valueBuf.writeLong(dataVersion.getCounter().get());
+            writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());
+        } finally {
+            keyBuf.release();
+            valueBuf.release();
+        }
+    }
+
+    public static void onDataVersionLoad(ByteBuf buf, DataVersion dataVersion) {
+        if (buf.readableBytes() == 8 /* state machine version */ + 8 /* timestamp */ + 8 /* counter */) {
+            long stateMachineVersion = buf.readLong();
+            long timestamp = buf.readLong();
+            long counter = buf.readLong();
+            dataVersion.setStateVersion(stateMachineVersion);
+            dataVersion.setTimestamp(timestamp);
+            dataVersion.setCounter(new AtomicLong(counter));
+        }
+        buf.release();
+    }
+
+    public static ByteBuf keyBufOf(TableId tableId, final String name) {
+        Preconditions.checkNotNull(name);
+        byte[] bytes = name.getBytes(StandardCharsets.UTF_8);
+        int keyLen = 1 /* table-prefix */ + 2 /* table-id */ + 1 /* record-type-prefix */ + 2 /* name-length */ + bytes.length;
+        ByteBuf keyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        keyBuf.writeByte(TablePrefix.TABLE.getValue());
+        keyBuf.writeShort(tableId.getValue());
+        keyBuf.writeByte(RecordPrefix.DATA.getValue());
+        keyBuf.writeShort(bytes.length);
+        keyBuf.writeBytes(bytes);
+        return keyBuf;
+    }
+
+    public static ByteBuf valueBufOf(final Object config, SerializationType serializationType) {
+        if (SerializationType.JSON == serializationType) {
+            byte[] payload = JSON.toJSONBytes(config);
+            ByteBuf valueBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(1 + payload.length);
+            valueBuf.writeByte(SerializationType.JSON.getValue());
+            valueBuf.writeBytes(payload);
+            return valueBuf;
+        }
+        throw new RuntimeException("Unsupported serialization type: " + serializationType);
+    }
+
+    public static byte[] readBytes(final ByteBuf buf) {
+        byte[] bytes = new byte[buf.readableBytes()];
+        buf.readBytes(bytes);
+        return bytes;
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java
new file mode 100644
index 00000000000..c4056d142fc
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import io.netty.buffer.PooledByteBufAllocatorMetric;
+import io.netty.util.internal.PlatformDependent;
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.config.AbstractRocksDBStorage;
+import org.apache.rocketmq.common.config.ConfigHelper;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.FlushOptions;
+import org.rocksdb.ReadOptions;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.Slice;
+import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteOptions;
+
+/**
+ * https://book.tidb.io/session1/chapter3/tidb-kv-to-relation.html
+ */
+public class ConfigStorage extends AbstractRocksDBStorage {
+
+    public static final String DATA_VERSION_KEY = "data_version";
+    public static final byte[] DATA_VERSION_KEY_BYTES = DATA_VERSION_KEY.getBytes(StandardCharsets.UTF_8);
+
+    private final ScheduledExecutorService scheduledExecutorService;
+
+    /**
+     * Number of write ops since previous flush.
+     */
+    private final AtomicInteger writeOpsCounter;
+
+    private final AtomicLong estimateWalFileSize = new AtomicLong(0L);
+
+    private final MessageStoreConfig messageStoreConfig;
+
+    private final FlushSyncService flushSyncService;
+
+    public ConfigStorage(MessageStoreConfig messageStoreConfig) {
+        super(messageStoreConfig.getStorePathRootDir() + File.separator + "config" + File.separator + "rdb");
+        this.messageStoreConfig = messageStoreConfig;
+        ThreadFactory threadFactory = new ThreadFactoryBuilder()
+            .setDaemon(true)
+            .setNameFormat("config-storage-%d")
+            .build();
+        scheduledExecutorService = new ScheduledThreadPoolExecutor(1, threadFactory);
+        writeOpsCounter = new AtomicInteger(0);
+        this.flushSyncService = new FlushSyncService();
+        this.flushSyncService.setDaemon(true);
+    }
+
+    private void statNettyMemory() {
+        PooledByteBufAllocatorMetric metric = AbstractRocksDBStorage.POOLED_ALLOCATOR.metric();
+        LOGGER.info("Netty Memory Usage: {}", metric);
+    }
+
+    @Override
+    public synchronized boolean start() {
+        boolean started = super.start();
+        if (started) {
+            scheduledExecutorService.scheduleWithFixedDelay(() -> statRocksdb(LOGGER), 1, 10, TimeUnit.SECONDS);
+            scheduledExecutorService.scheduleWithFixedDelay(this::statNettyMemory, 10, 10, TimeUnit.SECONDS);
+            this.flushSyncService.start();
+        } else {
+            LOGGER.error("Failed to start config storage");
+        }
+        return started;
+    }
+
+    @Override
+    protected boolean postLoad() {
+        if (!PlatformDependent.hasUnsafe()) {
+            LOGGER.error("Unsafe not available and POOLED_ALLOCATOR cannot work correctly");
+            return false;
+        }
+        try {
+            UtilAll.ensureDirOK(this.dbPath);
+            initOptions();
+            List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList<>();
+
+            ColumnFamilyOptions defaultOptions = ConfigHelper.createConfigColumnFamilyOptions();
+            this.cfOptions.add(defaultOptions);
+            cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions));
+
+            // Start RocksDB instance
+            open(cfDescriptors);
+
+            this.defaultCFHandle = cfHandles.get(0);
+        } catch (Exception e) {
+            AbstractRocksDBStorage.LOGGER.error("postLoad Failed. {}", this.dbPath, e);
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    protected void preShutdown() {
+        scheduledExecutorService.shutdown();
+        flushSyncService.shutdown();
+    }
+
+    protected void initOptions() {
+        this.options = ConfigHelper.createConfigDBOptions();
+        super.initOptions();
+    }
+
+    @Override
+    protected void initAbleWalWriteOptions() {
+        this.ableWalWriteOptions = new WriteOptions();
+
+        // Given that fdatasync is kind of expensive, sync-WAL for every write cannot be afforded.
+        this.ableWalWriteOptions.setSync(false);
+
+        // We need WAL for config changes
+        this.ableWalWriteOptions.setDisableWAL(false);
+
+        // No fast failure on block, wait synchronously even if there is wait for the write request
+        this.ableWalWriteOptions.setNoSlowdown(false);
+    }
+
+    public byte[] get(ByteBuffer key) throws RocksDBException {
+        byte[] keyBytes = new byte[key.remaining()];
+        key.get(keyBytes);
+        return super.get(getDefaultCFHandle(), totalOrderReadOptions, keyBytes);
+    }
+
+    public void write(WriteBatch writeBatch) throws RocksDBException {
+        db.write(ableWalWriteOptions, writeBatch);
+        accountWriteOps(writeBatch.getDataSize());
+    }
+
+    private void accountWriteOps(long dataSize) {
+        writeOpsCounter.incrementAndGet();
+        estimateWalFileSize.addAndGet(dataSize);
+    }
+
+    public RocksIterator iterate(ByteBuffer beginKey, ByteBuffer endKey) {
+        try (ReadOptions readOptions = new ReadOptions()) {
+            readOptions.setTotalOrderSeek(true);
+            readOptions.setTailing(false);
+            readOptions.setAutoPrefixMode(true);
+            // Use DirectSlice till the follow issue is fixed:
+            // https://github.com/facebook/rocksdb/issues/13098
+            //
+            // readOptions.setIterateUpperBound(new DirectSlice(endKey));
+            byte[] buf = new byte[endKey.remaining()];
+            endKey.slice().get(buf);
+            readOptions.setIterateUpperBound(new Slice(buf));
+
+            RocksIterator iterator = db.newIterator(defaultCFHandle, readOptions);
+            iterator.seek(beginKey.slice());
+            return iterator;
+        }
+    }
+
+    /**
+     * RocksDB writes contain 3 stages: application memory buffer --> OS Page Cache --> Disk.
+     * Given that we are having DBOptions::manual_wal_flush, we need to manually call DB::FlushWAL and DB::SyncWAL
+     * Note: DB::FlushWAL(true) will internally call DB::SyncWAL.
+     * <p>
+     * See <a href="https://rocksdb.org/blog/2017/08/25/flushwal.html">Flush And Sync WAL</a>
+     */
+    class FlushSyncService extends ServiceThread {
+
+        private long lastSyncTime = 0;
+
+        private static final long MAX_SYNC_INTERVAL_IN_MILLIS = 100;
+
+        private final Stopwatch stopwatch = Stopwatch.createUnstarted();
+
+        private final FlushOptions flushOptions = new FlushOptions();
+
+        @Override
+        public String getServiceName() {
+            return "FlushSyncService";
+        }
+
+        @Override
+        public void run() {
+            flushOptions.setAllowWriteStall(false);
+            flushOptions.setWaitForFlush(true);
+            log.info("{} service started", this.getServiceName());
+            while (!this.isStopped()) {
+                try {
+                    this.waitForRunning(10);
+                    this.flushAndSyncWAL(false);
+                } catch (Exception e) {
+                    log.warn("{} service has exception. ", this.getServiceName(), e);
+                }
+            }
+            try {
+                flushAndSyncWAL(true);
+            } catch (Exception e) {
+                log.warn("{} raised an exception while performing flush-and-sync WAL on exit",
+                    this.getServiceName(), e);
+            }
+            flushOptions.close();
+            log.info("{} service end", this.getServiceName());
+        }
+
+        private void flushAndSyncWAL(boolean onExit) throws RocksDBException {
+            int writeOps = writeOpsCounter.get();
+            if (0 == writeOps) {
+                // No write ops to flush
+                return;
+            }
+
+            /*
+             * Normally, when MemTables become full then immutable, RocksDB threads will automatically flush them to L0
+             * SST files. The use case here is different: the MemTable may never get full and immutable given that the
+             * volume of data involved is relatively small. Further, we are constantly modifying the key-value pairs and
+             * generating WAL entries. The WAL file size can grow up to dozens of gigabytes without manual triggering of
+             * flush.
+             */
+            if (ConfigStorage.this.estimateWalFileSize.get() >= messageStoreConfig.getRocksdbWalFileRollingThreshold()) {
+                ConfigStorage.this.flush(flushOptions);
+                estimateWalFileSize.set(0L);
+            }
+
+            // Flush and Sync WAL if we have committed enough writes
+            if (writeOps >= messageStoreConfig.getRocksdbFlushWalFrequency() || onExit) {
+                stopwatch.reset().start();
+                ConfigStorage.this.db.flushWal(true);
+                long elapsed = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
+                writeOpsCounter.getAndAdd(-writeOps);
+                lastSyncTime = System.currentTimeMillis();
+                LOGGER.debug("Flush and Sync WAL of RocksDB[{}] costs {}ms, write-ops={}", dbPath, elapsed, writeOps);
+                return;
+            }
+            // Flush and Sync WAL if some writes are out there for a period of time
+            long elapsedTime = System.currentTimeMillis() - lastSyncTime;
+            if (elapsedTime > MAX_SYNC_INTERVAL_IN_MILLIS) {
+                stopwatch.reset().start();
+                ConfigStorage.this.db.flushWal(true);
+                long elapsed = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
+                LOGGER.debug("Flush and Sync WAL of RocksDB[{}] costs {}ms, write-ops={}", dbPath, elapsed, writeOps);
+                writeOpsCounter.getAndAdd(-writeOps);
+                lastSyncTime = System.currentTimeMillis();
+            }
+        }
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java
new file mode 100644
index 00000000000..1821c801cbc
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java
@@ -0,0 +1,428 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.util.internal.PlatformDependent;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.config.AbstractRocksDBStorage;
+import org.apache.rocketmq.store.MessageStore;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.WriteBatch;
+
+/**
+ * <p>
+ * Layout of consumer offset key:
+ * [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte]
+ * [topic-len, 2 bytes][topic bytes][CTRL_1, 1 byte][queue-id, 4 bytes]
+ * </p>
+ *
+ * <p>
+ * Layout of consumer offset value: [offset, 8 bytes]
+ * </p>
+ */
+public class ConsumerOffsetManagerV2 extends ConsumerOffsetManager {
+
+    private final ConfigStorage configStorage;
+
+    public ConsumerOffsetManagerV2(BrokerController brokerController, ConfigStorage configStorage) {
+        super(brokerController);
+        this.configStorage = configStorage;
+    }
+
+    @Override
+    protected void removeConsumerOffset(String topicAtGroup) {
+        if (!MixAll.isLmq(topicAtGroup)) {
+            super.removeConsumerOffset(topicAtGroup);
+        }
+
+        String[] topicGroup = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
+        if (topicGroup.length != 2) {
+            LOG.error("Invalid topic group: {}", topicAtGroup);
+            return;
+        }
+
+        byte[] topicBytes = topicGroup[0].getBytes(StandardCharsets.UTF_8);
+        byte[] groupBytes = topicGroup[1].getBytes(StandardCharsets.UTF_8);
+
+        int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */
+            + Short.BYTES /* group-len */ + groupBytes.length + 1 /* CTRL_1 */
+            + Short.BYTES + topicBytes.length + 1;
+        // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group-bytes][CTRL_1, 1 byte]
+        // [topic-len, 2 bytes][topic-bytes][CTRL_1]
+        ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        beginKey.writeByte(TablePrefix.TABLE.getValue());
+        beginKey.writeShort(TableId.CONSUMER_OFFSET.getValue());
+        beginKey.writeByte(RecordPrefix.DATA.getValue());
+        beginKey.writeShort(groupBytes.length);
+        beginKey.writeBytes(groupBytes);
+        beginKey.writeByte(AbstractRocksDBStorage.CTRL_1);
+        beginKey.writeShort(topicBytes.length);
+        beginKey.writeBytes(topicBytes);
+        beginKey.writeByte(AbstractRocksDBStorage.CTRL_1);
+
+        ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        endKey.writeByte(TablePrefix.TABLE.getValue());
+        endKey.writeShort(TableId.CONSUMER_OFFSET.getValue());
+        endKey.writeByte(RecordPrefix.DATA.getValue());
+        endKey.writeShort(groupBytes.length);
+        endKey.writeBytes(groupBytes);
+        endKey.writeByte(AbstractRocksDBStorage.CTRL_1);
+        endKey.writeShort(topicBytes.length);
+        endKey.writeBytes(topicBytes);
+        endKey.writeByte(AbstractRocksDBStorage.CTRL_2);
+
+        try (WriteBatch writeBatch = new WriteBatch()) {
+            // TODO: we have to make a copy here as WriteBatch lacks ByteBuffer API here
+            writeBatch.deleteRange(ConfigHelper.readBytes(beginKey), ConfigHelper.readBytes(endKey));
+            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+            ConfigHelper.stampDataVersion(writeBatch, TableId.CONSUMER_OFFSET, dataVersion, stateMachineVersion);
+            configStorage.write(writeBatch);
+        } catch (RocksDBException e) {
+            LOG.error("Failed to removeConsumerOffset, topicAtGroup={}", topicAtGroup, e);
+        } finally {
+            beginKey.release();
+            endKey.release();
+        }
+    }
+
+    @Override
+    public void removeOffset(String group) {
+        if (!MixAll.isLmq(group)) {
+            super.removeOffset(group);
+        }
+
+        byte[] groupBytes = group.getBytes(StandardCharsets.UTF_8);
+        int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */
+            + Short.BYTES /* group-len */ + groupBytes.length + 1 /* CTRL_1 */;
+
+        // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte]
+        ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        beginKey.writeByte(TablePrefix.TABLE.getValue());
+        beginKey.writeShort(TableId.CONSUMER_OFFSET.getValue());
+        beginKey.writeByte(RecordPrefix.DATA.getValue());
+        beginKey.writeShort(groupBytes.length);
+        beginKey.writeBytes(groupBytes);
+        beginKey.writeByte(AbstractRocksDBStorage.CTRL_1);
+
+        ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        endKey.writeByte(TablePrefix.TABLE.getValue());
+        endKey.writeShort(TableId.CONSUMER_OFFSET.getValue());
+        endKey.writeByte(RecordPrefix.DATA.getValue());
+        endKey.writeShort(groupBytes.length);
+        endKey.writeBytes(groupBytes);
+        endKey.writeByte(AbstractRocksDBStorage.CTRL_2);
+        try (WriteBatch writeBatch = new WriteBatch()) {
+            // TODO: we have to make a copy here as WriteBatch lacks ByteBuffer API here
+            writeBatch.deleteRange(ConfigHelper.readBytes(beginKey), ConfigHelper.readBytes(endKey));
+            MessageStore messageStore = brokerController.getMessageStore();
+            long stateMachineVersion = messageStore != null ? messageStore.getStateMachineVersion() : 0;
+            ConfigHelper.stampDataVersion(writeBatch, TableId.CONSUMER_OFFSET, dataVersion, stateMachineVersion);
+            configStorage.write(writeBatch);
+        } catch (RocksDBException e) {
+            LOG.error("Failed to consumer offsets by group={}", group, e);
+        } finally {
+            beginKey.release();
+            endKey.release();
+        }
+    }
+
+    /**
+     * <p>
+     * Layout of consumer offset key:
+     * [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte]
+     * [topic-len, 2 bytes][topic bytes][CTRL_1, 1 byte][queue-id, 4 bytes]
+     * </p>
+     *
+     * <p>
+     * Layout of consumer offset value:
+     * [offset, 8 bytes]
+     * </p>
+     *
+     * @param clientHost The client that submits consumer offsets
+     * @param group      Group name
+     * @param topic      Topic name
+     * @param queueId    Queue ID
+     * @param offset     Consumer offset of the specified queue
+     */
+    @Override
+    public void commitOffset(String clientHost, String group, String topic, int queueId, long offset) {
+        String key = topic + TOPIC_GROUP_SEPARATOR + group;
+
+        // We maintain a copy of classic consumer offset table in memory as they take very limited memory footprint.
+        // For LMQ offsets, given the volume and number of these type of records, they are maintained in RocksDB
+        // directly. Frequently used LMQ consumer offsets should reside either in block-cache or MemTable, so read/write
+        // should be blazingly fast.
+        if (!MixAll.isLmq(topic)) {
+            if (offsetTable.containsKey(key)) {
+                offsetTable.get(key).put(queueId, offset);
+            } else {
+                ConcurrentMap<Integer, Long> map = new ConcurrentHashMap<>();
+                ConcurrentMap<Integer, Long> prev = offsetTable.putIfAbsent(key, map);
+                if (null != prev) {
+                    map = prev;
+                }
+                map.put(queueId, offset);
+            }
+        }
+
+        ByteBuf keyBuf = keyOfConsumerOffset(group, topic, queueId);
+        ByteBuf valueBuf = ConfigStorage.POOLED_ALLOCATOR.buffer(Long.BYTES);
+        try (WriteBatch writeBatch = new WriteBatch()) {
+            valueBuf.writeLong(offset);
+            writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());
+            MessageStore messageStore = brokerController.getMessageStore();
+            long stateMachineVersion = messageStore != null ? messageStore.getStateMachineVersion() : 0;
+            ConfigHelper.stampDataVersion(writeBatch, TableId.CONSUMER_OFFSET, dataVersion, stateMachineVersion);
+            configStorage.write(writeBatch);
+        } catch (RocksDBException e) {
+            LOG.error("Failed to commit consumer offset", e);
+        } finally {
+            keyBuf.release();
+            valueBuf.release();
+        }
+    }
+
+    private ByteBuf keyOfConsumerOffset(String group, String topic, int queueId) {
+        byte[] groupBytes = group.getBytes(StandardCharsets.UTF_8);
+        byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
+        int keyLen = 1 /*table prefix*/ + Short.BYTES /*table-id*/ + 1 /*record-prefix*/
+            + Short.BYTES /*group-len*/ + groupBytes.length + 1 /*CTRL_1*/
+            + 2 /*topic-len*/ + topicBytes.length + 1 /* CTRL_1*/
+            + Integer.BYTES /*queue-id*/;
+        ByteBuf keyBuf = ConfigStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        keyBuf.writeByte(TablePrefix.TABLE.getValue());
+        keyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue());
+        keyBuf.writeByte(RecordPrefix.DATA.getValue());
+        keyBuf.writeShort(groupBytes.length);
+        keyBuf.writeBytes(groupBytes);
+        keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1);
+        keyBuf.writeShort(topicBytes.length);
+        keyBuf.writeBytes(topicBytes);
+        keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1);
+        keyBuf.writeInt(queueId);
+        return keyBuf;
+    }
+
+    private ByteBuf keyOfPullOffset(String group, String topic, int queueId) {
+        byte[] groupBytes = group.getBytes(StandardCharsets.UTF_8);
+        byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
+        int keyLen = 1 /*table prefix*/ + Short.BYTES /*table-id*/ + 1 /*record-prefix*/
+            + Short.BYTES /*group-len*/ + groupBytes.length + 1 /*CTRL_1*/
+            + 2 /*topic-len*/ + topicBytes.length + 1 /* CTRL_1*/
+            + Integer.BYTES /*queue-id*/;
+        ByteBuf keyBuf = ConfigStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        keyBuf.writeByte(TablePrefix.TABLE.getValue());
+        keyBuf.writeShort(TableId.PULL_OFFSET.getValue());
+        keyBuf.writeByte(RecordPrefix.DATA.getValue());
+        keyBuf.writeShort(groupBytes.length);
+        keyBuf.writeBytes(groupBytes);
+        keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1);
+        keyBuf.writeShort(topicBytes.length);
+        keyBuf.writeBytes(topicBytes);
+        keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1);
+        keyBuf.writeInt(queueId);
+        return keyBuf;
+    }
+
+    @Override
+    public boolean load() {
+        return loadDataVersion() && loadConsumerOffsets();
+    }
+
+    @Override
+    public synchronized void persist() {
+        try {
+            configStorage.flushWAL();
+        } catch (RocksDBException e) {
+            LOG.error("Failed to flush RocksDB config instance WAL", e);
+        }
+    }
+
+    /**
+     * <p>
+     * Layout of data version key:
+     * [table-prefix, 1 byte][table-id, 2 byte][record-prefix, 1 byte][data-version-bytes]
+     * </p>
+     *
+     * <p>
+     * Layout of data version value:
+     * [state-machine-version, 8 bytes][timestamp, 8 bytes][sequence counter, 8 bytes]
+     * </p>
+     */
+    public boolean loadDataVersion() {
+        try {
+            ConfigHelper.loadDataVersion(configStorage, TableId.CONSUMER_OFFSET)
+                .ifPresent(buf -> ConfigHelper.onDataVersionLoad(buf, dataVersion));
+        } catch (RocksDBException e) {
+            LOG.error("Failed to load RocksDB config", e);
+            return false;
+        }
+        return true;
+    }
+
+    private boolean loadConsumerOffsets() {
+        // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte]
+        ByteBuf beginKeyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(4);
+        beginKeyBuf.writeByte(TablePrefix.TABLE.getValue());
+        beginKeyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue());
+        beginKeyBuf.writeByte(RecordPrefix.DATA.getValue());
+
+        ByteBuf endKeyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(4);
+        endKeyBuf.writeByte(TablePrefix.TABLE.getValue());
+        endKeyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue());
+        endKeyBuf.writeByte(RecordPrefix.DATA.getValue() + 1);
+
+        try (RocksIterator iterator = configStorage.iterate(beginKeyBuf.nioBuffer(), endKeyBuf.nioBuffer())) {
+            int keyCapacity = 256;
+            // We may iterate millions of LMQ consumer offsets here, use direct byte buffers here to avoid memory
+            // fragment
+            ByteBuffer keyBuffer = ByteBuffer.allocateDirect(keyCapacity);
+            ByteBuffer valueBuffer = ByteBuffer.allocateDirect(Long.BYTES);
+            while (iterator.isValid()) {
+                keyBuffer.clear();
+                valueBuffer.clear();
+
+                int len = iterator.key(keyBuffer);
+                if (len > keyCapacity) {
+                    keyCapacity = len;
+                    PlatformDependent.freeDirectBuffer(keyBuffer);
+                    // Reserve more space for key
+                    keyBuffer = ByteBuffer.allocateDirect(keyCapacity);
+                    continue;
+                }
+                len = iterator.value(valueBuffer);
+                assert len == Long.BYTES;
+
+                // skip table-prefix, table-id, record-prefix
+                keyBuffer.position(1 + 2 + 1);
+                short groupLen = keyBuffer.getShort();
+                byte[] groupBytes = new byte[groupLen];
+                keyBuffer.get(groupBytes);
+                byte ctrl = keyBuffer.get();
+                assert ctrl == AbstractRocksDBStorage.CTRL_1;
+
+                short topicLen = keyBuffer.getShort();
+                byte[] topicBytes = new byte[topicLen];
+                keyBuffer.get(topicBytes);
+                String topic = new String(topicBytes, StandardCharsets.UTF_8);
+                ctrl = keyBuffer.get();
+                assert ctrl == AbstractRocksDBStorage.CTRL_1;
+
+                int queueId = keyBuffer.getInt();
+
+                long offset = valueBuffer.getLong();
+
+                if (!MixAll.isLmq(topic)) {
+                    String group = new String(groupBytes, StandardCharsets.UTF_8);
+                    onConsumerOffsetRecordLoad(topic, group, queueId, offset);
+                }
+                iterator.next();
+            }
+            PlatformDependent.freeDirectBuffer(keyBuffer);
+            PlatformDependent.freeDirectBuffer(valueBuffer);
+        } finally {
+            beginKeyBuf.release();
+            endKeyBuf.release();
+        }
+        return true;
+    }
+
+    private void onConsumerOffsetRecordLoad(String topic, String group, int queueId, long offset) {
+        if (MixAll.isLmq(topic)) {
+            return;
+        }
+        String key = topic + TOPIC_GROUP_SEPARATOR + group;
+        if (!offsetTable.containsKey(key)) {
+            ConcurrentMap<Integer, Long> map = new ConcurrentHashMap<>();
+            offsetTable.putIfAbsent(key, map);
+        }
+        offsetTable.get(key).put(queueId, offset);
+    }
+
+    @Override
+    public long queryOffset(String group, String topic, int queueId) {
+        if (!MixAll.isLmq(topic)) {
+            return super.queryOffset(group, topic, queueId);
+        }
+
+        ByteBuf keyBuf = keyOfConsumerOffset(group, topic, queueId);
+        try {
+            byte[] slice = configStorage.get(keyBuf.nioBuffer());
+            if (null == slice) {
+                return -1;
+            }
+            assert slice.length == Long.BYTES;
+            return ByteBuffer.wrap(slice).getLong();
+        } catch (RocksDBException e) {
+            throw new RuntimeException(e);
+        } finally {
+            keyBuf.release();
+        }
+    }
+
+    @Override
+    public void commitPullOffset(String clientHost, String group, String topic, int queueId, long offset) {
+        if (!MixAll.isLmq(topic)) {
+            super.commitPullOffset(clientHost, group, topic, queueId, offset);
+        }
+
+        ByteBuf keyBuf = keyOfPullOffset(group, topic, queueId);
+        ByteBuf valueBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(8);
+        valueBuf.writeLong(offset);
+        try (WriteBatch writeBatch = new WriteBatch()) {
+            writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());
+            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+            ConfigHelper.stampDataVersion(writeBatch, TableId.PULL_OFFSET, dataVersion, stateMachineVersion);
+            configStorage.write(writeBatch);
+        } catch (RocksDBException e) {
+            LOG.error("Failed to commit pull offset. group={}, topic={}, queueId={}, offset={}",
+                group, topic, queueId, offset);
+        } finally {
+            keyBuf.release();
+            valueBuf.release();
+        }
+    }
+
+    @Override
+    public long queryPullOffset(String group, String topic, int queueId) {
+        if (!MixAll.isLmq(topic)) {
+            return super.queryPullOffset(group, topic, queueId);
+        }
+
+        ByteBuf keyBuf = keyOfPullOffset(group, topic, queueId);
+        try {
+            byte[] valueBytes = configStorage.get(keyBuf.nioBuffer());
+            if (null == valueBytes) {
+                return -1;
+            }
+            return ByteBuffer.wrap(valueBytes).getLong();
+        } catch (RocksDBException e) {
+            LOG.error("Failed to queryPullOffset. group={}, topic={}, queueId={}", group, topic, queueId);
+        } finally {
+            keyBuf.release();
+        }
+        return -1;
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/CQItemBufferUtil.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/RecordPrefix.java
similarity index 65%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/CQItemBufferUtil.java
rename to broker/src/main/java/org/apache/rocketmq/broker/config/v2/RecordPrefix.java
index 2acc133d830..750d454d4ee 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/CQItemBufferUtil.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/RecordPrefix.java
@@ -14,20 +14,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.tieredstore.util;
+package org.apache.rocketmq.broker.config.v2;
 
-import java.nio.ByteBuffer;
+public enum RecordPrefix {
+    UNSPECIFIED((byte)0),
+    DATA_VERSION((byte)1),
+    DATA((byte)2);
 
-public class CQItemBufferUtil {
-    public static long getCommitLogOffset(ByteBuffer cqItem) {
-        return cqItem.getLong(cqItem.position());
-    }
+    private final byte value;
 
-    public static int getSize(ByteBuffer cqItem) {
-        return cqItem.getInt(cqItem.position() + 8);
+    RecordPrefix(byte value) {
+        this.value = value;
     }
 
-    public static long getTagCode(ByteBuffer cqItem) {
-        return cqItem.getLong(cqItem.position() + 12);
+    public byte getValue() {
+        return value;
     }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SerializationType.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SerializationType.java
new file mode 100644
index 00000000000..2ee157fdc8d
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SerializationType.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+public enum SerializationType {
+    UNSPECIFIED((byte) 0),
+
+    JSON((byte) 1),
+
+    PROTOBUF((byte) 2),
+
+    FLAT_BUFFERS((byte) 3);
+
+    private final byte value;
+
+    SerializationType(byte value) {
+        this.value = value;
+    }
+
+    public byte getValue() {
+        return value;
+    }
+
+    public static SerializationType valueOf(byte value) {
+        for (SerializationType type : SerializationType.values()) {
+            if (type.getValue() == value) {
+                return type;
+            }
+        }
+        return SerializationType.UNSPECIFIED;
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java
new file mode 100644
index 00000000000..dd67871f184
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+import com.alibaba.fastjson2.JSON;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import java.nio.charset.StandardCharsets;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.config.AbstractRocksDBStorage;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.WriteBatch;
+
+public class SubscriptionGroupManagerV2 extends SubscriptionGroupManager {
+
+    private final ConfigStorage configStorage;
+
+    public SubscriptionGroupManagerV2(BrokerController brokerController, ConfigStorage configStorage) {
+        super(brokerController);
+        this.configStorage = configStorage;
+    }
+
+    @Override
+    public boolean load() {
+        return loadDataVersion() && loadSubscriptions();
+    }
+
+    public boolean loadDataVersion() {
+        try {
+            ConfigHelper.loadDataVersion(configStorage, TableId.SUBSCRIPTION_GROUP)
+                .ifPresent(buf -> {
+                    ConfigHelper.onDataVersionLoad(buf, dataVersion);
+                });
+        } catch (RocksDBException e) {
+            log.error("loadDataVersion error", e);
+            return false;
+        }
+        return true;
+    }
+
+    private boolean loadSubscriptions() {
+        int keyLen = 1 /* table prefix */ + 2 /* table-id */ + 1 /* record-type-prefix */;
+        ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        beginKey.writeByte(TablePrefix.TABLE.getValue());
+        beginKey.writeShort(TableId.SUBSCRIPTION_GROUP.getValue());
+        beginKey.writeByte(RecordPrefix.DATA.getValue());
+
+        ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        endKey.writeByte(TablePrefix.TABLE.getValue());
+        endKey.writeShort(TableId.SUBSCRIPTION_GROUP.getValue());
+        endKey.writeByte(RecordPrefix.DATA.getValue() + 1);
+
+        try (RocksIterator iterator = configStorage.iterate(beginKey.nioBuffer(), endKey.nioBuffer())) {
+            while (iterator.isValid()) {
+                SubscriptionGroupConfig subscriptionGroupConfig = parseSubscription(iterator.key(), iterator.value());
+                if (null != subscriptionGroupConfig) {
+                    super.putSubscriptionGroupConfig(subscriptionGroupConfig);
+                }
+                iterator.next();
+            }
+        } finally {
+            beginKey.release();
+            endKey.release();
+        }
+        return true;
+    }
+
+    private SubscriptionGroupConfig parseSubscription(byte[] key, byte[] value) {
+        ByteBuf keyBuf = Unpooled.wrappedBuffer(key);
+        ByteBuf valueBuf = Unpooled.wrappedBuffer(value);
+        try {
+            // Skip table-prefix, table-id, record-type-prefix
+            keyBuf.readerIndex(4);
+            short groupNameLen = keyBuf.readShort();
+            assert groupNameLen == keyBuf.readableBytes();
+            CharSequence groupName = keyBuf.readCharSequence(groupNameLen, StandardCharsets.UTF_8);
+            assert null != groupName;
+            byte serializationType = valueBuf.readByte();
+            if (SerializationType.JSON == SerializationType.valueOf(serializationType)) {
+                CharSequence json = valueBuf.readCharSequence(valueBuf.readableBytes(), StandardCharsets.UTF_8);
+                SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(json.toString(), SubscriptionGroupConfig.class);
+                assert subscriptionGroupConfig != null;
+                assert groupName.equals(subscriptionGroupConfig.getGroupName());
+                return subscriptionGroupConfig;
+            }
+        } finally {
+            keyBuf.release();
+            valueBuf.release();
+        }
+        return null;
+    }
+
+    @Override
+    public synchronized void persist() {
+        try {
+            configStorage.flushWAL();
+        } catch (RocksDBException e) {
+            log.error("Failed to flush RocksDB WAL", e);
+        }
+    }
+
+    @Override
+    public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) {
+        if (MixAll.isLmq(group)) {
+            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
+            subscriptionGroupConfig.setGroupName(group);
+            return subscriptionGroupConfig;
+        }
+        return super.findSubscriptionGroupConfig(group);
+    }
+
+    @Override
+    public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) {
+        if (config == null || MixAll.isLmq(config.getGroupName())) {
+            return;
+        }
+        ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.SUBSCRIPTION_GROUP, config.getGroupName());
+        ByteBuf valueBuf = ConfigHelper.valueBufOf(config, SerializationType.JSON);
+        try (WriteBatch writeBatch = new WriteBatch()) {
+            writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());
+            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+            ConfigHelper.stampDataVersion(writeBatch, TableId.SUBSCRIPTION_GROUP, dataVersion, stateMachineVersion);
+            configStorage.write(writeBatch);
+            // fdatasync on core metadata change
+            persist();
+        } catch (RocksDBException e) {
+            log.error("update subscription group config error", e);
+        } finally {
+            keyBuf.release();
+            valueBuf.release();
+        }
+        super.updateSubscriptionGroupConfigWithoutPersist(config);
+    }
+
+    @Override
+    public boolean containsSubscriptionGroup(String group) {
+        if (MixAll.isLmq(group)) {
+            return true;
+        } else {
+            return super.containsSubscriptionGroup(group);
+        }
+    }
+
+    @Override
+    protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) {
+        ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.SUBSCRIPTION_GROUP, groupName);
+        try (WriteBatch writeBatch = new WriteBatch()) {
+            writeBatch.delete(ConfigHelper.readBytes(keyBuf));
+            long stateMachineVersion = brokerController.getMessageStore().getStateMachineVersion();
+            ConfigHelper.stampDataVersion(writeBatch, TableId.SUBSCRIPTION_GROUP, dataVersion, stateMachineVersion);
+            configStorage.write(writeBatch);
+        } catch (RocksDBException e) {
+            log.error("Failed to remove subscription group config by group-name={}", groupName, e);
+        }
+        return super.removeSubscriptionGroupConfig(groupName);
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TableId.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TableId.java
new file mode 100644
index 00000000000..7a61899371e
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TableId.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+/**
+ * See <a href="https://book.tidb.io/session1/chapter3/tidb-kv-to-relation.html">Table, Key Value Mapping</a>
+ */
+public enum TableId {
+    UNSPECIFIED((short) 0),
+    CONSUMER_OFFSET((short) 1),
+    PULL_OFFSET((short) 2),
+    TOPIC((short) 3),
+    SUBSCRIPTION_GROUP((short) 4);
+
+    private final short value;
+
+    TableId(short value) {
+        this.value = value;
+    }
+
+    public short getValue() {
+        return value;
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TablePrefix.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TablePrefix.java
new file mode 100644
index 00000000000..d16c14d275a
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TablePrefix.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+public enum TablePrefix {
+    UNSPECIFIED((byte) 0),
+    TABLE((byte) 1);
+
+    private final byte value;
+
+    TablePrefix(byte value) {
+        this.value = value;
+    }
+
+    public byte getValue() {
+        return value;
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java
new file mode 100644
index 00000000000..7991d704459
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+import com.alibaba.fastjson2.JSON;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import java.nio.charset.StandardCharsets;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.config.AbstractRocksDBStorage;
+import org.apache.rocketmq.common.constant.PermName;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.WriteBatch;
+
+/**
+ * Key layout: [table-prefix, 1 byte][table-id, 2 bytes][record-type-prefix, 1 byte][topic-len, 2 bytes][topic-bytes]
+ * Value layout: [serialization-type, 1 byte][topic-config-bytes]
+ */
+public class TopicConfigManagerV2 extends TopicConfigManager {
+    private final ConfigStorage configStorage;
+
+    public TopicConfigManagerV2(BrokerController brokerController, ConfigStorage configStorage) {
+        super(brokerController);
+        this.configStorage = configStorage;
+    }
+
+    @Override
+    public boolean load() {
+        return loadDataVersion() && loadTopicConfig();
+    }
+
+    public boolean loadDataVersion() {
+        try {
+            ConfigHelper.loadDataVersion(configStorage, TableId.TOPIC)
+                .ifPresent(buf -> ConfigHelper.onDataVersionLoad(buf, dataVersion));
+        } catch (RocksDBException e) {
+            log.error("Failed to load data version of topic", e);
+            return false;
+        }
+        return true;
+    }
+
+    private boolean loadTopicConfig() {
+        int keyLen = 1 /* table-prefix */ + 2 /* table-id */ + 1 /* record-type-prefix */;
+        ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        beginKey.writeByte(TablePrefix.TABLE.getValue());
+        beginKey.writeShort(TableId.TOPIC.getValue());
+        beginKey.writeByte(RecordPrefix.DATA.getValue());
+
+        ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+        endKey.writeByte(TablePrefix.TABLE.getValue());
+        endKey.writeShort(TableId.TOPIC.getValue());
+        endKey.writeByte(RecordPrefix.DATA.getValue() + 1);
+
+        try (RocksIterator iterator = configStorage.iterate(beginKey.nioBuffer(), endKey.nioBuffer())) {
+            while (iterator.isValid()) {
+                byte[] key = iterator.key();
+                byte[] value = iterator.value();
+                TopicConfig topicConfig = parseTopicConfig(key, value);
+                if (null != topicConfig) {
+                    super.putTopicConfig(topicConfig);
+                }
+                iterator.next();
+            }
+        } finally {
+            beginKey.release();
+            endKey.release();
+        }
+        return true;
+    }
+
+    /**
+     * Key layout: [table-prefix, 1 byte][table-id, 2 bytes][record-type-prefix, 1 byte][topic-len, 2 bytes][topic-bytes]
+     * Value layout: [serialization-type, 1 byte][topic-config-bytes]
+     *
+     * @param key   Topic config key representation in RocksDB
+     * @param value Topic config value representation in RocksDB
+     * @return decoded topic config
+     */
+    private TopicConfig parseTopicConfig(byte[] key, byte[] value) {
+        ByteBuf keyBuf = Unpooled.wrappedBuffer(key);
+        ByteBuf valueBuf = Unpooled.wrappedBuffer(value);
+        try {
+            // Skip table-prefix, table-id, record-type-prefix
+            keyBuf.readerIndex(4);
+            short topicLen = keyBuf.readShort();
+            assert topicLen == keyBuf.readableBytes();
+            CharSequence topic = keyBuf.readCharSequence(topicLen, StandardCharsets.UTF_8);
+            assert null != topic;
+
+            byte serializationType = valueBuf.readByte();
+            if (SerializationType.JSON == SerializationType.valueOf(serializationType)) {
+                CharSequence json = valueBuf.readCharSequence(valueBuf.readableBytes(), StandardCharsets.UTF_8);
+                TopicConfig topicConfig = JSON.parseObject(json.toString(), TopicConfig.class);
+                assert topicConfig != null;
+                assert topic.equals(topicConfig.getTopicName());
+                return topicConfig;
+            }
+        } finally {
+            keyBuf.release();
+            valueBuf.release();
+        }
+
+        return null;
+    }
+
+    @Override
+    public synchronized void persist() {
+        try {
+            configStorage.flushWAL();
+        } catch (RocksDBException e) {
+            log.error("Failed to flush WAL", e);
+        }
+    }
+
+    @Override
+    public TopicConfig selectTopicConfig(final String topic) {
+        if (MixAll.isLmq(topic)) {
+            return simpleLmqTopicConfig(topic);
+        }
+        return super.selectTopicConfig(topic);
+    }
+
+    @Override
+    public void updateTopicConfig(final TopicConfig topicConfig) {
+        if (topicConfig == null || MixAll.isLmq(topicConfig.getTopicName())) {
+            return;
+        }
+        super.updateSingleTopicConfigWithoutPersist(topicConfig);
+
+        ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.TOPIC, topicConfig.getTopicName());
+        ByteBuf valueBuf = ConfigHelper.valueBufOf(topicConfig, SerializationType.JSON);
+        try (WriteBatch writeBatch = new WriteBatch()) {
+            writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());
+            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+            ConfigHelper.stampDataVersion(writeBatch, TableId.TOPIC, dataVersion, stateMachineVersion);
+            configStorage.write(writeBatch);
+            // fdatasync on core metadata change
+            this.persist();
+        } catch (RocksDBException e) {
+            log.error("Failed to update topic config", e);
+        } finally {
+            keyBuf.release();
+            valueBuf.release();
+        }
+    }
+
+    @Override
+    protected TopicConfig removeTopicConfig(String topicName) {
+        ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.TOPIC, topicName);
+        try (WriteBatch writeBatch = new WriteBatch()) {
+            writeBatch.delete(keyBuf.nioBuffer());
+            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+            ConfigHelper.stampDataVersion(writeBatch, TableId.TOPIC, dataVersion, stateMachineVersion);
+            configStorage.write(writeBatch);
+        } catch (RocksDBException e) {
+            log.error("Failed to delete topic config by topicName={}", topicName, e);
+        } finally {
+            keyBuf.release();
+        }
+        return super.removeTopicConfig(topicName);
+    }
+
+    @Override
+    public boolean containsTopic(String topic) {
+        if (MixAll.isLmq(topic)) {
+            return true;
+        }
+        return super.containsTopic(topic);
+    }
+
+    private TopicConfig simpleLmqTopicConfig(String topic) {
+        return new TopicConfig(topic, 1, 1, PermName.PERM_READ | PermName.PERM_WRITE);
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/package-info.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/package-info.java
new file mode 100644
index 00000000000..1ea216193c3
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/package-info.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+/*
+ * <strong>Endian</strong>: we use network byte order for all integrals, aka, always big endian.
+ *
+ * Unlike v1 config managers, implementations in this package prioritize data integrity and reliability.
+ * As a result,RocksDB write-ahead-log is always on and changes are immediately flushed. Another significant
+ * difference is that heap-based cache is removed because it is not necessary and duplicated to RocksDB
+ * MemTable/BlockCache.
+ */
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
index a1d711cb275..f22f22a12bd 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
@@ -30,7 +30,7 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
-
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.out.BrokerOuterAPI;
@@ -525,7 +525,7 @@ private boolean applyBrokerId() {
             return true;
 
         } catch (Exception e) {
-            LOGGER.error("fail to apply broker id: {}", e, tempBrokerMetadata.getBrokerId());
+            LOGGER.error("fail to apply broker id: {}", tempBrokerMetadata.getBrokerId(), e);
             return false;
         }
     }
@@ -686,7 +686,7 @@ private void schedulingSyncBrokerMetadata() {
     }
 
     /**
-     * Scheduling sync controller medata.
+     * Scheduling sync controller metadata.
      */
     private boolean schedulingSyncControllerMetadata() {
         // Get controller metadata first.
@@ -803,7 +803,10 @@ private void scanAvailableControllerAddresses() {
 
     private void updateControllerAddr() {
         if (brokerConfig.isFetchControllerAddrByDnsLookup()) {
-            this.controllerAddresses = brokerOuterAPI.dnsLookupAddressByDomain(this.brokerConfig.getControllerAddr());
+            List<String> adders = brokerOuterAPI.dnsLookupAddressByDomain(this.brokerConfig.getControllerAddr());
+            if (CollectionUtils.isNotEmpty(adders)) {
+                this.controllerAddresses = adders;
+            }
         } else {
             final String controllerPaths = this.brokerConfig.getControllerAddr();
             final String[] controllers = controllerPaths.split(";");
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java
index 6a081748014..dd37f42b2c5 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java
@@ -25,7 +25,9 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Triple;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil;
 import org.apache.rocketmq.client.consumer.PullStatus;
@@ -34,7 +36,6 @@
 import org.apache.rocketmq.client.producer.SendResult;
 import org.apache.rocketmq.client.producer.SendStatus;
 import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.common.Pair;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.message.MessageConst;
@@ -51,6 +52,7 @@
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.tieredstore.TieredMessageStore;
 
 public class EscapeBridge {
     protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -99,7 +101,7 @@ public PutMessageResult putMessage(MessageExtBrokerInner messageExt) {
 
             try {
                 messageExt.setWaitStoreMsgOK(false);
-                final SendResult sendResult = putMessageToRemoteBroker(messageExt);
+                final SendResult sendResult = putMessageToRemoteBroker(messageExt, null);
                 return transformSendResult2PutResult(sendResult);
             } catch (Exception e) {
                 LOG.error("sendMessageInFailover to remote failed", e);
@@ -112,7 +114,10 @@ public PutMessageResult putMessage(MessageExtBrokerInner messageExt) {
         }
     }
 
-    private SendResult putMessageToRemoteBroker(MessageExtBrokerInner messageExt) {
+    public SendResult putMessageToRemoteBroker(MessageExtBrokerInner messageExt, String brokerNameToSend) {
+        if (this.brokerController.getBrokerConfig().getBrokerName().equals(brokerNameToSend)) { // not remote broker
+            return null;
+        }
         final boolean isTransHalfMessage = TransactionalMessageUtil.buildHalfTopic().equals(messageExt.getTopic());
         MessageExtBrokerInner messageToPut = messageExt;
         if (isTransHalfMessage) {
@@ -125,12 +130,26 @@ private SendResult putMessageToRemoteBroker(MessageExtBrokerInner messageExt) {
             return null;
         }
 
-        final MessageQueue mqSelected = topicPublishInfo.selectOneMessageQueue();
-
-        messageToPut.setQueueId(mqSelected.getQueueId());
+        final MessageQueue mqSelected;
+        if (StringUtils.isEmpty(brokerNameToSend)) {
+            mqSelected = topicPublishInfo.selectOneMessageQueue(this.brokerController.getBrokerConfig().getBrokerName());
+            messageToPut.setQueueId(mqSelected.getQueueId());
+            brokerNameToSend = mqSelected.getBrokerName();
+            if (this.brokerController.getBrokerConfig().getBrokerName().equals(brokerNameToSend)) {
+                LOG.warn("putMessageToRemoteBroker failed, remote broker not found. Topic: {}, MsgId: {}, Broker: {}",
+                    messageExt.getTopic(), messageExt.getMsgId(), brokerNameToSend);
+                return null;
+            }
+        } else {
+            mqSelected = new MessageQueue(messageExt.getTopic(), brokerNameToSend, messageExt.getQueueId());
+        }
 
-        final String brokerNameToSend = mqSelected.getBrokerName();
         final String brokerAddrToSend = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInPublish(brokerNameToSend);
+        if (null == brokerAddrToSend) {
+            LOG.warn("putMessageToRemoteBroker failed, remote broker address not found. Topic: {}, MsgId: {}, Broker: {}",
+                messageExt.getTopic(), messageExt.getMsgId(), brokerNameToSend);
+            return null;
+        }
 
         final long beginTimestamp = System.currentTimeMillis();
         try {
@@ -178,7 +197,7 @@ public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner
                     producerGroup, SEND_TIMEOUT);
 
                 return future.exceptionally(throwable -> null)
-                    .thenApplyAsync(sendResult -> transformSendResult2PutResult(sendResult), this.defaultAsyncSenderExecutor)
+                    .thenApplyAsync(this::transformSendResult2PutResult, this.defaultAsyncSenderExecutor)
                     .exceptionally(throwable -> transformSendResult2PutResult(null));
 
             } catch (Exception e) {
@@ -192,7 +211,6 @@ public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner
         }
     }
 
-
     private String getProducerGroup(MessageExtBrokerInner messageExt) {
         if (null == messageExt) {
             return this.innerProducerGroupName;
@@ -204,12 +222,29 @@ private String getProducerGroup(MessageExtBrokerInner messageExt) {
         return producerGroup;
     }
 
-
     public PutMessageResult putMessageToSpecificQueue(MessageExtBrokerInner messageExt) {
         BrokerController masterBroker = this.brokerController.peekMasterBroker();
         if (masterBroker != null) {
             return masterBroker.getMessageStore().putMessage(messageExt);
-        } else if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster()
+        }
+        try {
+            return asyncRemotePutMessageToSpecificQueue(messageExt).get(SEND_TIMEOUT, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            LOG.error("Put message to specific queue error", e);
+            return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null, true);
+        }
+    }
+
+    public CompletableFuture<PutMessageResult> asyncPutMessageToSpecificQueue(MessageExtBrokerInner messageExt) {
+        BrokerController masterBroker = this.brokerController.peekMasterBroker();
+        if (masterBroker != null) {
+            return masterBroker.getMessageStore().asyncPutMessage(messageExt);
+        }
+        return asyncRemotePutMessageToSpecificQueue(messageExt);
+    }
+
+    public CompletableFuture<PutMessageResult> asyncRemotePutMessageToSpecificQueue(MessageExtBrokerInner messageExt) {
+        if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster()
             && this.brokerController.getBrokerConfig().isEnableRemoteEscape()) {
             try {
                 messageExt.setWaitStoreMsgOK(false);
@@ -218,7 +253,7 @@ public PutMessageResult putMessageToSpecificQueue(MessageExtBrokerInner messageE
                 List<MessageQueue> mqs = topicPublishInfo.getMessageQueueList();
 
                 if (null == mqs || mqs.isEmpty()) {
-                    return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true);
+                    return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true));
                 }
 
                 String id = messageExt.getTopic() + messageExt.getStoreHost();
@@ -229,19 +264,17 @@ public PutMessageResult putMessageToSpecificQueue(MessageExtBrokerInner messageE
 
                 String brokerNameToSend = mq.getBrokerName();
                 String brokerAddrToSend = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInPublish(brokerNameToSend);
-                final SendResult sendResult = this.brokerController.getBrokerOuterAPI().sendMessageToSpecificBroker(
+                return this.brokerController.getBrokerOuterAPI().sendMessageToSpecificBrokerAsync(
                     brokerAddrToSend, brokerNameToSend,
-                    messageExt, this.getProducerGroup(messageExt), SEND_TIMEOUT);
-
-                return transformSendResult2PutResult(sendResult);
+                    messageExt, this.getProducerGroup(messageExt), SEND_TIMEOUT).thenCompose(sendResult -> CompletableFuture.completedFuture(transformSendResult2PutResult(sendResult)));
             } catch (Exception e) {
                 LOG.error("sendMessageInFailover to remote failed", e);
-                return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true);
+                return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true));
             }
         } else {
             LOG.warn("Put message to specific queue failed, enableSlaveActingMaster={}, enableRemoteEscape={}.",
                 this.brokerController.getBrokerConfig().isEnableSlaveActingMaster(), this.brokerController.getBrokerConfig().isEnableRemoteEscape());
-            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);
+            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null));
         }
     }
 
@@ -263,34 +296,35 @@ private PutMessageResult transformSendResult2PutResult(SendResult sendResult) {
         }
     }
 
-    public Pair<GetMessageStatus, MessageExt> getMessage(String topic, long offset, int queueId, String brokerName, boolean deCompressBody) {
+    public Triple<MessageExt, String, Boolean> getMessage(String topic, long offset, int queueId, String brokerName,
+        boolean deCompressBody) {
         return getMessageAsync(topic, offset, queueId, brokerName, deCompressBody).join();
     }
 
-    public CompletableFuture<Pair<GetMessageStatus, MessageExt>> getMessageAsync(String topic, long offset, int queueId, String brokerName, boolean deCompressBody) {
+    // Triple<MessageExt, info, needRetry>, check info and retry if and only if MessageExt is null
+    public CompletableFuture<Triple<MessageExt, String, Boolean>> getMessageAsync(String topic, long offset,
+        int queueId, String brokerName, boolean deCompressBody) {
         MessageStore messageStore = brokerController.getMessageStoreByBrokerName(brokerName);
         if (messageStore != null) {
             return messageStore.getMessageAsync(innerConsumerGroupName, topic, queueId, offset, 1, null)
                 .thenApply(result -> {
                     if (result == null) {
                         LOG.warn("getMessageResult is null , innerConsumerGroupName {}, topic {}, offset {}, queueId {}", innerConsumerGroupName, topic, offset, queueId);
-                        return new Pair<>(GetMessageStatus.MESSAGE_WAS_REMOVING, null);
+                        return Triple.of(null, "getMessageResult is null", false); // local store, so no retry
                     }
                     List<MessageExt> list = decodeMsgList(result, deCompressBody);
                     if (list == null || list.isEmpty()) {
-                        LOG.warn("Can not get msg , topic {}, offset {}, queueId {}, result is {}", topic, offset, queueId, result);
-                        return new Pair<>(result.getStatus(), null);
+                        // OFFSET_FOUND_NULL returned by TieredMessageStore indicates exception occurred
+                        boolean needRetry = GetMessageStatus.OFFSET_FOUND_NULL.equals(result.getStatus())
+                            && messageStore instanceof TieredMessageStore;
+                        LOG.warn("Can not get msg , topic {}, offset {}, queueId {}, needRetry {}, result is {}",
+                            topic, offset, queueId, needRetry, result);
+                        return Triple.of(null, "Can not get msg", needRetry);
                     }
-                    return new Pair<>(result.getStatus(), list.get(0));
+                    return Triple.of(list.get(0), "", false);
                 });
         } else {
-            return getMessageFromRemoteAsync(topic, offset, queueId, brokerName)
-                .thenApply(msg -> {
-                    if (msg == null) {
-                        return new Pair<>(GetMessageStatus.MESSAGE_WAS_REMOVING, null);
-                    }
-                    return new Pair<>(GetMessageStatus.FOUND, msg);
-                });
+            return getMessageFromRemoteAsync(topic, offset, queueId, brokerName);
         }
     }
 
@@ -322,11 +356,14 @@ protected List<MessageExt> decodeMsgList(GetMessageResult getMessageResult, bool
         return foundList;
     }
 
-    protected MessageExt getMessageFromRemote(String topic, long offset, int queueId, String brokerName) {
+    protected Triple<MessageExt, String, Boolean> getMessageFromRemote(String topic, long offset, int queueId,
+        String brokerName) {
         return getMessageFromRemoteAsync(topic, offset, queueId, brokerName).join();
     }
 
-    protected CompletableFuture<MessageExt> getMessageFromRemoteAsync(String topic, long offset, int queueId, String brokerName) {
+    // Triple<MessageExt, info, needRetry>, check info and retry if and only if MessageExt is null
+    protected CompletableFuture<Triple<MessageExt, String, Boolean>> getMessageFromRemoteAsync(String topic,
+        long offset, int queueId, String brokerName) {
         try {
             String brokerAddr = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, false);
             if (null == brokerAddr) {
@@ -334,23 +371,25 @@ protected CompletableFuture<MessageExt> getMessageFromRemoteAsync(String topic,
                 brokerAddr = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, false);
 
                 if (null == brokerAddr) {
-                    LOG.warn("can't find broker address for topic {}", topic);
-                    return CompletableFuture.completedFuture(null);
+                    LOG.warn("can't find broker address for topic {}, {}", topic, brokerName);
+                    return CompletableFuture.completedFuture(Triple.of(null, "brokerAddress not found", true)); // maybe offline temporarily, so need retry
                 }
             }
 
             return this.brokerController.getBrokerOuterAPI().pullMessageFromSpecificBrokerAsync(brokerName,
-                brokerAddr, this.innerConsumerGroupName, topic, queueId, offset, 1, DEFAULT_PULL_TIMEOUT_MILLIS)
+                    brokerAddr, this.innerConsumerGroupName, topic, queueId, offset, 1, DEFAULT_PULL_TIMEOUT_MILLIS)
                 .thenApply(pullResult -> {
-                    if (pullResult.getPullStatus().equals(PullStatus.FOUND) && !pullResult.getMsgFoundList().isEmpty()) {
-                        return pullResult.getMsgFoundList().get(0);
+                    if (pullResult.getLeft() != null
+                        && PullStatus.FOUND.equals(pullResult.getLeft().getPullStatus())
+                        && CollectionUtils.isNotEmpty(pullResult.getLeft().getMsgFoundList())) {
+                        return Triple.of(pullResult.getLeft().getMsgFoundList().get(0), "", false);
                     }
-                    return null;
+                    return Triple.of(null, pullResult.getMiddle(), pullResult.getRight());
                 });
         } catch (Exception e) {
-            LOG.error("Get message from remote failed.", e);
+            LOG.error("Get message from remote failed. {}, {}, {}, {}", topic, offset, queueId, brokerName, e);
         }
 
-        return CompletableFuture.completedFuture(null);
+        return CompletableFuture.completedFuture(Triple.of(null, "Get message from remote failed", true)); // need retry
     }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java
index 3b6e9dc676e..ce8fdd88579 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java
@@ -16,11 +16,15 @@
  */
 package org.apache.rocketmq.broker.latency;
 
+import java.util.List;
+import java.util.ArrayList;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.AbstractBrokerRunnable;
+import org.apache.rocketmq.common.Pair;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.constant.LoggerName;
@@ -42,13 +46,26 @@ public class BrokerFastFailure {
 
     private volatile long jstackTime = System.currentTimeMillis();
 
+    private final List<Pair<BlockingQueue<Runnable>, Supplier<Long>>> cleanExpiredRequestQueueList = new ArrayList<>();
+
     public BrokerFastFailure(final BrokerController brokerController) {
         this.brokerController = brokerController;
+        initCleanExpiredRequestQueueList();
         this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,
             new ThreadFactoryImpl("BrokerFastFailureScheduledThread", true,
                 brokerController == null ? null : brokerController.getBrokerConfig()));
     }
 
+    private void initCleanExpiredRequestQueueList() {
+        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getSendThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInSendQueue()));
+        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getPullThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue()));
+        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getLitePullThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInLitePullQueue()));
+        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getHeartbeatThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInHeartbeatQueue()));
+        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getEndTransactionThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInTransactionQueue()));
+        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getAckThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInAckQueue()));
+        cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getAdminBrokerThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInAdminBrokerQueue()));
+    }
+
     public static RequestTask castRunnable(final Runnable runnable) {
         try {
             if (runnable instanceof FutureTaskExt) {
@@ -98,23 +115,9 @@ private void cleanExpiredRequest() {
             }
         }
 
-        cleanExpiredRequestInQueue(this.brokerController.getSendThreadPoolQueue(),
-            this.brokerController.getBrokerConfig().getWaitTimeMillsInSendQueue());
-
-        cleanExpiredRequestInQueue(this.brokerController.getPullThreadPoolQueue(),
-            this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue());
-
-        cleanExpiredRequestInQueue(this.brokerController.getLitePullThreadPoolQueue(),
-            this.brokerController.getBrokerConfig().getWaitTimeMillsInLitePullQueue());
-
-        cleanExpiredRequestInQueue(this.brokerController.getHeartbeatThreadPoolQueue(),
-            this.brokerController.getBrokerConfig().getWaitTimeMillsInHeartbeatQueue());
-
-        cleanExpiredRequestInQueue(this.brokerController.getEndTransactionThreadPoolQueue(), this
-            .brokerController.getBrokerConfig().getWaitTimeMillsInTransactionQueue());
-
-        cleanExpiredRequestInQueue(this.brokerController.getAckThreadPoolQueue(),
-            brokerController.getBrokerConfig().getWaitTimeMillsInAckQueue());
+        for (Pair<BlockingQueue<Runnable>, Supplier<Long>> pair : cleanExpiredRequestQueueList) {
+            cleanExpiredRequestInQueue(pair.getObject1(), pair.getObject2().get());
+        }
     }
 
     void cleanExpiredRequestInQueue(final BlockingQueue<Runnable> blockingQueue, final long maxWaitTimeMillsInQueue) {
@@ -151,6 +154,11 @@ void cleanExpiredRequestInQueue(final BlockingQueue<Runnable> blockingQueue, fin
         }
     }
 
+    public synchronized void addCleanExpiredRequestQueue(BlockingQueue<Runnable> cleanExpiredRequestQueue,
+        Supplier<Long> maxWaitTimeMillsInQueueSupplier) {
+        cleanExpiredRequestQueueList.add(new Pair<>(cleanExpiredRequestQueue, maxWaitTimeMillsInQueueSupplier));
+    }
+
     public void shutdown() {
         this.scheduledExecutorService.shutdown();
     }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java
index 88e74fd6e5a..eddaee706a9 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java
@@ -22,7 +22,6 @@
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 
-
 public class LmqPullRequestHoldService extends PullRequestHoldService {
     private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
 
@@ -48,8 +47,8 @@ public void checkHoldRequest() {
             }
             String topic = key.substring(0, idx);
             int queueId = Integer.parseInt(key.substring(idx + 1));
-            final long offset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
             try {
+                final long offset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
                 this.notifyMessageArriving(topic, queueId, offset);
             } catch (Throwable e) {
                 LOGGER.error("check hold request failed. topic={}, queueId={}", topic, queueId, e);
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java
index e55ed2778ac..9c0ee89e4db 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java
@@ -36,9 +36,12 @@ public NotifyMessageArrivingListener(final PullRequestHoldService pullRequestHol
     @Override
     public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
                          long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
-        this.pullRequestHoldService.notifyMessageArriving(topic, queueId, logicOffset, tagsCode,
-            msgStoreTime, filterBitMap, properties);
-        this.popMessageProcessor.notifyMessageArriving(topic, queueId);
-        this.notificationProcessor.notifyMessageArriving(topic, queueId);
+
+        this.pullRequestHoldService.notifyMessageArriving(
+            topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties);
+        this.popMessageProcessor.notifyMessageArriving(
+            topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties);
+        this.notificationProcessor.notifyMessageArriving(
+            topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties);
     }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopCommandCallback.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopCommandCallback.java
new file mode 100644
index 00000000000..2e190e20f92
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopCommandCallback.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.longpolling;
+
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import org.apache.rocketmq.broker.metrics.ConsumerLagCalculator;
+import org.apache.rocketmq.remoting.CommandCallback;
+
+public class PopCommandCallback implements CommandCallback {
+
+    private final BiConsumer<ConsumerLagCalculator.ProcessGroupInfo,
+        Consumer<ConsumerLagCalculator.CalculateLagResult>> biConsumer;
+
+    private final ConsumerLagCalculator.ProcessGroupInfo info;
+    private final Consumer<ConsumerLagCalculator.CalculateLagResult> lagRecorder;
+
+
+    public PopCommandCallback(
+        BiConsumer<ConsumerLagCalculator.ProcessGroupInfo,
+                    Consumer<ConsumerLagCalculator.CalculateLagResult>> biConsumer,
+        ConsumerLagCalculator.ProcessGroupInfo info,
+        Consumer<ConsumerLagCalculator.CalculateLagResult> lagRecorder) {
+
+        this.biConsumer = biConsumer;
+        this.info = info;
+        this.lagRecorder = lagRecorder;
+    }
+
+    @Override
+    public void accept() {
+        biConsumer.accept(info, lagRecorder);
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java
index f1bc9adc463..e87a8e803fd 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java
@@ -19,6 +19,7 @@
 
 import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
 import io.netty.channel.ChannelHandlerContext;
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -31,10 +32,14 @@
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.CommandCallback;
 import org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;
 import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.netty.RequestTask;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.store.ConsumeQueueExt;
+import org.apache.rocketmq.store.MessageFilter;
 
 import static org.apache.rocketmq.broker.longpolling.PollingResult.NOT_POLLING;
 import static org.apache.rocketmq.broker.longpolling.PollingResult.POLLING_FULL;
@@ -42,23 +47,27 @@
 import static org.apache.rocketmq.broker.longpolling.PollingResult.POLLING_TIMEOUT;
 
 public class PopLongPollingService extends ServiceThread {
+
     private static final Logger POP_LOGGER =
         LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
     private final BrokerController brokerController;
     private final NettyRequestProcessor processor;
-    private final ConcurrentHashMap<String, ConcurrentHashMap<String, Byte>> topicCidMap;
+    private final ConcurrentLinkedHashMap<String, ConcurrentHashMap<String, Byte>> topicCidMap;
     private final ConcurrentLinkedHashMap<String, ConcurrentSkipListSet<PopRequest>> pollingMap;
     private long lastCleanTime = 0;
 
     private final AtomicLong totalPollingNum = new AtomicLong(0);
+    private final boolean notifyLast;
 
-    public PopLongPollingService(BrokerController brokerController, NettyRequestProcessor processor) {
+    public PopLongPollingService(BrokerController brokerController, NettyRequestProcessor processor, boolean notifyLast) {
         this.brokerController = brokerController;
         this.processor = processor;
         // 100000 topic default,  100000 lru topic + cid + qid
-        this.topicCidMap = new ConcurrentHashMap<>(brokerController.getBrokerConfig().getPopPollingMapSize());
+        this.topicCidMap = new ConcurrentLinkedHashMap.Builder<String, ConcurrentHashMap<String, Byte>>()
+            .maximumWeightedCapacity(this.brokerController.getBrokerConfig().getPopPollingMapSize() * 2L).build();
         this.pollingMap = new ConcurrentLinkedHashMap.Builder<String, ConcurrentSkipListSet<PopRequest>>()
             .maximumWeightedCapacity(this.brokerController.getBrokerConfig().getPopPollingMapSize()).build();
+        this.notifyLast = notifyLast;
     }
 
     @Override
@@ -145,57 +154,98 @@ public void run() {
     }
 
     public void notifyMessageArrivingWithRetryTopic(final String topic, final int queueId) {
+        this.notifyMessageArrivingWithRetryTopic(topic, queueId, -1L, null, 0L, null, null);
+    }
+
+    public void notifyMessageArrivingWithRetryTopic(final String topic, final int queueId, long offset,
+        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
         String notifyTopic;
         if (KeyBuilder.isPopRetryTopicV2(topic)) {
             notifyTopic = KeyBuilder.parseNormalTopic(topic);
         } else {
             notifyTopic = topic;
         }
-        notifyMessageArriving(notifyTopic, queueId);
+        notifyMessageArriving(notifyTopic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);
     }
 
-    public void notifyMessageArriving(final String topic, final int queueId) {
+    public void notifyMessageArriving(final String topic, final int queueId, long offset,
+        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
         ConcurrentHashMap<String, Byte> cids = topicCidMap.get(topic);
         if (cids == null) {
             return;
         }
+        long interval = brokerController.getBrokerConfig().getPopLongPollingForceNotifyInterval();
+        boolean force = interval > 0L && offset % interval == 0L;
         for (Map.Entry<String, Byte> cid : cids.entrySet()) {
             if (queueId >= 0) {
-                notifyMessageArriving(topic, cid.getKey(), -1);
+                notifyMessageArriving(topic, -1, cid.getKey(), force, tagsCode, msgStoreTime, filterBitMap, properties);
             }
-            notifyMessageArriving(topic, cid.getKey(), queueId);
+            notifyMessageArriving(topic, queueId, cid.getKey(), force, tagsCode, msgStoreTime, filterBitMap, properties);
         }
     }
 
-    public boolean notifyMessageArriving(final String topic, final String cid, final int queueId) {
+    public boolean notifyMessageArriving(final String topic, final int queueId, final String cid,
+        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
+        return notifyMessageArriving(topic, queueId, cid, false, tagsCode, msgStoreTime, filterBitMap, properties, null);
+    }
+
+    public boolean notifyMessageArriving(final String topic, final int queueId, final String cid, boolean force,
+        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
+        return notifyMessageArriving(topic, queueId, cid, force, tagsCode, msgStoreTime, filterBitMap, properties, null);
+    }
+
+    public boolean notifyMessageArriving(final String topic, final int queueId, final String cid, boolean force,
+        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties, CommandCallback callback) {
         ConcurrentSkipListSet<PopRequest> remotingCommands = pollingMap.get(KeyBuilder.buildPollingKey(topic, cid, queueId));
         if (remotingCommands == null || remotingCommands.isEmpty()) {
             return false;
         }
-        PopRequest popRequest = remotingCommands.pollFirst();
-        //clean inactive channel
-        while (popRequest != null && !popRequest.getChannel().isActive()) {
-            totalPollingNum.decrementAndGet();
-            popRequest = remotingCommands.pollFirst();
-        }
 
+        PopRequest popRequest = pollRemotingCommands(remotingCommands);
         if (popRequest == null) {
             return false;
         }
-        totalPollingNum.decrementAndGet();
+
+        if (!force && popRequest.getMessageFilter() != null && popRequest.getSubscriptionData() != null) {
+            boolean match = popRequest.getMessageFilter().isMatchedByConsumeQueue(tagsCode,
+                new ConsumeQueueExt.CqExtUnit(tagsCode, msgStoreTime, filterBitMap));
+            if (match && properties != null) {
+                match = popRequest.getMessageFilter().isMatchedByCommitLog(null, properties);
+            }
+            if (!match) {
+                remotingCommands.add(popRequest);
+                totalPollingNum.incrementAndGet();
+                return false;
+            }
+        }
+
         if (brokerController.getBrokerConfig().isEnablePopLog()) {
-            POP_LOGGER.info("lock release , new msg arrive , wakeUp : {}", popRequest);
+            POP_LOGGER.info("lock release, new msg arrive, wakeUp: {}", popRequest);
         }
-        return wakeUp(popRequest);
+
+        return wakeUp(popRequest, callback);
     }
 
     public boolean wakeUp(final PopRequest request) {
+        return wakeUp(request, null);
+    }
+
+    public boolean wakeUp(final PopRequest request, CommandCallback callback) {
         if (request == null || !request.complete()) {
             return false;
         }
+
+        if (callback != null && request.getRemotingCommand() != null) {
+            if (request.getRemotingCommand().getCallbackList() == null) {
+                request.getRemotingCommand().setCallbackList(new ArrayList<>());
+            }
+            request.getRemotingCommand().getCallbackList().add(callback);
+        }
+
         if (!request.getCtx().channel().isActive()) {
             return false;
         }
+
         Runnable run = () -> {
             try {
                 final RemotingCommand response = processor.processRequest(request.getCtx(), request.getRemotingCommand());
@@ -214,7 +264,9 @@ public boolean wakeUp(final PopRequest request) {
                 POP_LOGGER.error("ExecuteRequestWhenWakeup run", e1);
             }
         };
-        this.brokerController.getPullMessageExecutor().submit(new RequestTask(run, request.getChannel(), request.getRemotingCommand()));
+
+        this.brokerController.getPullMessageExecutor().submit(
+            new RequestTask(run, request.getChannel(), request.getRemotingCommand()));
         return true;
     }
 
@@ -226,6 +278,11 @@ public boolean wakeUp(final PopRequest request) {
      */
     public PollingResult polling(final ChannelHandlerContext ctx, RemotingCommand remotingCommand,
         final PollingHeader requestHeader) {
+        return this.polling(ctx, remotingCommand, requestHeader, null, null);
+    }
+
+    public PollingResult polling(final ChannelHandlerContext ctx, RemotingCommand remotingCommand,
+        final PollingHeader requestHeader, SubscriptionData subscriptionData, MessageFilter messageFilter) {
         if (requestHeader.getPollTime() <= 0 || this.isStopped()) {
             return NOT_POLLING;
         }
@@ -239,7 +296,7 @@ public PollingResult polling(final ChannelHandlerContext ctx, RemotingCommand re
         }
         cids.putIfAbsent(requestHeader.getConsumerGroup(), Byte.MIN_VALUE);
         long expired = requestHeader.getBornTime() + requestHeader.getPollTime();
-        final PopRequest request = new PopRequest(remotingCommand, ctx, expired);
+        final PopRequest request = new PopRequest(remotingCommand, ctx, expired, subscriptionData, messageFilter);
         boolean isFull = totalPollingNum.get() >= this.brokerController.getBrokerConfig().getMaxPopPollingSize();
         if (isFull) {
             POP_LOGGER.info("polling {}, result POLLING_FULL, total:{}", remotingCommand, totalPollingNum.get());
@@ -294,7 +351,7 @@ private void cleanUnusedResource() {
                     Map.Entry<String, ConcurrentHashMap<String, Byte>> entry = topicCidMapIter.next();
                     String topic = entry.getKey();
                     if (brokerController.getTopicConfigManager().selectTopicConfig(topic) == null) {
-                        POP_LOGGER.info("remove not exit topic {} in topicCidMap!", topic);
+                        POP_LOGGER.info("remove nonexistent topic {} in topicCidMap!", topic);
                         topicCidMapIter.remove();
                         continue;
                     }
@@ -302,8 +359,8 @@ private void cleanUnusedResource() {
                     while (cidMapIter.hasNext()) {
                         Map.Entry<String, Byte> cidEntry = cidMapIter.next();
                         String cid = cidEntry.getKey();
-                        if (!brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().containsKey(cid)) {
-                            POP_LOGGER.info("remove not exit sub {} of topic {} in topicCidMap!", cid, topic);
+                        if (!brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(cid)) {
+                            POP_LOGGER.info("remove nonexistent subscription group {} of topic {} in topicCidMap!", cid, topic);
                             cidMapIter.remove();
                         }
                     }
@@ -324,12 +381,12 @@ private void cleanUnusedResource() {
                     String topic = keyArray[0];
                     String cid = keyArray[1];
                     if (brokerController.getTopicConfigManager().selectTopicConfig(topic) == null) {
-                        POP_LOGGER.info("remove not exit topic {} in pollingMap!", topic);
+                        POP_LOGGER.info("remove nonexistent topic {} in pollingMap!", topic);
                         pollingMapIter.remove();
                         continue;
                     }
-                    if (!brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().containsKey(cid)) {
-                        POP_LOGGER.info("remove not exit sub {} of topic {} in pollingMap!", cid, topic);
+                    if (!brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(cid)) {
+                        POP_LOGGER.info("remove nonexistent subscription group {} of topic {} in pollingMap!", cid, topic);
                         pollingMapIter.remove();
                     }
                 }
@@ -340,4 +397,22 @@ private void cleanUnusedResource() {
 
         lastCleanTime = System.currentTimeMillis();
     }
+
+    private PopRequest pollRemotingCommands(ConcurrentSkipListSet<PopRequest> remotingCommands) {
+        if (remotingCommands == null || remotingCommands.isEmpty()) {
+            return null;
+        }
+
+        PopRequest popRequest;
+        do {
+            if (notifyLast) {
+                popRequest = remotingCommands.pollLast();
+            } else {
+                popRequest = remotingCommands.pollFirst();
+            }
+            totalPollingNum.decrementAndGet();
+        } while (popRequest != null && !popRequest.getChannel().isActive());
+
+        return popRequest;
+    }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java
index a45bcce9f60..0419dbf637d 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java
@@ -16,28 +16,35 @@
  */
 package org.apache.rocketmq.broker.longpolling;
 
+import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
 import java.util.Comparator;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
-
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-
-import io.netty.channel.Channel;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.store.MessageFilter;
 
 public class PopRequest {
     private static final AtomicLong COUNTER = new AtomicLong(Long.MIN_VALUE);
 
     private final RemotingCommand remotingCommand;
     private final ChannelHandlerContext ctx;
-    private final long expired;
     private final AtomicBoolean complete = new AtomicBoolean(false);
     private final long op = COUNTER.getAndIncrement();
 
-    public PopRequest(RemotingCommand remotingCommand, ChannelHandlerContext ctx, long expired) {
+    private final long expired;
+    private final SubscriptionData subscriptionData;
+    private final MessageFilter messageFilter;
+
+    public PopRequest(RemotingCommand remotingCommand, ChannelHandlerContext ctx,
+        long expired, SubscriptionData subscriptionData, MessageFilter messageFilter) {
+
         this.ctx = ctx;
         this.remotingCommand = remotingCommand;
         this.expired = expired;
+        this.subscriptionData = subscriptionData;
+        this.messageFilter = messageFilter;
     }
 
     public Channel getChannel() {
@@ -64,6 +71,14 @@ public long getExpired() {
         return expired;
     }
 
+    public SubscriptionData getSubscriptionData() {
+        return subscriptionData;
+    }
+
+    public MessageFilter getMessageFilter() {
+        return messageFilter;
+    }
+
     @Override
     public String toString() {
         final StringBuilder sb = new StringBuilder("PopRequest{");
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
index e8da9d0c47c..7dbc9e4fd86 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
@@ -28,6 +28,7 @@
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.store.ConsumeQueueExt;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 
 public class PullRequestHoldService extends ServiceThread {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -103,8 +104,8 @@ protected void checkHoldRequest() {
             if (2 == kArray.length) {
                 String topic = kArray[0];
                 int queueId = Integer.parseInt(kArray[1]);
-                final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
                 try {
+                    final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
                     this.notifyMessageArriving(topic, queueId, offset);
                 } catch (Throwable e) {
                     log.error(
@@ -131,7 +132,12 @@ public void notifyMessageArriving(final String topic, final int queueId, final l
                 for (PullRequest request : requestList) {
                     long newestOffset = maxOffset;
                     if (newestOffset <= request.getPullFromThisOffset()) {
-                        newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
+                        try {
+                            newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
+                        } catch (ConsumeQueueException e) {
+                            log.error("Failed tp get max offset in queue", e);
+                            continue;
+                        }
                     }
 
                     if (newestOffset > request.getPullFromThisOffset()) {
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java
index 5733aa40bac..4b319f12f6f 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java
@@ -21,12 +21,16 @@ public class BrokerMetricsConstant {
 
     public static final String GAUGE_PROCESSOR_WATERMARK = "rocketmq_processor_watermark";
     public static final String GAUGE_BROKER_PERMISSION = "rocketmq_broker_permission";
+    public static final String GAUGE_TOPIC_NUM = "rocketmq_topic_number";
+    public static final String GAUGE_CONSUMER_GROUP_NUM = "rocketmq_consumer_group_number";
 
     public static final String COUNTER_MESSAGES_IN_TOTAL = "rocketmq_messages_in_total";
     public static final String COUNTER_MESSAGES_OUT_TOTAL = "rocketmq_messages_out_total";
     public static final String COUNTER_THROUGHPUT_IN_TOTAL = "rocketmq_throughput_in_total";
     public static final String COUNTER_THROUGHPUT_OUT_TOTAL = "rocketmq_throughput_out_total";
     public static final String HISTOGRAM_MESSAGE_SIZE = "rocketmq_message_size";
+    public static final String HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME = "rocketmq_topic_create_execution_time";
+    public static final String HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME = "rocketmq_consumer_group_create_execution_time";
 
     public static final String GAUGE_PRODUCER_CONNECTIONS = "rocketmq_producer_connections";
     public static final String GAUGE_CONSUMER_CONNECTIONS = "rocketmq_consumer_connections";
@@ -52,6 +56,7 @@ public class BrokerMetricsConstant {
     public static final String LABEL_PROCESSOR = "processor";
 
     public static final String LABEL_TOPIC = "topic";
+    public static final String LABEL_INVOCATION_STATUS = "invocation_status";
     public static final String LABEL_IS_RETRY = "is_retry";
     public static final String LABEL_IS_SYSTEM = "is_system";
     public static final String LABEL_CONSUMER_GROUP = "consumer_group";
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java
index fc7e97bda95..d8d94f8e69a 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java
@@ -64,6 +64,7 @@
 import org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant;
 import org.slf4j.bridge.SLF4JBridgeHandler;
 
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -80,6 +81,8 @@
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_ROLLBACK_MESSAGES_TOTAL;
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_THROUGHPUT_IN_TOTAL;
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_THROUGHPUT_OUT_TOTAL;
+import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_TOPIC_NUM;
+import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_GROUP_NUM;
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_BROKER_PERMISSION;
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_CONNECTIONS;
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_INFLIGHT_MESSAGES;
@@ -92,6 +95,8 @@
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_PRODUCER_CONNECTIONS;
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_FINISH_MSG_LATENCY;
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_MESSAGE_SIZE;
+import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME;
+import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME;
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_AGGREGATION;
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CLUSTER_NAME;
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;
@@ -128,6 +133,9 @@ public class BrokerMetricsManager {
     // broker stats metrics
     public static ObservableLongGauge processorWatermark = new NopObservableLongGauge();
     public static ObservableLongGauge brokerPermission = new NopObservableLongGauge();
+    public static ObservableLongGauge topicNum = new NopObservableLongGauge();
+    public static ObservableLongGauge consumerGroupNum = new NopObservableLongGauge();
+
 
     // request metrics
     public static LongCounter messagesInTotal = new NopLongCounter();
@@ -135,6 +143,8 @@ public class BrokerMetricsManager {
     public static LongCounter throughputInTotal = new NopLongCounter();
     public static LongCounter throughputOutTotal = new NopLongCounter();
     public static LongHistogram messageSize = new NopLongHistogram();
+    public static LongHistogram topicCreateExecuteTime = new NopLongHistogram();
+    public static LongHistogram consumerGroupCreateExecuteTime = new NopLongHistogram();
 
     // client connection metrics
     public static ObservableLongGauge producerConnection = new NopObservableLongGauge();
@@ -381,6 +391,14 @@ private void registerMetricsView(SdkMeterProviderBuilder providerBuilder) {
                 1d * 12 * 60 * 60, //12h
                 1d * 24 * 60 * 60 //24h
         );
+
+        List<Double> createTimeBuckets = Arrays.asList(
+                (double) Duration.ofMillis(10).toMillis(), //10ms
+                (double) Duration.ofMillis(100).toMillis(), //100ms
+                (double) Duration.ofSeconds(1).toMillis(), //1s
+                (double) Duration.ofSeconds(3).toMillis(), //3s
+                (double) Duration.ofSeconds(5).toMillis() //5s
+        );
         InstrumentSelector messageSizeSelector = InstrumentSelector.builder()
             .setType(InstrumentType.HISTOGRAM)
             .setName(HISTOGRAM_MESSAGE_SIZE)
@@ -401,6 +419,24 @@ private void registerMetricsView(SdkMeterProviderBuilder providerBuilder) {
         SdkMeterProviderUtil.setCardinalityLimit(commitLatencyViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
         providerBuilder.registerView(commitLatencySelector, commitLatencyViewBuilder.build());
 
+        InstrumentSelector createTopicTimeSelector = InstrumentSelector.builder()
+                .setType(InstrumentType.HISTOGRAM)
+                .setName(HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME)
+                .build();
+        InstrumentSelector createSubGroupTimeSelector = InstrumentSelector.builder()
+                .setType(InstrumentType.HISTOGRAM)
+                .setName(HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME)
+                .build();
+        ViewBuilder createTopicTimeViewBuilder = View.builder()
+                .setAggregation(Aggregation.explicitBucketHistogram(createTimeBuckets));
+        ViewBuilder createSubGroupTimeViewBuilder = View.builder()
+                .setAggregation(Aggregation.explicitBucketHistogram(createTimeBuckets));
+        // To config the cardinalityLimit for openTelemetry metrics exporting.
+        SdkMeterProviderUtil.setCardinalityLimit(createTopicTimeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
+        providerBuilder.registerView(createTopicTimeSelector, createTopicTimeViewBuilder.build());
+        SdkMeterProviderUtil.setCardinalityLimit(createSubGroupTimeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
+        providerBuilder.registerView(createSubGroupTimeSelector, createSubGroupTimeViewBuilder.build());
+
         for (Pair<InstrumentSelector, ViewBuilder> selectorViewPair : RemotingMetricsManager.getMetricsView()) {
             ViewBuilder viewBuilder = selectorViewPair.getObject2();
             SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
@@ -459,6 +495,16 @@ private void initStatsMetrics() {
             .setDescription("Broker permission")
             .ofLongs()
             .buildWithCallback(measurement -> measurement.record(brokerConfig.getBrokerPermission(), newAttributesBuilder().build()));
+
+        topicNum = brokerMeter.gaugeBuilder(GAUGE_TOPIC_NUM)
+            .setDescription("Active topic number")
+            .ofLongs()
+            .buildWithCallback(measurement -> measurement.record(brokerController.getTopicConfigManager().getTopicConfigTable().size(), newAttributesBuilder().build()));
+
+        consumerGroupNum = brokerMeter.gaugeBuilder(GAUGE_CONSUMER_GROUP_NUM)
+            .setDescription("Active subscription group number")
+            .ofLongs()
+            .buildWithCallback(measurement -> measurement.record(brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().size(), newAttributesBuilder().build()));
     }
 
     private void initRequestMetrics() {
@@ -482,6 +528,18 @@ private void initRequestMetrics() {
             .setDescription("Incoming messages size")
             .ofLongs()
             .build();
+
+        topicCreateExecuteTime = brokerMeter.histogramBuilder(HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME)
+                .setDescription("The distribution of create topic time")
+                .ofLongs()
+                .setUnit("milliseconds")
+                .build();
+
+        consumerGroupCreateExecuteTime = brokerMeter.histogramBuilder(HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME)
+                .setDescription("The distribution of create subscription time")
+                .ofLongs()
+                .setUnit("milliseconds")
+                .build();
     }
 
     private void initConnectionMetrics() {
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java
index 1930d0dfcb6..35519c1d1cb 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java
@@ -26,6 +26,8 @@
 import org.apache.rocketmq.broker.filter.ConsumerFilterData;
 import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
 import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
+import org.apache.rocketmq.broker.longpolling.PopCommandCallback;
+import org.apache.rocketmq.broker.longpolling.PopLongPollingService;
 import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
 import org.apache.rocketmq.broker.processor.PopBufferMergeService;
 import org.apache.rocketmq.broker.processor.PopInflightMessageCounter;
@@ -48,8 +50,10 @@
 import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
 import org.apache.rocketmq.store.DefaultMessageFilter;
 import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 
 public class ConsumerLagCalculator {
+
     private final BrokerConfig brokerConfig;
     private final TopicConfigManager topicConfigManager;
     private final ConsumerManager consumerManager;
@@ -58,6 +62,7 @@ public class ConsumerLagCalculator {
     private final SubscriptionGroupManager subscriptionGroupManager;
     private final MessageStore messageStore;
     private final PopBufferMergeService popBufferMergeService;
+    private final PopLongPollingService popLongPollingService;
     private final PopInflightMessageCounter popInflightMessageCounter;
 
     private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -71,10 +76,11 @@ public ConsumerLagCalculator(BrokerController brokerController) {
         this.subscriptionGroupManager = brokerController.getSubscriptionGroupManager();
         this.messageStore = brokerController.getMessageStore();
         this.popBufferMergeService = brokerController.getPopMessageProcessor().getPopBufferMergeService();
+        this.popLongPollingService = brokerController.getPopMessageProcessor().getPopLongPollingService();
         this.popInflightMessageCounter = brokerController.getPopInflightMessageCounter();
     }
 
-    private static class ProcessGroupInfo {
+    public static class ProcessGroupInfo {
         public String group;
         public String topic;
         public boolean isPop;
@@ -210,16 +216,32 @@ public void calculateLag(Consumer<CalculateLagResult> lagRecorder) {
                 return;
             }
 
-            CalculateLagResult result = new CalculateLagResult(info.group, info.topic, false);
+            if (info.isPop && brokerConfig.isEnableNotifyBeforePopCalculateLag()) {
+                if (popLongPollingService.notifyMessageArriving(info.topic, -1, info.group,
+                    true, null, 0, null, null, new PopCommandCallback(this::calculate, info, lagRecorder))) {
+                    return;
+                }
+            }
+
+            calculate(info, lagRecorder);
+        });
+    }
 
+    public void calculate(ProcessGroupInfo info, Consumer<CalculateLagResult> lagRecorder) {
+        CalculateLagResult result = new CalculateLagResult(info.group, info.topic, false);
+        try {
             Pair<Long, Long> lag = getConsumerLagStats(info.group, info.topic, info.isPop);
             if (lag != null) {
                 result.lag = lag.getObject1();
                 result.earliestUnconsumedTimestamp = lag.getObject2();
             }
             lagRecorder.accept(result);
+        } catch (ConsumeQueueException e) {
+            LOGGER.error("Failed to get lag stats", e);
+        }
 
-            if (info.isPop) {
+        if (info.isPop) {
+            try {
                 Pair<Long, Long> retryLag = getConsumerLagStats(info.group, info.retryTopic, true);
 
                 result = new CalculateLagResult(info.group, info.topic, true);
@@ -228,29 +250,39 @@ public void calculateLag(Consumer<CalculateLagResult> lagRecorder) {
                     result.earliestUnconsumedTimestamp = retryLag.getObject2();
                 }
                 lagRecorder.accept(result);
+            } catch (ConsumeQueueException e) {
+                LOGGER.error("Failed to get lag stats", e);
             }
-        });
+        }
     }
 
     public void calculateInflight(Consumer<CalculateInflightResult> inflightRecorder) {
         processAllGroup(info -> {
             CalculateInflightResult result = new CalculateInflightResult(info.group, info.topic, false);
-            Pair<Long, Long> inFlight = getInFlightMsgStats(info.group, info.topic, info.isPop);
-            if (inFlight != null) {
-                result.inFlight = inFlight.getObject1();
-                result.earliestUnPulledTimestamp = inFlight.getObject2();
+            try {
+                Pair<Long, Long> inFlight = getInFlightMsgStats(info.group, info.topic, info.isPop);
+                if (inFlight != null) {
+                    result.inFlight = inFlight.getObject1();
+                    result.earliestUnPulledTimestamp = inFlight.getObject2();
+                }
+                inflightRecorder.accept(result);
+            } catch (ConsumeQueueException e) {
+                LOGGER.error("Failed to get inflight message stats", e);
             }
-            inflightRecorder.accept(result);
 
             if (info.isPop) {
-                Pair<Long, Long> retryInFlight = getInFlightMsgStats(info.group, info.retryTopic, true);
+                try {
+                    Pair<Long, Long> retryInFlight = getInFlightMsgStats(info.group, info.retryTopic, true);
 
-                result = new CalculateInflightResult(info.group, info.topic, true);
-                if (retryInFlight != null) {
-                    result.inFlight = retryInFlight.getObject1();
-                    result.earliestUnPulledTimestamp = retryInFlight.getObject2();
+                    result = new CalculateInflightResult(info.group, info.topic, true);
+                    if (retryInFlight != null) {
+                        result.inFlight = retryInFlight.getObject1();
+                        result.earliestUnPulledTimestamp = retryInFlight.getObject2();
+                    }
+                    inflightRecorder.accept(result);
+                } catch (ConsumeQueueException e) {
+                    LOGGER.error("Failed to get inflight message stats", e);
                 }
-                inflightRecorder.accept(result);
             }
         });
     }
@@ -259,20 +291,28 @@ public void calculateAvailable(Consumer<CalculateAvailableResult> availableRecor
         processAllGroup(info -> {
             CalculateAvailableResult result = new CalculateAvailableResult(info.group, info.topic, false);
 
-            result.available = getAvailableMsgCount(info.group, info.topic, info.isPop);
-            availableRecorder.accept(result);
+            try {
+                result.available = getAvailableMsgCount(info.group, info.topic, info.isPop);
+                availableRecorder.accept(result);
+            } catch (ConsumeQueueException e) {
+                LOGGER.error("Failed to get available message count", e);
+            }
 
-            if (info.isPop) {
-                long retryAvailable = getAvailableMsgCount(info.group, info.retryTopic, true);
 
-                result = new CalculateAvailableResult(info.group, info.topic, true);
-                result.available = retryAvailable;
-                availableRecorder.accept(result);
+            if (info.isPop) {
+                try {
+                    long retryAvailable = getAvailableMsgCount(info.group, info.retryTopic, true);
+                    result = new CalculateAvailableResult(info.group, info.topic, true);
+                    result.available = retryAvailable;
+                    availableRecorder.accept(result);
+                } catch (ConsumeQueueException e) {
+                    LOGGER.error("Failed to get available message count", e);
+                }
             }
         });
     }
 
-    public Pair<Long, Long> getConsumerLagStats(String group, String topic, boolean isPop) {
+    public Pair<Long, Long> getConsumerLagStats(String group, String topic, boolean isPop) throws ConsumeQueueException {
         long total = 0L;
         long earliestUnconsumedTimestamp = Long.MAX_VALUE;
 
@@ -295,16 +335,20 @@ public Pair<Long, Long> getConsumerLagStats(String group, String topic, boolean
             earliestUnconsumedTimestamp = 0L;
         }
 
+        LOGGER.debug("GetConsumerLagStats, topic={}, group={}, lag={}, latency={}", topic, group, total,
+            earliestUnconsumedTimestamp > 0 ? System.currentTimeMillis() - earliestUnconsumedTimestamp : 0);
+
         return new Pair<>(total, earliestUnconsumedTimestamp);
     }
 
-    public Pair<Long, Long> getConsumerLagStats(String group, String topic, int queueId, boolean isPop) {
+    public Pair<Long, Long> getConsumerLagStats(String group, String topic, int queueId, boolean isPop)
+        throws ConsumeQueueException {
         long brokerOffset = messageStore.getMaxOffsetInQueue(topic, queueId);
         if (brokerOffset < 0) {
             brokerOffset = 0;
         }
 
-        if (isPop) {
+        if (isPop && !brokerConfig.isPopConsumerKVServiceEnable()) {
             long pullOffset = popBufferMergeService.getLatestOffset(topic, group, queueId);
             if (pullOffset < 0) {
                 pullOffset = offsetManager.queryOffset(group, topic, queueId);
@@ -329,7 +373,7 @@ public Pair<Long, Long> getConsumerLagStats(String group, String topic, int queu
         return new Pair<>(lag, consumerStoreTimeStamp);
     }
 
-    public Pair<Long, Long> getInFlightMsgStats(String group, String topic, boolean isPop) {
+    public Pair<Long, Long> getInFlightMsgStats(String group, String topic, boolean isPop) throws ConsumeQueueException {
         long total = 0L;
         long earliestUnPulledTimestamp = Long.MAX_VALUE;
 
@@ -355,8 +399,9 @@ public Pair<Long, Long> getInFlightMsgStats(String group, String topic, boolean
         return new Pair<>(total, earliestUnPulledTimestamp);
     }
 
-    public Pair<Long, Long> getInFlightMsgStats(String group, String topic, int queueId, boolean isPop) {
-        if (isPop) {
+    public Pair<Long, Long> getInFlightMsgStats(String group, String topic, int queueId, boolean isPop)
+        throws ConsumeQueueException {
+        if (isPop && !brokerConfig.isPopConsumerKVServiceEnable()) {
             long inflight = popInflightMessageCounter.getGroupPopInFlightMessageNum(topic, group, queueId);
             long pullOffset = popBufferMergeService.getLatestOffset(topic, group, queueId);
             if (pullOffset < 0) {
@@ -384,7 +429,7 @@ public Pair<Long, Long> getInFlightMsgStats(String group, String topic, int queu
         return new Pair<>(inflight, pullStoreTimeStamp);
     }
 
-    public long getAvailableMsgCount(String group, String topic, boolean isPop) {
+    public long getAvailableMsgCount(String group, String topic, boolean isPop) throws ConsumeQueueException {
         long total = 0L;
 
         if (group == null || topic == null) {
@@ -403,21 +448,19 @@ public long getAvailableMsgCount(String group, String topic, boolean isPop) {
         return total;
     }
 
-    public long getAvailableMsgCount(String group, String topic, int queueId, boolean isPop) {
+    public long getAvailableMsgCount(String group, String topic, int queueId, boolean isPop)
+        throws ConsumeQueueException {
         long brokerOffset = messageStore.getMaxOffsetInQueue(topic, queueId);
         if (brokerOffset < 0) {
             brokerOffset = 0;
         }
 
         long pullOffset;
-        if (isPop) {
+        if (isPop && !brokerConfig.isPopConsumerKVServiceEnable()) {
             pullOffset = popBufferMergeService.getLatestOffset(topic, group, queueId);
             if (pullOffset < 0) {
                 pullOffset = offsetManager.queryOffset(group, topic, queueId);
             }
-            if (pullOffset < 0) {
-                pullOffset = brokerOffset;
-            }
         } else {
             pullOffset = offsetManager.queryPullOffset(group, topic, queueId);
         }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/InvocationStatus.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/InvocationStatus.java
new file mode 100644
index 00000000000..c7501e53d96
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/InvocationStatus.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.metrics;
+
+public enum InvocationStatus {
+    SUCCESS("success"),
+    FAILURE("failure");
+
+    private final String name;
+
+    InvocationStatus(String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+}
\ No newline at end of file
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java
index 2de220da166..6e87cb0b69e 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java
@@ -39,8 +39,11 @@
 import org.apache.rocketmq.common.metrics.NopLongCounter;
 import org.apache.rocketmq.common.metrics.NopLongHistogram;
 import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.pop.AckMsg;
 import org.apache.rocketmq.store.pop.PopCheckPoint;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;
@@ -57,6 +60,7 @@
 import static org.apache.rocketmq.broker.metrics.PopMetricsConstant.LABEL_REVIVE_MESSAGE_TYPE;
 
 public class PopMetricsManager {
+    private static final Logger log = LoggerFactory.getLogger(PopMetricsManager.class);
     public static Supplier<AttributesBuilder> attributesBuilderSupplier;
 
     private static LongHistogram popBufferScanTimeConsume = new NopLongHistogram();
@@ -138,9 +142,13 @@ private static void calculatePopReviveLatency(BrokerController brokerController,
         ObservableLongMeasurement measurement) {
         PopReviveService[] popReviveServices = brokerController.getAckMessageProcessor().getPopReviveServices();
         for (PopReviveService popReviveService : popReviveServices) {
-            measurement.record(popReviveService.getReviveBehindMillis(), newAttributesBuilder()
-                .put(LABEL_QUEUE_ID, popReviveService.getQueueId())
-                .build());
+            try {
+                measurement.record(popReviveService.getReviveBehindMillis(), newAttributesBuilder()
+                    .put(LABEL_QUEUE_ID, popReviveService.getQueueId())
+                    .build());
+            } catch (ConsumeQueueException e) {
+                log.error("Failed to get revive behind duration", e);
+            }
         }
     }
 
@@ -148,9 +156,13 @@ private static void calculatePopReviveLag(BrokerController brokerController,
         ObservableLongMeasurement measurement) {
         PopReviveService[] popReviveServices = brokerController.getAckMessageProcessor().getPopReviveServices();
         for (PopReviveService popReviveService : popReviveServices) {
-            measurement.record(popReviveService.getReviveBehindMessages(), newAttributesBuilder()
-                .put(LABEL_QUEUE_ID, popReviveService.getQueueId())
-                .build());
+            try {
+                measurement.record(popReviveService.getReviveBehindMessages(), newAttributesBuilder()
+                    .put(LABEL_QUEUE_ID, popReviveService.getQueueId())
+                    .build());
+            } catch (ConsumeQueueException e) {
+                log.error("Failed to get revive behind message count", e);
+            }
         }
     }
 
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/ConsumeMessageContext.java b/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/ConsumeMessageContext.java
index ed7bfba06d6..e45f48fe5ac 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/ConsumeMessageContext.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/ConsumeMessageContext.java
@@ -46,6 +46,7 @@ public class ConsumeMessageContext {
     private BrokerStatsManager.StatsType commercialRcvStats;
     private int commercialRcvTimes;
     private int commercialRcvSize;
+    private int filterMessageCount;
 
     private String namespace;
     public String getConsumerGroup() {
@@ -231,4 +232,12 @@ public String getNamespace() {
     public void setNamespace(String namespace) {
         this.namespace = namespace;
     }
+
+    public int getFilterMessageCount() {
+        return filterMessageCount;
+    }
+
+    public void setFilterMessageCount(int filterMessageCount) {
+        this.filterMessageCount = filterMessageCount;
+    }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java
index 9896735dd1c..79bb0c771d6 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java
@@ -25,6 +25,7 @@
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 
 /**
  * manage the offset of broadcast.
@@ -72,7 +73,7 @@ public void updateOffset(String topic, String group, int queueId, long offset, S
      * @return -1 means no init offset, use the queueOffset in pullRequestHeader
      */
     public Long queryInitOffset(String topic, String groupId, int queueId, String clientId, long requestOffset,
-        boolean fromProxy) {
+        boolean fromProxy) throws ConsumeQueueException {
 
         BroadcastOffsetData broadcastOffsetData = offsetStoreMap.get(buildKey(topic, groupId));
         if (broadcastOffsetData == null) {
@@ -84,29 +85,26 @@ public Long queryInitOffset(String topic, String groupId, int queueId, String cl
         }
 
         final AtomicLong offset = new AtomicLong(-1L);
-        broadcastOffsetData.clientOffsetStore.compute(clientId, (clientIdK, offsetStore) -> {
-            if (offsetStore == null) {
-                offsetStore = new BroadcastTimedOffsetStore(fromProxy);
-            }
+        BroadcastTimedOffsetStore offsetStore = broadcastOffsetData.clientOffsetStore.get(clientId);
+        if (offsetStore == null) {
+            offsetStore = new BroadcastTimedOffsetStore(fromProxy);
+            broadcastOffsetData.clientOffsetStore.put(clientId, offsetStore);
+        }
 
-            if (offsetStore.fromProxy && requestOffset < 0) {
-                // when from proxy and requestOffset is -1
-                // means proxy need a init offset to pull message
+        if (offsetStore.fromProxy && requestOffset < 0) {
+            // when from proxy and requestOffset is -1
+            // means proxy need a init offset to pull message
+            offset.set(getOffset(offsetStore, topic, groupId, queueId));
+        } else {
+            if (offsetStore.fromProxy != fromProxy) {
                 offset.set(getOffset(offsetStore, topic, groupId, queueId));
-                return offsetStore;
-            }
-
-            if (offsetStore.fromProxy == fromProxy) {
-                return offsetStore;
             }
-
-            offset.set(getOffset(offsetStore, topic, groupId, queueId));
-            return offsetStore;
-        });
+        }
         return offset.get();
     }
 
-    private long getOffset(BroadcastTimedOffsetStore offsetStore, String topic, String groupId, int queueId) {
+    private long getOffset(BroadcastTimedOffsetStore offsetStore, String topic, String groupId, int queueId)
+        throws ConsumeQueueException {
         long storeOffset = -1;
         if (offsetStore != null) {
             storeOffset = offsetStore.offsetStore.readOffset(queueId);
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
index 21f20dde325..140604f5217 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
@@ -16,6 +16,7 @@
  */
 package org.apache.rocketmq.broker.offset;
 
+import com.google.common.collect.Maps;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -28,9 +29,11 @@
 
 import com.google.common.base.Strings;
 
+import java.util.function.Function;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.BrokerPathConfigHelper;
 import org.apache.rocketmq.common.ConfigManager;
+import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
@@ -42,7 +45,7 @@ public class ConsumerOffsetManager extends ConfigManager {
     protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     public static final String TOPIC_GROUP_SEPARATOR = "@";
 
-    private DataVersion dataVersion = new DataVersion();
+    protected DataVersion dataVersion = new DataVersion();
 
     protected ConcurrentMap<String/* topic@group */, ConcurrentMap<Integer, Long>> offsetTable =
         new ConcurrentHashMap<>(512);
@@ -280,6 +283,10 @@ public long queryPullOffset(final String group, final String topic, final int qu
         return offset;
     }
 
+    public void clearPullOffset(final String group, final String topic) {
+        this.pullOffsetTable.remove(topic + TOPIC_GROUP_SEPARATOR + group);
+    }
+
     @Override
     public String encode() {
         return this.encode(false);
@@ -373,22 +380,51 @@ public void setDataVersion(DataVersion dataVersion) {
         this.dataVersion = dataVersion;
     }
 
-    public void removeOffset(final String group) {
-        Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();
-        while (it.hasNext()) {
-            Entry<String, ConcurrentMap<Integer, Long>> next = it.next();
-            String topicAtGroup = next.getKey();
-            if (topicAtGroup.contains(group)) {
-                String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
-                if (arrays.length == 2 && group.equals(arrays[1])) {
-                    it.remove();
-                    removeConsumerOffset(topicAtGroup);
-                    LOG.warn("clean group offset {}", topicAtGroup);
+    public boolean loadDataVersion() {
+        String fileName = null;
+        try {
+            fileName = this.configFilePath();
+            String jsonString = MixAll.file2String(fileName);
+            if (jsonString != null) {
+                ConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, ConsumerOffsetManager.class);
+                if (obj != null) {
+                    this.dataVersion = obj.dataVersion;
                 }
+                LOG.info("load consumer offset dataVersion success,{},{} ", fileName, jsonString);
             }
+            return true;
+        } catch (Exception e) {
+            LOG.error("load consumer offset dataVersion failed " + fileName, e);
+            return false;
         }
     }
 
+    public void removeOffset(final String group) {
+        Function<Iterator<Entry<String, ConcurrentMap<Integer, Long>>>, Boolean> deleteFunction = it -> {
+            boolean removed = false;
+            while (it.hasNext()) {
+                Entry<String, ConcurrentMap<Integer, Long>> entry = it.next();
+                String topicAtGroup = entry.getKey();
+                if (topicAtGroup.contains(group)) {
+                    String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
+                    if (arrays.length == 2 && group.equals(arrays[1])) {
+                        it.remove();
+                        removeConsumerOffset(topicAtGroup);
+                        removed = true;
+                    }
+                }
+            }
+            return removed;
+        };
+
+        boolean clearOffset = deleteFunction.apply(this.offsetTable.entrySet().iterator());
+        boolean clearReset = deleteFunction.apply(this.resetOffsetTable.entrySet().iterator());
+        boolean clearPull = deleteFunction.apply(this.pullOffsetTable.entrySet().iterator());
+
+        LOG.info("Consumer offset manager clean group offset, groupName={}, " +
+            "offsetTable={}, resetOffsetTable={}, pullOffsetTable={}", group, clearOffset, clearReset, clearPull);
+    }
+
     public void assignResetOffset(String topic, String group, int queueId, long offset) {
         if (Strings.isNullOrEmpty(topic) || Strings.isNullOrEmpty(group) || queueId < 0 || offset < 0) {
             LOG.warn("Illegal arguments when assigning reset offset. Topic={}, group={}, queueId={}, offset={}",
@@ -397,27 +433,14 @@ public void assignResetOffset(String topic, String group, int queueId, long offs
         }
 
         String key = topic + TOPIC_GROUP_SEPARATOR + group;
-        ConcurrentMap<Integer, Long> map = resetOffsetTable.get(key);
-        if (null == map) {
-            map = new ConcurrentHashMap<Integer, Long>();
-            ConcurrentMap<Integer, Long> previous = resetOffsetTable.putIfAbsent(key, map);
-            if (null != previous) {
-                map = previous;
-            }
-        }
-
-        map.put(queueId, offset);
-        LOG.debug("Reset offset OK. Topic={}, group={}, queueId={}, resetOffset={}",
-            topic, group, queueId, offset);
+        resetOffsetTable.computeIfAbsent(key, k -> Maps.newConcurrentMap()).put(queueId, offset);
+        LOG.debug("Reset offset OK. Topic={}, group={}, queueId={}, resetOffset={}", topic, group, queueId, offset);
 
         // Two things are important here:
         // 1, currentOffsetMap might be null if there is no previous records;
         // 2, Our overriding here may get overridden by the client instantly in concurrent cases; But it still makes
         // sense in cases like clients are offline.
-        ConcurrentMap<Integer, Long> currentOffsetMap = offsetTable.get(key);
-        if (null != currentOffsetMap) {
-            currentOffsetMap.put(queueId, offset);
-        }
+        offsetTable.computeIfAbsent(key, k -> Maps.newConcurrentMap()).put(queueId, offset);
     }
 
     public boolean hasOffsetReset(String topic, String group, int queueId) {
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManager.java
index 4eccc6c0374..120f5b104c7 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManager.java
@@ -281,7 +281,7 @@ protected void autoClean() {
                 continue;
             }
 
-            if (this.brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().get(group) == null) {
+            if (!this.brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(group)) {
                 iterator.remove();
                 log.info("Group not exist, Clean order info, {}:{}", topicAtGroup, qs);
                 continue;
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java
index ce70b1a820f..53e9e2e0634 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java
@@ -17,6 +17,7 @@
 package org.apache.rocketmq.broker.offset;
 
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -110,4 +111,25 @@ public ConcurrentHashMap<String, Long> getLmqOffsetTable() {
     public void setLmqOffsetTable(ConcurrentHashMap<String, Long> lmqOffsetTable) {
         this.lmqOffsetTable = lmqOffsetTable;
     }
+
+    @Override
+    public void removeOffset(String group) {
+        if (!MixAll.isLmq(group)) {
+            super.removeOffset(group);
+            return;
+        }
+        Iterator<Map.Entry<String, Long>> it = this.lmqOffsetTable.entrySet().iterator();
+        while (it.hasNext()) {
+            Map.Entry<String, Long> next = it.next();
+            String topicAtGroup = next.getKey();
+            if (topicAtGroup.contains(group)) {
+                String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
+                if (arrays.length == 2 && group.equals(arrays[1])) {
+                    it.remove();
+                    removeConsumerOffset(topicAtGroup);
+                    LOG.warn("clean lmq group offset {}", topicAtGroup);
+                }
+            }
+        }
+    }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManager.java
deleted file mode 100644
index d0faa661406..00000000000
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManager.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.broker.offset;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.rocketmq.broker.BrokerController;
-import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
-
-public class RocksDBLmqConsumerOffsetManager extends RocksDBConsumerOffsetManager {
-    private ConcurrentHashMap<String, Long> lmqOffsetTable = new ConcurrentHashMap<>(512);
-
-    public RocksDBLmqConsumerOffsetManager(BrokerController brokerController) {
-        super(brokerController);
-    }
-
-    @Override
-    public long queryOffset(final String group, final String topic, final int queueId) {
-        if (!MixAll.isLmq(group)) {
-            return super.queryOffset(group, topic, queueId);
-        }
-        // topic@group
-        String key = topic + TOPIC_GROUP_SEPARATOR + group;
-        Long offset = lmqOffsetTable.get(key);
-        if (offset != null) {
-            return offset;
-        }
-        return -1;
-    }
-
-    @Override
-    public Map<Integer, Long> queryOffset(final String group, final String topic) {
-        if (!MixAll.isLmq(group)) {
-            return super.queryOffset(group, topic);
-        }
-        Map<Integer, Long> map = new HashMap<>();
-        // topic@group
-        String key = topic + TOPIC_GROUP_SEPARATOR + group;
-        Long offset = lmqOffsetTable.get(key);
-        if (offset != null) {
-            map.put(0, offset);
-        }
-        return map;
-    }
-
-    @Override
-    public void commitOffset(final String clientHost, final String group, final String topic, final int queueId,
-        final long offset) {
-        if (!MixAll.isLmq(group)) {
-            super.commitOffset(clientHost, group, topic, queueId, offset);
-            return;
-        }
-        // topic@group
-        String key = topic + TOPIC_GROUP_SEPARATOR + group;
-        lmqOffsetTable.put(key, offset);
-    }
-
-    @Override
-    public String encode() {
-        return this.encode(false);
-    }
-
-    @Override
-    public void decode(String jsonString) {
-        if (jsonString != null) {
-            RocksDBLmqConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, RocksDBLmqConsumerOffsetManager.class);
-            if (obj != null) {
-                super.setOffsetTable(obj.getOffsetTable());
-                this.lmqOffsetTable = obj.lmqOffsetTable;
-            }
-        }
-    }
-
-    @Override
-    public String encode(final boolean prettyFormat) {
-        return RemotingSerializable.toJson(this, prettyFormat);
-    }
-
-    public ConcurrentHashMap<String, Long> getLmqOffsetTable() {
-        return lmqOffsetTable;
-    }
-
-    public void setLmqOffsetTable(ConcurrentHashMap<String, Long> lmqOffsetTable) {
-        this.lmqOffsetTable = lmqOffsetTable;
-    }
-}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
index 01745a2b79d..83edd88408a 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
@@ -16,6 +16,7 @@
  */
 package org.apache.rocketmq.broker.out;
 
+import com.alibaba.fastjson2.JSON;
 import java.io.UnsupportedEncodingException;
 import java.net.InetAddress;
 import java.nio.ByteBuffer;
@@ -30,6 +31,10 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.auth.config.AuthConfig;
 import org.apache.rocketmq.client.consumer.PullResult;
 import org.apache.rocketmq.client.consumer.PullStatus;
 import org.apache.rocketmq.client.exception.MQBrokerException;
@@ -154,17 +159,30 @@ public class BrokerOuterAPI {
     private final RpcClient rpcClient;
     private String nameSrvAddr = null;
 
-    public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) {
-        this(nettyClientConfig, new DynamicalExtFieldRPCHook(), new ClientMetadata());
+    public BrokerOuterAPI(final NettyClientConfig nettyClientConfig, AuthConfig authConfig) {
+        this(nettyClientConfig, authConfig, new DynamicalExtFieldRPCHook(), new ClientMetadata());
     }
 
-    private BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook, ClientMetadata clientMetadata) {
+    private BrokerOuterAPI(final NettyClientConfig nettyClientConfig, AuthConfig authConfig, RPCHook rpcHook, ClientMetadata clientMetadata) {
         this.remotingClient = new NettyRemotingClient(nettyClientConfig);
         this.clientMetadata = clientMetadata;
         this.remotingClient.registerRPCHook(rpcHook);
+        this.remotingClient.registerRPCHook(newAclRPCHook(authConfig));
         this.rpcClient = new RpcClientImpl(this.clientMetadata, this.remotingClient);
     }
 
+    private RPCHook newAclRPCHook(AuthConfig config) {
+        if (config == null || StringUtils.isBlank(config.getInnerClientAuthenticationCredentials())) {
+            return null;
+        }
+        SessionCredentials sessionCredentials =
+            JSON.parseObject(config.getInnerClientAuthenticationCredentials(), SessionCredentials.class);
+        if (StringUtils.isBlank(sessionCredentials.getAccessKey()) || StringUtils.isBlank(sessionCredentials.getSecretKey())) {
+            return null;
+        }
+        return new AclClientRPCHook(sessionCredentials);
+    }
+
     public void start() {
         this.remotingClient.start();
     }
@@ -319,7 +337,7 @@ public void sendHeartbeatViaDataVersion(
         final DataVersion dataVersion,
         final boolean isInBrokerContainer) {
         List<String> nameServerAddressList = this.remotingClient.getAvailableNameSrvList();
-        if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
+        if (nameServerAddressList != null && !nameServerAddressList.isEmpty()) {
             final QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader();
             requestHeader.setBrokerAddr(brokerAddr);
             requestHeader.setBrokerName(brokerName);
@@ -388,7 +406,7 @@ public BrokerSyncInfo retrieveBrokerHaInfo(String masterBrokerAddr)
         assert response != null;
         switch (response.getCode()) {
             case ResponseCode.SUCCESS: {
-                ExchangeHAInfoResponseHeader responseHeader = (ExchangeHAInfoResponseHeader) response.decodeCommandCustomHeader(ExchangeHAInfoResponseHeader.class);
+                ExchangeHAInfoResponseHeader responseHeader = response.decodeCommandCustomHeader(ExchangeHAInfoResponseHeader.class);
                 return new BrokerSyncInfo(responseHeader.getMasterHaAddress(), responseHeader.getMasterFlushOffset(), responseHeader.getMasterAddress());
             }
             default:
@@ -557,8 +575,7 @@ private RegisterBrokerResult registerBroker(
         assert response != null;
         switch (response.getCode()) {
             case ResponseCode.SUCCESS: {
-                RegisterBrokerResponseHeader responseHeader =
-                    (RegisterBrokerResponseHeader) response.decodeCommandCustomHeader(RegisterBrokerResponseHeader.class);
+                RegisterBrokerResponseHeader responseHeader = response.decodeCommandCustomHeader(RegisterBrokerResponseHeader.class);
                 RegisterBrokerResult result = new RegisterBrokerResult();
                 result.setMasterAddr(responseHeader.getMasterAddr());
                 result.setHaServerAddr(responseHeader.getHaServerAddr());
@@ -708,7 +725,7 @@ public void run0() {
                             switch (response.getCode()) {
                                 case ResponseCode.SUCCESS: {
                                     QueryDataVersionResponseHeader queryDataVersionResponseHeader =
-                                        (QueryDataVersionResponseHeader) response.decodeCommandCustomHeader(QueryDataVersionResponseHeader.class);
+                                        response.decodeCommandCustomHeader(QueryDataVersionResponseHeader.class);
                                     changed = queryDataVersionResponseHeader.getChanged();
                                     byte[] body = response.getBody();
                                     if (body != null) {
@@ -870,7 +887,7 @@ public long getMaxOffset(final String addr, final String topic, final int queueI
         assert response != null;
         switch (response.getCode()) {
             case ResponseCode.SUCCESS: {
-                GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);
+                GetMaxOffsetResponseHeader responseHeader = response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);
 
                 return responseHeader.getOffset();
             }
@@ -892,7 +909,7 @@ public long getMinOffset(final String addr, final String topic, final int queueI
         assert response != null;
         switch (response.getCode()) {
             case ResponseCode.SUCCESS: {
-                GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);
+                GetMinOffsetResponseHeader responseHeader = response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);
 
                 return responseHeader.getOffset();
             }
@@ -1079,8 +1096,7 @@ private SendResult processSendResponse(
                 break;
         }
         if (sendStatus != null) {
-            SendMessageResponseHeader responseHeader =
-                (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class);
+            SendMessageResponseHeader responseHeader = response.decodeCommandCustomHeader(SendMessageResponseHeader.class);
 
             //If namespace not null , reset Topic without namespace.
             String topic = msg.getTopic();
@@ -1253,7 +1269,7 @@ public Pair<ElectMasterResponseHeader, Set<Long>> brokerElect(String controllerA
             // Only record success response.
             case CONTROLLER_MASTER_STILL_EXIST:
             case SUCCESS:
-                final ElectMasterResponseHeader responseHeader = (ElectMasterResponseHeader) response.decodeCommandCustomHeader(ElectMasterResponseHeader.class);
+                final ElectMasterResponseHeader responseHeader = response.decodeCommandCustomHeader(ElectMasterResponseHeader.class);
                 final ElectMasterResponseBody responseBody = RemotingSerializable.decode(response.getBody(), ElectMasterResponseBody.class);
                 return new Pair<>(responseHeader, responseBody.getSyncStateSet());
         }
@@ -1268,7 +1284,7 @@ public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, f
         final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);
         assert response != null;
         if (response.getCode() == SUCCESS) {
-            return (GetNextBrokerIdResponseHeader) response.decodeCommandCustomHeader(GetNextBrokerIdResponseHeader.class);
+            return response.decodeCommandCustomHeader(GetNextBrokerIdResponseHeader.class);
         }
         throw new MQBrokerException(response.getCode(), response.getRemark());
     }
@@ -1280,7 +1296,7 @@ public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final
         final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);
         assert response != null;
         if (response.getCode() == SUCCESS) {
-            return (ApplyBrokerIdResponseHeader) response.decodeCommandCustomHeader(ApplyBrokerIdResponseHeader.class);
+            return response.decodeCommandCustomHeader(ApplyBrokerIdResponseHeader.class);
         }
         throw new MQBrokerException(response.getCode(), response.getRemark());
     }
@@ -1293,7 +1309,7 @@ public Pair<RegisterBrokerToControllerResponseHeader, Set<Long>> registerBrokerT
         final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);
         assert response != null;
         if (response.getCode() == SUCCESS) {
-            RegisterBrokerToControllerResponseHeader responseHeader = (RegisterBrokerToControllerResponseHeader) response.decodeCommandCustomHeader(RegisterBrokerToControllerResponseHeader.class);
+            RegisterBrokerToControllerResponseHeader responseHeader = response.decodeCommandCustomHeader(RegisterBrokerToControllerResponseHeader.class);
             Set<Long> syncStateSet = RemotingSerializable.decode(response.getBody(), SyncStateSet.class).getSyncStateSet();
             return new Pair<>(responseHeader, syncStateSet);
         }
@@ -1311,7 +1327,7 @@ public Pair<GetReplicaInfoResponseHeader, SyncStateSet> getReplicaInfo(final Str
         assert response != null;
         switch (response.getCode()) {
             case SUCCESS: {
-                final GetReplicaInfoResponseHeader header = (GetReplicaInfoResponseHeader) response.decodeCommandCustomHeader(GetReplicaInfoResponseHeader.class);
+                final GetReplicaInfoResponseHeader header = response.decodeCommandCustomHeader(GetReplicaInfoResponseHeader.class);
                 assert response.getBody() != null;
                 final SyncStateSet stateSet = RemotingSerializable.decode(response.getBody(), SyncStateSet.class);
                 return new Pair<>(header, stateSet);
@@ -1363,7 +1379,8 @@ public void run0() {
         });
     }
 
-    public CompletableFuture<PullResult> pullMessageFromSpecificBrokerAsync(String brokerName, String brokerAddr,
+    // Triple<PullResult, info, needRetry>, should check info and retry if and only if PullResult is null
+    public CompletableFuture<Triple<PullResult, String, Boolean>> pullMessageFromSpecificBrokerAsync(String brokerName, String brokerAddr,
         String consumerGroup, String topic, int queueId, long offset,
         int maxNums, long timeoutMillis) throws RemotingException, InterruptedException {
         PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();
@@ -1382,7 +1399,7 @@ public CompletableFuture<PullResult> pullMessageFromSpecificBrokerAsync(String b
         requestHeader.setBrokerName(brokerName);
 
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader);
-        CompletableFuture<PullResult> pullResultFuture = new CompletableFuture<>();
+        CompletableFuture<Triple<PullResult, String, Boolean>> pullResultFuture = new CompletableFuture<>();
         this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {
             @Override
             public void operationComplete(ResponseFuture responseFuture) {
@@ -1394,15 +1411,16 @@ public void operationSucceed(RemotingCommand response) {
                 try {
                     PullResultExt pullResultExt = processPullResponse(response, brokerAddr);
                     processPullResult(pullResultExt, brokerName, queueId);
-                    pullResultFuture.complete(pullResultExt);
+                    pullResultFuture.complete(Triple.of(pullResultExt, pullResultExt.getPullStatus().name(), false)); // found or not found really, so no retry
                 } catch (Exception e) {
-                    pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>()));
+                    // retry when NO_PERMISSION, SUBSCRIPTION_GROUP_NOT_EXIST etc. even when TOPIC_NOT_EXIST
+                    pullResultFuture.complete(Triple.of(null, "Response Code:" + response.getCode(), true));
                 }
             }
 
             @Override
             public void operationFail(Throwable throwable) {
-                pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>()));
+                pullResultFuture.complete(Triple.of(null, throwable.getMessage(), true));
             }
         });
         return pullResultFuture;
@@ -1430,8 +1448,7 @@ private PullResultExt processPullResponse(
                 throw new MQBrokerException(response.getCode(), response.getRemark(), addr);
         }
 
-        PullMessageResponseHeader responseHeader =
-            (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class);
+        PullMessageResponseHeader responseHeader = response.decodeCommandCustomHeader(PullMessageResponseHeader.class);
 
         return new PullResultExt(pullStatus, responseHeader.getNextBeginOffset(), responseHeader.getMinOffset(),
             responseHeader.getMaxOffset(), null, responseHeader.getSuggestWhichBrokerId(), response.getBody(), responseHeader.getOffsetDelta());
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerCache.java b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerCache.java
new file mode 100644
index 00000000000..e7ce68e0193
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerCache.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PopConsumerCache extends ServiceThread {
+
+    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+    private static final long OFFSET_NOT_EXIST = -1L;
+
+    private final BrokerController brokerController;
+    private final PopConsumerKVStore consumerRecordStore;
+    private final PopConsumerLockService consumerLockService;
+    private final Consumer<PopConsumerRecord> reviveConsumer;
+
+    private final AtomicInteger estimateCacheSize;
+    private final ConcurrentMap<String, ConsumerRecords> consumerRecordTable;
+
+    public PopConsumerCache(BrokerController brokerController, PopConsumerKVStore consumerRecordStore,
+        PopConsumerLockService popConsumerLockService, Consumer<PopConsumerRecord> reviveConsumer) {
+
+        this.reviveConsumer = reviveConsumer;
+        this.brokerController = brokerController;
+        this.consumerRecordStore = consumerRecordStore;
+        this.consumerLockService = popConsumerLockService;
+        this.estimateCacheSize = new AtomicInteger();
+        this.consumerRecordTable = new ConcurrentHashMap<>();
+    }
+
+    public String getKey(String groupId, String topicId, int queueId) {
+        return groupId + "@" + topicId + "@" + queueId;
+    }
+
+    public String getKey(PopConsumerRecord consumerRecord) {
+        return consumerRecord.getGroupId() + "@" + consumerRecord.getTopicId() + "@" + consumerRecord.getQueueId();
+    }
+
+    public int getCacheKeySize() {
+        return this.consumerRecordTable.size();
+    }
+
+    public int getCacheSize() {
+        return this.estimateCacheSize.intValue();
+    }
+
+    public boolean isCacheFull() {
+        return this.estimateCacheSize.intValue() > brokerController.getBrokerConfig().getPopCkMaxBufferSize();
+    }
+
+    public long getMinOffsetInCache(String groupId, String topicId, int queueId) {
+        ConsumerRecords consumerRecords = consumerRecordTable.get(this.getKey(groupId, topicId, queueId));
+        return consumerRecords != null ? consumerRecords.getMinOffsetInBuffer() : OFFSET_NOT_EXIST;
+    }
+
+    public long getPopInFlightMessageCount(String groupId, String topicId, int queueId) {
+        ConsumerRecords consumerRecords = consumerRecordTable.get(this.getKey(groupId, topicId, queueId));
+        return consumerRecords != null ? consumerRecords.getInFlightRecordCount() : 0L;
+    }
+
+    public void writeRecords(List<PopConsumerRecord> consumerRecordList) {
+        this.estimateCacheSize.addAndGet(consumerRecordList.size());
+        consumerRecordList.forEach(consumerRecord -> {
+            ConsumerRecords consumerRecords = ConcurrentHashMapUtils.computeIfAbsent(consumerRecordTable,
+                this.getKey(consumerRecord), k -> new ConsumerRecords(brokerController.getBrokerConfig(),
+                    consumerRecord.getGroupId(), consumerRecord.getTopicId(), consumerRecord.getQueueId()));
+            assert consumerRecords != null;
+            consumerRecords.write(consumerRecord);
+        });
+    }
+
+    /**
+     * Remove the record from the input list then return the content that has not been deleted
+     */
+    public List<PopConsumerRecord> deleteRecords(List<PopConsumerRecord> consumerRecordList) {
+        int total = consumerRecordList.size();
+        List<PopConsumerRecord> remain = new ArrayList<>();
+        consumerRecordList.forEach(consumerRecord -> {
+            ConsumerRecords consumerRecords = consumerRecordTable.get(this.getKey(consumerRecord));
+            if (consumerRecords == null || !consumerRecords.delete(consumerRecord)) {
+                remain.add(consumerRecord);
+            }
+        });
+        this.estimateCacheSize.addAndGet(remain.size() - total);
+        return remain;
+    }
+
+    public int cleanupRecords(Consumer<PopConsumerRecord> consumer) {
+        int remain = 0;
+        Iterator<Map.Entry<String, ConsumerRecords>> iterator = consumerRecordTable.entrySet().iterator();
+        while (iterator.hasNext()) {
+            // revive or write record to store
+            ConsumerRecords records = iterator.next().getValue();
+            boolean timeout = consumerLockService.isLockTimeout(
+                records.getGroupId(), records.getTopicId());
+
+            if (timeout) {
+                List<PopConsumerRecord> removeExpiredRecords =
+                    records.removeExpiredRecords(Long.MAX_VALUE);
+                if (removeExpiredRecords != null) {
+                    consumerRecordStore.writeRecords(removeExpiredRecords);
+                }
+                log.info("PopConsumerOffline, so clean expire records, groupId={}, topic={}, queueId={}, records={}",
+                    records.getGroupId(), records.getTopicId(), records.getQueueId(),
+                    removeExpiredRecords != null ? removeExpiredRecords.size() : 0);
+                iterator.remove();
+                continue;
+            }
+
+            long currentTime = System.currentTimeMillis();
+            List<PopConsumerRecord> writeConsumerRecords = new ArrayList<>();
+            List<PopConsumerRecord> consumerRecords = records.removeExpiredRecords(currentTime);
+            if (consumerRecords != null) {
+                consumerRecords.forEach(consumerRecord -> {
+                    if (consumerRecord.getVisibilityTimeout() <= currentTime) {
+                        consumer.accept(consumerRecord);
+                    } else {
+                        writeConsumerRecords.add(consumerRecord);
+                    }
+                });
+            }
+
+            // write to store and handle it later
+            consumerRecordStore.writeRecords(writeConsumerRecords);
+
+            // commit min offset in buffer to offset store
+            long offset = records.getMinOffsetInBuffer();
+            if (offset > OFFSET_NOT_EXIST) {
+                this.commitOffset("PopConsumerCache",
+                    records.getGroupId(), records.getTopicId(), records.getQueueId(), offset);
+            }
+
+            remain += records.getInFlightRecordCount();
+        }
+        return remain;
+    }
+
+    public void commitOffset(String clientHost, String groupId, String topicId, int queueId, long offset) {
+        if (!consumerLockService.tryLock(groupId, topicId)) {
+            return;
+        }
+        try {
+            ConsumerOffsetManager consumerOffsetManager = brokerController.getConsumerOffsetManager();
+            long commit = consumerOffsetManager.queryOffset(groupId, topicId, queueId);
+            if (commit != OFFSET_NOT_EXIST && offset < commit) {
+                log.info("PopConsumerCache, consumer offset less than store, " +
+                    "groupId={}, topicId={}, queueId={}, offset={}", groupId, topicId, queueId, offset);
+            }
+            consumerOffsetManager.commitOffset(clientHost, groupId, topicId, queueId, offset);
+        } finally {
+            consumerLockService.unlock(groupId, topicId);
+        }
+    }
+
+    public void removeRecords(String groupId, String topicId, int queueId) {
+        this.consumerRecordTable.remove(this.getKey(groupId, topicId, queueId));
+    }
+
+    @Override
+    public String getServiceName() {
+        return PopConsumerCache.class.getSimpleName();
+    }
+
+    @Override
+    public void run() {
+        while (!this.isStopped()) {
+            try {
+                this.waitForRunning(TimeUnit.SECONDS.toMillis(1));
+                int cacheSize = this.cleanupRecords(reviveConsumer);
+                this.estimateCacheSize.set(cacheSize);
+            } catch (Exception e) {
+                log.error("PopConsumerCacheService revive error", e);
+            }
+        }
+    }
+
+    protected static class ConsumerRecords {
+
+        private final Lock lock;
+        private final String groupId;
+        private final String topicId;
+        private final int queueId;
+        private final BrokerConfig brokerConfig;
+        private final TreeMap<Long /* offset */, PopConsumerRecord> recordTreeMap;
+
+        public ConsumerRecords(BrokerConfig brokerConfig, String groupId, String topicId, int queueId) {
+            this.groupId = groupId;
+            this.topicId = topicId;
+            this.queueId = queueId;
+            this.lock = new ReentrantLock();
+            this.brokerConfig = brokerConfig;
+            this.recordTreeMap = new TreeMap<>();
+        }
+
+        public void write(PopConsumerRecord record) {
+            lock.lock();
+            try {
+                recordTreeMap.put(record.getOffset(), record);
+            } finally {
+                lock.unlock();
+            }
+        }
+
+        public boolean delete(PopConsumerRecord record) {
+            PopConsumerRecord popConsumerRecord;
+            lock.lock();
+            try {
+                popConsumerRecord = recordTreeMap.remove(record.getOffset());
+            } finally {
+                lock.unlock();
+            }
+            return popConsumerRecord != null;
+        }
+
+        public long getMinOffsetInBuffer() {
+            Map.Entry<Long, PopConsumerRecord> entry = recordTreeMap.firstEntry();
+            return entry != null ? entry.getKey() : OFFSET_NOT_EXIST;
+        }
+
+        public int getInFlightRecordCount() {
+            return recordTreeMap.size();
+        }
+
+        public List<PopConsumerRecord> removeExpiredRecords(long currentTime) {
+            List<PopConsumerRecord> result = null;
+            lock.lock();
+            try {
+                Iterator<Map.Entry<Long, PopConsumerRecord>> iterator = recordTreeMap.entrySet().iterator();
+                while (iterator.hasNext()) {
+                    Map.Entry<Long, PopConsumerRecord> entry = iterator.next();
+                    // org.apache.rocketmq.broker.processor.PopBufferMergeService.scan
+                    if (entry.getValue().getVisibilityTimeout() <= currentTime ||
+                        entry.getValue().getPopTime() + brokerConfig.getPopCkStayBufferTime() <= currentTime) {
+                        if (result == null) {
+                            result = new ArrayList<>();
+                        }
+                        result.add(entry.getValue());
+                        iterator.remove();
+                    }
+                }
+            } finally {
+                lock.unlock();
+            }
+            return result;
+        }
+
+        public String getGroupId() {
+            return groupId;
+        }
+
+        public String getTopicId() {
+            return topicId;
+        }
+
+        public int getQueueId() {
+            return queueId;
+        }
+
+        @Override
+        public String toString() {
+            return "ConsumerRecords{" +
+                "lock=" + lock +
+                ", topicId=" + topicId +
+                ", groupId=" + groupId +
+                ", queueId=" + queueId +
+                ", recordTreeMap=" + recordTreeMap.size() +
+                '}';
+        }
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerContext.java b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerContext.java
new file mode 100644
index 00000000000..0ad8bacab1c
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerContext.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+
+public class PopConsumerContext {
+
+    private final String clientHost;
+
+    private final long popTime;
+
+    private final long invisibleTime;
+
+    private final String groupId;
+
+    private final boolean fifo;
+
+    private final int initMode;
+
+    private final String attemptId;
+
+    private final AtomicLong restCount;
+
+    private final StringBuilder startOffsetInfo;
+
+    private final StringBuilder msgOffsetInfo;
+
+    private final StringBuilder orderCountInfo;
+
+    private List<GetMessageResult> getMessageResultList;
+
+    private List<PopConsumerRecord> popConsumerRecordList;
+
+    public PopConsumerContext(String clientHost,
+        long popTime, long invisibleTime, String groupId, boolean fifo, int initMode, String attemptId) {
+
+        this.clientHost = clientHost;
+        this.popTime = popTime;
+        this.invisibleTime = invisibleTime;
+        this.groupId = groupId;
+        this.fifo = fifo;
+        this.initMode = initMode;
+        this.attemptId = attemptId;
+        this.restCount = new AtomicLong(0);
+        this.startOffsetInfo = new StringBuilder();
+        this.msgOffsetInfo = new StringBuilder();
+        this.orderCountInfo = new StringBuilder();
+    }
+
+    public boolean isFound() {
+        return getMessageResultList != null && !getMessageResultList.isEmpty();
+    }
+
+    // offset is consumer last request offset
+    public void addGetMessageResult(GetMessageResult result,
+        String topicId, int queueId, PopConsumerRecord.RetryType retryType, long offset) {
+
+        if (result.getStatus() != GetMessageStatus.FOUND || result.getMessageQueueOffset().isEmpty()) {
+            return;
+        }
+
+        if (this.getMessageResultList == null) {
+            this.getMessageResultList = new ArrayList<>();
+        }
+
+        if (this.popConsumerRecordList == null) {
+            this.popConsumerRecordList = new ArrayList<>();
+        }
+
+        this.getMessageResultList.add(result);
+        this.addRestCount(result.getMaxOffset() - result.getNextBeginOffset());
+
+        for (int i = 0; i < result.getMessageQueueOffset().size(); i++) {
+            this.popConsumerRecordList.add(new PopConsumerRecord(popTime, groupId, topicId, queueId,
+                retryType.getCode(), invisibleTime, result.getMessageQueueOffset().get(i), attemptId));
+        }
+
+        ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topicId, queueId, offset);
+        ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topicId, queueId, result.getMessageQueueOffset());
+    }
+
+    public String getClientHost() {
+        return clientHost;
+    }
+
+    public String getGroupId() {
+        return groupId;
+    }
+
+    public void addRestCount(long delta) {
+        this.restCount.addAndGet(delta);
+    }
+
+    public long getRestCount() {
+        return restCount.get();
+    }
+
+    public long getPopTime() {
+        return popTime;
+    }
+
+    public boolean isFifo() {
+        return fifo;
+    }
+
+    public int getInitMode() {
+        return initMode;
+    }
+
+    public long getInvisibleTime() {
+        return invisibleTime;
+    }
+
+    public String getAttemptId() {
+        return attemptId;
+    }
+
+    public int getMessageCount() {
+        return getMessageResultList != null ?
+            getMessageResultList.stream().mapToInt(GetMessageResult::getMessageCount).sum() : 0;
+    }
+
+    public String getStartOffsetInfo() {
+        return startOffsetInfo.toString();
+    }
+
+    public String getMsgOffsetInfo() {
+        return msgOffsetInfo.toString();
+    }
+
+    public StringBuilder getOrderCountInfoBuilder() {
+        return orderCountInfo;
+    }
+
+    public String getOrderCountInfo() {
+        return orderCountInfo.toString();
+    }
+
+    public List<GetMessageResult> getGetMessageResultList() {
+        return getMessageResultList;
+    }
+
+    public List<PopConsumerRecord> getPopConsumerRecordList() {
+        return popConsumerRecordList;
+    }
+
+    @Override
+    public String toString() {
+        return "PopConsumerContext{" +
+            "clientHost=" + clientHost +
+            ", popTime=" + popTime +
+            ", invisibleTime=" + invisibleTime +
+            ", groupId=" + groupId +
+            ", isFifo=" + fifo +
+            ", attemptId=" + attemptId +
+            ", restCount=" + restCount +
+            ", startOffsetInfo=" + startOffsetInfo +
+            ", msgOffsetInfo=" + msgOffsetInfo +
+            ", orderCountInfo=" + orderCountInfo +
+            ", getMessageResultList=" + (getMessageResultList != null ? getMessageResultList.size() : 0) +
+            ", popConsumerRecordList=" + (popConsumerRecordList != null ? popConsumerRecordList.size() : 0) +
+            '}';
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerKVStore.java b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerKVStore.java
new file mode 100644
index 00000000000..33072d699b5
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerKVStore.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import java.util.List;
+
+public interface PopConsumerKVStore {
+
+    /**
+     * Starts the storage service.
+     */
+    boolean start();
+
+    /**
+     * Shutdown the storage service.
+     */
+    boolean shutdown();
+
+    /**
+     * Gets the file path of the storage.
+     * @return The file path of the storage.
+     */
+    String getFilePath();
+
+    /**
+     * Writes a list of consumer records to the storage.
+     * @param consumerRecordList The list of consumer records to be written.
+     */
+    void writeRecords(List<PopConsumerRecord> consumerRecordList);
+
+    /**
+     * Deletes a list of consumer records from the storage.
+     * @param consumerRecordList The list of consumer records to be deleted.
+     */
+    void deleteRecords(List<PopConsumerRecord> consumerRecordList);
+
+    /**
+     * Scans and returns a list of expired consumer records within the specified time range.
+     * @param lowerTime The start time (inclusive) of the time range to search, in milliseconds.
+     * @param upperTime The end time (exclusive) of the time range to search, in milliseconds.
+     * @param maxCount The maximum number of records to return.
+     *                 Even if more records match the criteria, only this many will be returned.
+     * @return A list of expired consumer records within the specified time range.
+     *         If no matching records are found, an empty list is returned.
+     */
+    List<PopConsumerRecord> scanExpiredRecords(long lowerTime, long upperTime, int maxCount);
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerLockService.java b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerLockService.java
new file mode 100644
index 00000000000..33221430492
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerLockService.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.PopAckConstants;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PopConsumerLockService {
+
+    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+
+    private final long timeout;
+    private final ConcurrentMap<String /* groupId@topicId */, TimedLock> lockTable;
+
+    public PopConsumerLockService(long timeout) {
+        this.timeout = timeout;
+        this.lockTable = new ConcurrentHashMap<>();
+    }
+
+    public boolean tryLock(String groupId, String topicId) {
+        return Objects.requireNonNull(ConcurrentHashMapUtils.computeIfAbsent(lockTable,
+            groupId + PopAckConstants.SPLIT + topicId, s -> new TimedLock())).tryLock();
+    }
+
+    public void unlock(String groupId, String topicId) {
+        TimedLock lock = lockTable.get(groupId + PopAckConstants.SPLIT + topicId);
+        if (lock != null) {
+            lock.unlock();
+        }
+    }
+
+    // For retry topics, should lock origin group and topic
+    public boolean isLockTimeout(String groupId, String topicId) {
+        topicId = KeyBuilder.parseNormalTopic(topicId, groupId);
+        TimedLock lock = lockTable.get(groupId + PopAckConstants.SPLIT + topicId);
+        return lock == null || System.currentTimeMillis() - lock.getLockTime() > timeout;
+    }
+
+    public void removeTimeout() {
+        Iterator<Map.Entry<String, TimedLock>> iterator = lockTable.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, TimedLock> entry = iterator.next();
+            if (System.currentTimeMillis() - entry.getValue().getLockTime() > timeout) {
+                log.info("PopConsumerLockService remove timeout lock, " +
+                    "key={}, locked={}", entry.getKey(), entry.getValue().lock.get());
+                iterator.remove();
+            }
+        }
+    }
+
+    static class TimedLock {
+        private volatile long lockTime;
+        private final AtomicBoolean lock;
+
+        public TimedLock() {
+            this.lockTime = System.currentTimeMillis();
+            this.lock = new AtomicBoolean(false);
+        }
+
+        public boolean tryLock() {
+            if (lock.compareAndSet(false, true)) {
+                this.lockTime = System.currentTimeMillis();
+                return true;
+            }
+            return false;
+        }
+
+        public void unlock() {
+            lock.set(false);
+        }
+
+        public long getLockTime() {
+            return lockTime;
+        }
+    }
+}
\ No newline at end of file
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerRecord.java b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerRecord.java
new file mode 100644
index 00000000000..1ee01fea1c8
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerRecord.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.annotation.JSONField;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+
+public class PopConsumerRecord {
+
+    public enum RetryType {
+
+        NORMAL_TOPIC(0),
+
+        RETRY_TOPIC_V1(1),
+
+        RETRY_TOPIC_V2(2);
+
+        private final int code;
+
+        RetryType(int code) {
+            this.code = code;
+        }
+
+        public int getCode() {
+            return code;
+        }
+    }
+
+    @JSONField()
+    private long popTime;
+
+    @JSONField(ordinal = 1)
+    private String groupId;
+
+    @JSONField(ordinal = 2)
+    private String topicId;
+
+    @JSONField(ordinal = 3)
+    private int queueId;
+
+    @JSONField(ordinal = 4)
+    private int retryFlag;
+
+    @JSONField(ordinal = 5)
+    private long invisibleTime;
+
+    @JSONField(ordinal = 6)
+    private long offset;
+
+    @JSONField(ordinal = 7)
+    private int attemptTimes;
+
+    @JSONField(ordinal = 8)
+    private String attemptId;
+
+    // used for test and fastjson
+    public PopConsumerRecord() {
+    }
+
+    public PopConsumerRecord(long popTime, String groupId, String topicId, int queueId,
+        int retryFlag, long invisibleTime, long offset, String attemptId) {
+
+        this.popTime = popTime;
+        this.groupId = groupId;
+        this.topicId = topicId;
+        this.queueId = queueId;
+        this.retryFlag = retryFlag;
+        this.invisibleTime = invisibleTime;
+        this.offset = offset;
+        this.attemptId = attemptId;
+    }
+
+    @JSONField(serialize = false)
+    public long getVisibilityTimeout() {
+        return popTime + invisibleTime;
+    }
+
+    /**
+     * Key: timestamp(8) + groupId + topicId + queueId + offset
+     */
+    @JSONField(serialize = false)
+    public byte[] getKeyBytes() {
+        int length = Long.BYTES + groupId.length() + 1 + topicId.length() + 1 + Integer.BYTES + 1 + Long.BYTES;
+        byte[] bytes = new byte[length];
+        ByteBuffer buffer = ByteBuffer.wrap(bytes);
+        buffer.putLong(this.getVisibilityTimeout());
+        buffer.put(groupId.getBytes(StandardCharsets.UTF_8)).put((byte) '@');
+        buffer.put(topicId.getBytes(StandardCharsets.UTF_8)).put((byte) '@');
+        buffer.putInt(queueId).put((byte) '@');
+        buffer.putLong(offset);
+        return bytes;
+    }
+
+    @JSONField(serialize = false)
+    public boolean isRetry() {
+        return retryFlag != 0;
+    }
+
+    @JSONField(serialize = false)
+    public byte[] getValueBytes() {
+        return JSON.toJSONBytes(this);
+    }
+
+    public static PopConsumerRecord decode(byte[] body) {
+        return JSONObject.parseObject(body, PopConsumerRecord.class);
+    }
+
+    public long getPopTime() {
+        return popTime;
+    }
+
+    public void setPopTime(long popTime) {
+        this.popTime = popTime;
+    }
+
+    public String getGroupId() {
+        return groupId;
+    }
+
+    public void setGroupId(String groupId) {
+        this.groupId = groupId;
+    }
+
+    public String getTopicId() {
+        return topicId;
+    }
+
+    public void setTopicId(String topicId) {
+        this.topicId = topicId;
+    }
+
+    public int getQueueId() {
+        return queueId;
+    }
+
+    public void setQueueId(int queueId) {
+        this.queueId = queueId;
+    }
+
+    public int getRetryFlag() {
+        return retryFlag;
+    }
+
+    public void setRetryFlag(int retryFlag) {
+        this.retryFlag = retryFlag;
+    }
+
+    public long getInvisibleTime() {
+        return invisibleTime;
+    }
+
+    public void setInvisibleTime(long invisibleTime) {
+        this.invisibleTime = invisibleTime;
+    }
+
+    public long getOffset() {
+        return offset;
+    }
+
+    public void setOffset(long offset) {
+        this.offset = offset;
+    }
+
+    public int getAttemptTimes() {
+        return attemptTimes;
+    }
+
+    public void setAttemptTimes(int attemptTimes) {
+        this.attemptTimes = attemptTimes;
+    }
+
+    public String getAttemptId() {
+        return attemptId;
+    }
+
+    public void setAttemptId(String attemptId) {
+        this.attemptId = attemptId;
+    }
+
+    @Override
+    public String toString() {
+        return "PopDeliveryRecord{" +
+            "popTime=" + popTime +
+            ", groupId='" + groupId + '\'' +
+            ", topicId='" + topicId + '\'' +
+            ", queueId=" + queueId +
+            ", retryFlag=" + retryFlag +
+            ", invisibleTime=" + invisibleTime +
+            ", offset=" + offset +
+            ", attemptTimes=" + attemptTimes +
+            ", attemptId='" + attemptId + '\'' +
+            '}';
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerRocksdbStore.java b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerRocksdbStore.java
new file mode 100644
index 00000000000..7ab276a4185
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerRocksdbStore.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.config.AbstractRocksDBStorage;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.store.rocksdb.RocksDBOptionsFactory;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.CompactRangeOptions;
+import org.rocksdb.ReadOptions;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.Slice;
+import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PopConsumerRocksdbStore extends AbstractRocksDBStorage implements PopConsumerKVStore {
+
+    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+    private static final byte[] COLUMN_FAMILY_NAME = "popState".getBytes(StandardCharsets.UTF_8);
+
+    private WriteOptions writeOptions;
+    private WriteOptions deleteOptions;
+    protected ColumnFamilyHandle columnFamilyHandle;
+
+    public PopConsumerRocksdbStore(String filePath) {
+        super(filePath);
+    }
+
+    // https://www.cnblogs.com/renjc/p/rocksdb-class-db.html
+    // https://github.com/johnzeng/rocksdb-doc-cn/blob/master/doc/RocksDB-Tuning-Guide.md
+    protected void initOptions() {
+        this.options = RocksDBOptionsFactory.createDBOptions();
+
+        this.writeOptions = new WriteOptions();
+        this.writeOptions.setSync(true);
+        this.writeOptions.setDisableWAL(false);
+        this.writeOptions.setNoSlowdown(false);
+
+        this.deleteOptions = new WriteOptions();
+        this.deleteOptions.setSync(true);
+        this.deleteOptions.setDisableWAL(false);
+        this.deleteOptions.setNoSlowdown(false);
+
+        this.compactRangeOptions = new CompactRangeOptions();
+        this.compactRangeOptions.setBottommostLevelCompaction(
+            CompactRangeOptions.BottommostLevelCompaction.kForce);
+        this.compactRangeOptions.setAllowWriteStall(true);
+        this.compactRangeOptions.setExclusiveManualCompaction(false);
+        this.compactRangeOptions.setChangeLevel(true);
+        this.compactRangeOptions.setTargetLevel(-1);
+        this.compactRangeOptions.setMaxSubcompactions(4);
+    }
+
+    @Override
+    protected boolean postLoad() {
+        try {
+            UtilAll.ensureDirOK(this.dbPath);
+            initOptions();
+
+            // init column family here
+            ColumnFamilyOptions defaultOptions = RocksDBOptionsFactory.createPopCFOptions();
+            ColumnFamilyOptions popStateOptions = RocksDBOptionsFactory.createPopCFOptions();
+            this.cfOptions.add(defaultOptions);
+            this.cfOptions.add(popStateOptions);
+
+            List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList<>();
+            cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions));
+            cfDescriptors.add(new ColumnFamilyDescriptor(COLUMN_FAMILY_NAME, popStateOptions));
+            this.open(cfDescriptors);
+            this.defaultCFHandle = cfHandles.get(0);
+            this.columnFamilyHandle = cfHandles.get(1);
+
+            log.debug("PopConsumerRocksdbStore init, filePath={}", this.dbPath);
+        } catch (final Exception e) {
+            log.error("PopConsumerRocksdbStore init error, filePath={}", this.dbPath, e);
+            return false;
+        }
+        return true;
+    }
+
+    public String getFilePath() {
+        return this.dbPath;
+    }
+
+    @Override
+    public void writeRecords(List<PopConsumerRecord> consumerRecordList) {
+        if (!consumerRecordList.isEmpty()) {
+            try (WriteBatch writeBatch = new WriteBatch()) {
+                for (PopConsumerRecord record : consumerRecordList) {
+                    writeBatch.put(columnFamilyHandle, record.getKeyBytes(), record.getValueBytes());
+                }
+                this.db.write(writeOptions, writeBatch);
+            } catch (RocksDBException e) {
+                throw new RuntimeException("Write record error", e);
+            }
+        }
+    }
+
+    @Override
+    public void deleteRecords(List<PopConsumerRecord> consumerRecordList) {
+        if (!consumerRecordList.isEmpty()) {
+            try (WriteBatch writeBatch = new WriteBatch()) {
+                for (PopConsumerRecord record : consumerRecordList) {
+                    writeBatch.delete(columnFamilyHandle, record.getKeyBytes());
+                }
+                this.db.write(deleteOptions, writeBatch);
+            } catch (RocksDBException e) {
+                throw new RuntimeException("Delete record error", e);
+            }
+        }
+    }
+
+    @Override
+    // https://github.com/facebook/rocksdb/issues/10300
+    public List<PopConsumerRecord> scanExpiredRecords(long lower, long upper, int maxCount) {
+        // In RocksDB, we can use SstPartitionerFixedPrefixFactory in cfOptions
+        // and new ColumnFamilyOptions().useFixedLengthPrefixExtractor() to
+        // configure prefix indexing to improve the performance of scans.
+        // However, in the current implementation, this is not the bottleneck.
+        List<PopConsumerRecord> consumerRecordList = new ArrayList<>();
+        try (ReadOptions scanOptions = new ReadOptions()
+            .setIterateLowerBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(lower).array()))
+            .setIterateUpperBound(new Slice(ByteBuffer.allocate(Long.BYTES).putLong(upper).array()));
+             RocksIterator iterator = db.newIterator(this.columnFamilyHandle, scanOptions)) {
+            iterator.seek(ByteBuffer.allocate(Long.BYTES).putLong(lower).array());
+            while (iterator.isValid() && consumerRecordList.size() < maxCount) {
+                consumerRecordList.add(PopConsumerRecord.decode(iterator.value()));
+                iterator.next();
+            }
+        }
+        return consumerRecordList;
+    }
+
+    @Override
+    protected void preShutdown() {
+        if (this.writeOptions != null) {
+            this.writeOptions.close();
+        }
+        if (this.deleteOptions != null) {
+            this.deleteOptions.close();
+        }
+        if (this.defaultCFHandle != null) {
+            this.defaultCFHandle.close();
+        }
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerService.java b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerService.java
new file mode 100644
index 00000000000..1138ff4afe9
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/pop/PopConsumerService.java
@@ -0,0 +1,747 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import com.alibaba.fastjson.JSON;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Stopwatch;
+import java.nio.ByteBuffer;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Objects;
+import java.util.Queue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.TopicFilterType;
+import org.apache.rocketmq.common.constant.ConsumeInitMode;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;
+import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
+import org.apache.rocketmq.store.AppendMessageStatus;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+import org.apache.rocketmq.store.MessageFilter;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
+import org.apache.rocketmq.store.pop.PopCheckPoint;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PopConsumerService extends ServiceThread {
+
+    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+    private static final long OFFSET_NOT_EXIST = -1L;
+    private static final String ROCKSDB_DIRECTORY = "kvStore";
+    private static final int[] REWRITE_INTERVALS_IN_SECONDS =
+        new int[] {10, 30, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600, 1200, 1800, 3600, 7200};
+
+    private final AtomicBoolean consumerRunning;
+    private final BrokerConfig brokerConfig;
+    private final BrokerController brokerController;
+    private final AtomicLong currentTime;
+    private final AtomicLong lastCleanupLockTime;
+    private final PopConsumerCache popConsumerCache;
+    private final PopConsumerKVStore popConsumerStore;
+    private final PopConsumerLockService consumerLockService;
+    private final ConcurrentMap<String /* groupId@topicId*/, AtomicLong> requestCountTable;
+
+    public PopConsumerService(BrokerController brokerController) {
+
+        this.brokerController = brokerController;
+        this.brokerConfig = brokerController.getBrokerConfig();
+
+        this.consumerRunning = new AtomicBoolean(false);
+        this.requestCountTable = new ConcurrentHashMap<>();
+        this.currentTime = new AtomicLong(TimeUnit.SECONDS.toMillis(3));
+        this.lastCleanupLockTime = new AtomicLong(System.currentTimeMillis());
+        this.consumerLockService = new PopConsumerLockService(TimeUnit.MINUTES.toMillis(2));
+        this.popConsumerStore = new PopConsumerRocksdbStore(Paths.get(
+            brokerController.getMessageStoreConfig().getStorePathRootDir(), ROCKSDB_DIRECTORY).toString());
+        this.popConsumerCache = brokerConfig.isEnablePopBufferMerge() ? new PopConsumerCache(
+            brokerController, this.popConsumerStore, this.consumerLockService, this::revive) : null;
+
+        log.info("PopConsumerService init, buffer={}, rocksdb filePath={}",
+            brokerConfig.isEnablePopBufferMerge(), this.popConsumerStore.getFilePath());
+    }
+
+    /**
+     * In-flight messages are those that have been received from a queue
+     * by a consumer but have not yet been deleted. For standard queues,
+     * there is a limit on the number of in-flight messages, depending on queue traffic and message backlog.
+     */
+    public boolean isPopShouldStop(String group, String topic, int queueId) {
+        return brokerConfig.isEnablePopMessageThreshold() && popConsumerCache != null &&
+            popConsumerCache.getPopInFlightMessageCount(group, topic, queueId) >=
+                brokerConfig.getPopInflightMessageThreshold();
+    }
+
+    public long getPendingFilterCount(String groupId, String topicId, int queueId) {
+        try {
+            long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topicId, queueId);
+            long consumeOffset = this.brokerController.getConsumerOffsetManager().queryOffset(groupId, topicId, queueId);
+            return maxOffset - consumeOffset;
+        } catch (ConsumeQueueException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public GetMessageResult recodeRetryMessage(GetMessageResult getMessageResult,
+        String topicId, long offset, long popTime, long invisibleTime) {
+
+        if (getMessageResult.getMessageCount() == 0 ||
+            getMessageResult.getMessageMapedList().isEmpty()) {
+            return getMessageResult;
+        }
+
+        GetMessageResult result = new GetMessageResult(getMessageResult.getMessageCount());
+        result.setStatus(GetMessageStatus.FOUND);
+        String brokerName = brokerConfig.getBrokerName();
+
+        for (SelectMappedBufferResult bufferResult : getMessageResult.getMessageMapedList()) {
+            List<MessageExt> messageExtList = MessageDecoder.decodesBatch(
+                bufferResult.getByteBuffer(), true, false, true);
+            bufferResult.release();
+            for (MessageExt messageExt : messageExtList) {
+                try {
+                    // When override retry message topic to origin topic,
+                    // need clear message store size to recode
+                    String ckInfo = ExtraInfoUtil.buildExtraInfo(offset, popTime, invisibleTime, 0,
+                        messageExt.getTopic(), brokerName, messageExt.getQueueId(), messageExt.getQueueOffset());
+                    messageExt.getProperties().putIfAbsent(MessageConst.PROPERTY_POP_CK, ckInfo);
+                    messageExt.setTopic(topicId);
+                    messageExt.setStoreSize(0);
+                    byte[] encode = MessageDecoder.encode(messageExt, false);
+                    ByteBuffer buffer = ByteBuffer.wrap(encode);
+                    SelectMappedBufferResult tmpResult = new SelectMappedBufferResult(
+                        bufferResult.getStartOffset(), buffer, encode.length, null);
+                    result.addMessage(tmpResult);
+                } catch (Exception e) {
+                    log.error("PopConsumerService exception in recode retry message, topic={}", topicId, e);
+                }
+            }
+        }
+
+        return result;
+    }
+
+    public PopConsumerContext handleGetMessageResult(PopConsumerContext context, GetMessageResult result,
+        String topicId, int queueId, PopConsumerRecord.RetryType retryType, long offset) {
+
+        if (GetMessageStatus.FOUND.equals(result.getStatus()) && !result.getMessageQueueOffset().isEmpty()) {
+            if (context.isFifo()) {
+                this.setFifoBlocked(context, context.getGroupId(), topicId, queueId, result.getMessageQueueOffset());
+            }
+            // build response header here
+            context.addGetMessageResult(result, topicId, queueId, retryType, offset);
+            if (brokerConfig.isPopConsumerKVServiceLog()) {
+                log.info("PopConsumerService pop, time={}, invisible={}, " +
+                        "groupId={}, topic={}, queueId={}, offset={}, attemptId={}",
+                    context.getPopTime(), context.getInvisibleTime(), context.getGroupId(),
+                    topicId, queueId, result.getMessageQueueOffset(), context.getAttemptId());
+            }
+        }
+
+        long commitOffset = offset;
+        if (context.isFifo()) {
+            if (!GetMessageStatus.FOUND.equals(result.getStatus())) {
+                commitOffset = result.getNextBeginOffset();
+            }
+        } else {
+            this.brokerController.getConsumerOffsetManager().commitPullOffset(
+                context.getClientHost(), context.getGroupId(), topicId, queueId, result.getNextBeginOffset());
+            if (brokerConfig.isEnablePopBufferMerge() && popConsumerCache != null) {
+                long minOffset = popConsumerCache.getMinOffsetInCache(context.getGroupId(), topicId, queueId);
+                if (minOffset != OFFSET_NOT_EXIST) {
+                    commitOffset = minOffset;
+                }
+            }
+        }
+        this.brokerController.getConsumerOffsetManager().commitOffset(
+            context.getClientHost(), context.getGroupId(), topicId, queueId, commitOffset);
+        return context;
+    }
+
+    public long getPopOffset(String groupId, String topicId, int queueId, int initMode) {
+        long offset = this.brokerController.getConsumerOffsetManager().queryPullOffset(groupId, topicId, queueId);
+        if (offset < 0L) {
+            try {
+                offset = this.brokerController.getPopMessageProcessor()
+                    .getInitOffset(topicId, groupId, queueId, initMode, true);
+                log.info("PopConsumerService init offset, groupId={}, topicId={}, queueId={}, init={}, offset={}",
+                    groupId, topicId, queueId, ConsumeInitMode.MIN == initMode ? "min" : "max", offset);
+            } catch (ConsumeQueueException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        Long resetOffset =
+            this.brokerController.getConsumerOffsetManager().queryThenEraseResetOffset(topicId, groupId, queueId);
+        if (resetOffset != null) {
+            this.clearCache(groupId, topicId, queueId);
+            this.brokerController.getConsumerOrderInfoManager().clearBlock(topicId, groupId, queueId);
+            this.brokerController.getConsumerOffsetManager()
+                .commitOffset("ResetPopOffset", groupId, topicId, queueId, resetOffset);
+        }
+        return resetOffset != null ? resetOffset : offset;
+    }
+
+    public CompletableFuture<GetMessageResult> getMessageAsync(String clientHost,
+        String groupId, String topicId, int queueId, long offset, int batchSize, MessageFilter filter) {
+
+        log.debug("PopConsumerService getMessageAsync, groupId={}, topicId={}, queueId={}, offset={}, batchSize={}, filter={}",
+            groupId, topicId, offset, queueId, batchSize, filter != null);
+
+        CompletableFuture<GetMessageResult> getMessageFuture =
+            brokerController.getMessageStore().getMessageAsync(groupId, topicId, queueId, offset, batchSize, filter);
+
+        // refer org.apache.rocketmq.broker.processor.PopMessageProcessor#popMsgFromQueue
+        return getMessageFuture.thenCompose(result -> {
+            if (result == null) {
+                return CompletableFuture.completedFuture(null);
+            }
+
+            // maybe store offset is not correct.
+            if (GetMessageStatus.OFFSET_TOO_SMALL.equals(result.getStatus()) ||
+                GetMessageStatus.OFFSET_OVERFLOW_BADLY.equals(result.getStatus()) ||
+                GetMessageStatus.OFFSET_FOUND_NULL.equals(result.getStatus())) {
+
+                // commit offset, because the offset is not correct
+                // If offset in store is greater than cq offset, it will cause duplicate messages,
+                // because offset in PopBuffer is not committed.
+                this.brokerController.getConsumerOffsetManager().commitOffset(
+                    clientHost, groupId, topicId, queueId, result.getNextBeginOffset());
+
+                log.warn("PopConsumerService getMessageAsync, initial offset because store is no correct, " +
+                        "groupId={}, topicId={}, queueId={}, batchSize={}, offset={}->{}",
+                    groupId, topicId, queueId, batchSize, offset, result.getNextBeginOffset());
+
+                return brokerController.getMessageStore().getMessageAsync(
+                    groupId, topicId, queueId, result.getNextBeginOffset(), batchSize, filter);
+            }
+
+            return CompletableFuture.completedFuture(result);
+
+        }).whenComplete((result, throwable) -> {
+            if (throwable != null) {
+                log.error("Pop getMessageAsync error", throwable);
+            }
+        });
+    }
+
+    /**
+     * Fifo message does not have retry feature in broker
+     */
+    public void setFifoBlocked(PopConsumerContext context,
+        String groupId, String topicId, int queueId, List<Long> queueOffsetList) {
+        brokerController.getConsumerOrderInfoManager().update(
+            context.getAttemptId(), false, topicId, groupId, queueId,
+            context.getPopTime(), context.getInvisibleTime(), queueOffsetList, context.getOrderCountInfoBuilder());
+    }
+
+    public boolean isFifoBlocked(PopConsumerContext context, String groupId, String topicId, int queueId) {
+        return brokerController.getConsumerOrderInfoManager().checkBlock(
+            context.getAttemptId(), topicId, groupId, queueId, context.getInvisibleTime());
+    }
+
+    protected CompletableFuture<PopConsumerContext> getMessageAsync(CompletableFuture<PopConsumerContext> future,
+        String clientHost, String groupId, String topicId, int queueId, int batchSize, MessageFilter filter,
+        PopConsumerRecord.RetryType retryType) {
+
+        return future.thenCompose(result -> {
+
+            // pop request too much, should not add rest count here
+            if (isPopShouldStop(groupId, topicId, queueId)) {
+                return CompletableFuture.completedFuture(result);
+            }
+
+            // Current requests would calculate the total number of messages
+            // waiting to be filtered for new message arrival notifications in
+            // the long-polling service, need disregarding the backlog in order
+            // consumption scenario. If rest message num including the blocked
+            // queue accumulation would lead to frequent unnecessary wake-ups
+            // of long-polling requests, resulting unnecessary CPU usage.
+            // When client ack message, long-polling request would be notifications
+            // by AckMessageProcessor.ackOrderly() and message will not be delayed.
+            if (result.isFifo() && isFifoBlocked(result, groupId, topicId, queueId)) {
+                // should not add accumulation(max offset - consumer offset) here
+                return CompletableFuture.completedFuture(result);
+            }
+
+            int remain = batchSize - result.getMessageCount();
+            if (remain <= 0) {
+                result.addRestCount(this.getPendingFilterCount(groupId, topicId, queueId));
+                return CompletableFuture.completedFuture(result);
+            } else {
+                final long consumeOffset = this.getPopOffset(groupId, topicId, queueId, result.getInitMode());
+                return getMessageAsync(clientHost, groupId, topicId, queueId, consumeOffset, remain, filter)
+                    .thenApply(getMessageResult -> handleGetMessageResult(
+                        result, getMessageResult, topicId, queueId, retryType, consumeOffset));
+            }
+        });
+    }
+
+    public CompletableFuture<PopConsumerContext> popAsync(String clientHost, long popTime, long invisibleTime,
+        String groupId, String topicId, int queueId, int batchSize, boolean fifo, String attemptId, int initMode,
+        MessageFilter filter) {
+
+        PopConsumerContext popConsumerContext =
+            new PopConsumerContext(clientHost, popTime, invisibleTime, groupId, fifo, initMode, attemptId);
+
+        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topicId);
+        if (topicConfig == null || !consumerLockService.tryLock(groupId, topicId)) {
+            return CompletableFuture.completedFuture(popConsumerContext);
+        }
+
+        log.debug("PopConsumerService popAsync, groupId={}, topicId={}, queueId={}, " +
+                "batchSize={}, invisibleTime={}, fifo={}, attemptId={}, filter={}",
+            groupId, topicId, queueId, batchSize, invisibleTime, fifo, attemptId, filter);
+
+        String requestKey = groupId + "@" + topicId;
+        String retryTopicV1 = KeyBuilder.buildPopRetryTopicV1(topicId, groupId);
+        String retryTopicV2 = KeyBuilder.buildPopRetryTopicV2(topicId, groupId);
+        long requestCount = Objects.requireNonNull(ConcurrentHashMapUtils.computeIfAbsent(
+            requestCountTable, requestKey, k -> new AtomicLong(0L))).getAndIncrement();
+        boolean preferRetry = requestCount % 5L == 0L;
+
+        CompletableFuture<PopConsumerContext> getMessageFuture =
+            CompletableFuture.completedFuture(popConsumerContext);
+
+        try {
+            if (!fifo && preferRetry) {
+                if (brokerConfig.isRetrieveMessageFromPopRetryTopicV1()) {
+                    getMessageFuture = this.getMessageAsync(getMessageFuture, clientHost, groupId,
+                        retryTopicV1, 0, batchSize, filter, PopConsumerRecord.RetryType.RETRY_TOPIC_V1);
+                }
+
+                if (brokerConfig.isEnableRetryTopicV2()) {
+                    getMessageFuture = this.getMessageAsync(getMessageFuture, clientHost, groupId,
+                        retryTopicV2, 0, batchSize, filter, PopConsumerRecord.RetryType.RETRY_TOPIC_V2);
+                }
+            }
+
+            if (queueId != -1) {
+                getMessageFuture = this.getMessageAsync(getMessageFuture, clientHost, groupId,
+                    topicId, queueId, batchSize, filter, PopConsumerRecord.RetryType.NORMAL_TOPIC);
+            } else {
+                for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
+                    int current = (int) ((requestCount + i) % topicConfig.getReadQueueNums());
+                    getMessageFuture = this.getMessageAsync(getMessageFuture, clientHost, groupId,
+                        topicId, current, batchSize, filter, PopConsumerRecord.RetryType.NORMAL_TOPIC);
+                }
+
+                if (!fifo && !preferRetry) {
+                    if (brokerConfig.isRetrieveMessageFromPopRetryTopicV1()) {
+                        getMessageFuture = this.getMessageAsync(getMessageFuture, clientHost, groupId,
+                            retryTopicV1, 0, batchSize, filter, PopConsumerRecord.RetryType.RETRY_TOPIC_V1);
+                    }
+
+                    if (brokerConfig.isEnableRetryTopicV2()) {
+                        getMessageFuture = this.getMessageAsync(getMessageFuture, clientHost, groupId,
+                            retryTopicV2, 0, batchSize, filter, PopConsumerRecord.RetryType.RETRY_TOPIC_V2);
+                    }
+                }
+            }
+
+            return getMessageFuture.thenCompose(result -> {
+                if (result.isFound() && !result.isFifo()) {
+                    if (brokerConfig.isEnablePopBufferMerge() &&
+                        popConsumerCache != null && !popConsumerCache.isCacheFull()) {
+                        this.popConsumerCache.writeRecords(result.getPopConsumerRecordList());
+                    } else {
+                        this.popConsumerStore.writeRecords(result.getPopConsumerRecordList());
+                    }
+
+                    for (int i = 0; i < result.getGetMessageResultList().size(); i++) {
+                        GetMessageResult getMessageResult = result.getGetMessageResultList().get(i);
+                        PopConsumerRecord popConsumerRecord = result.getPopConsumerRecordList().get(i);
+
+                        // If the buffer belong retries message, the message needs to be re-encoded.
+                        // The buffer should not be re-encoded when popResponseReturnActualRetryTopic
+                        // is true or the current topic is not a retry topic.
+                        boolean recode = brokerConfig.isPopResponseReturnActualRetryTopic();
+                        if (recode && popConsumerRecord.isRetry()) {
+                            result.getGetMessageResultList().set(i, this.recodeRetryMessage(
+                                getMessageResult, popConsumerRecord.getTopicId(),
+                                popConsumerRecord.getQueueId(), result.getPopTime(), invisibleTime));
+                        }
+                    }
+                }
+                return CompletableFuture.completedFuture(result);
+            }).whenComplete((result, throwable) -> {
+                try {
+                    if (throwable != null) {
+                        log.error("PopConsumerService popAsync get message error",
+                            throwable instanceof CompletionException ? throwable.getCause() : throwable);
+                    }
+                    if (result.getMessageCount() > 0) {
+                        log.debug("PopConsumerService popAsync result, found={}, groupId={}, topicId={}, queueId={}, " +
+                                "batchSize={}, invisibleTime={}, fifo={}, attemptId={}, filter={}", result.getMessageCount(),
+                            groupId, topicId, queueId, batchSize, invisibleTime, fifo, attemptId, filter);
+                    }
+                } finally {
+                    consumerLockService.unlock(groupId, topicId);
+                }
+            });
+        } catch (Throwable t) {
+            log.error("PopConsumerService popAsync error", t);
+        }
+
+        return getMessageFuture;
+    }
+
+    // Notify polling request when receive orderly ack
+    public CompletableFuture<Boolean> ackAsync(
+        long popTime, long invisibleTime, String groupId, String topicId, int queueId, long offset) {
+
+        if (brokerConfig.isPopConsumerKVServiceLog()) {
+            log.info("PopConsumerService ack, time={}, invisible={}, groupId={}, topic={}, queueId={}, offset={}",
+                popTime, invisibleTime, groupId, topicId, queueId, offset);
+        }
+
+        PopConsumerRecord record = new PopConsumerRecord(
+            popTime, groupId, topicId, queueId, 0, invisibleTime, offset, null);
+
+        if (brokerConfig.isEnablePopBufferMerge() && popConsumerCache != null) {
+            if (popConsumerCache.deleteRecords(Collections.singletonList(record)).isEmpty()) {
+                return CompletableFuture.completedFuture(true);
+            }
+        }
+
+        this.popConsumerStore.deleteRecords(Collections.singletonList(record));
+        return CompletableFuture.completedFuture(true);
+    }
+
+    // refer ChangeInvisibleTimeProcessor.appendCheckPointThenAckOrigin
+    public void changeInvisibilityDuration(long popTime, long invisibleTime,
+        long changedPopTime, long changedInvisibleTime, String groupId, String topicId, int queueId, long offset) {
+
+        if (brokerConfig.isPopConsumerKVServiceLog()) {
+            log.info("PopConsumerService change, time={}, invisible={}, " +
+                    "groupId={}, topic={}, queueId={}, offset={}, new time={}, new invisible={}",
+                popTime, invisibleTime, groupId, topicId, queueId, offset, changedPopTime, changedInvisibleTime);
+        }
+
+        PopConsumerRecord ckRecord = new PopConsumerRecord(
+            changedPopTime, groupId, topicId, queueId, 0, changedInvisibleTime, offset, null);
+
+        PopConsumerRecord ackRecord = new PopConsumerRecord(
+            popTime, groupId, topicId, queueId, 0, invisibleTime, offset, null);
+
+        this.popConsumerStore.writeRecords(Collections.singletonList(ckRecord));
+
+        if (brokerConfig.isEnablePopBufferMerge() && popConsumerCache != null) {
+            if (popConsumerCache.deleteRecords(Collections.singletonList(ackRecord)).isEmpty()) {
+                return;
+            }
+        }
+
+        this.popConsumerStore.deleteRecords(Collections.singletonList(ackRecord));
+    }
+
+    // Use broker escape bridge to support remote read
+    public CompletableFuture<Triple<MessageExt, String, Boolean>> getMessageAsync(PopConsumerRecord consumerRecord) {
+        return this.brokerController.getEscapeBridge().getMessageAsync(consumerRecord.getTopicId(),
+            consumerRecord.getOffset(), consumerRecord.getQueueId(), brokerConfig.getBrokerName(), false);
+    }
+
+    public CompletableFuture<Boolean> revive(PopConsumerRecord record) {
+        return this.getMessageAsync(record)
+            .thenCompose(result -> {
+                if (result == null) {
+                    log.error("PopConsumerService revive error, message may be lost, record={}", record);
+                    return CompletableFuture.completedFuture(false);
+                }
+                // true in triple right means get message needs to be retried
+                if (result.getLeft() == null) {
+                    log.info("PopConsumerService revive no need retry, record={}", record);
+                    return CompletableFuture.completedFuture(!result.getRight());
+                }
+                return CompletableFuture.completedFuture(this.reviveRetry(record, result.getLeft()));
+            });
+    }
+
+    public void clearCache(String groupId, String topicId, int queueId) {
+        while (consumerLockService.tryLock(groupId, topicId)) {
+        }
+        try {
+            if (popConsumerCache != null) {
+                popConsumerCache.removeRecords(groupId, topicId, queueId);
+            }
+        } finally {
+            consumerLockService.unlock(groupId, topicId);
+        }
+    }
+
+    public long revive(AtomicLong currentTime, int maxCount) {
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        long upperTime = System.currentTimeMillis() - 50L;
+        List<PopConsumerRecord> consumerRecords = this.popConsumerStore.scanExpiredRecords(
+                currentTime.get() - TimeUnit.SECONDS.toMillis(3), upperTime, maxCount);
+        long scanCostTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+        Queue<PopConsumerRecord> failureList = new LinkedBlockingQueue<>();
+        List<CompletableFuture<?>> futureList = new ArrayList<>(consumerRecords.size());
+
+        // could merge read operation here
+        for (PopConsumerRecord record : consumerRecords) {
+            futureList.add(this.revive(record).thenAccept(result -> {
+                if (!result) {
+                    if (record.getAttemptTimes() < brokerConfig.getPopReviveMaxAttemptTimes()) {
+                        long backoffInterval = 1000L * REWRITE_INTERVALS_IN_SECONDS[
+                            Math.min(REWRITE_INTERVALS_IN_SECONDS.length, record.getAttemptTimes())];
+                        long nextInvisibleTime = record.getInvisibleTime() + backoffInterval;
+                        PopConsumerRecord retryRecord = new PopConsumerRecord(System.currentTimeMillis(),
+                            record.getGroupId(), record.getTopicId(), record.getQueueId(),
+                            record.getRetryFlag(), nextInvisibleTime, record.getOffset(), record.getAttemptId());
+                        retryRecord.setAttemptTimes(record.getAttemptTimes() + 1);
+                        failureList.add(retryRecord);
+                        log.warn("PopConsumerService revive backoff retry, record={}", retryRecord);
+                    } else {
+                        log.error("PopConsumerService drop record, message may be lost, record={}", record);
+                    }
+                }
+            }));
+        }
+
+        CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join();
+        this.popConsumerStore.writeRecords(new ArrayList<>(failureList));
+        this.popConsumerStore.deleteRecords(consumerRecords);
+        currentTime.set(consumerRecords.isEmpty() ?
+            upperTime : consumerRecords.get(consumerRecords.size() - 1).getVisibilityTimeout());
+
+        if (brokerConfig.isEnablePopBufferMerge()) {
+            log.info("PopConsumerService, key size={}, cache size={}, revive count={}, failure count={}, " +
+                    "behindInMillis={}, scanInMillis={}, costInMillis={}",
+                popConsumerCache.getCacheKeySize(), popConsumerCache.getCacheSize(),
+                consumerRecords.size(), failureList.size(), upperTime - currentTime.get(),
+                scanCostTime, stopwatch.elapsed(TimeUnit.MILLISECONDS));
+        } else {
+            log.info("PopConsumerService, revive count={}, failure count={}, " +
+                    "behindInMillis={}, scanInMillis={}, costInMillis={}",
+                consumerRecords.size(), failureList.size(), upperTime - currentTime.get(),
+                scanCostTime, stopwatch.elapsed(TimeUnit.MILLISECONDS));
+        }
+
+        return consumerRecords.size();
+    }
+
+    public void createRetryTopicIfNeeded(String groupId, String topicId) {
+        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topicId);
+        if (topicConfig != null) {
+            return;
+        }
+
+        topicConfig = new TopicConfig(topicId, 1, 1,
+            PermName.PERM_READ | PermName.PERM_WRITE, 0);
+        topicConfig.setTopicFilterType(TopicFilterType.SINGLE_TAG);
+        brokerController.getTopicConfigManager().updateTopicConfig(topicConfig);
+
+        long offset = this.brokerController.getConsumerOffsetManager().queryOffset(groupId, topicId, 0);
+        if (offset < 0) {
+            this.brokerController.getConsumerOffsetManager().commitOffset(
+                "InitPopOffset", groupId, topicId, 0, 0);
+        }
+    }
+
+    @SuppressWarnings("DuplicatedCode")
+    // org.apache.rocketmq.broker.processor.PopReviveService#reviveRetry
+    public boolean reviveRetry(PopConsumerRecord record, MessageExt messageExt) {
+
+        if (brokerConfig.isPopConsumerKVServiceLog()) {
+            log.info("PopConsumerService revive, time={}, invisible={}, groupId={}, topic={}, queueId={}, offset={}",
+                record.getPopTime(), record.getInvisibleTime(), record.getGroupId(), record.getTopicId(),
+                record.getQueueId(), record.getOffset());
+        }
+
+        boolean retry = StringUtils.startsWith(record.getTopicId(), MixAll.RETRY_GROUP_TOPIC_PREFIX);
+        String retryTopic = retry ? record.getTopicId() : KeyBuilder.buildPopRetryTopic(
+            record.getTopicId(), record.getGroupId(), brokerConfig.isEnableRetryTopicV2());
+        this.createRetryTopicIfNeeded(record.getGroupId(), retryTopic);
+
+        // deep copy here
+        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
+        msgInner.setTopic(retryTopic);
+        msgInner.setBody(messageExt.getBody() != null ? messageExt.getBody() : new byte[] {});
+        msgInner.setQueueId(0);
+        if (messageExt.getTags() != null) {
+            msgInner.setTags(messageExt.getTags());
+        } else {
+            MessageAccessor.setProperties(msgInner, new HashMap<>());
+        }
+
+        msgInner.setBornTimestamp(messageExt.getBornTimestamp());
+        msgInner.setFlag(messageExt.getFlag());
+        msgInner.setSysFlag(messageExt.getSysFlag());
+        msgInner.setBornHost(brokerController.getStoreHost());
+        msgInner.setStoreHost(brokerController.getStoreHost());
+        msgInner.setReconsumeTimes(messageExt.getReconsumeTimes() + 1);
+        msgInner.getProperties().putAll(messageExt.getProperties());
+
+        // set first pop time here
+        if (messageExt.getReconsumeTimes() == 0 ||
+            msgInner.getProperties().get(MessageConst.PROPERTY_FIRST_POP_TIME) == null) {
+            msgInner.getProperties().put(MessageConst.PROPERTY_FIRST_POP_TIME, String.valueOf(record.getPopTime()));
+        }
+        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+
+        PutMessageResult putMessageResult =
+            brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+
+        if (putMessageResult.getAppendMessageResult() == null ||
+            putMessageResult.getAppendMessageResult().getStatus() != AppendMessageStatus.PUT_OK) {
+            log.error("PopConsumerService revive retry msg error, put status={}, ck={}, delay={}ms",
+                putMessageResult, JSON.toJSONString(record), System.currentTimeMillis() - record.getVisibilityTimeout());
+            return false;
+        }
+
+        if (this.brokerController.getBrokerStatsManager() != null) {
+            this.brokerController.getBrokerStatsManager().incBrokerPutNums(msgInner.getTopic(), 1);
+            this.brokerController.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic());
+            this.brokerController.getBrokerStatsManager().incTopicPutSize(
+                msgInner.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());
+        }
+        return true;
+    }
+
+    // Export kv store record to revive topic
+    @SuppressWarnings("ExtractMethodRecommender")
+    public synchronized void transferToFsStore() {
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        while (true) {
+            try {
+                List<PopConsumerRecord> consumerRecords = this.popConsumerStore.scanExpiredRecords(
+                    0, Long.MAX_VALUE, brokerConfig.getPopReviveMaxReturnSizePerRead());
+                if (consumerRecords == null || consumerRecords.isEmpty()) {
+                    break;
+                }
+                for (PopConsumerRecord record : consumerRecords) {
+                    PopCheckPoint ck = new PopCheckPoint();
+                    ck.setBitMap(0);
+                    ck.setNum((byte) 1);
+                    ck.setPopTime(record.getPopTime());
+                    ck.setInvisibleTime(record.getInvisibleTime());
+                    ck.setStartOffset(record.getOffset());
+                    ck.setCId(record.getGroupId());
+                    ck.setTopic(record.getTopicId());
+                    ck.setQueueId(record.getQueueId());
+                    ck.setBrokerName(brokerConfig.getBrokerName());
+                    ck.addDiff(0);
+                    ck.setRePutTimes(ck.getRePutTimes());
+                    int reviveQueueId = (int) record.getOffset() % brokerConfig.getReviveQueueNum();
+                    MessageExtBrokerInner ckMsg =
+                        brokerController.getPopMessageProcessor().buildCkMsg(ck, reviveQueueId);
+                    brokerController.getMessageStore().asyncPutMessage(ckMsg).join();
+                }
+                log.info("PopConsumerStore transfer from kvStore to fsStore, count={}", consumerRecords.size());
+                this.popConsumerStore.deleteRecords(consumerRecords);
+                this.waitForRunning(1);
+            } catch (Throwable t) {
+                log.error("PopConsumerStore transfer from kvStore to fsStore failure", t);
+            }
+        }
+        log.info("PopConsumerStore transfer to fsStore finish, cost={}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
+    }
+
+    @Override
+    public String getServiceName() {
+        return PopConsumerService.class.getSimpleName();
+    }
+
+    @VisibleForTesting
+    protected PopConsumerKVStore getPopConsumerStore() {
+        return popConsumerStore;
+    }
+
+    public PopConsumerLockService getConsumerLockService() {
+        return consumerLockService;
+    }
+
+    @Override
+    public void start() {
+        if (!this.popConsumerStore.start()) {
+            throw new RuntimeException("PopConsumerStore init error");
+        }
+        if (this.popConsumerCache != null) {
+            this.popConsumerCache.start();
+        }
+        super.start();
+    }
+
+    @Override
+    public void shutdown() {
+        // Block shutdown thread until write records finish
+        super.shutdown();
+        do {
+            this.waitForRunning(10);
+        }
+        while (consumerRunning.get());
+        if (this.popConsumerCache != null) {
+            this.popConsumerCache.shutdown();
+        }
+        if (this.popConsumerStore != null) {
+            this.popConsumerStore.shutdown();
+        }
+    }
+
+    @Override
+    public void run() {
+        this.consumerRunning.set(true);
+        while (!isStopped()) {
+            try {
+                // to prevent concurrency issues during read and write operations
+                long reviveCount = this.revive(this.currentTime,
+                    brokerConfig.getPopReviveMaxReturnSizePerRead());
+
+                long current = System.currentTimeMillis();
+                if (lastCleanupLockTime.get() + TimeUnit.MINUTES.toMillis(1) < current) {
+                    this.consumerLockService.removeTimeout();
+                    this.lastCleanupLockTime.set(current);
+                }
+
+                if (reviveCount < brokerConfig.getPopReviveMaxReturnSizePerRead()) {
+                    this.waitForRunning(500);
+                }
+            } catch (Exception e) {
+                log.error("PopConsumerService revive error", e);
+                this.waitForRunning(500);
+            }
+        }
+        this.consumerRunning.set(false);
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
index b348ecb8f09..16c137dd2e7 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
@@ -21,6 +21,7 @@
 import java.net.SocketAddress;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Random;
 import java.util.concurrent.ThreadLocalRandom;
 import org.apache.rocketmq.broker.BrokerController;
@@ -370,18 +371,22 @@ protected SendMessageContext buildMsgContext(ChannelHandlerContext ctx,
         sendMessageContext.setCommercialOwner(owner);
 
         Map<String, String> properties = MessageDecoder.string2messageProperties(requestHeader.getProperties());
-        String uniqueKey = properties.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);
         properties.put(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId());
         properties.put(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn()));
         requestHeader.setProperties(MessageDecoder.messageProperties2String(properties));
 
-        if (uniqueKey == null) {
-            uniqueKey = "";
-        }
-        sendMessageContext.setMsgUniqueKey(uniqueKey);
+        String uniqueKey = properties.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);
+        sendMessageContext.setMsgUniqueKey(Optional.ofNullable(uniqueKey).orElse(""));
 
         if (properties.containsKey(MessageConst.PROPERTY_SHARDING_KEY)) {
             sendMessageContext.setMsgType(MessageType.Order_Msg);
+        } else if (properties.containsKey(MessageConst.PROPERTY_DELAY_TIME_LEVEL)
+                || properties.containsKey(MessageConst.PROPERTY_TIMER_DELIVER_MS)
+                || properties.containsKey(MessageConst.PROPERTY_TIMER_DELAY_SEC)
+                || properties.containsKey(MessageConst.PROPERTY_TIMER_DELAY_MS)) {
+            sendMessageContext.setMsgType(MessageType.Delay_Msg);
+        } else if (Boolean.parseBoolean(properties.get(MessageConst.PROPERTY_TRANSACTION_PREPARED))) {
+            sendMessageContext.setMsgType(MessageType.Trans_Msg_Half);
         } else {
             sendMessageContext.setMsgType(MessageType.Normal_Msg);
         }
@@ -469,7 +474,7 @@ protected RemotingCommand msgCheck(final ChannelHandlerContext ctx,
 
         TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(requestHeader.getTopic());
         if (!result.isValid()) {
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.INVALID_PARAMETER);
             response.setRemark(result.getRemark());
             return response;
         }
@@ -524,7 +529,7 @@ protected RemotingCommand msgCheck(final ChannelHandlerContext ctx,
                 RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
 
             LOGGER.warn(errorInfo);
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.INVALID_PARAMETER);
             response.setRemark(errorInfo);
 
             return response;
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
index 9a56498632f..23a4f6167c6 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
@@ -20,8 +20,12 @@
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
 import java.util.BitSet;
+import java.nio.charset.StandardCharsets;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.metrics.PopMetricsManager;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.broker.offset.ConsumerOrderInfoManager;
+import org.apache.rocketmq.broker.pop.PopConsumerLockService;
 import org.apache.rocketmq.common.KeyBuilder;
 import org.apache.rocketmq.common.PopAckConstants;
 import org.apache.rocketmq.common.TopicConfig;
@@ -30,7 +34,6 @@
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExtBrokerInner;
-import org.apache.rocketmq.common.utils.DataConverter;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
@@ -45,10 +48,12 @@
 import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.pop.AckMsg;
 import org.apache.rocketmq.store.pop.BatchAckMsg;
 
 public class AckMessageProcessor implements NettyRequestProcessor {
+
     private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
     private final BrokerController brokerController;
     private final String reviveTopic;
@@ -56,7 +61,8 @@ public class AckMessageProcessor implements NettyRequestProcessor {
 
     public AckMessageProcessor(final BrokerController brokerController) {
         this.brokerController = brokerController;
-        this.reviveTopic = PopAckConstants.buildClusterReviveTopic(this.brokerController.getBrokerConfig().getBrokerClusterName());
+        this.reviveTopic = PopAckConstants.buildClusterReviveTopic(
+            this.brokerController.getBrokerConfig().getBrokerClusterName());
         this.popReviveServices = new PopReviveService[this.brokerController.getBrokerConfig().getReviveQueueNum()];
         for (int i = 0; i < this.brokerController.getBrokerConfig().getReviveQueueNum(); i++) {
             this.popReviveServices[i] = new PopReviveService(brokerController, reviveTopic, i);
@@ -98,7 +104,7 @@ public boolean isPopReviveServiceRunning() {
 
     @Override
     public RemotingCommand processRequest(final ChannelHandlerContext ctx,
-                                          RemotingCommand request) throws RemotingCommandException {
+        RemotingCommand request) throws RemotingCommandException {
         return this.processRequest(ctx.channel(), request, true);
     }
 
@@ -108,7 +114,7 @@ public boolean rejectRequest() {
     }
 
     private RemotingCommand processRequest(final Channel channel, RemotingCommand request,
-                                           boolean brokerAllowSuspend) throws RemotingCommandException {
+        boolean brokerAllowSuspend) throws RemotingCommandException {
         AckMessageRequestHeader requestHeader;
         BatchAckMessageRequestBody reqBody = null;
         final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);
@@ -126,7 +132,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
 
             if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums() || requestHeader.getQueueId() < 0) {
                 String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
-                        requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
+                    requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
                 POP_LOGGER.warn(errorInfo);
                 response.setCode(ResponseCode.MESSAGE_ILLEGAL);
                 response.setRemark(errorInfo);
@@ -134,17 +140,25 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
             }
 
             long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
-            long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+            long maxOffset;
+            try {
+                maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+            } catch (ConsumeQueueException e) {
+                throw new RemotingCommandException("Failed to get max offset", e);
+            }
             if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) {
                 String errorInfo = String.format("offset is illegal, key:%s@%d, commit:%d, store:%d~%d",
-                        requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getOffset(), minOffset, maxOffset);
+                    requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getOffset(), minOffset, maxOffset);
                 POP_LOGGER.warn(errorInfo);
                 response.setCode(ResponseCode.NO_MESSAGE);
                 response.setRemark(errorInfo);
                 return response;
             }
-
-            appendAck(requestHeader, null, response, channel, null);
+            if (brokerController.getBrokerConfig().isPopConsumerKVServiceEnable()) {
+                appendAckNew(requestHeader, null, response, channel, null);
+            } else {
+                appendAck(requestHeader, null, response, channel, null);
+            }
         } else if (request.getCode() == RequestCode.BATCH_ACK_MESSAGE) {
             if (request.getBody() != null) {
                 reqBody = BatchAckMessageRequestBody.decode(request.getBody(), BatchAckMessageRequestBody.class);
@@ -154,7 +168,11 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
                 return response;
             }
             for (BatchAck bAck : reqBody.getAcks()) {
-                appendAck(null, bAck, response, channel, reqBody.getBrokerName());
+                if (brokerController.getBrokerConfig().isPopConsumerKVServiceEnable()) {
+                    appendAckNew(null, bAck, response, channel, reqBody.getBrokerName());
+                } else {
+                    appendAck(null, bAck, response, channel, reqBody.getBrokerName());
+                }
             }
         } else {
             POP_LOGGER.error("AckMessageProcessor failed to process RequestCode: {}, consumer: {} ", request.getCode(), RemotingHelper.parseChannelRemoteAddr(channel));
@@ -165,7 +183,8 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
         return response;
     }
 
-    private void appendAck(final AckMessageRequestHeader requestHeader, final BatchAck batchAck, final RemotingCommand response, final Channel channel, String brokerName) {
+    private void appendAck(final AckMessageRequestHeader requestHeader, final BatchAck batchAck,
+        final RemotingCommand response, final Channel channel, String brokerName) throws RemotingCommandException {
         String[] extraInfo;
         String consumeGroup, topic;
         int qId, rqId;
@@ -205,7 +224,12 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA
             invisibleTime = batchAck.getInvisibleTime();
 
             long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, qId);
-            long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, qId);
+            long maxOffset;
+            try {
+                maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, qId);
+            } catch (ConsumeQueueException e) {
+                throw new RemotingCommandException("Failed to get max offset in queue", e);
+            }
             if (minOffset == -1 || maxOffset == -1) {
                 POP_LOGGER.error("Illegal topic or queue found when batch ack {}", batchAck);
                 return;
@@ -253,7 +277,7 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA
 
         MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
         msgInner.setTopic(reviveTopic);
-        msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));
+        msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(StandardCharsets.UTF_8));
         msgInner.setQueueId(rqId);
         if (ackMsg instanceof BatchAckMsg) {
             msgInner.setTags(PopAckConstants.BATCH_ACK_TAG);
@@ -268,18 +292,104 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA
         msgInner.setDeliverTimeMs(popTime + invisibleTime);
         msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));
         msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
-        PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+        if (brokerController.getBrokerConfig().isAppendAckAsync()) {
+            int finalAckCount = ackCount;
+            this.brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> {
+                handlePutMessageResult(putMessageResult, ackMsg, topic, consumeGroup, popTime, qId, finalAckCount);
+            }).exceptionally(throwable -> {
+                handlePutMessageResult(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null, false),
+                    ackMsg, topic, consumeGroup, popTime, qId, finalAckCount);
+                POP_LOGGER.error("put ack msg error ", throwable);
+                return null;
+            });
+        } else {
+            PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+            handlePutMessageResult(putMessageResult, ackMsg, topic, consumeGroup, popTime, qId, ackCount);
+        }
+    }
+
+    private void appendAckNew(final AckMessageRequestHeader requestHeader, final BatchAck batchAck,
+        final RemotingCommand response, final Channel channel, String brokerName) throws RemotingCommandException {
+
+        if (requestHeader != null && batchAck == null) {
+            String[] extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo());
+            String groupId = requestHeader.getConsumerGroup();
+            String topicId = requestHeader.getTopic();
+            int queueId = requestHeader.getQueueId();
+            long ackOffset = requestHeader.getOffset();
+            long popTime = ExtraInfoUtil.getPopTime(extraInfo);
+            long invisibleTime = ExtraInfoUtil.getInvisibleTime(extraInfo);
+
+            int reviveQueueId = ExtraInfoUtil.getReviveQid(extraInfo);
+            if (reviveQueueId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) {
+                ackOrderlyNew(topicId, groupId, queueId, ackOffset, popTime, invisibleTime, channel, response);
+            } else {
+                this.brokerController.getPopConsumerService().ackAsync(
+                    popTime, invisibleTime, groupId, topicId, queueId, ackOffset);
+            }
+
+            this.brokerController.getBrokerStatsManager().incBrokerAckNums(1);
+            this.brokerController.getBrokerStatsManager().incGroupAckNums(groupId, topicId, 1);
+        } else {
+            String groupId = batchAck.getConsumerGroup();
+            String topicId = ExtraInfoUtil.getRealTopic(
+                batchAck.getTopic(), batchAck.getConsumerGroup(), batchAck.getRetry());
+            int queueId = batchAck.getQueueId();
+            int reviveQueueId = batchAck.getReviveQueueId();
+            long startOffset = batchAck.getStartOffset();
+            long popTime = batchAck.getPopTime();
+            long invisibleTime = batchAck.getInvisibleTime();
+
+            try {
+                long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(topicId, queueId);
+                long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topicId, queueId);
+                if (minOffset == -1 || maxOffset == -1) {
+                    POP_LOGGER.error("Illegal topic or queue found when batch ack {}", batchAck);
+                    return;
+                }
+
+                int ackCount = 0;
+                // Maintain consistency with the old implementation code style
+                BitSet bitSet = batchAck.getBitSet();
+                for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
+                    if (i == Integer.MAX_VALUE) {
+                        break;
+                    }
+                    long offset = startOffset + i;
+                    if (offset < minOffset || offset > maxOffset) {
+                        continue;
+                    }
+                    if (reviveQueueId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) {
+                        ackOrderlyNew(topicId, groupId, queueId, offset, popTime, invisibleTime, channel, response);
+                    } else {
+                        this.brokerController.getPopConsumerService().ackAsync(
+                            popTime, invisibleTime, groupId, topicId, queueId, offset);
+                    }
+                    ackCount++;
+                }
+
+                this.brokerController.getBrokerStatsManager().incBrokerAckNums(ackCount);
+                this.brokerController.getBrokerStatsManager().incGroupAckNums(groupId, topicId, ackCount);
+            } catch (ConsumeQueueException e) {
+                throw new RemotingCommandException("Failed to ack message", e);
+            }
+        }
+    }
+
+    private void handlePutMessageResult(PutMessageResult putMessageResult, AckMsg ackMsg, String topic,
+        String consumeGroup, long popTime, int qId, int ackCount) {
         if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
-                && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
-                && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
-                && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
             POP_LOGGER.error("put ack msg error:" + putMessageResult);
         }
         PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus());
         brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount);
     }
 
-    protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOffset, long popTime, long invisibleTime, Channel channel, RemotingCommand response) {
+    protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOffset, long popTime,
+        long invisibleTime, Channel channel, RemotingCommand response) {
         String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId;
         long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId);
         if (ackOffset < oldOffset) {
@@ -293,19 +403,14 @@ protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOf
                 return;
             }
             long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext(
-                topic, consumeGroup,
-                qId, ackOffset,
-                popTime);
+                topic, consumeGroup, qId, ackOffset, popTime);
             if (nextOffset > -1) {
-                if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset(
-                    topic, consumeGroup, qId)) {
-                    this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(),
-                        consumeGroup, topic, qId, nextOffset);
+                if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset(topic, consumeGroup, qId)) {
+                    this.brokerController.getConsumerOffsetManager().commitOffset(
+                        channel.remoteAddress().toString(), consumeGroup, topic, qId, nextOffset);
                 }
-                if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic,
-                    consumeGroup, qId, invisibleTime)) {
-                    this.brokerController.getPopMessageProcessor().notifyMessageArriving(
-                        topic, consumeGroup, qId);
+                if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, consumeGroup, qId, invisibleTime)) {
+                    this.brokerController.getPopMessageProcessor().notifyMessageArriving(topic, qId, consumeGroup);
                 }
             } else if (nextOffset == -1) {
                 String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s",
@@ -320,4 +425,55 @@ protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOf
         }
         brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, 1);
     }
+
+    protected void ackOrderlyNew(String topic, String consumeGroup, int qId, long ackOffset, long popTime,
+        long invisibleTime, Channel channel, RemotingCommand response) {
+
+        ConsumerOffsetManager consumerOffsetManager = this.brokerController.getConsumerOffsetManager();
+        ConsumerOrderInfoManager consumerOrderInfoManager = brokerController.getConsumerOrderInfoManager();
+        PopConsumerLockService consumerLockService = this.brokerController.getPopConsumerService().getConsumerLockService();
+
+        long oldOffset = consumerOffsetManager.queryOffset(consumeGroup, topic, qId);
+        if (ackOffset < oldOffset) {
+            return;
+        }
+
+        while (!consumerLockService.tryLock(consumeGroup, topic)) {
+        }
+
+        try {
+            // double check
+            oldOffset = consumerOffsetManager.queryOffset(consumeGroup, topic, qId);
+            if (ackOffset < oldOffset) {
+                return;
+            }
+
+            long nextOffset = consumerOrderInfoManager.commitAndNext(topic, consumeGroup, qId, ackOffset, popTime);
+            if (brokerController.getBrokerConfig().isPopConsumerKVServiceLog()) {
+                POP_LOGGER.info("PopConsumerService ack orderly, time={}, topicId={}, groupId={}, queueId={}, " +
+                        "offset={}, next={}", popTime, topic, consumeGroup, qId, ackOffset, nextOffset);
+            }
+
+            if (nextOffset > -1L) {
+                if (!consumerOffsetManager.hasOffsetReset(topic, consumeGroup, qId)) {
+                    String remoteAddress = RemotingHelper.parseSocketAddressAddr(channel.remoteAddress());
+                    consumerOffsetManager.commitOffset(remoteAddress, consumeGroup, topic, qId, nextOffset);
+                }
+                if (!consumerOrderInfoManager.checkBlock(null, topic, consumeGroup, qId, invisibleTime)) {
+                    this.brokerController.getPopMessageProcessor().notifyMessageArriving(topic, qId, consumeGroup);
+                }
+                return;
+            }
+
+            if (nextOffset == -1) {
+                String errorInfo = String.format("offset is illegal, key:%s %s %s, old:%d, commit:%d, next:%d, %s",
+                    consumeGroup, topic, qId, oldOffset, ackOffset, nextOffset, channel.remoteAddress());
+                POP_LOGGER.warn(errorInfo);
+                response.setCode(ResponseCode.MESSAGE_ILLEGAL);
+                response.setRemark(errorInfo);
+            }
+        } finally {
+            consumerLockService.unlock(consumeGroup, topic);
+        }
+    }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
index 67a4d45447c..812ca90e82b 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
@@ -21,6 +21,7 @@
 import com.google.common.collect.Sets;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
+import io.opentelemetry.api.common.Attributes;
 import java.io.UnsupportedEncodingException;
 import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
@@ -34,28 +35,49 @@
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.acl.AccessValidator;
-import org.apache.rocketmq.acl.plain.PlainAccessValidator;
+import org.apache.rocketmq.auth.authentication.enums.UserType;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authorization.enums.PolicyType;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.authorization.model.Resource;
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.auth.converter.AclConverter;
+import org.apache.rocketmq.broker.auth.converter.UserConverter;
 import org.apache.rocketmq.broker.client.ClientChannelInfo;
 import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
+import org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager;
+import org.apache.rocketmq.broker.config.v1.RocksDBSubscriptionGroupManager;
+import org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager;
 import org.apache.rocketmq.broker.controller.ReplicasManager;
 import org.apache.rocketmq.broker.filter.ConsumerFilterData;
 import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
+import org.apache.rocketmq.broker.metrics.BrokerMetricsManager;
+import org.apache.rocketmq.broker.metrics.InvocationStatus;
 import org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin;
 import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
 import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil;
+import org.apache.rocketmq.common.BoundaryType;
 import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.CheckRocksdbCqWriteResult;
 import org.apache.rocketmq.common.KeyBuilder;
 import org.apache.rocketmq.common.LockCallback;
 import org.apache.rocketmq.common.MQVersion;
 import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.common.PlainAccessConfig;
+import org.apache.rocketmq.common.Pair;
+import org.apache.rocketmq.common.TopicAttributes;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.UnlockCallback;
 import org.apache.rocketmq.common.UtilAll;
@@ -75,13 +97,13 @@
 import org.apache.rocketmq.common.stats.StatsItem;
 import org.apache.rocketmq.common.stats.StatsSnapshot;
 import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.common.utils.ExceptionUtils;
 import org.apache.rocketmq.filter.util.BitsArray;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
-import org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;
 import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.protocol.LanguageCode;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
@@ -93,6 +115,7 @@
 import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper;
 import org.apache.rocketmq.remoting.protocol.admin.TopicOffset;
 import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;
+import org.apache.rocketmq.remoting.protocol.body.AclInfo;
 import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;
 import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData;
 import org.apache.rocketmq.remoting.protocol.body.BrokerStatsItem;
@@ -100,6 +123,7 @@
 import org.apache.rocketmq.remoting.protocol.body.ConsumeQueueData;
 import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList;
 import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection;
+import org.apache.rocketmq.remoting.protocol.body.CreateTopicListRequestBody;
 import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache;
 import org.apache.rocketmq.remoting.protocol.body.GroupList;
 import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
@@ -114,22 +138,28 @@
 import org.apache.rocketmq.remoting.protocol.body.QuerySubscriptionResponseBody;
 import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;
 import org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;
+import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupList;
 import org.apache.rocketmq.remoting.protocol.body.SyncStateSet;
 import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;
 import org.apache.rocketmq.remoting.protocol.body.TopicList;
 import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;
+import org.apache.rocketmq.remoting.protocol.body.UserInfo;
+import org.apache.rocketmq.remoting.protocol.header.CheckRocksdbCqWriteProgressRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.CloneGroupOffsetRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.CreateAccessConfigRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.CreateAclRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.DeleteAccessConfigRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.CreateUserRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.DeleteAclRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.DeleteSubscriptionGroupRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.DeleteUserRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ExchangeHAInfoRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ExchangeHAInfoResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.ExportRocksDBConfigToJsonRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetAclRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetAllProducerInfoRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetAllTopicConfigResponseHeader;
-import org.apache.rocketmq.remoting.protocol.header.GetBrokerAclConfigResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetBrokerConfigResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsInBrokerHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsRequestHeader;
@@ -146,6 +176,9 @@
 import org.apache.rocketmq.remoting.protocol.header.GetSubscriptionGroupConfigRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetTopicConfigRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetUserRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ListAclsRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ListUsersRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.NotifyBrokerRoleChangedRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.NotifyMinBrokerIdChangeRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.QueryConsumeQueueRequestHeader;
@@ -159,8 +192,9 @@
 import org.apache.rocketmq.remoting.protocol.header.ResumeCheckHalfMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SearchOffsetRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader;
-import org.apache.rocketmq.remoting.protocol.header.UpdateGlobalWhiteAddrsConfigRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateAclRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateUserRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ViewBrokerStatsDataRequestHeader;
 import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
 import org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem;
@@ -180,8 +214,12 @@
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.RocksDBMessageStore;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.StoreType;
 import org.apache.rocketmq.store.config.BrokerRole;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
+import org.apache.rocketmq.store.plugin.AbstractPluginMessageStore;
 import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
 import org.apache.rocketmq.store.queue.CqUnit;
 import org.apache.rocketmq.store.queue.ReferredIterator;
@@ -189,12 +227,15 @@
 import org.apache.rocketmq.store.timer.TimerMessageStore;
 import org.apache.rocketmq.store.util.LibC;
 
+import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_INVOCATION_STATUS;
+import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;
 import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
 
 public class AdminBrokerProcessor implements NettyRequestProcessor {
     private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     protected final BrokerController brokerController;
     protected Set<String> configBlackList = new HashSet<>();
+    private final ExecutorService asyncExecuteWorker = new ThreadPoolExecutor(0, 4, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
 
     public AdminBrokerProcessor(final BrokerController brokerController) {
         this.brokerController = brokerController;
@@ -215,6 +256,8 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx,
         switch (request.getCode()) {
             case RequestCode.UPDATE_AND_CREATE_TOPIC:
                 return this.updateAndCreateTopic(ctx, request);
+            case RequestCode.UPDATE_AND_CREATE_TOPIC_LIST:
+                return this.updateAndCreateTopicList(ctx, request);
             case RequestCode.DELETE_TOPIC_IN_BROKER:
                 return this.deleteTopic(ctx, request);
             case RequestCode.GET_ALL_TOPIC_CONFIG:
@@ -251,6 +294,8 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx,
                 return this.unlockBatchMQ(ctx, request);
             case RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP:
                 return this.updateAndCreateSubscriptionGroup(ctx, request);
+            case RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP_LIST:
+                return this.updateAndCreateSubscriptionGroupList(ctx, request);
             case RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG:
                 return this.getAllSubscriptionGroup(ctx, request);
             case RequestCode.DELETE_SUBSCRIPTIONGROUP:
@@ -305,18 +350,14 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx,
                 return fetchAllConsumeStatsInBroker(ctx, request);
             case RequestCode.QUERY_CONSUME_QUEUE:
                 return queryConsumeQueue(ctx, request);
+            case RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS:
+                return this.checkRocksdbCqWriteProgress(ctx, request);
+            case RequestCode.EXPORT_ROCKSDB_CONFIG_TO_JSON:
+                return this.exportRocksDBConfigToJson(ctx, request);
             case RequestCode.UPDATE_AND_GET_GROUP_FORBIDDEN:
                 return this.updateAndGetGroupForbidden(ctx, request);
             case RequestCode.GET_SUBSCRIPTIONGROUP_CONFIG:
                 return this.getSubscriptionGroup(ctx, request);
-            case RequestCode.UPDATE_AND_CREATE_ACL_CONFIG:
-                return updateAndCreateAccessConfig(ctx, request);
-            case RequestCode.DELETE_ACL_CONFIG:
-                return deleteAccessConfig(ctx, request);
-            case RequestCode.GET_BROKER_CLUSTER_ACL_INFO:
-                return getBrokerAclConfigVersion(ctx, request);
-            case RequestCode.UPDATE_GLOBAL_WHITE_ADDRS_CONFIG:
-                return updateGlobalWhiteAddrsConfig(ctx, request);
             case RequestCode.RESUME_CHECK_HALF_MESSAGE:
                 return resumeCheckHalfMessage(ctx, request);
             case RequestCode.GET_TOPIC_CONFIG:
@@ -335,6 +376,28 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx,
                 return this.getBrokerEpochCache(ctx, request);
             case RequestCode.NOTIFY_BROKER_ROLE_CHANGED:
                 return this.notifyBrokerRoleChanged(ctx, request);
+            case RequestCode.AUTH_CREATE_USER:
+                return this.createUser(ctx, request);
+            case RequestCode.AUTH_UPDATE_USER:
+                return this.updateUser(ctx, request);
+            case RequestCode.AUTH_DELETE_USER:
+                return this.deleteUser(ctx, request);
+            case RequestCode.AUTH_GET_USER:
+                return this.getUser(ctx, request);
+            case RequestCode.AUTH_LIST_USER:
+                return this.listUser(ctx, request);
+            case RequestCode.AUTH_CREATE_ACL:
+                return this.createAcl(ctx, request);
+            case RequestCode.AUTH_UPDATE_ACL:
+                return this.updateAcl(ctx, request);
+            case RequestCode.AUTH_DELETE_ACL:
+                return this.deleteAcl(ctx, request);
+            case RequestCode.AUTH_GET_ACL:
+                return this.getAcl(ctx, request);
+            case RequestCode.AUTH_LIST_ACL:
+                return this.listAcl(ctx, request);
+            case RequestCode.POP_ROLLBACK:
+                return this.transferPopToFsStore(ctx, request);
             default:
                 return getUnknownCmdResponse(ctx, request);
         }
@@ -351,10 +414,10 @@ private RemotingCommand getSubscriptionGroup(ChannelHandlerContext ctx,
         GetSubscriptionGroupConfigRequestHeader requestHeader = (GetSubscriptionGroupConfigRequestHeader) request.decodeCommandCustomHeader(GetSubscriptionGroupConfigRequestHeader.class);
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
 
-        SubscriptionGroupConfig groupConfig = this.brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().get(requestHeader.getGroup());
+        SubscriptionGroupConfig groupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getGroup());
         if (groupConfig == null) {
             LOGGER.error("No group in this broker, client: {} group: {}", ctx.channel().remoteAddress(), requestHeader.getGroup());
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
             response.setRemark("No group in this broker");
             return response;
         }
@@ -404,6 +467,69 @@ private RemotingCommand updateAndGetGroupForbidden(ChannelHandlerContext ctx, Re
         return response;
     }
 
+    private RemotingCommand checkRocksdbCqWriteProgress(ChannelHandlerContext ctx, RemotingCommand request) {
+        CheckRocksdbCqWriteResult result = new CheckRocksdbCqWriteResult();
+        result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_IN_PROGRESS.getValue());
+        Runnable runnable = () -> {
+            try {
+                CheckRocksdbCqWriteResult checkResult = doCheckRocksdbCqWriteProgress(ctx, request);
+                LOGGER.info("checkRocksdbCqWriteProgress result: {}", JSON.toJSONString(checkResult));
+            } catch (Exception e) {
+                LOGGER.error("checkRocksdbCqWriteProgress error", e);
+            }
+        };
+        asyncExecuteWorker.submit(runnable);
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(ResponseCode.SUCCESS);
+        response.setBody(JSON.toJSONBytes(result));
+        return response;
+    }
+
+    private RemotingCommand exportRocksDBConfigToJson(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        ExportRocksDBConfigToJsonRequestHeader requestHeader = request.decodeCommandCustomHeader(ExportRocksDBConfigToJsonRequestHeader.class);
+        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configTypes = requestHeader.fetchConfigType();
+        List<CompletableFuture<Void>> futureList = new ArrayList<>(configTypes.size());
+        for (ExportRocksDBConfigToJsonRequestHeader.ConfigType type : configTypes) {
+            switch (type) {
+                case TOPICS:
+                    if (this.brokerController.getTopicConfigManager() instanceof RocksDBTopicConfigManager) {
+                        RocksDBTopicConfigManager rocksDBTopicConfigManager = (RocksDBTopicConfigManager) this.brokerController.getTopicConfigManager();
+                        futureList.add(CompletableFuture.runAsync(rocksDBTopicConfigManager::exportToJson, asyncExecuteWorker));
+                    }
+                    break;
+                case SUBSCRIPTION_GROUPS:
+                    if (this.brokerController.getSubscriptionGroupManager() instanceof RocksDBSubscriptionGroupManager) {
+                        RocksDBSubscriptionGroupManager rocksDBSubscriptionGroupManager = (RocksDBSubscriptionGroupManager) this.brokerController.getSubscriptionGroupManager();
+                        futureList.add(CompletableFuture.runAsync(rocksDBSubscriptionGroupManager::exportToJson, asyncExecuteWorker));
+                    }
+                    break;
+                case CONSUMER_OFFSETS:
+                    if (this.brokerController.getConsumerOffsetManager() instanceof RocksDBConsumerOffsetManager) {
+                        RocksDBConsumerOffsetManager rocksDBConsumerOffsetManager = (RocksDBConsumerOffsetManager) this.brokerController.getConsumerOffsetManager();
+                        futureList.add(CompletableFuture.runAsync(rocksDBConsumerOffsetManager::exportToJson, asyncExecuteWorker));
+                    }
+                    break;
+                default:
+                    break;
+            }
+        }
+
+        try {
+            CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join();
+        } catch (CompletionException e) {
+            RemotingCommand response = RemotingCommand.createResponseCommand(null);
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark(String.valueOf(e));
+            return response;
+        }
+
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(ResponseCode.SUCCESS);
+        response.setRemark("export done.");
+        return response;
+    }
+
     @Override
     public boolean rejectRequest() {
         return false;
@@ -411,6 +537,7 @@ public boolean rejectRequest() {
 
     private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx,
         RemotingCommand request) throws RemotingCommandException {
+        long startTime = System.currentTimeMillis();
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
         final CreateTopicRequestHeader requestHeader =
             (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class);
@@ -420,45 +547,50 @@ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext
 
         String topic = requestHeader.getTopic();
 
-        TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic);
-        if (!result.isValid()) {
-            response.setCode(ResponseCode.SYSTEM_ERROR);
-            response.setRemark(result.getRemark());
-            return response;
-        }
-        if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {
-            if (TopicValidator.isSystemTopic(topic)) {
-                response.setCode(ResponseCode.SYSTEM_ERROR);
-                response.setRemark("The topic[" + topic + "] is conflict with system topic.");
+        long executionTime;
+        try {
+            TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic);
+            if (!result.isValid()) {
+                response.setCode(ResponseCode.INVALID_PARAMETER);
+                response.setRemark(result.getRemark());
                 return response;
             }
-        }
-
-        TopicConfig topicConfig = new TopicConfig(topic);
-        topicConfig.setReadQueueNums(requestHeader.getReadQueueNums());
-        topicConfig.setWriteQueueNums(requestHeader.getWriteQueueNums());
-        topicConfig.setTopicFilterType(requestHeader.getTopicFilterTypeEnum());
-        topicConfig.setPerm(requestHeader.getPerm());
-        topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag());
-        topicConfig.setOrder(requestHeader.getOrder());
-        String attributesModification = requestHeader.getAttributes();
-        topicConfig.setAttributes(AttributeParser.parseToMap(attributesModification));
+            if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {
+                if (TopicValidator.isSystemTopic(topic)) {
+                    response.setCode(ResponseCode.INVALID_PARAMETER);
+                    response.setRemark("The topic[" + topic + "] is conflict with system topic.");
+                    return response;
+                }
+            }
 
-        if (topicConfig.getTopicMessageType() == TopicMessageType.MIXED
-            && !brokerController.getBrokerConfig().isEnableMixedMessageType()) {
-            response.setCode(ResponseCode.SYSTEM_ERROR);
-            response.setRemark("MIXED message type is not supported.");
-            return response;
-        }
+            TopicConfig topicConfig = new TopicConfig(topic);
+            topicConfig.setReadQueueNums(requestHeader.getReadQueueNums());
+            topicConfig.setWriteQueueNums(requestHeader.getWriteQueueNums());
+            topicConfig.setTopicFilterType(requestHeader.getTopicFilterTypeEnum());
+            topicConfig.setPerm(requestHeader.getPerm());
+            topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag());
+            topicConfig.setOrder(requestHeader.getOrder());
+            String attributesModification = requestHeader.getAttributes();
+            topicConfig.setAttributes(AttributeParser.parseToMap(attributesModification));
+
+            if (!brokerController.getBrokerConfig().isEnableMixedMessageType() && topicConfig.getAttributes() != null) {
+                // Get attribute by key with prefix sign
+                String msgTypeAttrKey = AttributeParser.ATTR_ADD_PLUS_SIGN + TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName();
+                String msgTypeAttrValue = topicConfig.getAttributes().get(msgTypeAttrKey);
+                if (msgTypeAttrValue != null && msgTypeAttrValue.equals(TopicMessageType.MIXED.getValue())) {
+                    response.setCode(ResponseCode.INVALID_PARAMETER);
+                    response.setRemark("MIXED message type is not supported.");
+                    return response;
+                }
+            }
 
-        if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) {
-            LOGGER.info("Broker receive request to update or create topic={}, but topicConfig has  no changes , so idempotent, caller address={}",
-                requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
-            response.setCode(ResponseCode.SUCCESS);
-            return response;
-        }
+            if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) {
+                LOGGER.info("Broker receive request to update or create topic={}, but topicConfig has  no changes , so idempotent, caller address={}",
+                    requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+                response.setCode(ResponseCode.SUCCESS);
+                return response;
+            }
 
-        try {
             this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig);
             if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
                 this.brokerController.registerSingleTopicAll(topicConfig);
@@ -470,8 +602,99 @@ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext
             LOGGER.error("Update / create topic failed for [{}]", request, e);
             response.setCode(ResponseCode.SYSTEM_ERROR);
             response.setRemark(e.getMessage());
+            return response;
+        } finally {
+            executionTime = System.currentTimeMillis() - startTime;
+            InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ?
+                InvocationStatus.SUCCESS : InvocationStatus.FAILURE;
+            Attributes attributes = BrokerMetricsManager.newAttributesBuilder()
+                .put(LABEL_INVOCATION_STATUS, status.getName())
+                .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topic))
+                .build();
+            BrokerMetricsManager.topicCreateExecuteTime.record(executionTime, attributes);
+        }
+        LOGGER.info("executionTime of create topic:{} is {} ms", topic, executionTime);
+        return response;
+    }
+
+    private synchronized RemotingCommand updateAndCreateTopicList(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        long startTime = System.currentTimeMillis();
+
+        final CreateTopicListRequestBody requestBody = CreateTopicListRequestBody.decode(request.getBody(), CreateTopicListRequestBody.class);
+        List<TopicConfig> topicConfigList = requestBody.getTopicConfigList();
+
+        StringBuilder builder = new StringBuilder();
+        for (TopicConfig topicConfig : topicConfigList) {
+            builder.append(topicConfig.getTopicName()).append(";");
         }
+        String topicNames = builder.toString();
+        LOGGER.info("AdminBrokerProcessor#updateAndCreateTopicList: topicNames: {}, called by {}", topicNames, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+        long executionTime;
 
+        try {
+            // Valid topics
+            for (TopicConfig topicConfig : topicConfigList) {
+                String topic = topicConfig.getTopicName();
+                TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic);
+                if (!result.isValid()) {
+                    response.setCode(ResponseCode.INVALID_PARAMETER);
+                    response.setRemark(result.getRemark());
+                    return response;
+                }
+                if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {
+                    if (TopicValidator.isSystemTopic(topic)) {
+                        response.setCode(ResponseCode.INVALID_PARAMETER);
+                        response.setRemark("The topic[" + topic + "] is conflict with system topic.");
+                        return response;
+                    }
+                }
+                if (!brokerController.getBrokerConfig().isEnableMixedMessageType() && topicConfig.getAttributes() != null) {
+                    // Get attribute by key with prefix sign
+                    String msgTypeAttrKey = AttributeParser.ATTR_ADD_PLUS_SIGN + TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName();
+                    String msgTypeAttrValue = topicConfig.getAttributes().get(msgTypeAttrKey);
+                    if (msgTypeAttrValue != null && msgTypeAttrValue.equals(TopicMessageType.MIXED.getValue())) {
+                        response.setCode(ResponseCode.INVALID_PARAMETER);
+                        response.setRemark("MIXED message type is not supported.");
+                        return response;
+                    }
+                }
+                if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) {
+                    LOGGER.info("Broker receive request to update or create topic={}, but topicConfig has  no changes , so idempotent, caller address={}",
+                        topic, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+                    response.setCode(ResponseCode.SUCCESS);
+                    return response;
+                }
+            }
+
+            this.brokerController.getTopicConfigManager().updateTopicConfigList(topicConfigList);
+            if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
+                for (TopicConfig topicConfig : topicConfigList) {
+                    this.brokerController.registerSingleTopicAll(topicConfig);
+                }
+            } else {
+                this.brokerController.registerIncrementBrokerData(topicConfigList, this.brokerController.getTopicConfigManager().getDataVersion());
+            }
+            response.setCode(ResponseCode.SUCCESS);
+        } catch (Exception e) {
+            LOGGER.error("Update / create topic failed for [{}]", request, e);
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark(e.getMessage());
+            return response;
+        } finally {
+            executionTime = System.currentTimeMillis() - startTime;
+            InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ?
+                InvocationStatus.SUCCESS : InvocationStatus.FAILURE;
+            Attributes attributes = BrokerMetricsManager.newAttributesBuilder()
+                .put(LABEL_INVOCATION_STATUS, status.getName())
+                .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topicNames))
+                .build();
+            BrokerMetricsManager.topicCreateExecuteTime.record(executionTime, attributes);
+        }
+        LOGGER.info("executionTime of all topics:{} is {} ms", topicNames, executionTime);
         return response;
     }
 
@@ -488,21 +711,18 @@ private synchronized RemotingCommand updateAndCreateStaticTopic(ChannelHandlerCo
 
         TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic);
         if (!result.isValid()) {
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.INVALID_PARAMETER);
             response.setRemark(result.getRemark());
             return response;
         }
         if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {
             if (TopicValidator.isSystemTopic(topic)) {
-                response.setCode(ResponseCode.SYSTEM_ERROR);
+                response.setCode(ResponseCode.INVALID_PARAMETER);
                 response.setRemark("The topic[" + topic + "] is conflict with system topic.");
                 return response;
             }
         }
-        boolean force = false;
-        if (requestHeader.getForce() != null && requestHeader.getForce()) {
-            force = true;
-        }
+        boolean force = requestHeader.getForce() != null && requestHeader.getForce();
 
         TopicConfig topicConfig = new TopicConfig(topic);
         topicConfig.setReadQueueNums(requestHeader.getReadQueueNums());
@@ -538,34 +758,41 @@ private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx,
         String topic = requestHeader.getTopic();
 
         if (UtilAll.isBlank(topic)) {
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.INVALID_PARAMETER);
             response.setRemark("The specified topic is blank.");
             return response;
         }
 
         if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {
             if (TopicValidator.isSystemTopic(topic)) {
-                response.setCode(ResponseCode.SYSTEM_ERROR);
+                response.setCode(ResponseCode.INVALID_PARAMETER);
                 response.setRemark("The topic[" + topic + "] is conflict with system topic.");
                 return response;
             }
         }
 
-        final Set<String> groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic);
-        // delete pop retry topics first
-        try {
+        List<String> topicsToClean = new ArrayList<>();
+        topicsToClean.add(topic);
+
+        if (brokerController.getBrokerConfig().isClearRetryTopicWhenDeleteTopic()) {
+            final Set<String> groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic);
             for (String group : groups) {
                 final String popRetryTopicV2 = KeyBuilder.buildPopRetryTopic(topic, group, true);
                 if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopicV2) != null) {
-                    deleteTopicInBroker(popRetryTopicV2);
+                    topicsToClean.add(popRetryTopicV2);
                 }
                 final String popRetryTopicV1 = KeyBuilder.buildPopRetryTopicV1(topic, group);
                 if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopicV1) != null) {
-                    deleteTopicInBroker(popRetryTopicV1);
+                    topicsToClean.add(popRetryTopicV1);
                 }
             }
-            // delete topic
-            deleteTopicInBroker(topic);
+        }
+
+        try {
+            for (String topicToClean : topicsToClean) {
+                // delete topic
+                deleteTopicInBroker(topicToClean);
+            }
         } catch (Throwable t) {
             return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
         }
@@ -580,141 +807,7 @@ private void deleteTopicInBroker(String topic) {
         this.brokerController.getConsumerOffsetManager().cleanOffsetByTopic(topic);
         this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByTopicName(topic);
         this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(topic));
-    }
-
-    private synchronized RemotingCommand updateAndCreateAccessConfig(ChannelHandlerContext ctx,
-        RemotingCommand request) throws RemotingCommandException {
-        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
-
-        final CreateAccessConfigRequestHeader requestHeader =
-            (CreateAccessConfigRequestHeader) request.decodeCommandCustomHeader(CreateAccessConfigRequestHeader.class);
-
-        PlainAccessConfig accessConfig = new PlainAccessConfig();
-        accessConfig.setAccessKey(requestHeader.getAccessKey());
-        accessConfig.setSecretKey(requestHeader.getSecretKey());
-        accessConfig.setWhiteRemoteAddress(requestHeader.getWhiteRemoteAddress());
-        accessConfig.setDefaultTopicPerm(requestHeader.getDefaultTopicPerm());
-        accessConfig.setDefaultGroupPerm(requestHeader.getDefaultGroupPerm());
-        accessConfig.setTopicPerms(UtilAll.split(requestHeader.getTopicPerms(), ","));
-        accessConfig.setGroupPerms(UtilAll.split(requestHeader.getGroupPerms(), ","));
-        accessConfig.setAdmin(requestHeader.isAdmin());
-        try {
-
-            AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class);
-            if (accessValidator.updateAccessConfig(accessConfig)) {
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                response.markResponseType();
-                response.setRemark(null);
-                NettyRemotingAbstract.writeResponse(ctx.channel(), request, response);
-            } else {
-                String errorMsg = "The accessKey[" + requestHeader.getAccessKey() + "] corresponding to accessConfig has been updated failed.";
-                LOGGER.warn(errorMsg);
-                response.setCode(ResponseCode.UPDATE_AND_CREATE_ACL_CONFIG_FAILED);
-                response.setRemark(errorMsg);
-                return response;
-            }
-        } catch (Exception e) {
-            LOGGER.error("Failed to generate a proper update accessValidator response", e);
-            response.setCode(ResponseCode.UPDATE_AND_CREATE_ACL_CONFIG_FAILED);
-            response.setRemark(e.getMessage());
-            return response;
-        }
-
-        return null;
-    }
-
-    private synchronized RemotingCommand deleteAccessConfig(ChannelHandlerContext ctx,
-        RemotingCommand request) throws RemotingCommandException {
-        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
-
-        final DeleteAccessConfigRequestHeader requestHeader =
-            (DeleteAccessConfigRequestHeader) request.decodeCommandCustomHeader(DeleteAccessConfigRequestHeader.class);
-        LOGGER.info("DeleteAccessConfig called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
-
-        try {
-            String accessKey = requestHeader.getAccessKey();
-            AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class);
-            if (accessValidator.deleteAccessConfig(accessKey)) {
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                response.markResponseType();
-                response.setRemark(null);
-                NettyRemotingAbstract.writeResponse(ctx.channel(), request, response);
-            } else {
-                String errorMsg = "The accessKey[" + requestHeader.getAccessKey() + "] corresponding to accessConfig has been deleted failed.";
-                LOGGER.warn(errorMsg);
-                response.setCode(ResponseCode.DELETE_ACL_CONFIG_FAILED);
-                response.setRemark(errorMsg);
-                return response;
-            }
-
-        } catch (Exception e) {
-            LOGGER.error("Failed to generate a proper delete accessValidator response", e);
-            response.setCode(ResponseCode.DELETE_ACL_CONFIG_FAILED);
-            response.setRemark(e.getMessage());
-            return response;
-        }
-
-        return null;
-    }
-
-    private synchronized RemotingCommand updateGlobalWhiteAddrsConfig(ChannelHandlerContext ctx,
-        RemotingCommand request) throws RemotingCommandException {
-
-        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
-
-        final UpdateGlobalWhiteAddrsConfigRequestHeader requestHeader =
-            (UpdateGlobalWhiteAddrsConfigRequestHeader) request.decodeCommandCustomHeader(UpdateGlobalWhiteAddrsConfigRequestHeader.class);
-
-        try {
-            AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class);
-            if (accessValidator.updateGlobalWhiteAddrsConfig(UtilAll.split(requestHeader.getGlobalWhiteAddrs(), ","),
-                requestHeader.getAclFileFullPath())) {
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                response.markResponseType();
-                response.setRemark(null);
-                NettyRemotingAbstract.writeResponse(ctx.channel(), request, response);
-            } else {
-                String errorMsg = "The globalWhiteAddresses[" + requestHeader.getGlobalWhiteAddrs() + "] has been updated failed.";
-                LOGGER.warn(errorMsg);
-                response.setCode(ResponseCode.UPDATE_GLOBAL_WHITE_ADDRS_CONFIG_FAILED);
-                response.setRemark(errorMsg);
-                return response;
-            }
-        } catch (Exception e) {
-            LOGGER.error("Failed to generate a proper update globalWhiteAddresses response", e);
-            response.setCode(ResponseCode.UPDATE_GLOBAL_WHITE_ADDRS_CONFIG_FAILED);
-            response.setRemark(e.getMessage());
-            return response;
-        }
-
-        return null;
-    }
-
-    private RemotingCommand getBrokerAclConfigVersion(ChannelHandlerContext ctx, RemotingCommand request) {
-
-        final RemotingCommand response = RemotingCommand.createResponseCommand(GetBrokerAclConfigResponseHeader.class);
-
-        final GetBrokerAclConfigResponseHeader responseHeader = (GetBrokerAclConfigResponseHeader) response.readCustomHeader();
-
-        try {
-            AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class);
-
-            responseHeader.setVersion(accessValidator.getAclConfigVersion());
-            responseHeader.setBrokerAddr(this.brokerController.getBrokerAddr());
-            responseHeader.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
-            responseHeader.setClusterName(this.brokerController.getBrokerConfig().getBrokerClusterName());
-
-            response.setCode(ResponseCode.SUCCESS);
-            response.setRemark(null);
-            return response;
-        } catch (Exception e) {
-            LOGGER.error("Failed to generate a proper getBrokerAclConfigVersion response", e);
-        }
-
-        return null;
+        this.brokerController.getMessageStore().getTimerMessageStore().getTimerMetrics().removeTimingCount(topic);
     }
 
     private RemotingCommand getUnknownCmdResponse(ChannelHandlerContext ctx, RemotingCommand request) {
@@ -803,13 +896,15 @@ private synchronized RemotingCommand updateColdDataFlowCtrGroupConfig(ChannelHan
                 Properties properties = MixAll.string2Properties(bodyStr);
                 if (properties != null) {
                     LOGGER.info("updateColdDataFlowCtrGroupConfig new config: {}, client: {}", properties, ctx.channel().remoteAddress());
-                    properties.entrySet().stream().forEach(i -> {
+                    properties.forEach((key, value) -> {
                         try {
-                            String consumerGroup = String.valueOf(i.getKey());
-                            Long threshold = Long.valueOf(String.valueOf(i.getValue()));
-                            this.brokerController.getColdDataCgCtrService().addOrUpdateGroupConfig(consumerGroup, threshold);
+                            String consumerGroup = String.valueOf(key);
+                            Long threshold = Long.valueOf(String.valueOf(value));
+                            this.brokerController.getColdDataCgCtrService()
+                                .addOrUpdateGroupConfig(consumerGroup, threshold);
                         } catch (Exception e) {
-                            LOGGER.error("updateColdDataFlowCtrGroupConfig properties on entry error, key: {}, val: {}", i.getKey(), i.getValue(), e);
+                            LOGGER.error("updateColdDataFlowCtrGroupConfig properties on entry error, key: {}, val: {}",
+                                key, value, e);
                         }
                     });
                 } else {
@@ -893,7 +988,7 @@ private RemotingCommand setCommitLogReadaheadMode(ChannelHandlerContext ctx, Rem
             }
             int mode = Integer.parseInt(extFields.get(FIleReadaheadMode.READ_AHEAD_MODE));
             if (mode != LibC.MADV_RANDOM && mode != LibC.MADV_NORMAL) {
-                response.setCode(ResponseCode.SYSTEM_ERROR);
+                response.setCode(ResponseCode.INVALID_PARAMETER);
                 response.setRemark("set commitlog readahead mode param value error");
                 return response;
             }
@@ -1119,8 +1214,7 @@ private RemotingCommand getMaxOffset(ChannelHandlerContext ctx,
         RemotingCommand request) throws RemotingCommandException {
         final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class);
         final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader();
-        final GetMaxOffsetRequestHeader requestHeader =
-            (GetMaxOffsetRequestHeader) request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class);
+        final GetMaxOffsetRequestHeader requestHeader = request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class);
 
         TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader);
         RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext);
@@ -1128,10 +1222,12 @@ private RemotingCommand getMaxOffset(ChannelHandlerContext ctx,
             return rewriteResult;
         }
 
-        long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
-
-        responseHeader.setOffset(offset);
-
+        try {
+            long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+            responseHeader.setOffset(offset);
+        } catch (ConsumeQueueException e) {
+            throw new RemotingCommandException("Failed to get max offset in queue", e);
+        }
         response.setCode(ResponseCode.SUCCESS);
         response.setRemark(null);
         return response;
@@ -1262,7 +1358,8 @@ private RemotingCommand getEarliestMsgStoretime(ChannelHandlerContext ctx,
         return response;
     }
 
-    private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request) {
+    private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request)
+        throws RemotingCommandException {
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
 
         HashMap<String, String> runtimeInfo = this.prepareRuntimeInfo();
@@ -1406,6 +1503,7 @@ public void onException(Throwable e) {
 
     private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request)
         throws RemotingCommandException {
+        long startTime = System.currentTimeMillis();
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
 
         LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroup called by {}",
@@ -1418,10 +1516,53 @@ private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext c
 
         response.setCode(ResponseCode.SUCCESS);
         response.setRemark(null);
+        long executionTime = System.currentTimeMillis() - startTime;
+        LOGGER.info("executionTime of create subscriptionGroup:{} is {} ms", config.getGroupName(), executionTime);
+        InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ?
+            InvocationStatus.SUCCESS : InvocationStatus.FAILURE;
+        Attributes attributes = BrokerMetricsManager.newAttributesBuilder()
+            .put(LABEL_INVOCATION_STATUS, status.getName())
+            .build();
+        BrokerMetricsManager.consumerGroupCreateExecuteTime.record(executionTime, attributes);
         return response;
     }
 
-    private void initConsumerOffset(String clientHost, String groupName, int mode, TopicConfig topicConfig) {
+    private RemotingCommand updateAndCreateSubscriptionGroupList(ChannelHandlerContext ctx, RemotingCommand request) {
+        final long startTime = System.nanoTime();
+
+        final SubscriptionGroupList subscriptionGroupList = SubscriptionGroupList.decode(request.getBody(), SubscriptionGroupList.class);
+        final List<SubscriptionGroupConfig> groupConfigList = subscriptionGroupList.getGroupConfigList();
+
+        final StringBuilder builder = new StringBuilder();
+        for (SubscriptionGroupConfig config : groupConfigList) {
+            builder.append(config.getGroupName()).append(";");
+        }
+        final String groupNames = builder.toString();
+        LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroupList: groupNames: {}, called by {}",
+            groupNames,
+            RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        try {
+            this.brokerController.getSubscriptionGroupManager().updateSubscriptionGroupConfigList(groupConfigList);
+            response.setCode(ResponseCode.SUCCESS);
+            response.setRemark(null);
+        } finally {
+            long executionTime = (System.nanoTime() - startTime) / 1000000L;
+            LOGGER.info("executionTime of create updateAndCreateSubscriptionGroupList: {} is {} ms", groupNames, executionTime);
+            InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ?
+                InvocationStatus.SUCCESS : InvocationStatus.FAILURE;
+            Attributes attributes = BrokerMetricsManager.newAttributesBuilder()
+                .put(LABEL_INVOCATION_STATUS, status.getName())
+                .build();
+            BrokerMetricsManager.consumerGroupCreateExecuteTime.record(executionTime, attributes);
+        }
+
+        return response;
+    }
+
+    private void initConsumerOffset(String clientHost, String groupName, int mode, TopicConfig topicConfig)
+        throws ConsumeQueueException {
         String topic = topicConfig.getTopicName();
         for (int queueId = 0; queueId < topicConfig.getReadQueueNums(); queueId++) {
             if (this.brokerController.getConsumerOffsetManager().queryOffset(groupName, topic, queueId) > -1) {
@@ -1495,8 +1636,7 @@ private RemotingCommand deleteSubscriptionGroup(ChannelHandlerContext ctx,
     private RemotingCommand getTopicStatsInfo(ChannelHandlerContext ctx,
         RemotingCommand request) throws RemotingCommandException {
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
-        final GetTopicStatsInfoRequestHeader requestHeader =
-            (GetTopicStatsInfoRequestHeader) request.decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class);
+        final GetTopicStatsInfoRequestHeader requestHeader = request.decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class);
 
         final String topic = requestHeader.getTopic();
         TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
@@ -1507,39 +1647,48 @@ private RemotingCommand getTopicStatsInfo(ChannelHandlerContext ctx,
         }
 
         TopicStatsTable topicStatsTable = new TopicStatsTable();
-        for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) {
-            MessageQueue mq = new MessageQueue();
-            mq.setTopic(topic);
-            mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
-            mq.setQueueId(i);
 
-            TopicOffset topicOffset = new TopicOffset();
-            long min = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, i);
-            if (min < 0) {
-                min = 0;
-            }
+        int maxQueueNums = Math.max(topicConfig.getWriteQueueNums(), topicConfig.getReadQueueNums());
+        try {
+            for (int i = 0; i < maxQueueNums; i++) {
+                MessageQueue mq = new MessageQueue();
+                mq.setTopic(topic);
+                mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
+                mq.setQueueId(i);
 
-            long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
-            if (max < 0) {
-                max = 0;
-            }
+                TopicOffset topicOffset = new TopicOffset();
+                long min = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, i);
+                if (min < 0) {
+                    min = 0;
+                }
 
-            long timestamp = 0;
-            if (max > 0) {
-                timestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1);
-            }
+                long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+                if (max < 0) {
+                    max = 0;
+                }
+
+                long timestamp = 0;
+                if (max > 0) {
+                    timestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1);
+                }
+
+                topicOffset.setMinOffset(min);
+                topicOffset.setMaxOffset(max);
+                topicOffset.setLastUpdateTimestamp(timestamp);
 
-            topicOffset.setMinOffset(min);
-            topicOffset.setMaxOffset(max);
-            topicOffset.setLastUpdateTimestamp(timestamp);
+                topicStatsTable.getOffsetTable().put(mq, topicOffset);
+            }
 
-            topicStatsTable.getOffsetTable().put(mq, topicOffset);
+            topicStatsTable.setTopicPutTps(this.brokerController.getBrokerStatsManager().tpsTopicPutNums(requestHeader.getTopic()));
+            byte[] body = topicStatsTable.encode();
+            response.setBody(body);
+            response.setCode(ResponseCode.SUCCESS);
+            response.setRemark(null);
+        } catch (ConsumeQueueException e) {
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark(e.getMessage());
         }
 
-        byte[] body = topicStatsTable.encode();
-        response.setBody(body);
-        response.setCode(ResponseCode.SUCCESS);
-        response.setRemark(null);
         return response;
     }
 
@@ -1639,96 +1788,115 @@ private RemotingCommand getProducerConnectionList(ChannelHandlerContext ctx,
     private RemotingCommand getConsumeStats(ChannelHandlerContext ctx,
         RemotingCommand request) throws RemotingCommandException {
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
-        final GetConsumeStatsRequestHeader requestHeader =
-            (GetConsumeStatsRequestHeader) request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class);
-
-        ConsumeStats consumeStats = new ConsumeStats();
-
-        Set<String> topics = new HashSet<>();
-        if (UtilAll.isBlank(requestHeader.getTopic())) {
-            topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getConsumerGroup());
-        } else {
-            topics.add(requestHeader.getTopic());
-        }
-
-        for (String topic : topics) {
-            TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
-            if (null == topicConfig) {
-                LOGGER.warn("AdminBrokerProcessor#getConsumeStats: topic config does not exist, topic={}", topic);
-                continue;
-            }
-
-            TopicQueueMappingDetail mappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(topic);
+        try {
+            final GetConsumeStatsRequestHeader requestHeader = request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class);
+            List<String> topicListProvided = requestHeader.fetchTopicList();
+            String topicProvided = requestHeader.getTopic();
+            String group = requestHeader.getConsumerGroup();
 
-            {
-                SubscriptionData findSubscriptionData =
-                    this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic);
+            ConsumeStats consumeStats = new ConsumeStats();
+            Set<String> topicsForCollecting = getTopicsForCollectingConsumeStats(topicListProvided, topicProvided, group);
 
-                if (null == findSubscriptionData
-                    && this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getConsumerGroup()) > 0) {
-                    LOGGER.warn(
-                        "AdminBrokerProcessor#getConsumeStats: topic does not exist in consumer group's subscription, "
-                            + "topic={}, consumer group={}", topic, requestHeader.getConsumerGroup());
+            for (String topic : topicsForCollecting) {
+                TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
+                if (null == topicConfig) {
+                    LOGGER.warn("AdminBrokerProcessor#getConsumeStats: topic config does not exist, topic={}", topic);
                     continue;
                 }
-            }
-
-            for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
-                MessageQueue mq = new MessageQueue();
-                mq.setTopic(topic);
-                mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
-                mq.setQueueId(i);
 
-                OffsetWrapper offsetWrapper = new OffsetWrapper();
+                TopicQueueMappingDetail mappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(topic);
+                for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
+                    MessageQueue mq = new MessageQueue();
+                    mq.setTopic(topic);
+                    mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
+                    mq.setQueueId(i);
 
-                long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
-                if (brokerOffset < 0) {
-                    brokerOffset = 0;
-                }
+                    OffsetWrapper offsetWrapper = new OffsetWrapper();
 
-                long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(
-                    requestHeader.getConsumerGroup(), topic, i);
+                    long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+                    if (brokerOffset < 0) {
+                        brokerOffset = 0;
+                    }
 
-                // the consumerOffset cannot be zero for static topic because of the "double read check" strategy
-                // just remain the logic for dynamic topic
-                // maybe we should remove it in the future
-                if (mappingDetail == null) {
-                    if (consumerOffset < 0) {
-                        consumerOffset = 0;
+                    long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(
+                        requestHeader.getConsumerGroup(), topic, i);
+
+                    // the consumerOffset cannot be zero for static topic because of the "double read check" strategy
+                    // just remain the logic for dynamic topic
+                    // maybe we should remove it in the future
+                    if (mappingDetail == null) {
+                        if (consumerOffset < 0) {
+                            consumerOffset = 0;
+                        }
                     }
-                }
 
-                long pullOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset(
-                    requestHeader.getConsumerGroup(), topic, i);
+                    long pullOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset(
+                        requestHeader.getConsumerGroup(), topic, i);
 
-                offsetWrapper.setBrokerOffset(brokerOffset);
-                offsetWrapper.setConsumerOffset(consumerOffset);
-                offsetWrapper.setPullOffset(Math.max(consumerOffset, pullOffset));
+                    offsetWrapper.setBrokerOffset(brokerOffset);
+                    offsetWrapper.setConsumerOffset(consumerOffset);
+                    offsetWrapper.setPullOffset(Math.max(consumerOffset, pullOffset));
 
-                long timeOffset = consumerOffset - 1;
-                if (timeOffset >= 0) {
-                    long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset);
-                    if (lastTimestamp > 0) {
-                        offsetWrapper.setLastTimestamp(lastTimestamp);
+                    long timeOffset = consumerOffset - 1;
+                    if (timeOffset >= 0) {
+                        long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset);
+                        if (lastTimestamp > 0) {
+                            offsetWrapper.setLastTimestamp(lastTimestamp);
+                        }
                     }
+
+                    consumeStats.getOffsetTable().put(mq, offsetWrapper);
                 }
 
-                consumeStats.getOffsetTable().put(mq, offsetWrapper);
-            }
+                double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic);
 
-            double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic);
+                consumeTps += consumeStats.getConsumeTps();
+                consumeStats.setConsumeTps(consumeTps);
+            }
 
-            consumeTps += consumeStats.getConsumeTps();
-            consumeStats.setConsumeTps(consumeTps);
+            byte[] body = consumeStats.encode();
+            response.setBody(body);
+            response.setCode(ResponseCode.SUCCESS);
+            response.setRemark(null);
+        } catch (ConsumeQueueException e) {
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark(e.getMessage());
         }
-
-        byte[] body = consumeStats.encode();
-        response.setBody(body);
-        response.setCode(ResponseCode.SUCCESS);
-        response.setRemark(null);
         return response;
     }
 
+    private Set<String> getTopicsForCollectingConsumeStats(List<String> topicListProvided, String topicProvided,
+        String group) {
+        Set<String> topicsForCollecting = new HashSet<>();
+        if (!topicListProvided.isEmpty()) {
+            // if topic list is provided, only collect the topics in the list
+            // and ignore subscription check
+            topicsForCollecting.addAll(topicListProvided);
+        } else {
+            // In order to be compatible with the old logic,
+            // even if the topic has been provided here, the subscription will be checked.
+            if (UtilAll.isBlank(topicProvided)) {
+                topicsForCollecting.addAll(
+                    this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(group));
+            } else {
+                topicsForCollecting.add(topicProvided);
+            }
+            int subscriptionCount = this.brokerController.getConsumerManager().findSubscriptionDataCount(group);
+            Iterator<String> iterator = topicsForCollecting.iterator();
+            while (iterator.hasNext()) {
+                String topic = iterator.next();
+                SubscriptionData findSubscriptionData = this.brokerController.getConsumerManager().findSubscriptionData(group, topic);
+                if (findSubscriptionData == null && subscriptionCount > 0) {
+                    LOGGER.warn(
+                        "AdminBrokerProcessor#getConsumeStats: topic does not exist in consumer group's subscription, topic={}, consumer group={}",
+                        topic, group);
+                    iterator.remove();
+                }
+            }
+        }
+        return topicsForCollecting;
+    }
+
     private RemotingCommand getAllConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request) {
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
 
@@ -1789,7 +1957,7 @@ private RemotingCommand getAllMessageRequestMode(ChannelHandlerContext ctx, Remo
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
 
         String content = this.brokerController.getQueryAssignmentProcessor().getMessageRequestModeManager().encode();
-        if (content != null && content.length() > 0) {
+        if (content != null && !content.isEmpty()) {
             try {
                 response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));
             } catch (UnsupportedEncodingException e) {
@@ -1840,7 +2008,7 @@ public RemotingCommand resetOffset(ChannelHandlerContext ctx,
             requestHeader.getTimestamp(), requestHeader.isForce(), isC);
     }
 
-    private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp) {
+    private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp) throws ConsumeQueueException {
         if (timestamp < 0) {
             return brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
         } else {
@@ -1872,7 +2040,7 @@ private RemotingCommand resetOffsetInner(String topic, String group, int queueId
         Map<Integer, Long> queueOffsetMap = new HashMap<>();
 
         // Reset offset for all queues belonging to the specified topic
-        TopicConfig topicConfig = brokerController.getTopicConfigManager().getTopicConfigTable().get(topic);
+        TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic);
         if (null == topicConfig) {
             response.setCode(ResponseCode.TOPIC_NOT_EXIST);
             response.setRemark("Topic " + topic + " does not exist");
@@ -1887,25 +2055,31 @@ private RemotingCommand resetOffsetInner(String topic, String group, int queueId
             return response;
         }
 
-        if (queueId >= 0) {
-            if (null != offset && -1 != offset) {
-                long min = brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
-                long max = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
-                if (min >= 0 && offset < min || offset > max + 1) {
-                    response.setCode(ResponseCode.SYSTEM_ERROR);
-                    response.setRemark(
-                        String.format("Target offset %d not in consume queue range [%d-%d]", offset, min, max));
-                    return response;
+        try {
+            if (queueId >= 0) {
+                if (null != offset && -1 != offset) {
+                    long min = brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
+                    long max = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
+                    if (min >= 0 && offset < min || offset > max + 1) {
+                        response.setCode(ResponseCode.SYSTEM_ERROR);
+                        response.setRemark(
+                            String.format("Target offset %d not in consume queue range [%d-%d]", offset, min, max));
+                        return response;
+                    }
+                } else {
+                    offset = searchOffsetByTimestamp(topic, queueId, timestamp);
                 }
+                queueOffsetMap.put(queueId, offset);
             } else {
-                offset = searchOffsetByTimestamp(topic, queueId, timestamp);
-            }
-            queueOffsetMap.put(queueId, offset);
-        } else {
-            for (int index = 0; index < topicConfig.getReadQueueNums(); index++) {
-                offset = searchOffsetByTimestamp(topic, index, timestamp);
-                queueOffsetMap.put(index, offset);
+                for (int index = 0; index < topicConfig.getReadQueueNums(); index++) {
+                    offset = searchOffsetByTimestamp(topic, index, timestamp);
+                    queueOffsetMap.put(index, offset);
+                }
             }
+        } catch (ConsumeQueueException e) {
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark(e.getMessage());
+            return response;
         }
 
         if (queueOffsetMap.isEmpty()) {
@@ -1924,7 +2098,13 @@ private RemotingCommand resetOffsetInner(String topic, String group, int queueId
         ResetOffsetBody body = new ResetOffsetBody();
         String brokerName = brokerController.getBrokerConfig().getBrokerName();
         for (Map.Entry<Integer, Long> entry : queueOffsetMap.entrySet()) {
-            brokerController.getPopInflightMessageCounter().clearInFlightMessageNum(topic, group, entry.getKey());
+            if (brokerController.getPopInflightMessageCounter() != null) {
+                brokerController.getPopInflightMessageCounter().clearInFlightMessageNum(topic, group, entry.getKey());
+            }
+            if (brokerController.getBrokerConfig().isPopConsumerKVServiceEnable()) {
+                brokerController.getPopConsumerService().clearCache(group, topic, entry.getKey());
+                brokerController.getConsumerOffsetManager().clearPullOffset(group, topic);
+            }
             body.getOffsetTable().put(new MessageQueue(topic, brokerName, entry.getKey()), entry.getValue());
         }
 
@@ -2012,8 +2192,7 @@ private RemotingCommand querySubscriptionByConsumer(ChannelHandlerContext ctx,
     private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx,
         RemotingCommand request) throws RemotingCommandException {
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
-        QueryConsumeTimeSpanRequestHeader requestHeader =
-            (QueryConsumeTimeSpanRequestHeader) request.decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class);
+        QueryConsumeTimeSpanRequestHeader requestHeader = request.decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class);
 
         final String topic = requestHeader.getTopic();
         TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
@@ -2035,7 +2214,12 @@ private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx,
             long minTime = this.brokerController.getMessageStore().getEarliestMessageTime(topic, i);
             timeSpan.setMinTimeStamp(minTime);
 
-            long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+            long max;
+            try {
+                max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+            } catch (ConsumeQueueException e) {
+                throw new RemotingCommandException("Failed to get max offset in queue", e);
+            }
             long maxTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1);
             timeSpan.setMaxTimeStamp(maxTime);
 
@@ -2049,7 +2233,12 @@ private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx,
             }
             timeSpan.setConsumeTimeStamp(consumeTime);
 
-            long maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), i);
+            long maxBrokerOffset;
+            try {
+                maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), i);
+            } catch (ConsumeQueueException e) {
+                throw new RemotingCommandException("Failed to get max offset in queue", e);
+            }
             if (consumerOffset < maxBrokerOffset) {
                 long nextTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, consumerOffset);
                 timeSpan.setDelayTime(System.currentTimeMillis() - nextTime);
@@ -2165,7 +2354,7 @@ private RemotingCommand consumeMessageDirectly(ChannelHandlerContext ctx,
         }
         // groupSysFlag
         if (StringUtils.isNotEmpty(requestHeader.getConsumerGroup())) {
-            SubscriptionGroupConfig groupConfig = brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().get(requestHeader.getConsumerGroup());
+            SubscriptionGroupConfig groupConfig = brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());
             if (groupConfig != null) {
                 request.addExtField("groupSysFlag", String.valueOf(groupConfig.getGroupSysFlag()));
             }
@@ -2284,8 +2473,7 @@ private RemotingCommand ViewBrokerStatsData(ChannelHandlerContext ctx,
     private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, RemotingCommand request)
         throws RemotingCommandException {
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
-        GetConsumeStatsInBrokerHeader requestHeader =
-            (GetConsumeStatsInBrokerHeader) request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class);
+        GetConsumeStatsInBrokerHeader requestHeader = request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class);
         boolean isOrder = requestHeader.isOrder();
         ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroups =
             brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable();
@@ -2331,7 +2519,12 @@ private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx,
                     mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
                     mq.setQueueId(i);
                     OffsetWrapper offsetWrapper = new OffsetWrapper();
-                    long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+                    long brokerOffset;
+                    try {
+                        brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+                    } catch (ConsumeQueueException e) {
+                        throw new RemotingCommandException("Failed to get max offset", e);
+                    }
                     if (brokerOffset < 0) {
                         brokerOffset = 0;
                     }
@@ -2375,7 +2568,7 @@ private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx,
         return response;
     }
 
-    private HashMap<String, String> prepareRuntimeInfo() {
+    private HashMap<String, String> prepareRuntimeInfo() throws RemotingCommandException {
         HashMap<String, String> runtimeInfo = this.brokerController.getMessageStore().getRuntimeInfo();
 
         for (BrokerAttachedPlugin brokerAttachedPlugin : brokerController.getBrokerAttachedPlugins()) {
@@ -2384,7 +2577,11 @@ private HashMap<String, String> prepareRuntimeInfo() {
             }
         }
 
-        this.brokerController.getScheduleMessageService().buildRunningStats(runtimeInfo);
+        try {
+            this.brokerController.getScheduleMessageService().buildRunningStats(runtimeInfo);
+        } catch (ConsumeQueueException e) {
+            throw new RemotingCommandException("Failed to get max offset in queue", e);
+        }
         runtimeInfo.put("brokerActive", String.valueOf(this.brokerController.isSpecialServiceRunning()));
         runtimeInfo.put("brokerVersionDesc", MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION));
         runtimeInfo.put("brokerVersion", String.valueOf(MQVersion.CURRENT_VERSION));
@@ -2446,10 +2643,15 @@ private HashMap<String, String> prepareRuntimeInfo() {
         runtimeInfo.put("queryThreadPoolQueueCapacity",
             String.valueOf(this.brokerController.getBrokerConfig().getQueryThreadPoolQueueCapacity()));
 
+        runtimeInfo.put("ackThreadPoolQueueSize", String.valueOf(this.brokerController.getAckThreadPoolQueue().size()));
+        runtimeInfo.put("ackThreadPoolQueueCapacity",
+            String.valueOf(this.brokerController.getBrokerConfig().getAckThreadPoolQueueCapacity()));
+
         runtimeInfo.put("sendThreadPoolQueueHeadWaitTimeMills", String.valueOf(this.brokerController.headSlowTimeMills4SendThreadPoolQueue()));
         runtimeInfo.put("pullThreadPoolQueueHeadWaitTimeMills", String.valueOf(brokerController.headSlowTimeMills4PullThreadPoolQueue()));
         runtimeInfo.put("queryThreadPoolQueueHeadWaitTimeMills", String.valueOf(this.brokerController.headSlowTimeMills4QueryThreadPoolQueue()));
         runtimeInfo.put("litePullThreadPoolQueueHeadWaitTimeMills", String.valueOf(brokerController.headSlowTimeMills4LitePullThreadPoolQueue()));
+        runtimeInfo.put("ackThreadPoolQueueHeadWaitTimeMills", String.valueOf(brokerController.headSlowTimeMills4AckThreadPoolQueue()));
 
         runtimeInfo.put("EndTransactionQueueSize", String.valueOf(this.brokerController.getEndTransactionThreadPoolQueue().size()));
         runtimeInfo.put("EndTransactionThreadPoolQueueCapacity",
@@ -2646,7 +2848,7 @@ private RemotingCommand getTopicConfig(ChannelHandlerContext ctx,
         GetTopicConfigRequestHeader requestHeader = (GetTopicConfigRequestHeader) request.decodeCommandCustomHeader(GetTopicConfigRequestHeader.class);
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
 
-        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(requestHeader.getTopic());
+        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
         if (topicConfig == null) {
             LOGGER.error("No topic in this broker, client: {} topic: {}", ctx.channel().remoteAddress(), requestHeader.getTopic());
             //be care of the response code, should set "not-exist" explicitly
@@ -2797,6 +2999,306 @@ private RemotingCommand notifyBrokerRoleChanged(ChannelHandlerContext ctx,
         return response;
     }
 
+    private RemotingCommand createUser(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+        CreateUserRequestHeader requestHeader = request.decodeCommandCustomHeader(CreateUserRequestHeader.class);
+        if (StringUtils.isEmpty(requestHeader.getUsername())) {
+            response.setCode(ResponseCode.INVALID_PARAMETER);
+            response.setRemark("The username is blank");
+            return response;
+        }
+
+        UserInfo userInfo = RemotingSerializable.decode(request.getBody(), UserInfo.class);
+        userInfo.setUsername(requestHeader.getUsername());
+        User user = UserConverter.convertUser(userInfo);
+
+        if (user.getUserType() == UserType.SUPER && isNotSuperUserLogin(request)) {
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark("The super user can only be create by super user");
+            return response;
+        }
+
+        this.brokerController.getAuthenticationMetadataManager().createUser(user)
+            .thenAccept(nil -> response.setCode(ResponseCode.SUCCESS))
+            .exceptionally(ex -> {
+                LOGGER.error("create user {} error", user.getUsername(), ex);
+                return handleAuthException(response, ex);
+            })
+            .join();
+
+        return response;
+    }
+
+    private RemotingCommand updateUser(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+        UpdateUserRequestHeader requestHeader = request.decodeCommandCustomHeader(UpdateUserRequestHeader.class);
+        if (StringUtils.isEmpty(requestHeader.getUsername())) {
+            response.setCode(ResponseCode.INVALID_PARAMETER);
+            response.setRemark("The username is blank");
+            return response;
+        }
+
+        UserInfo userInfo = RemotingSerializable.decode(request.getBody(), UserInfo.class);
+        userInfo.setUsername(requestHeader.getUsername());
+        User user = UserConverter.convertUser(userInfo);
+
+        if (user.getUserType() == UserType.SUPER && isNotSuperUserLogin(request)) {
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark("The super user can only be update by super user");
+            return response;
+        }
+
+        this.brokerController.getAuthenticationMetadataManager().getUser(requestHeader.getUsername())
+            .thenCompose(old -> {
+                if (old == null) {
+                    throw new AuthenticationException("The user is not exist");
+                }
+                if (old.getUserType() == UserType.SUPER && isNotSuperUserLogin(request)) {
+                    throw new AuthenticationException("The super user can only be update by super user");
+                }
+                return this.brokerController.getAuthenticationMetadataManager().updateUser(user);
+            }).thenAccept(nil -> response.setCode(ResponseCode.SUCCESS))
+            .exceptionally(ex -> {
+                LOGGER.error("update user {} error", requestHeader.getUsername(), ex);
+                return handleAuthException(response, ex);
+            })
+            .join();
+        return response;
+    }
+
+    private RemotingCommand deleteUser(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+        DeleteUserRequestHeader requestHeader = request.decodeCommandCustomHeader(DeleteUserRequestHeader.class);
+
+        this.brokerController.getAuthenticationMetadataManager().getUser(requestHeader.getUsername())
+            .thenCompose(user -> {
+                if (user == null) {
+                    return CompletableFuture.completedFuture(null);
+                }
+                if (user.getUserType() == UserType.SUPER && isNotSuperUserLogin(request)) {
+                    throw new AuthenticationException("The super user can only be update by super user");
+                }
+                return this.brokerController.getAuthenticationMetadataManager().deleteUser(requestHeader.getUsername());
+            }).thenAccept(nil -> response.setCode(ResponseCode.SUCCESS))
+            .exceptionally(ex -> {
+                LOGGER.error("delete user {} error", requestHeader.getUsername(), ex);
+                return handleAuthException(response, ex);
+            })
+            .join();
+        return response;
+    }
+
+    private RemotingCommand getUser(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+        GetUserRequestHeader requestHeader = request.decodeCommandCustomHeader(GetUserRequestHeader.class);
+
+        if (StringUtils.isBlank(requestHeader.getUsername())) {
+            response.setCode(ResponseCode.INVALID_PARAMETER);
+            response.setRemark("The username is blank");
+            return response;
+        }
+
+        this.brokerController.getAuthenticationMetadataManager().getUser(requestHeader.getUsername())
+            .thenAccept(user -> {
+                response.setCode(ResponseCode.SUCCESS);
+                if (user != null) {
+                    UserInfo userInfo = UserConverter.convertUser(user);
+                    response.setBody(JSON.toJSONString(userInfo).getBytes(StandardCharsets.UTF_8));
+                }
+            })
+            .exceptionally(ex -> {
+                LOGGER.error("get user {} error", requestHeader.getUsername(), ex);
+                return handleAuthException(response, ex);
+            })
+            .join();
+
+        return response;
+    }
+
+    private RemotingCommand listUser(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+        ListUsersRequestHeader requestHeader = request.decodeCommandCustomHeader(ListUsersRequestHeader.class);
+
+        this.brokerController.getAuthenticationMetadataManager().listUser(requestHeader.getFilter())
+            .thenAccept(users -> {
+                response.setCode(ResponseCode.SUCCESS);
+                if (CollectionUtils.isNotEmpty(users)) {
+                    List<UserInfo> userInfos = UserConverter.convertUsers(users);
+                    response.setBody(JSON.toJSONString(userInfos).getBytes(StandardCharsets.UTF_8));
+                }
+            })
+            .exceptionally(ex -> {
+                LOGGER.error("list user by {} error", requestHeader.getFilter(), ex);
+                return handleAuthException(response, ex);
+            })
+            .join();
+
+        return response;
+    }
+
+    private RemotingCommand createAcl(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+        CreateAclRequestHeader requestHeader = request.decodeCommandCustomHeader(CreateAclRequestHeader.class);
+        Subject subject = Subject.of(requestHeader.getSubject());
+
+        AclInfo aclInfo = RemotingSerializable.decode(request.getBody(), AclInfo.class);
+        if (aclInfo == null || CollectionUtils.isEmpty(aclInfo.getPolicies())) {
+            throw new AuthorizationException("The body of acl is null");
+        }
+
+        Acl acl = AclConverter.convertAcl(aclInfo);
+        if (acl != null && acl.getSubject() == null) {
+            acl.setSubject(subject);
+        }
+
+        this.brokerController.getAuthorizationMetadataManager().createAcl(acl)
+            .thenAccept(nil -> response.setCode(ResponseCode.SUCCESS))
+            .exceptionally(ex -> {
+                LOGGER.error("create acl for {} error", requestHeader.getSubject(), ex);
+                return handleAuthException(response, ex);
+            })
+            .join();
+        return response;
+    }
+
+    private RemotingCommand updateAcl(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+        UpdateAclRequestHeader requestHeader = request.decodeCommandCustomHeader(UpdateAclRequestHeader.class);
+        Subject subject = Subject.of(requestHeader.getSubject());
+
+        AclInfo aclInfo = RemotingSerializable.decode(request.getBody(), AclInfo.class);
+        if (aclInfo == null || CollectionUtils.isEmpty(aclInfo.getPolicies())) {
+            throw new AuthorizationException("The body of acl is null");
+        }
+
+        Acl acl = AclConverter.convertAcl(aclInfo);
+        if (acl != null && acl.getSubject() == null) {
+            acl.setSubject(subject);
+        }
+
+        this.brokerController.getAuthorizationMetadataManager().updateAcl(acl)
+            .thenAccept(nil -> response.setCode(ResponseCode.SUCCESS))
+            .exceptionally(ex -> {
+                LOGGER.error("update acl for {} error", requestHeader.getSubject(), ex);
+                return handleAuthException(response, ex);
+            })
+            .join();
+
+        return response;
+    }
+
+    private RemotingCommand deleteAcl(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+        DeleteAclRequestHeader requestHeader = request.decodeCommandCustomHeader(DeleteAclRequestHeader.class);
+
+        Subject subject = Subject.of(requestHeader.getSubject());
+
+        PolicyType policyType = PolicyType.getByName(requestHeader.getPolicyType());
+
+        Resource resource = Resource.of(requestHeader.getResource());
+
+        this.brokerController.getAuthorizationMetadataManager().deleteAcl(subject, policyType, resource)
+            .thenAccept(nil -> {
+                response.setCode(ResponseCode.SUCCESS);
+            })
+            .exceptionally(ex -> {
+                LOGGER.error("delete acl for {} error", requestHeader.getSubject(), ex);
+                return handleAuthException(response, ex);
+            })
+            .join();
+
+        return response;
+    }
+
+    private RemotingCommand getAcl(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException {
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+        GetAclRequestHeader requestHeader = request.decodeCommandCustomHeader(GetAclRequestHeader.class);
+
+        Subject subject = Subject.of(requestHeader.getSubject());
+
+        this.brokerController.getAuthorizationMetadataManager().getAcl(subject)
+            .thenAccept(acl -> {
+                response.setCode(ResponseCode.SUCCESS);
+                if (acl != null) {
+                    AclInfo aclInfo = AclConverter.convertAcl(acl);
+                    String body = JSON.toJSONString(aclInfo);
+                    response.setBody(body.getBytes(StandardCharsets.UTF_8));
+                }
+            })
+            .exceptionally(ex -> {
+                LOGGER.error("get acl for {} error", requestHeader.getSubject(), ex);
+                return handleAuthException(response, ex);
+            })
+            .join();
+
+        return response;
+    }
+
+    private RemotingCommand listAcl(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+        ListAclsRequestHeader requestHeader = request.decodeCommandCustomHeader(ListAclsRequestHeader.class);
+
+        this.brokerController.getAuthorizationMetadataManager()
+            .listAcl(requestHeader.getSubjectFilter(), requestHeader.getResourceFilter())
+            .thenAccept(acls -> {
+                response.setCode(ResponseCode.SUCCESS);
+                if (CollectionUtils.isNotEmpty(acls)) {
+                    List<AclInfo> aclInfos = AclConverter.convertAcls(acls);
+                    String body = JSON.toJSONString(aclInfos);
+                    response.setBody(body.getBytes(StandardCharsets.UTF_8));
+                }
+            })
+            .exceptionally(ex -> {
+                LOGGER.error("list acl error, subjectFilter:{}, resourceFilter:{}", requestHeader.getSubjectFilter(), requestHeader.getResourceFilter(), ex);
+                return handleAuthException(response, ex);
+            })
+            .join();
+
+        return response;
+    }
+
+    private boolean isNotSuperUserLogin(RemotingCommand request) {
+        String accessKey = request.getExtFields().get("AccessKey");
+        // if accessKey is null, it may be authentication is not enabled.
+        if (StringUtils.isEmpty(accessKey)) {
+            return false;
+        }
+        return !this.brokerController.getAuthenticationMetadataManager()
+            .isSuperUser(accessKey).join();
+    }
+
+    private Void handleAuthException(RemotingCommand response, Throwable ex) {
+        Throwable throwable = ExceptionUtils.getRealException(ex);
+        if (throwable instanceof AuthenticationException || throwable instanceof AuthorizationException) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark(throwable.getMessage());
+        } else {
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark("An system error occurred, please try again later.");
+            LOGGER.error("An system error occurred when processing auth admin request.", ex);
+        }
+        return null;
+    }
+
     private boolean validateSlave(RemotingCommand response) {
         if (this.brokerController.getMessageStoreConfig().getBrokerRole().equals(BrokerRole.SLAVE)) {
             response.setCode(ResponseCode.SYSTEM_ERROR);
@@ -2808,11 +3310,154 @@ private boolean validateSlave(RemotingCommand response) {
     }
 
     private boolean validateBlackListConfigExist(Properties properties) {
-        for (String blackConfig:configBlackList) {
+        for (String blackConfig : configBlackList) {
             if (properties.containsKey(blackConfig)) {
                 return true;
             }
         }
         return false;
     }
+
+    private CheckRocksdbCqWriteResult doCheckRocksdbCqWriteProgress(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        CheckRocksdbCqWriteProgressRequestHeader requestHeader = request.decodeCommandCustomHeader(CheckRocksdbCqWriteProgressRequestHeader.class);
+        String requestTopic = requestHeader.getTopic();
+        MessageStore messageStore = brokerController.getMessageStore();
+        DefaultMessageStore defaultMessageStore;
+        if (messageStore instanceof AbstractPluginMessageStore) {
+            defaultMessageStore = (DefaultMessageStore) ((AbstractPluginMessageStore) messageStore).getNext();
+        } else {
+            defaultMessageStore = (DefaultMessageStore) messageStore;
+        }
+        RocksDBMessageStore rocksDBMessageStore = defaultMessageStore.getRocksDBMessageStore();
+        CheckRocksdbCqWriteResult result = new CheckRocksdbCqWriteResult();
+
+        if (defaultMessageStore.getMessageStoreConfig().getStoreType().equals(StoreType.DEFAULT_ROCKSDB.getStoreType())) {
+            result.setCheckResult("storeType is DEFAULT_ROCKSDB, no need check");
+            result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue());
+            return result;
+        }
+
+        if (!defaultMessageStore.getMessageStoreConfig().isRocksdbCQDoubleWriteEnable()) {
+            result.setCheckResult("rocksdbCQWriteEnable is false, checkRocksdbCqWriteProgressCommand is invalid");
+            result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_NOT_OK.getValue());
+            return result;
+        }
+
+        ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueueInterface>> cqTable = defaultMessageStore.getConsumeQueueTable();
+        StringBuilder diffResult = new StringBuilder();
+        try {
+            if (StringUtils.isNotBlank(requestTopic)) {
+                boolean checkResult = processConsumeQueuesForTopic(cqTable.get(requestTopic), requestTopic, rocksDBMessageStore, diffResult, true, requestHeader.getCheckStoreTime());
+                result.setCheckResult(diffResult.toString());
+                result.setCheckStatus(checkResult ? CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue() : CheckRocksdbCqWriteResult.CheckStatus.CHECK_NOT_OK.getValue());
+                return result;
+            }
+            int successNum = 0;
+            int checkSize = 0;
+            for (Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>> topicEntry : cqTable.entrySet()) {
+                boolean checkResult = processConsumeQueuesForTopic(topicEntry.getValue(), topicEntry.getKey(), rocksDBMessageStore, diffResult, false, requestHeader.getCheckStoreTime());
+                successNum += checkResult ? 1 : 0;
+                checkSize++;
+            }
+            // check all topic finish, all topic is ready, checkSize: 100, currentQueueNum: 110      -> ready  (The currentQueueNum means when we do checking, new topics are added.)
+            // check all topic finish, success/all : 89/100, currentQueueNum: 110                    -> not ready
+            boolean checkReady = successNum == checkSize;
+            String checkResultString = checkReady ? String.format("all topic is ready, checkSize: %s, currentQueueNum: %s", checkSize, cqTable.size()) :
+                String.format("success/all : %s/%s, currentQueueNum: %s", successNum, checkSize, cqTable.size());
+            diffResult.append("check all topic finish, ").append(checkResultString);
+            result.setCheckResult(diffResult.toString());
+            result.setCheckStatus(checkReady ? CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue() : CheckRocksdbCqWriteResult.CheckStatus.CHECK_NOT_OK.getValue());
+        } catch (Exception e) {
+            LOGGER.error("CheckRocksdbCqWriteProgressCommand error", e);
+            result.setCheckResult(e.getMessage() + Arrays.toString(e.getStackTrace()));
+            result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_ERROR.getValue());
+        }
+        return result;
+    }
+
+    private boolean processConsumeQueuesForTopic(ConcurrentMap<Integer, ConsumeQueueInterface> queueMap, String topic,
+        RocksDBMessageStore rocksDBMessageStore, StringBuilder diffResult, boolean printDetail,
+        long checkpointByStoreTime) {
+        boolean processResult = true;
+        for (Map.Entry<Integer, ConsumeQueueInterface> queueEntry : queueMap.entrySet()) {
+            Integer queueId = queueEntry.getKey();
+            ConsumeQueueInterface jsonCq = queueEntry.getValue();
+            ConsumeQueueInterface kvCq = rocksDBMessageStore.getConsumeQueue(topic, queueId);
+            if (printDetail) {
+                String format = String.format("[topic: %s, queue:  %s] \n  kvEarliest : %s |  kvLatest : %s \n fileEarliest: %s | fileEarliest: %s ",
+                    topic, queueId, kvCq.getEarliestUnit(), kvCq.getLatestUnit(), jsonCq.getEarliestUnit(), jsonCq.getLatestUnit());
+                diffResult.append(format).append("\n");
+            }
+
+            long minOffsetByTime = 0L;
+            try {
+                minOffsetByTime = rocksDBMessageStore.getConsumeQueueStore().getOffsetInQueueByTime(topic, queueId, checkpointByStoreTime, BoundaryType.UPPER);
+            } catch (Exception e) {
+                // ignore
+            }
+            long minOffsetInQueue = kvCq.getMinOffsetInQueue();
+            long checkFrom = Math.max(minOffsetInQueue, minOffsetByTime);
+            long checkTo = jsonCq.getMaxOffsetInQueue() - 1;
+            /*
+                                                            checkTo(maxOffsetInQueue - 1)
+                                                                        v
+        fileCq   +------------------------------------------------------+
+        kvCq             +----------------------------------------------+
+                         ^                ^
+                   minOffsetInQueue   minOffsetByTime
+                                   ^
+                        checkFrom = max(minOffsetInQueue, minOffsetByTime)
+             */
+            // The latest message is earlier than the check time
+            Pair<CqUnit, Long> fileLatestCq = jsonCq.getCqUnitAndStoreTime(checkTo);
+            if (fileLatestCq != null) {
+                if (fileLatestCq.getObject2() < checkpointByStoreTime) {
+                    continue;
+                }
+            }
+            for (long i = checkFrom; i <= checkTo; i++) {
+                Pair<CqUnit, Long> fileCqUnit = jsonCq.getCqUnitAndStoreTime(i);
+                Pair<CqUnit, Long> kvCqUnit = kvCq.getCqUnitAndStoreTime(i);
+                if (fileCqUnit == null || kvCqUnit == null || !checkCqUnitEqual(kvCqUnit.getObject1(), fileCqUnit.getObject1())) {
+                    LOGGER.error(String.format("[topic: %s, queue: %s, offset: %s] \n file : %s  \n  kv : %s \n",
+                        topic, queueId, i, kvCqUnit != null ? kvCqUnit.getObject1() : "null", fileCqUnit != null ? fileCqUnit.getObject1() : "null"));
+                    processResult = false;
+                    break;
+                }
+            }
+        }
+        return processResult;
+    }
+
+    private boolean checkCqUnitEqual(CqUnit cqUnit1, CqUnit cqUnit2) {
+        if (cqUnit1.getQueueOffset() != cqUnit2.getQueueOffset()) {
+            return false;
+        }
+        if (cqUnit1.getSize() != cqUnit2.getSize()) {
+            return false;
+        }
+        if (cqUnit1.getPos() != cqUnit2.getPos()) {
+            return false;
+        }
+        if (cqUnit1.getBatchNum() != cqUnit2.getBatchNum()) {
+            return false;
+        }
+        return cqUnit1.getTagsCode() == cqUnit2.getTagsCode();
+    }
+
+    private RemotingCommand transferPopToFsStore(ChannelHandlerContext ctx, RemotingCommand request) {
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        try {
+            if (brokerController.getPopConsumerService() != null) {
+                brokerController.getPopConsumerService().transferToFsStore();
+            }
+            response.setCode(ResponseCode.SUCCESS);
+        } catch (Exception e) {
+            LOGGER.error("PopConsumerStore transfer from kvStore to fsStore finish [{}]", request, e);
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark(e.getMessage());
+        }
+        return response;
+    }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java
index bdfffff096a..de72ee7baff 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java
@@ -19,8 +19,14 @@
 import com.alibaba.fastjson.JSON;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.nio.charset.StandardCharsets;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.metrics.PopMetricsManager;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.broker.offset.ConsumerOrderInfoManager;
+import org.apache.rocketmq.broker.pop.PopConsumerLockService;
 import org.apache.rocketmq.common.PopAckConstants;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.constant.LoggerName;
@@ -28,19 +34,19 @@
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExtBrokerInner;
-import org.apache.rocketmq.common.utils.DataConverter;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;
 import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
 import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
-import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.pop.AckMsg;
 import org.apache.rocketmq.store.pop.PopCheckPoint;
 
@@ -67,6 +73,35 @@ public boolean rejectRequest() {
 
     private RemotingCommand processRequest(final Channel channel, RemotingCommand request,
         boolean brokerAllowSuspend) throws RemotingCommandException {
+
+        CompletableFuture<RemotingCommand> responseFuture = processRequestAsync(channel, request, brokerAllowSuspend);
+
+        if (brokerController.getBrokerConfig().isAppendCkAsync() && brokerController.getBrokerConfig().isAppendAckAsync()) {
+            responseFuture.thenAccept(response -> doResponse(channel, request, response)).exceptionally(throwable -> {
+                RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);
+                response.setCode(ResponseCode.SYSTEM_ERROR);
+                response.setOpaque(request.getOpaque());
+                doResponse(channel, request, response);
+                POP_LOGGER.error("append checkpoint or ack origin failed", throwable);
+                return null;
+            });
+        } else {
+            RemotingCommand response;
+            try {
+                response = responseFuture.get(3000, TimeUnit.MILLISECONDS);
+            } catch (Exception e) {
+                response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);
+                response.setCode(ResponseCode.SYSTEM_ERROR);
+                response.setOpaque(request.getOpaque());
+                POP_LOGGER.error("append checkpoint or ack origin failed", e);
+            }
+            return response;
+        }
+        return null;
+    }
+
+    public CompletableFuture<RemotingCommand> processRequestAsync(final Channel channel, RemotingCommand request,
+        boolean brokerAllowSuspend) throws RemotingCommandException {
         final ChangeInvisibleTimeRequestHeader requestHeader = (ChangeInvisibleTimeRequestHeader) request.decodeCommandCustomHeader(ChangeInvisibleTimeRequestHeader.class);
         RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);
         response.setCode(ResponseCode.SUCCESS);
@@ -77,7 +112,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
             POP_LOGGER.error("The topic {} not exist, consumer: {} ", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));
             response.setCode(ResponseCode.TOPIC_NOT_EXIST);
             response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
-            return response;
+            return CompletableFuture.completedFuture(response);
         }
 
         if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums() || requestHeader.getQueueId() < 0) {
@@ -86,46 +121,105 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
             POP_LOGGER.warn(errorInfo);
             response.setCode(ResponseCode.MESSAGE_ILLEGAL);
             response.setRemark(errorInfo);
-            return response;
+            return CompletableFuture.completedFuture(response);
         }
         long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
-        long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
-        if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) {
+        long maxOffset;
+        try {
+            maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+        } catch (ConsumeQueueException e) {
+            throw new RemotingCommandException("Failed to get max consume offset", e);
+        }
+        if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() >= maxOffset) {
             response.setCode(ResponseCode.NO_MESSAGE);
-            return response;
+            return CompletableFuture.completedFuture(response);
         }
 
         String[] extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo());
+        if (brokerController.getBrokerConfig().isPopConsumerKVServiceEnable()) {
+            if (ExtraInfoUtil.isOrder(extraInfo)) {
+                return this.processChangeInvisibleTimeForOrderNew(
+                    requestHeader, extraInfo, response, responseHeader);
+            }
+            try {
+                long current = System.currentTimeMillis();
+                brokerController.getPopConsumerService().changeInvisibilityDuration(
+                    ExtraInfoUtil.getPopTime(extraInfo), ExtraInfoUtil.getInvisibleTime(extraInfo), current,
+                    requestHeader.getInvisibleTime(), requestHeader.getConsumerGroup(), requestHeader.getTopic(),
+                    requestHeader.getQueueId(), requestHeader.getOffset());
+                responseHeader.setInvisibleTime(requestHeader.getInvisibleTime());
+                responseHeader.setPopTime(current);
+                responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo));
+            } catch (Exception e) {
+                response.setCode(ResponseCode.SYSTEM_ERROR);
+            }
+            return CompletableFuture.completedFuture(response);
+        }
 
         if (ExtraInfoUtil.isOrder(extraInfo)) {
-            return processChangeInvisibleTimeForOrder(requestHeader, extraInfo, response, responseHeader);
+            return CompletableFuture.completedFuture(
+                processChangeInvisibleTimeForOrder(requestHeader, extraInfo, response, responseHeader));
         }
 
         // add new ck
         long now = System.currentTimeMillis();
-        PutMessageResult ckResult = appendCheckPoint(requestHeader, ExtraInfoUtil.getReviveQid(extraInfo), requestHeader.getQueueId(), requestHeader.getOffset(), now, ExtraInfoUtil.getBrokerName(extraInfo));
-
-        if (ckResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
-            && ckResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
-            && ckResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
-            && ckResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
-            POP_LOGGER.error("change Invisible, put new ck error: {}", ckResult);
-            response.setCode(ResponseCode.SYSTEM_ERROR);
-            return response;
+        CompletableFuture<Boolean> futureResult = appendCheckPointThenAckOrigin(requestHeader,
+            ExtraInfoUtil.getReviveQid(extraInfo), requestHeader.getQueueId(), requestHeader.getOffset(), now, extraInfo);
+
+        return futureResult.thenCompose(result -> {
+            if (result) {
+                responseHeader.setInvisibleTime(requestHeader.getInvisibleTime());
+                responseHeader.setPopTime(now);
+                responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo));
+            } else {
+                response.setCode(ResponseCode.SYSTEM_ERROR);
+            }
+            return CompletableFuture.completedFuture(response);
+        });
+    }
+
+    @SuppressWarnings({"StatementWithEmptyBody", "DuplicatedCode"})
+    public CompletableFuture<RemotingCommand> processChangeInvisibleTimeForOrderNew(
+        ChangeInvisibleTimeRequestHeader requestHeader, String[] extraInfo,
+        RemotingCommand response, ChangeInvisibleTimeResponseHeader responseHeader) {
+
+        String groupId = requestHeader.getConsumerGroup();
+        String topicId = requestHeader.getTopic();
+        Integer queueId = requestHeader.getQueueId();
+        long popTime = ExtraInfoUtil.getPopTime(extraInfo);
+
+        PopConsumerLockService consumerLockService =
+            this.brokerController.getPopConsumerService().getConsumerLockService();
+        ConsumerOffsetManager consumerOffsetManager = this.brokerController.getConsumerOffsetManager();
+        ConsumerOrderInfoManager consumerOrderInfoManager = brokerController.getConsumerOrderInfoManager();
+
+        long oldOffset = consumerOffsetManager.queryOffset(groupId, topicId, queueId);
+        if (requestHeader.getOffset() < oldOffset) {
+            return CompletableFuture.completedFuture(response);
+        }
+
+        while (!consumerLockService.tryLock(groupId, topicId)) {
         }
 
-        // ack old msg.
         try {
-            ackOrigin(requestHeader, extraInfo);
-        } catch (Throwable e) {
-            POP_LOGGER.error("change Invisible, put ack msg error: {}, {}", requestHeader.getExtraInfo(), e.getMessage());
-            // cancel new ck?
+            // double check
+            oldOffset = consumerOffsetManager.queryOffset(groupId, topicId, queueId);
+            if (requestHeader.getOffset() < oldOffset) {
+                return CompletableFuture.completedFuture(response);
+            }
+
+            long visibilityTimeout = System.currentTimeMillis() + requestHeader.getInvisibleTime();
+            consumerOrderInfoManager.updateNextVisibleTime(
+                topicId, groupId, queueId, requestHeader.getOffset(), popTime, visibilityTimeout);
+
+            responseHeader.setInvisibleTime(visibilityTimeout - popTime);
+            responseHeader.setPopTime(popTime);
+            responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo));
+        } finally {
+            consumerLockService.unlock(groupId, topicId);
         }
 
-        responseHeader.setInvisibleTime(requestHeader.getInvisibleTime());
-        responseHeader.setPopTime(now);
-        responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo));
-        return response;
+        return CompletableFuture.completedFuture(response);
     }
 
     protected RemotingCommand processChangeInvisibleTimeForOrder(ChangeInvisibleTimeRequestHeader requestHeader,
@@ -158,7 +252,8 @@ protected RemotingCommand processChangeInvisibleTimeForOrder(ChangeInvisibleTime
         return response;
     }
 
-    private void ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, String[] extraInfo) {
+    private CompletableFuture<Boolean> ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader,
+        String[] extraInfo) {
         MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
         AckMsg ackMsg = new AckMsg();
 
@@ -176,11 +271,11 @@ private void ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, Str
         this.brokerController.getBrokerStatsManager().incGroupAckNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);
 
         if (brokerController.getPopMessageProcessor().getPopBufferMergeService().addAk(rqId, ackMsg)) {
-            return;
+            return CompletableFuture.completedFuture(true);
         }
 
         msgInner.setTopic(reviveTopic);
-        msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));
+        msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(StandardCharsets.UTF_8));
         msgInner.setQueueId(rqId);
         msgInner.setTags(PopAckConstants.ACK_TAG);
         msgInner.setBornTimestamp(System.currentTimeMillis());
@@ -189,18 +284,25 @@ private void ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, Str
         msgInner.setDeliverTimeMs(ExtraInfoUtil.getPopTime(extraInfo) + ExtraInfoUtil.getInvisibleTime(extraInfo));
         msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));
         msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
-        PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
-        if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
-            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
-            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
-            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
-            POP_LOGGER.error("change Invisible, put ack msg fail: {}, {}", ackMsg, putMessageResult);
-        }
-        PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus());
+        return this.brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenCompose(putMessageResult -> {
+            if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
+                && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+                && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+                && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+                POP_LOGGER.error("change Invisible, put ack msg fail: {}, {}", ackMsg, putMessageResult);
+            }
+            PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus());
+            return CompletableFuture.completedFuture(true);
+        }).exceptionally(e -> {
+            POP_LOGGER.error("change Invisible, put ack msg error: {}, {}", requestHeader.getExtraInfo(), e.getMessage());
+            return false;
+        });
     }
 
-    private PutMessageResult appendCheckPoint(final ChangeInvisibleTimeRequestHeader requestHeader, int reviveQid,
-        int queueId, long offset, long popTime, String brokerName) {
+    private CompletableFuture<Boolean> appendCheckPointThenAckOrigin(
+        final ChangeInvisibleTimeRequestHeader requestHeader,
+        int reviveQid,
+        int queueId, long offset, long popTime, String[] extraInfo) {
         // add check point msg to revive log
         MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
         msgInner.setTopic(reviveTopic);
@@ -214,9 +316,9 @@ private PutMessageResult appendCheckPoint(final ChangeInvisibleTimeRequestHeader
         ck.setTopic(requestHeader.getTopic());
         ck.setQueueId(queueId);
         ck.addDiff(0);
-        ck.setBrokerName(brokerName);
+        ck.setBrokerName(ExtraInfoUtil.getBrokerName(extraInfo));
 
-        msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8));
+        msgInner.setBody(JSON.toJSONString(ck).getBytes(StandardCharsets.UTF_8));
         msgInner.setQueueId(reviveQid);
         msgInner.setTags(PopAckConstants.CK_TAG);
         msgInner.setBornTimestamp(System.currentTimeMillis());
@@ -225,21 +327,36 @@ private PutMessageResult appendCheckPoint(final ChangeInvisibleTimeRequestHeader
         msgInner.setDeliverTimeMs(ck.getReviveTime() - PopAckConstants.ackTimeInterval);
         msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genCkUniqueId(ck));
         msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
-        PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
-
-        if (brokerController.getBrokerConfig().isEnablePopLog()) {
-            POP_LOGGER.info("change Invisible , appendCheckPoint, topic {}, queueId {},reviveId {}, cid {}, startOffset {}, rt {}, result {}", requestHeader.getTopic(), queueId, reviveQid, requestHeader.getConsumerGroup(), offset,
-                ck.getReviveTime(), putMessageResult);
-        }
+        return this.brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenCompose(putMessageResult -> {
+            if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                POP_LOGGER.info("change Invisible, appendCheckPoint, topic {}, queueId {},reviveId {}, cid {}, startOffset {}, rt {}, result {}", requestHeader.getTopic(), queueId, reviveQid, requestHeader.getConsumerGroup(), offset,
+                    ck.getReviveTime(), putMessageResult);
+            }
 
-        if (putMessageResult != null) {
-            PopMetricsManager.incPopReviveCkPutCount(ck, putMessageResult.getPutMessageStatus());
-            if (putMessageResult.isOk()) {
-                this.brokerController.getBrokerStatsManager().incBrokerCkNums(1);
-                this.brokerController.getBrokerStatsManager().incGroupCkNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);
+            if (putMessageResult != null) {
+                PopMetricsManager.incPopReviveCkPutCount(ck, putMessageResult.getPutMessageStatus());
+                if (putMessageResult.isOk()) {
+                    this.brokerController.getBrokerStatsManager().incBrokerCkNums(1);
+                    this.brokerController.getBrokerStatsManager().incGroupCkNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);
+                }
             }
-        }
+            if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
+                && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+                && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+                && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+                POP_LOGGER.error("change invisible, put new ck error: {}", putMessageResult);
+                return CompletableFuture.completedFuture(false);
+            } else {
+                return ackOrigin(requestHeader, extraInfo);
+            }
+        }).exceptionally(throwable -> {
+            POP_LOGGER.error("change invisible, put new ck error", throwable);
+            return null;
+        });
+    }
 
-        return putMessageResult;
+    protected void doResponse(Channel channel, RemotingCommand request,
+        final RemotingCommand response) {
+        NettyRemotingAbstract.writeResponse(channel, request, response);
     }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
index e16a1e9090f..dfa755d7c44 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
@@ -164,6 +164,12 @@ private RemotingCommand updateConsumerOffset(ChannelHandlerContext ctx, Remoting
         Integer queueId = requestHeader.getQueueId();
         Long offset = requestHeader.getCommitOffset();
 
+        if (!this.brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(group)) {
+            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
+            response.setRemark("Group " + group + " not exist!");
+            return response;
+        }
+
         if (!this.brokerController.getTopicConfigManager().containsTopic(requestHeader.getTopic())) {
             response.setCode(ResponseCode.TOPIC_NOT_EXIST);
             response.setRemark("Topic " + topic + " not exist!");
@@ -171,13 +177,13 @@ private RemotingCommand updateConsumerOffset(ChannelHandlerContext ctx, Remoting
         }
 
         if (queueId == null) {
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.INVALID_PARAMETER);
             response.setRemark("QueueId is null, topic is " + topic);
             return response;
         }
 
         if (offset == null) {
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.INVALID_PARAMETER);
             response.setRemark("Offset is null, topic is " + topic);
             return response;
         }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java
index e812a53ba7c..468a8791d40 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java
@@ -279,6 +279,9 @@ private RemotingCommand sendFinalMessage(MessageExtBrokerInner msgInner) {
             switch (putMessageResult.getPutMessageStatus()) {
                 // Success
                 case PUT_OK:
+                    this.brokerController.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1);
+                    this.brokerController.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());
+                    this.brokerController.getBrokerStatsManager().incBrokerPutNums(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum());
                 case FLUSH_DISK_TIMEOUT:
                 case FLUSH_SLAVE_TIMEOUT:
                 case SLAVE_NOT_AVAILABLE:
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java
index 0deb3ee7077..2fe34649432 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java
@@ -18,6 +18,7 @@
 
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Random;
 import org.apache.rocketmq.broker.BrokerController;
@@ -40,6 +41,7 @@
 import org.apache.rocketmq.remoting.protocol.header.NotificationRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.NotificationResponseHeader;
 import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 
 public class NotificationProcessor implements NettyRequestProcessor {
     private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
@@ -50,7 +52,7 @@ public class NotificationProcessor implements NettyRequestProcessor {
 
     public NotificationProcessor(final BrokerController brokerController) {
         this.brokerController = brokerController;
-        this.popLongPollingService = new PopLongPollingService(brokerController, this);
+        this.popLongPollingService = new PopLongPollingService(brokerController, this, true);
     }
 
     @Override
@@ -58,8 +60,16 @@ public boolean rejectRequest() {
         return false;
     }
 
+    // When a new message is written to CommitLog, this method would be called.
+    // Suspended long polling will receive notification and be wakeup.
+    public void notifyMessageArriving(final String topic, final int queueId, long offset,
+        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
+        this.popLongPollingService.notifyMessageArrivingWithRetryTopic(
+            topic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);
+    }
+
     public void notifyMessageArriving(final String topic, final int queueId) {
-        popLongPollingService.notifyMessageArrivingWithRetryTopic(topic, queueId);
+        this.popLongPollingService.notifyMessageArrivingWithRetryTopic(topic, queueId);
     }
 
     @Override
@@ -102,7 +112,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx,
             String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
                 requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
             POP_LOGGER.warn(errorInfo);
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.INVALID_PARAMETER);
             response.setRemark(errorInfo);
             return response;
         }
@@ -151,8 +161,11 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx,
         }
 
         if (!hasMsg) {
-            if (popLongPollingService.polling(ctx, request, new PollingHeader(requestHeader)) == PollingResult.POLLING_SUC) {
+            PollingResult pollingResult = popLongPollingService.polling(ctx, request, new PollingHeader(requestHeader));
+            if (pollingResult == PollingResult.POLLING_SUC) {
                 return null;
+            } else if (pollingResult == PollingResult.POLLING_FULL) {
+                responseHeader.setPollingFull(true);
             }
         }
         response.setCode(ResponseCode.SUCCESS);
@@ -160,13 +173,14 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx,
         return response;
     }
 
-    private boolean hasMsgFromTopic(String topicName, int randomQ, NotificationRequestHeader requestHeader) {
-        boolean hasMsg;
+    private boolean hasMsgFromTopic(String topicName, int randomQ, NotificationRequestHeader requestHeader)
+        throws RemotingCommandException {
         TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topicName);
         return hasMsgFromTopic(topicConfig, randomQ, requestHeader);
     }
 
-    private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, NotificationRequestHeader requestHeader) {
+    private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, NotificationRequestHeader requestHeader)
+        throws RemotingCommandException {
         boolean hasMsg;
         if (topicConfig != null) {
             for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
@@ -180,15 +194,19 @@ private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, Notificati
         return false;
     }
 
-    private boolean hasMsgFromQueue(String targetTopic, NotificationRequestHeader requestHeader, int queueId) {
+    private boolean hasMsgFromQueue(String targetTopic, NotificationRequestHeader requestHeader, int queueId) throws RemotingCommandException {
         if (Boolean.TRUE.equals(requestHeader.getOrder())) {
             if (this.brokerController.getConsumerOrderInfoManager().checkBlock(requestHeader.getAttemptId(), requestHeader.getTopic(), requestHeader.getConsumerGroup(), queueId, 0)) {
                 return false;
             }
         }
         long offset = getPopOffset(targetTopic, requestHeader.getConsumerGroup(), queueId);
-        long restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(targetTopic, queueId) - offset;
-        return restNum > 0;
+        try {
+            long restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(targetTopic, queueId) - offset;
+            return restNum > 0;
+        } catch (ConsumeQueueException e) {
+            throw new RemotingCommandException("Failed tp get max offset in queue", e);
+        }
     }
 
     private long getPopOffset(String topic, String cid, int queueId) {
@@ -196,13 +214,16 @@ private long getPopOffset(String topic, String cid, int queueId) {
         if (offset < 0) {
             offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
         }
-        long bufferOffset = this.brokerController.getPopMessageProcessor().getPopBufferMergeService()
-            .getLatestOffset(topic, cid, queueId);
-        if (bufferOffset < 0) {
-            return offset;
+
+        long bufferOffset;
+        if (brokerController.getBrokerConfig().isPopConsumerKVServiceEnable()) {
+            bufferOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset(cid, topic, queueId);
         } else {
-            return bufferOffset > offset ? bufferOffset : offset;
+            bufferOffset = this.brokerController.getPopMessageProcessor()
+                .getPopBufferMergeService().getLatestOffset(topic, cid, queueId);
         }
+
+        return bufferOffset < 0L ? offset : Math.max(bufferOffset, offset);
     }
 
     public PopLongPollingService getPopLongPollingService() {
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java
index 55552003d80..40117b74a54 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java
@@ -51,6 +51,7 @@
 import org.apache.rocketmq.store.GetMessageResult;
 import org.apache.rocketmq.store.GetMessageStatus;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;
 import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;
@@ -113,7 +114,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
             String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
                 requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
             LOG.warn(errorInfo);
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.INVALID_PARAMETER);
             response.setRemark(errorInfo);
             return response;
         }
@@ -229,13 +230,18 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
 
     private long peekMsgFromQueue(boolean isRetry, GetMessageResult getMessageResult,
         PeekMessageRequestHeader requestHeader, int queueId, long restNum, int reviveQid, Channel channel,
-        long popTime) {
+        long popTime) throws RemotingCommandException {
         String topic = isRetry ?
             KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerController.getBrokerConfig().isEnableRetryTopicV2())
             : requestHeader.getTopic();
         GetMessageResult getMessageTmpResult;
         long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId);
-        restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+        try {
+            restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+        } catch (ConsumeQueueException e) {
+            LOG.error("Failed to get max offset in queue. topic={}, queue-id={}", topic, queueId, e);
+            throw new RemotingCommandException("Failed to get max offset in queue", e);
+        }
         if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) {
             return restNum;
         }
@@ -258,8 +264,8 @@ private long peekMsgFromQueue(boolean isRetry, GetMessageResult getMessageResult
                 BrokerMetricsManager.throughputOutTotal.add(getMessageResult.getBufferTotalSize(), attributes);
             }
 
-            for (SelectMappedBufferResult mapedBuffer : getMessageTmpResult.getMessageMapedList()) {
-                getMessageResult.addMessage(mapedBuffer);
+            for (SelectMappedBufferResult mappedBuffer : getMessageTmpResult.getMessageMapedList()) {
+                getMessageResult.addMessage(mappedBuffer);
             }
         }
         return restNum;
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java
index 65a4d7d7851..f7baac144e6 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java
@@ -89,7 +89,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
             String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
                 requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
             POP_LOGGER.warn(errorInfo);
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.INVALID_PARAMETER);
             response.setRemark(errorInfo);
             return response;
         }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java
index 8a85dd8fec8..820388b18d2 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java
@@ -62,7 +62,7 @@ public class PopBufferMergeService extends ServiceThread {
     private final int countOfSecond1 = (int) (1000 / interval);
     private final int countOfSecond30 = (int) (30 * 1000 / interval);
 
-    private final List<Byte> batchAckIndexList = new ArrayList(32);
+    private final List<Byte> batchAckIndexList = new ArrayList<>(32);
     private volatile boolean master = false;
 
     public PopBufferMergeService(BrokerController brokerController, PopMessageProcessor popMessageProcessor) {
@@ -197,12 +197,12 @@ private void scanGarbage() {
             String topic = keyArray[0];
             String cid = keyArray[1];
             if (brokerController.getTopicConfigManager().selectTopicConfig(topic) == null) {
-                POP_LOGGER.info("[PopBuffer]remove not exit topic {} in buffer!", topic);
+                POP_LOGGER.info("[PopBuffer]remove nonexistent topic {} in buffer!", topic);
                 iterator.remove();
                 continue;
             }
-            if (!brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().containsKey(cid)) {
-                POP_LOGGER.info("[PopBuffer]remove not exit sub {} of topic {} in buffer!", cid, topic);
+            if (!brokerController.getSubscriptionGroupManager().containsSubscriptionGroup(cid)) {
+                POP_LOGGER.info("[PopBuffer]remove nonexistent subscription group {} of topic {} in buffer!", cid, topic);
                 iterator.remove();
                 continue;
             }
@@ -216,7 +216,8 @@ private void scanGarbage() {
 
     private void scan() {
         long startTime = System.currentTimeMillis();
-        int count = 0, countCk = 0;
+        AtomicInteger count = new AtomicInteger(0);
+        int countCk = 0;
         Iterator<Map.Entry<String, PopCheckPointWrapper>> iterator = buffer.entrySet().iterator();
         while (iterator.hasNext()) {
             Map.Entry<String, PopCheckPointWrapper> entry = iterator.next();
@@ -257,14 +258,14 @@ private void scan() {
             } else if (pointWrapper.isJustOffset()) {
                 // just offset should be in store.
                 if (pointWrapper.getReviveQueueOffset() < 0) {
-                    putCkToStore(pointWrapper, false);
+                    putCkToStore(pointWrapper, this.brokerController.getBrokerConfig().isAppendCkAsync());
                     countCk++;
                 }
                 continue;
             } else if (removeCk) {
                 // put buffer ak to store
                 if (pointWrapper.getReviveQueueOffset() < 0) {
-                    putCkToStore(pointWrapper, false);
+                    putCkToStore(pointWrapper, this.brokerController.getBrokerConfig().isAppendCkAsync());
                     countCk++;
                 }
 
@@ -278,17 +279,12 @@ private void scan() {
                         for (byte i = 0; i < point.getNum(); i++) {
                             // reput buffer ak to store
                             if (DataConverter.getBit(pointWrapper.getBits().get(), i)
-                                    && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) {
+                                && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) {
                                 indexList.add(i);
                             }
                         }
                         if (indexList.size() > 0) {
-                            if (putBatchAckToStore(pointWrapper, indexList)) {
-                                count += indexList.size();
-                                for (Byte i : indexList) {
-                                    markBitCAS(pointWrapper.getToStoreBits(), i);
-                                }
-                            }
+                            putBatchAckToStore(pointWrapper, indexList, count);
                         }
                     } finally {
                         indexList.clear();
@@ -297,11 +293,8 @@ private void scan() {
                     for (byte i = 0; i < point.getNum(); i++) {
                         // reput buffer ak to store
                         if (DataConverter.getBit(pointWrapper.getBits().get(), i)
-                                && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) {
-                            if (putAckToStore(pointWrapper, i)) {
-                                count++;
-                                markBitCAS(pointWrapper.getToStoreBits(), i);
-                            }
+                            && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) {
+                            putAckToStore(pointWrapper, i, count);
                         }
                     }
                 }
@@ -312,7 +305,6 @@ private void scan() {
                     }
                     iterator.remove();
                     counter.decrementAndGet();
-                    continue;
                 }
             }
         }
@@ -323,13 +315,13 @@ private void scan() {
         if (eclipse > brokerController.getBrokerConfig().getPopCkStayBufferTimeOut() - 1000) {
             POP_LOGGER.warn("[PopBuffer]scan stop, because eclipse too long, PopBufferEclipse={}, " +
                     "PopBufferToStoreAck={}, PopBufferToStoreCk={}, PopBufferSize={}, PopBufferOffsetSize={}",
-                eclipse, count, countCk, counter.get(), offsetBufferSize);
+                eclipse, count.get(), countCk, counter.get(), offsetBufferSize);
             this.serving = false;
         } else {
             if (scanTimes % countOfSecond1 == 0) {
                 POP_LOGGER.info("[PopBuffer]scan, PopBufferEclipse={}, " +
                         "PopBufferToStoreAck={}, PopBufferToStoreCk={}, PopBufferSize={}, PopBufferOffsetSize={}",
-                    eclipse, count, countCk, counter.get(), offsetBufferSize);
+                    eclipse, count.get(), countCk, counter.get(), offsetBufferSize);
             }
         }
         PopMetricsManager.recordPopBufferScanTimeConsume(eclipse);
@@ -429,7 +421,8 @@ private boolean checkQueueOk(PopCheckPointWrapper pointWrapper) {
      * @param nextBeginOffset
      * @return
      */
-    public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long reviveQueueOffset, long nextBeginOffset) {
+    public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long reviveQueueOffset,
+        long nextBeginOffset) {
         PopCheckPointWrapper pointWrapper = new PopCheckPointWrapper(reviveQueueId, reviveQueueOffset, point, nextBeginOffset, true);
 
         if (this.buffer.containsKey(pointWrapper.getMergeKey())) {
@@ -439,7 +432,7 @@ public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long revi
             return false;
         }
 
-        this.putCkToStore(pointWrapper, !checkQueueOk(pointWrapper));
+        this.putCkToStore(pointWrapper, checkQueueOk(pointWrapper));
 
         putOffsetQueue(pointWrapper);
         this.buffer.put(pointWrapper.getMergeKey(), pointWrapper);
@@ -447,7 +440,7 @@ public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long revi
         if (brokerController.getBrokerConfig().isEnablePopLog()) {
             POP_LOGGER.info("[PopBuffer]add ck just offset, {}", pointWrapper);
         }
-        return  true;
+        return true;
     }
 
     public void addCkMock(String group, String topic, int queueId, long startOffset, long invisibleTime,
@@ -597,13 +590,32 @@ private void putCkToStore(final PopCheckPointWrapper pointWrapper, final boolean
         if (pointWrapper.getReviveQueueOffset() >= 0) {
             return;
         }
+
         MessageExtBrokerInner msgInner = popMessageProcessor.buildCkMsg(pointWrapper.getCk(), pointWrapper.getReviveQueueId());
-        PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+
+        // Indicates that ck message is storing
+        pointWrapper.setReviveQueueOffset(Long.MAX_VALUE);
+        if (brokerController.getBrokerConfig().isAppendCkAsync() && runInCurrent) {
+            brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> {
+                handleCkMessagePutResult(putMessageResult, pointWrapper);
+            }).exceptionally(throwable -> {
+                POP_LOGGER.error("[PopBuffer]put ck to store fail: {}", pointWrapper, throwable);
+                pointWrapper.setReviveQueueOffset(-1);
+                return null;
+            });
+        } else {
+            PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+            handleCkMessagePutResult(putMessageResult, pointWrapper);
+        }
+    }
+
+    private void handleCkMessagePutResult(PutMessageResult putMessageResult, final PopCheckPointWrapper pointWrapper) {
         PopMetricsManager.incPopReviveCkPutCount(pointWrapper.getCk(), putMessageResult.getPutMessageStatus());
         if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
             && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
             && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
             && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+            pointWrapper.setReviveQueueOffset(-1);
             POP_LOGGER.error("[PopBuffer]put ck to store fail: {}, {}", pointWrapper, putMessageResult);
             return;
         }
@@ -621,7 +633,7 @@ private void putCkToStore(final PopCheckPointWrapper pointWrapper, final boolean
         }
     }
 
-    private boolean putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgIndex) {
+    private void putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgIndex, AtomicInteger count) {
         PopCheckPoint point = pointWrapper.getCk();
         MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
         final AckMsg ackMsg = new AckMsg();
@@ -632,7 +644,8 @@ private boolean putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgI
         ackMsg.setTopic(point.getTopic());
         ackMsg.setQueueId(point.getQueueId());
         ackMsg.setPopTime(point.getPopTime());
-        msgInner.setTopic(popMessageProcessor.reviveTopic);
+        ackMsg.setBrokerName(point.getBrokerName());
+        msgInner.setTopic(popMessageProcessor.getReviveTopic());
         msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));
         msgInner.setQueueId(pointWrapper.getReviveQueueId());
         msgInner.setTags(PopAckConstants.ACK_TAG);
@@ -643,23 +656,39 @@ private boolean putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgI
         msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));
 
         msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
-        PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+
+        if (brokerController.getBrokerConfig().isAppendAckAsync()) {
+            brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> {
+                handleAckPutMessageResult(ackMsg, putMessageResult, pointWrapper, count, msgIndex);
+            }).exceptionally(throwable -> {
+                POP_LOGGER.error("[PopBuffer]put ack to store fail: {}, {}", pointWrapper, ackMsg, throwable);
+                return null;
+            });
+        } else {
+            PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+            handleAckPutMessageResult(ackMsg, putMessageResult, pointWrapper, count, msgIndex);
+        }
+    }
+
+    private void handleAckPutMessageResult(AckMsg ackMsg, PutMessageResult putMessageResult,
+        PopCheckPointWrapper pointWrapper, AtomicInteger count, byte msgIndex) {
         PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus());
         if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
             && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
             && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
             && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
             POP_LOGGER.error("[PopBuffer]put ack to store fail: {}, {}, {}", pointWrapper, ackMsg, putMessageResult);
-            return false;
+            return;
         }
         if (brokerController.getBrokerConfig().isEnablePopLog()) {
             POP_LOGGER.info("[PopBuffer]put ack to store ok: {}, {}, {}", pointWrapper, ackMsg, putMessageResult);
         }
-
-        return true;
+        count.incrementAndGet();
+        markBitCAS(pointWrapper.getToStoreBits(), msgIndex);
     }
 
-    private boolean putBatchAckToStore(final PopCheckPointWrapper pointWrapper, final List<Byte> msgIndexList) {
+    private void putBatchAckToStore(final PopCheckPointWrapper pointWrapper, final List<Byte> msgIndexList,
+        AtomicInteger count) {
         PopCheckPoint point = pointWrapper.getCk();
         MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
         final BatchAckMsg batchAckMsg = new BatchAckMsg();
@@ -672,7 +701,7 @@ private boolean putBatchAckToStore(final PopCheckPointWrapper pointWrapper, fina
         batchAckMsg.setTopic(point.getTopic());
         batchAckMsg.setQueueId(point.getQueueId());
         batchAckMsg.setPopTime(point.getPopTime());
-        msgInner.setTopic(popMessageProcessor.reviveTopic);
+        msgInner.setTopic(popMessageProcessor.getReviveTopic());
         msgInner.setBody(JSON.toJSONString(batchAckMsg).getBytes(DataConverter.CHARSET_UTF8));
         msgInner.setQueueId(pointWrapper.getReviveQueueId());
         msgInner.setTags(PopAckConstants.BATCH_ACK_TAG);
@@ -683,19 +712,36 @@ private boolean putBatchAckToStore(final PopCheckPointWrapper pointWrapper, fina
         msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genBatchAckUniqueId(batchAckMsg));
 
         msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
-        PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+        if (brokerController.getBrokerConfig().isAppendAckAsync()) {
+            brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> {
+                handleBatchAckPutMessageResult(batchAckMsg, putMessageResult, pointWrapper, count, msgIndexList);
+            }).exceptionally(throwable -> {
+                POP_LOGGER.error("[PopBuffer]put batchAckMsg to store fail: {}, {}", pointWrapper, batchAckMsg, throwable);
+                return null;
+            });
+        } else {
+            PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+            handleBatchAckPutMessageResult(batchAckMsg, putMessageResult, pointWrapper, count, msgIndexList);
+        }
+    }
+
+    private void handleBatchAckPutMessageResult(BatchAckMsg batchAckMsg, PutMessageResult putMessageResult,
+        PopCheckPointWrapper pointWrapper, AtomicInteger count, List<Byte> msgIndexList) {
         if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
-                && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
-                && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
-                && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
             POP_LOGGER.error("[PopBuffer]put batch ack to store fail: {}, {}, {}", pointWrapper, batchAckMsg, putMessageResult);
-            return false;
+            return;
         }
         if (brokerController.getBrokerConfig().isEnablePopLog()) {
             POP_LOGGER.info("[PopBuffer]put batch ack to store ok: {}, {}, {}", pointWrapper, batchAckMsg, putMessageResult);
         }
 
-        return true;
+        count.addAndGet(msgIndexList.size());
+        for (Byte i : msgIndexList) {
+            markBitCAS(pointWrapper.getToStoreBits(), i);
+        }
     }
 
     private boolean cancelCkTimer(final PopCheckPointWrapper pointWrapper) {
@@ -705,7 +751,7 @@ private boolean cancelCkTimer(final PopCheckPointWrapper pointWrapper) {
         }
         PopCheckPoint point = pointWrapper.getCk();
         MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
-        msgInner.setTopic(popMessageProcessor.reviveTopic);
+        msgInner.setTopic(popMessageProcessor.getReviveTopic());
         msgInner.setBody((pointWrapper.getReviveQueueId() + "-" + pointWrapper.getReviveQueueOffset()).getBytes(StandardCharsets.UTF_8));
         msgInner.setQueueId(pointWrapper.getReviveQueueId());
         msgInner.setTags(PopAckConstants.CK_TAG);
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
index 59ff2e0fd52..d73acc84df6 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
@@ -24,8 +24,10 @@
 import io.netty.channel.FileRegion;
 import io.opentelemetry.api.common.Attributes;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Random;
@@ -45,6 +47,7 @@
 import org.apache.rocketmq.broker.longpolling.PopRequest;
 import org.apache.rocketmq.broker.metrics.BrokerMetricsManager;
 import org.apache.rocketmq.broker.pagecache.ManyMessageTransfer;
+import org.apache.rocketmq.broker.pop.PopConsumerContext;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.KeyBuilder;
 import org.apache.rocketmq.common.MixAll;
@@ -61,9 +64,10 @@
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.common.topic.TopicValidator;
-import org.apache.rocketmq.common.utils.DataConverter;
+import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.CommandCallback;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.metrics.RemotingMetricsManager;
@@ -82,6 +86,7 @@
 import org.apache.rocketmq.store.GetMessageResult;
 import org.apache.rocketmq.store.GetMessageStatus;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.pop.AckMsg;
 import org.apache.rocketmq.store.pop.BatchAckMsg;
 import org.apache.rocketmq.store.pop.PopCheckPoint;
@@ -95,12 +100,13 @@
 import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESULT;
 
 public class PopMessageProcessor implements NettyRequestProcessor {
-    private static final Logger POP_LOGGER =
-        LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+
+    private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+    private static final String BORN_TIME = "bornTime";
+
     private final BrokerController brokerController;
     private final Random random = new Random(System.currentTimeMillis());
-    String reviveTopic;
-    private static final String BORN_TIME = "bornTime";
+    private final String reviveTopic;
 
     private final PopLongPollingService popLongPollingService;
     private final PopBufferMergeService popBufferMergeService;
@@ -109,13 +115,18 @@ public class PopMessageProcessor implements NettyRequestProcessor {
 
     public PopMessageProcessor(final BrokerController brokerController) {
         this.brokerController = brokerController;
-        this.reviveTopic = PopAckConstants.buildClusterReviveTopic(this.brokerController.getBrokerConfig().getBrokerClusterName());
-        this.popLongPollingService = new PopLongPollingService(brokerController, this);
+        this.reviveTopic = PopAckConstants.buildClusterReviveTopic(
+            this.brokerController.getBrokerConfig().getBrokerClusterName());
+        this.popLongPollingService = new PopLongPollingService(brokerController, this, false);
         this.queueLockManager = new QueueLockManager();
         this.popBufferMergeService = new PopBufferMergeService(this.brokerController, this);
         this.ckMessageNumber = new AtomicLong();
     }
 
+    protected String getReviveTopic() {
+        return reviveTopic;
+    }
+
     public PopLongPollingService getPopLongPollingService() {
         return popLongPollingService;
     }
@@ -140,11 +151,11 @@ public static String genAckUniqueId(AckMsg ackMsg) {
 
     public static String genBatchAckUniqueId(BatchAckMsg batchAckMsg) {
         return batchAckMsg.getTopic()
-                + PopAckConstants.SPLIT + batchAckMsg.getQueueId()
-                + PopAckConstants.SPLIT + batchAckMsg.getAckOffsetList().toString()
-                + PopAckConstants.SPLIT + batchAckMsg.getConsumerGroup()
-                + PopAckConstants.SPLIT + batchAckMsg.getPopTime()
-                + PopAckConstants.SPLIT + PopAckConstants.BATCH_ACK_TAG;
+            + PopAckConstants.SPLIT + batchAckMsg.getQueueId()
+            + PopAckConstants.SPLIT + batchAckMsg.getAckOffsetList().toString()
+            + PopAckConstants.SPLIT + batchAckMsg.getConsumerGroup()
+            + PopAckConstants.SPLIT + batchAckMsg.getPopTime()
+            + PopAckConstants.SPLIT + PopAckConstants.BATCH_ACK_TAG;
     }
 
     public static String genCkUniqueId(PopCheckPoint ck) {
@@ -166,16 +177,25 @@ public ConcurrentLinkedHashMap<String, ConcurrentSkipListSet<PopRequest>> getPol
         return popLongPollingService.getPollingMap();
     }
 
-    public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId) {
+    public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId) throws ConsumeQueueException {
+        this.notifyLongPollingRequestIfNeed(
+            topic, group, queueId, null, 0L, null, null);
+    }
+
+    public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId,
+        Long tagsCode, long msgStoreTime, byte[] filterBitMap,
+        Map<String, String> properties) throws ConsumeQueueException {
         long popBufferOffset = this.brokerController.getPopMessageProcessor().getPopBufferMergeService().getLatestOffset(topic, group, queueId);
         long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId);
         long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
         long offset = Math.max(popBufferOffset, consumerOffset);
         if (maxOffset > offset) {
-            boolean notifySuccess = popLongPollingService.notifyMessageArriving(topic, group, -1);
+            boolean notifySuccess = popLongPollingService.notifyMessageArriving(
+                topic, -1, group, tagsCode, msgStoreTime, filterBitMap, properties);
             if (!notifySuccess) {
                 // notify pop queue
-                notifySuccess = popLongPollingService.notifyMessageArriving(topic, group, queueId);
+                notifySuccess = popLongPollingService.notifyMessageArriving(
+                    topic, queueId, group, tagsCode, msgStoreTime, filterBitMap, properties);
             }
             this.brokerController.getNotificationProcessor().notifyMessageArriving(topic, queueId);
             if (this.brokerController.getBrokerConfig().isEnablePopLog()) {
@@ -185,39 +205,40 @@ public void notifyLongPollingRequestIfNeed(String topic, String group, int queue
         }
     }
 
-    public void notifyMessageArriving(final String topic, final int queueId) {
-        popLongPollingService.notifyMessageArrivingWithRetryTopic(topic, queueId);
+    public void notifyMessageArriving(final String topic, final int queueId, long offset,
+        Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
+        popLongPollingService.notifyMessageArrivingWithRetryTopic(
+            topic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);
     }
 
-    public boolean notifyMessageArriving(final String topic, final String cid, final int queueId) {
-        return popLongPollingService.notifyMessageArriving(topic, cid, queueId);
+    public void notifyMessageArriving(final String topic, final int queueId, final String cid) {
+        popLongPollingService.notifyMessageArriving(
+            topic, queueId, cid, false, null, 0L, null, null);
     }
 
     @Override
     public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingCommand request)
         throws RemotingCommandException {
+
         final long beginTimeMills = this.brokerController.getMessageStore().now();
+
+        // fill bron time to properties if not exist, why we need this?
         request.addExtFieldIfNotExist(BORN_TIME, String.valueOf(System.currentTimeMillis()));
         if (Objects.equals(request.getExtFields().get(BORN_TIME), "0")) {
             request.addExtField(BORN_TIME, String.valueOf(System.currentTimeMillis()));
         }
-        Channel channel = ctx.channel();
 
+        Channel channel = ctx.channel();
         RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);
-        final PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();
-        final PopMessageRequestHeader requestHeader =
-            (PopMessageRequestHeader) request.decodeCommandCustomHeader(PopMessageRequestHeader.class, true);
-        StringBuilder startOffsetInfo = new StringBuilder(64);
-        StringBuilder msgOffsetInfo = new StringBuilder(64);
-        StringBuilder orderCountInfo = null;
-        if (requestHeader.isOrder()) {
-            orderCountInfo = new StringBuilder(64);
-        }
+        response.setOpaque(request.getOpaque());
 
-        brokerController.getConsumerManager().compensateBasicConsumerInfo(requestHeader.getConsumerGroup(),
-            ConsumeType.CONSUME_POP, MessageModel.CLUSTERING);
+        final PopMessageRequestHeader requestHeader =
+            request.decodeCommandCustomHeader(PopMessageRequestHeader.class, true);
+        final PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();
 
-        response.setOpaque(request.getOpaque());
+        // Pop mode only supports consumption in cluster load balancing mode
+        brokerController.getConsumerManager().compensateBasicConsumerInfo(
+            requestHeader.getConsumerGroup(), ConsumeType.CONSUME_POP, MessageModel.CLUSTERING);
 
         if (brokerController.getBrokerConfig().isEnablePopLog()) {
             POP_LOGGER.info("receive PopMessage request command, {}", request);
@@ -229,14 +250,16 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
                 this.brokerController.getBrokerConfig().getBrokerIP1()));
             return response;
         }
+
         if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {
             response.setCode(ResponseCode.NO_PERMISSION);
             response.setRemark(String.format("the broker[%s] pop message is forbidden",
                 this.brokerController.getBrokerConfig().getBrokerIP1()));
             return response;
         }
+
         if (requestHeader.getMaxMsgNums() > 32) {
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.INVALID_PARAMETER);
             response.setRemark(String.format("the broker[%s] pop message's num is greater than 32",
                 this.brokerController.getBrokerConfig().getBrokerIP1()));
             return response;
@@ -272,10 +295,11 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
                 requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(),
                 channel.remoteAddress());
             POP_LOGGER.warn(errorInfo);
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.INVALID_PARAMETER);
             response.setRemark(errorInfo);
             return response;
         }
+
         SubscriptionGroupConfig subscriptionGroupConfig =
             this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());
         if (null == subscriptionGroupConfig) {
@@ -292,24 +316,29 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
         }
 
         BrokerConfig brokerConfig = brokerController.getBrokerConfig();
+        SubscriptionData subscriptionData = null;
         ExpressionMessageFilter messageFilter = null;
-        if (requestHeader.getExp() != null && requestHeader.getExp().length() > 0) {
+        if (requestHeader.getExp() != null && !requestHeader.getExp().isEmpty()) {
             try {
-                SubscriptionData subscriptionData = FilterAPI.build(requestHeader.getTopic(), requestHeader.getExp(), requestHeader.getExpType());
-                brokerController.getConsumerManager().compensateSubscribeData(requestHeader.getConsumerGroup(),
-                    requestHeader.getTopic(), subscriptionData);
-
-                String retryTopic = KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerConfig.isEnableRetryTopicV2());
-                SubscriptionData retrySubscriptionData = FilterAPI.build(retryTopic, SubscriptionData.SUB_ALL, requestHeader.getExpType());
-                brokerController.getConsumerManager().compensateSubscribeData(requestHeader.getConsumerGroup(),
-                    retryTopic, retrySubscriptionData);
+                // origin topic
+                subscriptionData = FilterAPI.build(
+                    requestHeader.getTopic(), requestHeader.getExp(), requestHeader.getExpType());
+                brokerController.getConsumerManager().compensateSubscribeData(
+                    requestHeader.getConsumerGroup(), requestHeader.getTopic(), subscriptionData);
+
+                // retry topic
+                String retryTopic = KeyBuilder.buildPopRetryTopic(
+                    requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerConfig.isEnableRetryTopicV2());
+                SubscriptionData retrySubscriptionData = FilterAPI.build(
+                    retryTopic, SubscriptionData.SUB_ALL, requestHeader.getExpType());
+                brokerController.getConsumerManager().compensateSubscribeData(
+                    requestHeader.getConsumerGroup(), retryTopic, retrySubscriptionData);
 
                 ConsumerFilterData consumerFilterData = null;
                 if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {
                     consumerFilterData = ConsumerFilterManager.build(
                         requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getExp(),
-                        requestHeader.getExpType(), System.currentTimeMillis()
-                    );
+                        requestHeader.getExpType(), System.currentTimeMillis());
                     if (consumerFilterData == null) {
                         POP_LOGGER.warn("Parse the consumer's subscription[{}] failed, group: {}",
                             requestHeader.getExp(), requestHeader.getConsumerGroup());
@@ -318,8 +347,8 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
                         return response;
                     }
                 }
-                messageFilter = new ExpressionMessageFilter(subscriptionData, consumerFilterData,
-                    brokerController.getConsumerFilterManager());
+                messageFilter = new ExpressionMessageFilter(
+                    subscriptionData, consumerFilterData, brokerController.getConsumerFilterManager());
             } catch (Exception e) {
                 POP_LOGGER.warn("Parse the consumer's subscription[{}] error, group: {}", requestHeader.getExp(),
                     requestHeader.getConsumerGroup());
@@ -329,30 +358,140 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
             }
         } else {
             try {
-                SubscriptionData subscriptionData = FilterAPI.build(requestHeader.getTopic(), "*", ExpressionType.TAG);
-                brokerController.getConsumerManager().compensateSubscribeData(requestHeader.getConsumerGroup(),
-                    requestHeader.getTopic(), subscriptionData);
-
-                String retryTopic = KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerConfig.isEnableRetryTopicV2());
+                // origin topic
+                subscriptionData = FilterAPI.build(requestHeader.getTopic(), "*", ExpressionType.TAG);
+                brokerController.getConsumerManager().compensateSubscribeData(
+                    requestHeader.getConsumerGroup(), requestHeader.getTopic(), subscriptionData);
+
+                // retry topic
+                String retryTopic = KeyBuilder.buildPopRetryTopic(
+                    requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerConfig.isEnableRetryTopicV2());
                 SubscriptionData retrySubscriptionData = FilterAPI.build(retryTopic, "*", ExpressionType.TAG);
-                brokerController.getConsumerManager().compensateSubscribeData(requestHeader.getConsumerGroup(),
-                    retryTopic, retrySubscriptionData);
+                brokerController.getConsumerManager().compensateSubscribeData(
+                    requestHeader.getConsumerGroup(), retryTopic, retrySubscriptionData);
             } catch (Exception e) {
                 POP_LOGGER.warn("Build default subscription error, group: {}", requestHeader.getConsumerGroup());
             }
         }
 
+        GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums());
+        ExpressionMessageFilter finalMessageFilter = messageFilter;
+        SubscriptionData finalSubscriptionData = subscriptionData;
+
+        if (brokerConfig.isPopConsumerKVServiceEnable()) {
+
+            CompletableFuture<PopConsumerContext> popAsyncFuture = brokerController.getPopConsumerService().popAsync(
+                RemotingHelper.parseChannelRemoteAddr(channel), beginTimeMills, requestHeader.getInvisibleTime(),
+                requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(),
+                requestHeader.getMaxMsgNums(), requestHeader.isOrder(),
+                requestHeader.getAttemptId(), requestHeader.getInitMode(), messageFilter);
+
+            popAsyncFuture.thenApply(result -> {
+                if (result.isFound()) {
+                    response.setCode(ResponseCode.SUCCESS);
+                    getMessageResult.setStatus(GetMessageStatus.FOUND);
+                    // recursive processing
+                    if (result.getRestCount() > 0) {
+                        popLongPollingService.notifyMessageArriving(
+                            requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getConsumerGroup(),
+                            null, 0L, null, null);
+                    }
+                } else {
+                    POP_LOGGER.debug("Processor not found, polling request, popTime={}, restCount={}",
+                        result.getPopTime(), result.getRestCount());
+
+                    PollingResult pollingResult = popLongPollingService.polling(
+                        ctx, request, new PollingHeader(requestHeader), finalSubscriptionData, finalMessageFilter);
+
+                    if (PollingResult.POLLING_SUC == pollingResult) {
+                        // recursive processing
+                        if (result.getRestCount() > 0) {
+                            popLongPollingService.notifyMessageArriving(
+                                requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getConsumerGroup(),
+                                null, 0L, null, null);
+                        }
+                        return null;
+                    } else if (PollingResult.POLLING_FULL == pollingResult) {
+                        response.setCode(ResponseCode.POLLING_FULL);
+                    } else {
+                        response.setCode(ResponseCode.POLLING_TIMEOUT);
+                    }
+                    getMessageResult.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);
+                }
+
+                responseHeader.setPopTime(result.getPopTime());
+                responseHeader.setInvisibleTime(result.getInvisibleTime());
+                responseHeader.setReviveQid(
+                    requestHeader.isOrder() ? KeyBuilder.POP_ORDER_REVIVE_QUEUE : 0);
+                responseHeader.setRestNum(result.getRestCount());
+                responseHeader.setStartOffsetInfo(result.getStartOffsetInfo());
+                responseHeader.setMsgOffsetInfo(result.getMsgOffsetInfo());
+                if (requestHeader.isOrder() && !result.getOrderCountInfo().isEmpty()) {
+                    responseHeader.setOrderCountInfo(result.getOrderCountInfo());
+                }
+
+                response.setRemark(getMessageResult.getStatus().name());
+                if (response.getCode() != ResponseCode.SUCCESS) {
+                    return response;
+                }
+
+                // add message
+                result.getGetMessageResultList().forEach(temp -> {
+                    for (int i = 0; i < temp.getMessageMapedList().size(); i++) {
+                        getMessageResult.addMessage(temp.getMessageMapedList().get(i));
+                    }
+                });
+
+                if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) {
+                    final byte[] r = this.readGetMessageResult(getMessageResult,
+                        requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId());
+                    this.brokerController.getBrokerStatsManager().incGroupGetLatency(
+                        requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(),
+                        (int) (this.brokerController.getMessageStore().now() - beginTimeMills));
+                    response.setBody(r);
+                } else {
+                    final GetMessageResult tmpGetMessageResult = getMessageResult;
+                    try {
+                        FileRegion fileRegion = new ManyMessageTransfer(
+                            response.encodeHeader(getMessageResult.getBufferTotalSize()), getMessageResult);
+                        channel.writeAndFlush(fileRegion)
+                            .addListener((ChannelFutureListener) future -> {
+                                tmpGetMessageResult.release();
+                                Attributes attributes = RemotingMetricsManager.newAttributesBuilder()
+                                    .put(LABEL_REQUEST_CODE, RemotingHelper.getRequestCodeDesc(request.getCode()))
+                                    .put(LABEL_RESPONSE_CODE, RemotingHelper.getResponseCodeDesc(response.getCode()))
+                                    .put(LABEL_RESULT, RemotingMetricsManager.getWriteAndFlushResult(future))
+                                    .build();
+                                RemotingMetricsManager.rpcLatency.record(
+                                    request.getProcessTimer().elapsed(TimeUnit.MILLISECONDS), attributes);
+                                if (!future.isSuccess()) {
+                                    POP_LOGGER.error("Fail to transfer messages from page cache to {}",
+                                        channel.remoteAddress(), future.cause());
+                                }
+                            });
+                    } catch (Throwable e) {
+                        POP_LOGGER.error("Error occurred when transferring messages from page cache", e);
+                        getMessageResult.release();
+                    }
+                    return null;
+                }
+                return response;
+            }).thenAccept(result -> NettyRemotingAbstract.writeResponse(channel, request, result));
+            return null;
+        }
+
         int randomQ = random.nextInt(100);
         int reviveQid;
         if (requestHeader.isOrder()) {
             reviveQid = KeyBuilder.POP_ORDER_REVIVE_QUEUE;
         } else {
-            reviveQid = (int) Math.abs(ckMessageNumber.getAndIncrement() % this.brokerController.getBrokerConfig().getReviveQueueNum());
+            reviveQid = (int) Math.abs(ckMessageNumber.getAndIncrement() %
+                this.brokerController.getBrokerConfig().getReviveQueueNum());
         }
 
-        GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums());
-        ExpressionMessageFilter finalMessageFilter = messageFilter;
-        StringBuilder finalOrderCountInfo = orderCountInfo;
+        StringBuilder startOffsetInfo = new StringBuilder(64);
+        StringBuilder msgOffsetInfo = new StringBuilder(64);
+        StringBuilder orderCountInfo = requestHeader.isOrder() ? new StringBuilder(64) : null;
 
         // Due to the design of the fields startOffsetInfo, msgOffsetInfo, and orderCountInfo,
         // a single POP request could only invoke the popMsgFromQueue method once
@@ -360,7 +499,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
         // considered the same type because they share the same retry flag in previous fields.
         // Therefore, needRetryV1 is designed as a subset of needRetry, and within a single request,
         // only one type of retry topic is able to call popMsgFromQueue.
-        boolean needRetry = randomQ % 5 == 0;
+        boolean needRetry = randomQ < brokerConfig.getPopFromRetryProbability();
         boolean needRetryV1 = false;
         if (brokerConfig.isEnableRetryTopicV2() && brokerConfig.isRetrieveMessageFromPopRetryTopicV1()) {
             needRetryV1 = randomQ % 2 == 0;
@@ -387,7 +526,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
             getMessageFuture = getMessageFuture.thenCompose(restNum ->
                 popMsgFromQueue(topicConfig.getTopicName(), requestHeader.getAttemptId(), false,
                     getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime, finalMessageFilter,
-                    startOffsetInfo, msgOffsetInfo, finalOrderCountInfo));
+                    startOffsetInfo, msgOffsetInfo, orderCountInfo));
         }
         // if not full , fetch retry again
         if (!needRetry && getMessageResult.getMessageMapedList().size() < requestHeader.getMaxMsgNums() && !requestHeader.isOrder()) {
@@ -404,17 +543,33 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
 
         final RemotingCommand finalResponse = response;
         getMessageFuture.thenApply(restNum -> {
+            try {
+                if (request.getCallbackList() != null) {
+                    request.getCallbackList().forEach(CommandCallback::accept);
+                    request.getCallbackList().clear();
+                }
+            } catch (Throwable t) {
+                POP_LOGGER.error("PopProcessor execute callback error", t);
+            }
+
             if (!getMessageResult.getMessageBufferList().isEmpty()) {
                 finalResponse.setCode(ResponseCode.SUCCESS);
                 getMessageResult.setStatus(GetMessageStatus.FOUND);
                 if (restNum > 0) {
                     // all queue pop can not notify specified queue pop, and vice versa
-                    popLongPollingService.notifyMessageArriving(requestHeader.getTopic(), requestHeader.getConsumerGroup(),
-                        requestHeader.getQueueId());
+                    popLongPollingService.notifyMessageArriving(
+                        requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getConsumerGroup(),
+                        null, 0L, null, null);
                 }
             } else {
-                PollingResult pollingResult = popLongPollingService.polling(ctx, request, new PollingHeader(requestHeader));
+                PollingResult pollingResult = popLongPollingService.polling(
+                    ctx, request, new PollingHeader(requestHeader), finalSubscriptionData, finalMessageFilter);
                 if (PollingResult.POLLING_SUC == pollingResult) {
+                    if (restNum > 0) {
+                        popLongPollingService.notifyMessageArriving(
+                            requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getConsumerGroup(),
+                            null, 0L, null, null);
+                    }
                     return null;
                 } else if (PollingResult.POLLING_FULL == pollingResult) {
                     finalResponse.setCode(ResponseCode.POLLING_FULL);
@@ -429,8 +584,8 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
             responseHeader.setRestNum(restNum);
             responseHeader.setStartOffsetInfo(startOffsetInfo.toString());
             responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString());
-            if (requestHeader.isOrder() && finalOrderCountInfo != null) {
-                responseHeader.setOrderCountInfo(finalOrderCountInfo.toString());
+            if (requestHeader.isOrder() && orderCountInfo != null) {
+                responseHeader.setOrderCountInfo(orderCountInfo.toString());
             }
             finalResponse.setRemark(getMessageResult.getStatus().name());
             switch (finalResponse.getCode()) {
@@ -503,45 +658,72 @@ private CompletableFuture<Long> popMsgFromTopic(String topic, boolean isRetry, G
             messageFilter, startOffsetInfo, msgOffsetInfo, orderCountInfo, randomQ, getMessageFuture);
     }
 
-    private CompletableFuture<Long> popMsgFromQueue(String topic, String attemptId, boolean isRetry, GetMessageResult getMessageResult,
+    private CompletableFuture<Long> popMsgFromQueue(String topic, String attemptId, boolean isRetry,
+        GetMessageResult getMessageResult,
         PopMessageRequestHeader requestHeader, int queueId, long restNum, int reviveQid,
         Channel channel, long popTime, ExpressionMessageFilter messageFilter, StringBuilder startOffsetInfo,
         StringBuilder msgOffsetInfo, StringBuilder orderCountInfo) {
+
         String lockKey =
             topic + PopAckConstants.SPLIT + requestHeader.getConsumerGroup() + PopAckConstants.SPLIT + queueId;
         boolean isOrder = requestHeader.isOrder();
-        long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(),
-            false, lockKey, false);
+        long offset;
+        try {
+            offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(),
+                false, lockKey, false);
+        } catch (ConsumeQueueException e) {
+            CompletableFuture<Long> failure = new CompletableFuture<>();
+            failure.completeExceptionally(e);
+            return failure;
+        }
+
         CompletableFuture<Long> future = new CompletableFuture<>();
         if (!queueLockManager.tryLock(lockKey)) {
-            restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
-            future.complete(restNum);
+            try {
+                if (!requestHeader.isOrder()) {
+                    restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+                }
+                future.complete(restNum);
+            } catch (ConsumeQueueException e) {
+                future.completeExceptionally(e);
+            }
             return future;
         }
 
+        future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey));
         if (isPopShouldStop(topic, requestHeader.getConsumerGroup(), queueId)) {
-            POP_LOGGER.warn("Too much msgs unacked, then stop poping. topic={}, group={}, queueId={}", topic, requestHeader.getConsumerGroup(), queueId);
-            restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
-            future.complete(restNum);
+            POP_LOGGER.warn("Too much msgs unacked, then stop popping. topic={}, group={}, queueId={}",
+                topic, requestHeader.getConsumerGroup(), queueId);
+            try {
+                restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+                future.complete(restNum);
+            } catch (ConsumeQueueException e) {
+                future.completeExceptionally(e);
+            }
             return future;
         }
 
         try {
-            future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey));
             offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(),
                 true, lockKey, true);
-            if (isOrder && brokerController.getConsumerOrderInfoManager().checkBlock(attemptId, topic,
-                requestHeader.getConsumerGroup(), queueId, requestHeader.getInvisibleTime())) {
-                future.complete(this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum);
-                return future;
-            }
 
+            // Current requests would calculate the total number of messages
+            // waiting to be filtered for new message arrival notifications in
+            // the long-polling service, need disregarding the backlog in order
+            // consumption scenario. If rest message num including the blocked
+            // queue accumulation would lead to frequent unnecessary wake-ups
+            // of long-polling requests, resulting unnecessary CPU usage.
+            // When client ack message, long-polling request would be notifications
+            // by AckMessageProcessor.ackOrderly() and message will not be delayed.
             if (isOrder) {
+                if (brokerController.getConsumerOrderInfoManager().checkBlock(
+                    attemptId, topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInvisibleTime())) {
+                    // should not add accumulation(max offset - consumer offset) here
+                    future.complete(restNum);
+                    return future;
+                }
                 this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNum(
-                    topic,
-                    requestHeader.getConsumerGroup(),
-                    queueId
-                );
+                    topic, requestHeader.getConsumerGroup(), queueId);
             }
 
             if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) {
@@ -583,7 +765,11 @@ private CompletableFuture<Long> popMsgFromQueue(String topic, String attemptId,
                 return CompletableFuture.completedFuture(result);
             }).thenApply(result -> {
                 if (result == null) {
-                    atomicRestNum.set(brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - atomicOffset.get() + atomicRestNum.get());
+                    try {
+                        atomicRestNum.set(brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - atomicOffset.get() + atomicRestNum.get());
+                    } catch (ConsumeQueueException e) {
+                        POP_LOGGER.error("Failed to get max offset in queue", e);
+                    }
                     return atomicRestNum.get();
                 }
                 if (!result.getMessageMapedList().isEmpty()) {
@@ -622,10 +808,13 @@ private CompletableFuture<Long> popMsgFromQueue(String topic, String attemptId,
                     || GetMessageStatus.MESSAGE_WAS_REMOVING.equals(result.getStatus())
                     || GetMessageStatus.NO_MATCHED_LOGIC_QUEUE.equals(result.getStatus()))
                     && result.getNextBeginOffset() > -1) {
-                    popBufferMergeService.addCkMock(requestHeader.getConsumerGroup(), topic, queueId, finalOffset,
-                        requestHeader.getInvisibleTime(), popTime, reviveQid, result.getNextBeginOffset(), brokerController.getBrokerConfig().getBrokerName());
-//                this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), requestHeader.getConsumerGroup(), topic,
-//                        queueId, getMessageTmpResult.getNextBeginOffset());
+                    if (isOrder) {
+                        this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), requestHeader.getConsumerGroup(), topic,
+                            queueId, result.getNextBeginOffset());
+                    } else {
+                        popBufferMergeService.addCkMock(requestHeader.getConsumerGroup(), topic, queueId, finalOffset,
+                            requestHeader.getInvisibleTime(), popTime, reviveQid, result.getNextBeginOffset(), brokerController.getBrokerConfig().getBrokerName());
+                    }
                 }
 
                 atomicRestNum.set(result.getMaxOffset() - result.getNextBeginOffset() + atomicRestNum.get());
@@ -676,11 +865,11 @@ private CompletableFuture<Long> popMsgFromQueue(String topic, String attemptId,
 
     private boolean isPopShouldStop(String topic, String group, int queueId) {
         return brokerController.getBrokerConfig().isEnablePopMessageThreshold() &&
-                brokerController.getPopInflightMessageCounter().getGroupPopInFlightMessageNum(topic, group, queueId) > brokerController.getBrokerConfig().getPopInflightMessageThreshold();
+            brokerController.getPopInflightMessageCounter().getGroupPopInFlightMessageNum(topic, group, queueId) > brokerController.getBrokerConfig().getPopInflightMessageThreshold();
     }
 
     private long getPopOffset(String topic, String group, int queueId, int initMode, boolean init, String lockKey,
-        boolean checkResetOffset) {
+        boolean checkResetOffset) throws ConsumeQueueException {
 
         long offset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId);
         if (offset < 0) {
@@ -702,10 +891,11 @@ private long getPopOffset(String topic, String group, int queueId, int initMode,
         }
     }
 
-    private long getInitOffset(String topic, String group, int queueId, int initMode, boolean init) {
+    public long getInitOffset(String topic, String group, int queueId, int initMode, boolean init)
+        throws ConsumeQueueException {
         long offset;
-        if (ConsumeInitMode.MIN == initMode) {
-            return this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
+        if (ConsumeInitMode.MIN == initMode || topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
+            offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
         } else {
             if (this.brokerController.getBrokerConfig().isInitPopOffsetByCheckMsgInMem() &&
                 this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId) <= 0 &&
@@ -719,19 +909,19 @@ private long getInitOffset(String topic, String group, int queueId, int initMode
                     offset = 0;
                 }
             }
-            if (init) {
-                this.brokerController.getConsumerOffsetManager().commitOffset(
-                    "getPopOffset", group, topic, queueId, offset);
-            }
+        }
+        if (init) { // whichever initMode
+            this.brokerController.getConsumerOffsetManager().commitOffset(
+                "getPopOffset", group, topic, queueId, offset);
         }
         return offset;
     }
 
-    public final MessageExtBrokerInner buildCkMsg(final PopCheckPoint ck, final int reviveQid) {
+    public MessageExtBrokerInner buildCkMsg(final PopCheckPoint ck, final int reviveQid) {
         MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
 
         msgInner.setTopic(reviveTopic);
-        msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8));
+        msgInner.setBody(JSON.toJSONString(ck).getBytes(StandardCharsets.UTF_8));
         msgInner.setQueueId(reviveQid);
         msgInner.setTags(PopAckConstants.CK_TAG);
         msgInner.setBornTimestamp(System.currentTimeMillis());
@@ -762,6 +952,9 @@ private boolean appendCheckPoint(final PopMessageRequestHeader requestHeader,
             ck.addDiff((int) (msgQueueOffset - offset));
         }
 
+        this.brokerController.getBrokerStatsManager().incBrokerCkNums(1);
+        this.brokerController.getBrokerStatsManager().incGroupCkNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);
+
         final boolean addBufferSuc = this.popBufferMergeService.addCk(
             ck, reviveQid, -1, getMessageTmpResult.getNextBeginOffset()
         );
@@ -813,12 +1006,13 @@ static class TimedLock {
         private volatile long lockTime;
 
         public TimedLock() {
-            this.lock = new AtomicBoolean(true);
+            // init lock status, false means not locked
+            this.lock = new AtomicBoolean(false);
             this.lockTime = System.currentTimeMillis();
         }
 
         public boolean tryLock() {
-            boolean ret = lock.compareAndSet(true, false);
+            boolean ret = lock.compareAndSet(false, true);
             if (ret) {
                 this.lockTime = System.currentTimeMillis();
                 return true;
@@ -828,11 +1022,11 @@ public boolean tryLock() {
         }
 
         public void unLock() {
-            lock.set(true);
+            lock.set(false);
         }
 
         public boolean isLock() {
-            return !lock.get();
+            return lock.get();
         }
 
         public long getLockTime() {
@@ -852,21 +1046,7 @@ public boolean tryLock(String topic, String consumerGroup, int queueId) {
         }
 
         public boolean tryLock(String key) {
-            TimedLock timedLock = expiredLocalCache.get(key);
-
-            if (timedLock == null) {
-                TimedLock old = expiredLocalCache.putIfAbsent(key, new TimedLock());
-                if (old != null) {
-                    return false;
-                } else {
-                    timedLock = expiredLocalCache.get(key);
-                }
-            }
-
-            if (timedLock == null) {
-                return false;
-            }
-
+            TimedLock timedLock = ConcurrentHashMapUtils.computeIfAbsent(expiredLocalCache, key, k -> new TimedLock());
             return timedLock.tryLock();
         }
 
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
index 104b78d4413..e1ead86169b 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
@@ -19,6 +19,7 @@
 import com.alibaba.fastjson.JSON;
 import io.opentelemetry.api.common.Attributes;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -27,6 +28,8 @@
 import java.util.NavigableMap;
 import java.util.TreeMap;
 import java.util.concurrent.CompletableFuture;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Triple;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.metrics.BrokerMetricsManager;
 import org.apache.rocketmq.broker.metrics.PopMetricsManager;
@@ -34,6 +37,7 @@
 import org.apache.rocketmq.client.consumer.PullStatus;
 import org.apache.rocketmq.common.KeyBuilder;
 import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.Pair;
 import org.apache.rocketmq.common.PopAckConstants;
 import org.apache.rocketmq.common.ServiceThread;
@@ -51,8 +55,8 @@
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.store.AppendMessageStatus;
 import org.apache.rocketmq.store.GetMessageResult;
-import org.apache.rocketmq.store.GetMessageStatus;
 import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.pop.AckMsg;
 import org.apache.rocketmq.store.pop.BatchAckMsg;
 import org.apache.rocketmq.store.pop.PopCheckPoint;
@@ -63,6 +67,7 @@
 
 public class PopReviveService extends ServiceThread {
     private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+    private final int[] ckRewriteIntervalsInSeconds = new int[] { 10, 20, 30, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600, 1200, 1800, 3600, 7200 };
 
     private int queueId;
     private BrokerController brokerController;
@@ -125,7 +130,7 @@ private boolean reviveRetry(PopCheckPoint popCheckPoint, MessageExt messageExt)
             msgInner.getProperties().put(MessageConst.PROPERTY_FIRST_POP_TIME, String.valueOf(popCheckPoint.getPopTime()));
         }
         msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
-        addRetryTopicIfNoExit(msgInner.getTopic(), popCheckPoint.getCId());
+        addRetryTopicIfNotExist(msgInner.getTopic(), popCheckPoint.getCId());
         PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
         PopMetricsManager.incPopReviveRetryMessageCount(popCheckPoint, putMessageResult.getPutMessageStatus());
         if (brokerController.getBrokerConfig().isEnablePopLog()) {
@@ -153,7 +158,7 @@ private void initPopRetryOffset(String topic, String consumerGroup) {
         }
     }
 
-    private void addRetryTopicIfNoExit(String topic, String consumerGroup) {
+    public void addRetryTopicIfNotExist(String topic, String consumerGroup) {
         if (brokerController != null) {
             TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic);
             if (topicConfig != null) {
@@ -196,9 +201,9 @@ private boolean reachTail(PullResult pullResult, long offset) {
             || pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL && offset == pullResult.getMaxOffset();
     }
 
-    private CompletableFuture<Pair<GetMessageStatus, MessageExt>> getBizMessage(String topic, long offset, int queueId,
-        String brokerName) {
-        return this.brokerController.getEscapeBridge().getMessageAsync(topic, offset, queueId, brokerName, false);
+    // Triple<MessageExt, info, needRetry>
+    public CompletableFuture<Triple<MessageExt, String, Boolean>> getBizMessage(PopCheckPoint popCheckPoint, long offset) {
+        return this.brokerController.getEscapeBridge().getMessageAsync(popCheckPoint.getTopic(), offset, popCheckPoint.getQueueId(), popCheckPoint.getBrokerName(), false);
     }
 
     public PullResult getMessage(String group, String topic, int queueId, long offset, int nums,
@@ -258,10 +263,14 @@ public PullResult getMessage(String group, String topic, int queueId, long offse
                 getMessageResult.getMaxOffset(), foundList);
 
         } else {
-            long maxQueueOffset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
-            if (maxQueueOffset > offset) {
-                POP_LOGGER.error("get message from store return null. topic={}, groupId={}, requestOffset={}, maxQueueOffset={}",
-                    topic, group, offset, maxQueueOffset);
+            try {
+                long maxQueueOffset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
+                if (maxQueueOffset > offset) {
+                    POP_LOGGER.error("get message from store return null. topic={}, groupId={}, requestOffset={}, maxQueueOffset={}",
+                        topic, group, offset, maxQueueOffset);
+                }
+            } catch (ConsumeQueueException e) {
+                POP_LOGGER.error("Failed to get max offset in queue", e);
             }
             return null;
         }
@@ -322,7 +331,7 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) {
                 if (endTime != 0 && System.currentTimeMillis() - endTime > 3 * PopAckConstants.SECOND && timerDelay <= 0 && commitLogDelay <= 0) {
                     endTime = System.currentTimeMillis();
                 }
-                POP_LOGGER.info("reviveQueueId={}, offset is {}, can not get new msg, old endTime {}, new endTime {}, timerDelay={}, commitLogDelay={} ",
+                POP_LOGGER.debug("reviveQueueId={}, offset is {}, can not get new msg, old endTime {}, new endTime {}, timerDelay={}, commitLogDelay={} ",
                     queueId, offset, old, endTime, timerDelay, commitLogDelay);
                 if (endTime - firstRt > PopAckConstants.ackTimeInterval + PopAckConstants.SECOND) {
                     break;
@@ -355,20 +364,22 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) {
                     if (point.getTopic() == null || point.getCId() == null) {
                         continue;
                     }
-                    map.put(point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime(), point);
+                    map.put(point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime() + point.getBrokerName(), point);
                     PopMetricsManager.incPopReviveCkGetCount(point, queueId);
                     point.setReviveOffset(messageExt.getQueueOffset());
                     if (firstRt == 0) {
                         firstRt = point.getReviveTime();
                     }
                 } else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) {
-                    String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8);
+                    String raw = new String(messageExt.getBody(), StandardCharsets.UTF_8);
                     if (brokerController.getBrokerConfig().isEnablePopLog()) {
                         POP_LOGGER.info("reviveQueueId={}, find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);
                     }
                     AckMsg ackMsg = JSON.parseObject(raw, AckMsg.class);
                     PopMetricsManager.incPopReviveAckGetCount(ackMsg, queueId);
-                    String mergeKey = ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime();
+                    String brokerName = StringUtils.isNotBlank(ackMsg.getBrokerName()) ?
+                        ackMsg.getBrokerName() : brokerController.getBrokerConfig().getBrokerName();
+                    String mergeKey = ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime() + brokerName;
                     PopCheckPoint point = map.get(mergeKey);
                     if (point == null) {
                         if (!brokerController.getBrokerConfig().isEnableSkipLongAwaitingAck()) {
@@ -386,14 +397,16 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) {
                         }
                     }
                 } else if (PopAckConstants.BATCH_ACK_TAG.equals(messageExt.getTags())) {
-                    String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8);
+                    String raw = new String(messageExt.getBody(), StandardCharsets.UTF_8);
                     if (brokerController.getBrokerConfig().isEnablePopLog()) {
                         POP_LOGGER.info("reviveQueueId={}, find batch ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);
                     }
 
                     BatchAckMsg bAckMsg = JSON.parseObject(raw, BatchAckMsg.class);
                     PopMetricsManager.incPopReviveAckGetCount(bAckMsg, queueId);
-                    String mergeKey = bAckMsg.getTopic() + bAckMsg.getConsumerGroup() + bAckMsg.getQueueId() + bAckMsg.getStartOffset() + bAckMsg.getPopTime();
+                    String brokerName = StringUtils.isNotBlank(bAckMsg.getBrokerName()) ?
+                        bAckMsg.getBrokerName() : brokerController.getBrokerConfig().getBrokerName();
+                    String mergeKey = bAckMsg.getTopic() + bAckMsg.getConsumerGroup() + bAckMsg.getQueueId() + bAckMsg.getStartOffset() + bAckMsg.getPopTime() + brokerName;
                     PopCheckPoint point = map.get(mergeKey);
                     if (point == null) {
                         if (!brokerController.getBrokerConfig().isEnableSkipLongAwaitingAck()) {
@@ -491,6 +504,8 @@ protected void mergeAndRevive(ConsumeReviveObj consumeReviveObj) throws Throwabl
                     PopCheckPoint oldCK = inflightReviveRequestMap.firstKey();
                     rePutCK(oldCK, pair);
                     inflightReviveRequestMap.remove(oldCK);
+                    POP_LOGGER.warn("stay too long, remove from reviveRequestMap, {}, {}, {}, {}", popCheckPoint.getTopic(),
+                            popCheckPoint.getBrokerName(), popCheckPoint.getQueueId(), popCheckPoint.getStartOffset());
                 }
             }
 
@@ -523,28 +538,13 @@ private void reviveMsgFromCk(PopCheckPoint popCheckPoint) {
 
             // retry msg
             long msgOffset = popCheckPoint.ackOffsetByIndex((byte) j);
-            CompletableFuture<Pair<Long, Boolean>> future = getBizMessage(popCheckPoint.getTopic(), msgOffset, popCheckPoint.getQueueId(), popCheckPoint.getBrokerName())
-                .thenApply(resultPair -> {
-                    GetMessageStatus getMessageStatus = resultPair.getObject1();
-                    MessageExt message = resultPair.getObject2();
+            CompletableFuture<Pair<Long, Boolean>> future = getBizMessage(popCheckPoint, msgOffset)
+                .thenApply(rst -> {
+                    MessageExt message = rst.getLeft();
                     if (message == null) {
-                        POP_LOGGER.warn("reviveQueueId={}, can not get biz msg topic is {}, offset is {}, then continue",
-                            queueId, popCheckPoint.getTopic(), msgOffset);
-                        switch (getMessageStatus) {
-                            case MESSAGE_WAS_REMOVING:
-                            case OFFSET_TOO_SMALL:
-                            case NO_MATCHED_LOGIC_QUEUE:
-                            case NO_MESSAGE_IN_QUEUE:
-                                return new Pair<>(msgOffset, true);
-                            default:
-                                return new Pair<>(msgOffset, false);
-
-                        }
-                    }
-                    //skip ck from last epoch
-                    if (popCheckPoint.getPopTime() < message.getStoreTimestamp()) {
-                        POP_LOGGER.warn("reviveQueueId={}, skip ck from last epoch {}", queueId, popCheckPoint);
-                        return new Pair<>(msgOffset, true);
+                        POP_LOGGER.info("reviveQueueId={}, can not get biz msg, topic:{}, qid:{}, offset:{}, brokerName:{}, info:{}, retry:{}, then continue",
+                            queueId, popCheckPoint.getTopic(), popCheckPoint.getQueueId(), msgOffset, popCheckPoint.getBrokerName(), UtilAll.frontStringAtLeast(rst.getMiddle(), 60), rst.getRight());
+                        return new Pair<>(msgOffset, !rst.getRight()); // Pair.object2 means OK or not, Triple.right value means needRetry
                     }
                     boolean result = reviveRetry(popCheckPoint, message);
                     return new Pair<>(msgOffset, result);
@@ -577,6 +577,13 @@ private void reviveMsgFromCk(PopCheckPoint popCheckPoint) {
     }
 
     private void rePutCK(PopCheckPoint oldCK, Pair<Long, Boolean> pair) {
+        int rePutTimes = oldCK.parseRePutTimes();
+        if (rePutTimes >= ckRewriteIntervalsInSeconds.length && brokerController.getBrokerConfig().isSkipWhenCKRePutReachMaxTimes()) {
+            POP_LOGGER.warn("rePut CK reach max times, drop it. {}, {}, {}, {}-{}, {}, {}, {}", oldCK.getTopic(), oldCK.getCId(),
+                    oldCK.getBrokerName(), oldCK.getQueueId(), pair.getObject1(), oldCK.getPopTime(), oldCK.getInvisibleTime(), rePutTimes);
+            return;
+        }
+
         PopCheckPoint newCk = new PopCheckPoint();
         newCk.setBitMap(0);
         newCk.setNum((byte) 1);
@@ -588,11 +595,17 @@ private void rePutCK(PopCheckPoint oldCK, Pair<Long, Boolean> pair) {
         newCk.setQueueId(oldCK.getQueueId());
         newCk.setBrokerName(oldCK.getBrokerName());
         newCk.addDiff(0);
+        newCk.setRePutTimes(String.valueOf(rePutTimes + 1)); // always increment even if removed from reviveRequestMap
+        if (oldCK.getReviveTime() <= System.currentTimeMillis()) {
+            // never expect an ACK matched in the future, we just use it to rewrite CK and try to revive retry message next time
+            int intervalIndex = rePutTimes >= ckRewriteIntervalsInSeconds.length ? ckRewriteIntervalsInSeconds.length - 1 : rePutTimes;
+            newCk.setInvisibleTime(oldCK.getInvisibleTime() + ckRewriteIntervalsInSeconds[intervalIndex] * 1000);
+        }
         MessageExtBrokerInner ckMsg = brokerController.getPopMessageProcessor().buildCkMsg(newCk, queueId);
         brokerController.getMessageStore().putMessage(ckMsg);
     }
 
-    public long getReviveBehindMillis() {
+    public long getReviveBehindMillis() throws ConsumeQueueException {
         if (currentReviveMessageTimestamp <= 0) {
             return 0;
         }
@@ -603,7 +616,7 @@ public long getReviveBehindMillis() {
         return 0;
     }
 
-    public long getReviveBehindMessages() {
+    public long getReviveBehindMessages() throws ConsumeQueueException {
         if (currentReviveMessageTimestamp <= 0) {
             return 0;
         }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
index d53454f215d..5d947fd088f 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
@@ -73,6 +73,7 @@
 import org.apache.rocketmq.store.MessageFilter;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.config.BrokerRole;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
 import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
@@ -298,7 +299,8 @@ public boolean rejectRequest() {
         return false;
     }
 
-    private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend, boolean brokerAllowFlowCtrSuspend)
+    private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend,
+        boolean brokerAllowFlowCtrSuspend)
         throws RemotingCommandException {
         final long beginTimeMills = this.brokerController.getMessageStore().now();
         RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class);
@@ -369,7 +371,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
             String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
                 requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
             LOGGER.warn(errorInfo);
-            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setCode(ResponseCode.INVALID_PARAMETER);
             response.setRemark(errorInfo);
             return response;
         }
@@ -487,9 +489,22 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
                 this.brokerController.getConsumerFilterManager());
         }
 
+        if (brokerController.getBrokerConfig().isRejectPullConsumerEnable()) {
+            ConsumerGroupInfo consumerGroupInfo =
+                    this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup());
+            if (null == consumerGroupInfo || ConsumeType.CONSUME_ACTIVELY == consumerGroupInfo.getConsumeType()) {
+                if ((null == consumerGroupInfo || null == consumerGroupInfo.findChannel(channel))
+                        && !MixAll.isSysConsumerGroupPullMessage(requestHeader.getConsumerGroup())) {
+                    response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);
+                    response.setRemark("the consumer's group info not exist, or the pull consumer is rejected by server." + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST));
+                    return response;
+                }
+            }
+        }
+
         final MessageStore messageStore = brokerController.getMessageStore();
         if (this.brokerController.getMessageStore() instanceof DefaultMessageStore) {
-            DefaultMessageStore defaultMessageStore = (DefaultMessageStore)this.brokerController.getMessageStore();
+            DefaultMessageStore defaultMessageStore = (DefaultMessageStore) this.brokerController.getMessageStore();
             boolean cgNeedColdDataFlowCtr = brokerController.getColdDataCgCtrService().isCgNeedColdDataFlowCtr(requestHeader.getConsumerGroup());
             if (cgNeedColdDataFlowCtr) {
                 boolean isMsgLogicCold = defaultMessageStore.getCommitLog()
@@ -526,7 +541,11 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
             getMessageResult.setStatus(GetMessageStatus.OFFSET_RESET);
             getMessageResult.setNextBeginOffset(resetOffset);
             getMessageResult.setMinOffset(messageStore.getMinOffsetInQueue(topic, queueId));
-            getMessageResult.setMaxOffset(messageStore.getMaxOffsetInQueue(topic, queueId));
+            try {
+                getMessageResult.setMaxOffset(messageStore.getMaxOffsetInQueue(topic, queueId));
+            } catch (ConsumeQueueException e) {
+                throw new RemotingCommandException("Failed tp get max offset in queue", e);
+            }
             getMessageResult.setSuggestPullingFromSlave(false);
         } else {
             long broadcastInitOffset = queryBroadcastPullInitOffset(topic, group, queueId, requestHeader, channel);
@@ -589,12 +608,13 @@ public boolean hasConsumeMessageHook() {
 
     /**
      * Composes the header of the response message to be sent back to the client
-     * @param requestHeader - the header of the request message
-     * @param getMessageResult - the result of the GetMessage request
-     * @param topicSysFlag - the system flag of the topic
+     *
+     * @param requestHeader           - the header of the request message
+     * @param getMessageResult        - the result of the GetMessage request
+     * @param topicSysFlag            - the system flag of the topic
      * @param subscriptionGroupConfig - configuration of the subscription group
-     * @param response - the response message to be sent back to the client
-     * @param clientAddress - the address of the client
+     * @param response                - the response message to be sent back to the client
+     * @param clientAddress           - the address of the client
      */
     protected void composeResponseHeader(PullMessageRequestHeader requestHeader, GetMessageResult getMessageResult,
         int topicSysFlag, SubscriptionGroupConfig subscriptionGroupConfig, RemotingCommand response,
@@ -706,6 +726,7 @@ protected void executeConsumeMessageHookBefore(RemotingCommand request, PullMess
             context.setAccountOwnerParent(ownerParent);
             context.setAccountOwnerSelf(ownerSelf);
             context.setNamespace(NamespaceUtil.getNamespaceFromResource(requestHeader.getTopic()));
+            context.setFilterMessageCount(getMessageResult.getFilterMessageCount());
 
             switch (responseCode) {
                 case ResponseCode.SUCCESS:
@@ -799,7 +820,7 @@ public void executeRequestWhenWakeup(final Channel channel, final RemotingComman
                     }
                 }
             } catch (RemotingCommandException e1) {
-                LOGGER.error("excuteRequestWhenWakeup run", e1);
+                LOGGER.error("executeRequestWhenWakeup run", e1);
             }
         };
         this.brokerController.getPullMessageExecutor().submit(new RequestTask(run, channel, request));
@@ -855,7 +876,7 @@ protected void updateBroadcastPulledOffset(String topic, String group, int queue
      * When pull request is not broadcast or not return -1
      */
     protected long queryBroadcastPullInitOffset(String topic, String group, int queueId,
-        PullMessageRequestHeader requestHeader, Channel channel) {
+        PullMessageRequestHeader requestHeader, Channel channel) throws RemotingCommandException {
 
         if (!this.brokerController.getBrokerConfig().isEnableBroadcastOffsetStore()) {
             return -1L;
@@ -877,8 +898,12 @@ protected long queryBroadcastPullInitOffset(String topic, String group, int queu
                 clientId = clientChannelInfo.getClientId();
             }
 
-            return this.brokerController.getBroadcastOffsetManager()
-                .queryInitOffset(topic, group, queueId, clientId, requestHeader.getQueueOffset(), proxyPullBroadcast);
+            try {
+                return this.brokerController.getBroadcastOffsetManager()
+                    .queryInitOffset(topic, group, queueId, clientId, requestHeader.getQueueOffset(), proxyPullBroadcast);
+            } catch (ConsumeQueueException e) {
+                throw new RemotingCommandException("Failed to query initial offset", e);
+            }
         }
         return -1L;
     }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java
index d55f1b5b7fb..d29e3d0e069 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java
@@ -33,6 +33,7 @@
 import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;
 import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragelyByCircle;
 import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.message.MessageQueueAssignment;
@@ -49,6 +50,7 @@
 import org.apache.rocketmq.remoting.protocol.body.QueryAssignmentResponseBody;
 import org.apache.rocketmq.remoting.protocol.body.SetMessageRequestModeRequestBody;
 import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
 
 public class QueryAssignmentProcessor implements NettyRequestProcessor {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -174,7 +176,14 @@ private Set<MessageQueue> doLoadBalance(final String topic, final String consume
                 break;
             }
             case CLUSTERING: {
-                Set<MessageQueue> mqSet = topicRouteInfoManager.getTopicSubscribeInfo(topic);
+                Set<MessageQueue> mqSet;
+                if (MixAll.isLmq(topic)) {
+                    mqSet = new HashSet<>();
+                    mqSet.add(new MessageQueue(
+                        topic, brokerController.getBrokerConfig().getBrokerName(), (int)MixAll.LMQ_QUEUE_ID));
+                } else {
+                    mqSet = topicRouteInfoManager.getTopicSubscribeInfo(topic);
+                }
                 if (null == mqSet) {
                     if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
                         log.warn("QueryLoad: no assignment for group[{}], the topic[{}] does not exist.", consumerGroup, topic);
@@ -307,8 +316,20 @@ private RemotingCommand setMessageRequestMode(ChannelHandlerContext ctx,
             response.setRemark("retry topic is not allowed to set mode");
             return response;
         }
+        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
+        if (null == topicConfig) {
+            response.setCode(ResponseCode.TOPIC_NOT_EXIST);
+            response.setRemark("topic[" + topic + "] not exist");
+            return response;
+        }
 
         final String consumerGroup = requestBody.getConsumerGroup();
+        SubscriptionGroupConfig groupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(consumerGroup);
+        if (null == groupConfig) {
+            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
+            response.setRemark("subscription group does not exist");
+            return response;
+        }
 
         this.messageRequestModeManager.setMessageRequestMode(topic, consumerGroup, requestBody);
         this.messageRequestModeManager.persist();
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java
new file mode 100644
index 00000000000..372db0d36eb
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageClientIDSetter;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.config.BrokerRole;
+import org.apache.rocketmq.store.timer.TimerMessageStore;
+
+import java.nio.charset.StandardCharsets;
+
+public class RecallMessageProcessor implements NettyRequestProcessor {
+    private static final String RECALL_MESSAGE_TAG = "_RECALL_TAG_";
+    private final BrokerController brokerController;
+
+    public RecallMessageProcessor(final BrokerController brokerController) {
+        this.brokerController = brokerController;
+    }
+
+    @Override
+    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws
+            RemotingCommandException {
+        final RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class);
+        response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId());
+        final RecallMessageRequestHeader requestHeader =
+            request.decodeCommandCustomHeader(RecallMessageRequestHeader.class);
+
+        if (!brokerController.getBrokerConfig().isRecallMessageEnable()) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark("recall failed, operation is forbidden");
+            return response;
+        }
+
+        if (BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) {
+            response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE);
+            response.setRemark("recall failed, broker service not available");
+            return response;
+        }
+
+        final long startTimestamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp();
+        if (this.brokerController.getMessageStore().now() < startTimestamp) {
+            response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);
+            response.setRemark("recall failed, broker service not available");
+            return response;
+        }
+
+        if (!PermName.isWriteable(this.brokerController.getBrokerConfig().getBrokerPermission())
+            && !this.brokerController.getBrokerConfig().isAllowRecallWhenBrokerNotWriteable()) {
+            response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);
+            response.setRemark("recall failed, broker service not available");
+            return response;
+        }
+
+        TopicConfig topicConfig =
+            this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
+        if (null == topicConfig) {
+            response.setCode(ResponseCode.TOPIC_NOT_EXIST);
+            response.setRemark("recall failed, the topic[" + requestHeader.getTopic() + "] not exist");
+            return response;
+        }
+
+        RecallMessageHandle.HandleV1 handle;
+        try {
+            handle = (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(requestHeader.getRecallHandle());
+        } catch (DecoderException e) {
+            response.setCode(ResponseCode.ILLEGAL_OPERATION);
+            response.setRemark(e.getMessage());
+            return response;
+        }
+
+        if (!requestHeader.getTopic().equals(handle.getTopic())) {
+            response.setCode(ResponseCode.ILLEGAL_OPERATION);
+            response.setRemark("recall failed, topic not match");
+            return response;
+        }
+        if (!brokerController.getBrokerConfig().getBrokerName().equals(handle.getBrokerName())) {
+            response.setCode(ResponseCode.ILLEGAL_OPERATION);
+            response.setRemark("recall failed, broker service not available");
+            return response;
+        }
+
+        long timestamp = NumberUtils.toLong(handle.getTimestampStr(), -1);
+        long timeLeft = timestamp - System.currentTimeMillis();
+        if (timeLeft <= 0
+            || timeLeft >= brokerController.getMessageStoreConfig().getTimerMaxDelaySec() * 1000L) {
+            response.setCode(ResponseCode.ILLEGAL_OPERATION);
+            response.setRemark("recall failed, timestamp invalid");
+            return response;
+        }
+
+        MessageExtBrokerInner msgInner = buildMessage(ctx, requestHeader, handle);
+        long beginTimeMillis = this.brokerController.getMessageStore().now();
+        PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
+        handlePutMessageResult(putMessageResult, request, response, msgInner, ctx, beginTimeMillis);
+        return response;
+    }
+
+    public MessageExtBrokerInner buildMessage(ChannelHandlerContext ctx, RecallMessageRequestHeader requestHeader,
+        RecallMessageHandle.HandleV1 handle) {
+        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
+        msgInner.setTopic(handle.getTopic());
+        msgInner.setBody("0".getBytes(StandardCharsets.UTF_8));
+        msgInner.setTags(RECALL_MESSAGE_TAG);
+        msgInner.setTagsCode(RECALL_MESSAGE_TAG.hashCode());
+        msgInner.setQueueId(0);
+        MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TIMER_DEL_UNIQKEY,
+            TimerMessageStore.buildDeleteKey(handle.getTopic(), handle.getMessageId()));
+        MessageAccessor.putProperty(msgInner,
+            MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, handle.getMessageId());
+        MessageAccessor.putProperty(msgInner,
+            MessageConst.PROPERTY_TIMER_DELIVER_MS, String.valueOf(handle.getTimestampStr()));
+        MessageAccessor.putProperty(msgInner,
+            MessageConst.PROPERTY_BORN_TIMESTAMP, String.valueOf(System.currentTimeMillis()));
+        MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRACE_CONTEXT, "");
+        MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_PRODUCER_GROUP, requestHeader.getProducerGroup());
+        msgInner.setBornTimestamp(System.currentTimeMillis());
+        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+        msgInner.setBornHost(ctx.channel().remoteAddress());
+        msgInner.setStoreHost(this.brokerController.getStoreHost());
+        return msgInner;
+    }
+
+    public void handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand request,
+        RemotingCommand response, MessageExt message, ChannelHandlerContext ctx, long beginTimeMillis) {
+        if (null == putMessageResult) {
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark("recall failed, execute error");
+            return;
+        }
+        RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();
+        switch (putMessageResult.getPutMessageStatus()) {
+            case PUT_OK:
+                this.brokerController.getBrokerStatsManager().incTopicPutNums(
+                    message.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1); // system timer topic
+                this.brokerController.getBrokerStatsManager().incTopicPutSize(
+                    message.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());
+                this.brokerController.getBrokerStatsManager().incBrokerPutNums(
+                    message.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum());
+                this.brokerController.getBrokerStatsManager().incTopicPutLatency(
+                    message.getTopic(), 0, (int) (this.brokerController.getMessageStore().now() - beginTimeMillis));
+            case FLUSH_DISK_TIMEOUT:
+            case FLUSH_SLAVE_TIMEOUT:
+            case SLAVE_NOT_AVAILABLE:
+                response.setCode(ResponseCode.SUCCESS);
+                responseHeader.setMsgId(MessageClientIDSetter.getUniqID(message));
+                break;
+            default:
+                response.setCode(ResponseCode.SYSTEM_ERROR);
+                response.setRemark("recall failed, execute error");
+                break;
+        }
+    }
+
+    @Override
+    public boolean rejectRequest() {
+        return false;
+    }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
index d3bb048f75d..a70b48debe1 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
@@ -115,10 +115,10 @@ private RemotingCommand processReplyMessageRequest(final ChannelHandlerContext c
         response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn()));
 
         log.debug("receive SendReplyMessage request command, {}", request);
-        final long startTimstamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp();
-        if (this.brokerController.getMessageStore().now() < startTimstamp) {
+        final long startTimestamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp();
+        if (this.brokerController.getMessageStore().now() < startTimestamp) {
             response.setCode(ResponseCode.SYSTEM_ERROR);
-            response.setRemark(String.format("broker unable to service, until %s", UtilAll.timeMillisToHumanString2(startTimstamp)));
+            response.setRemark(String.format("broker unable to service, until %s", UtilAll.timeMillisToHumanString2(startTimestamp)));
             return response;
         }
 
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
index 912d502eab2..669cd5e6771 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
@@ -40,6 +40,7 @@
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageExtBatch;
 import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.common.utils.CleanupPolicyUtils;
@@ -430,7 +431,7 @@ private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult
                         "the broker's disk is full [" + diskUtil() + "], messages are put to the slave, message store has been shut down, etc.");
                 break;
             case OS_PAGE_CACHE_BUSY:
-                response.setCode(ResponseCode.SYSTEM_ERROR);
+                response.setCode(ResponseCode.SYSTEM_BUSY);
                 response.setRemark("[PC_SYNCHRONIZED]broker busy, start flow control for a while");
                 break;
             case LMQ_CONSUME_QUEUE_NUM_EXCEEDED:
@@ -483,6 +484,7 @@ private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult
             responseHeader.setQueueId(queueIdInt);
             responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset());
             responseHeader.setTransactionId(MessageClientIDSetter.getUniqID(msg));
+            attachRecallHandle(request, msg, responseHeader);
 
             RemotingCommand rewriteResult = rewriteResponseForStaticTopic(responseHeader, mappingContext);
             if (rewriteResult != null) {
@@ -647,6 +649,21 @@ private RemotingCommand sendBatchMessage(final ChannelHandlerContext ctx,
         }
     }
 
+    public void attachRecallHandle(RemotingCommand request, MessageExt msg, SendMessageResponseHeader responseHeader) {
+        if (RequestCode.SEND_BATCH_MESSAGE == request.getCode()
+            || RequestCode.CONSUMER_SEND_MSG_BACK == request.getCode()) {
+            return;
+        }
+        String timestampStr = msg.getProperty(MessageConst.PROPERTY_TIMER_OUT_MS);
+        String realTopic = msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC);
+        if (timestampStr != null && realTopic != null && !realTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
+            timestampStr = String.valueOf(Long.parseLong(timestampStr) + 1); // consider of floor
+            String recallHandle = RecallMessageHandle.HandleV1.buildHandle(realTopic,
+                brokerController.getBrokerConfig().getBrokerName(), timestampStr, MessageClientIDSetter.getUniqID(msg));
+            responseHeader.setRecallHandle(recallHandle);
+        }
+    }
+
     private String diskUtil() {
         double physicRatio = 100;
         String storePath;
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
index ef7e4f67894..a5b02c9a63c 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
@@ -53,6 +53,7 @@
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
 import org.apache.rocketmq.store.queue.CqUnit;
 import org.apache.rocketmq.store.queue.ReferredIterator;
@@ -103,7 +104,7 @@ public static int delayLevel2QueueId(final int delayLevel) {
         return delayLevel - 1;
     }
 
-    public void buildRunningStats(HashMap<String, String> stats) {
+    public void buildRunningStats(HashMap<String, String> stats) throws ConsumeQueueException {
         for (Map.Entry<Integer, Long> next : this.offsetTable.entrySet()) {
             int queueId = delayLevel2QueueId(next.getKey());
             long delayOffset = next.getValue();
@@ -224,7 +225,7 @@ public boolean load() {
         result = result && this.correctDelayOffset();
         return result;
     }
-    
+
     public boolean loadWhenSyncDelayOffset() {
         boolean result = super.load();
         result = result && this.parseDelayLevel();
@@ -377,7 +378,7 @@ public void run() {
                 if (isStarted()) {
                     this.executeOnTimeUp();
                 }
-            } catch (Exception e) {
+            } catch (Throwable e) {
                 // XXX: warn and notify me
                 log.error("ScheduleMessageService, executeOnTimeUp exception", e);
                 this.scheduleNextTimerTask(this.offset, DELAY_FOR_A_PERIOD);
@@ -473,7 +474,7 @@ public void executeOnTimeUp() {
                     }
 
                     if (!deliverSuc) {
-                        this.scheduleNextTimerTask(nextOffset, DELAY_FOR_A_WHILE);
+                        this.scheduleNextTimerTask(currOffset, DELAY_FOR_A_WHILE);
                         return;
                     }
                 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java
index 7f802adb938..f75fd216104 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java
@@ -19,12 +19,14 @@
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.loadbalance.MessageRequestModeManager;
 import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.constant.LoggerName;
@@ -32,8 +34,10 @@
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.protocol.body.ConsumerOffsetSerializeWrapper;
 import org.apache.rocketmq.remoting.protocol.body.MessageRequestModeSerializeWrapper;
+import org.apache.rocketmq.remoting.protocol.body.SetMessageRequestModeRequestBody;
 import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;
 import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
 import org.apache.rocketmq.store.timer.TimerCheckpoint;
 import org.apache.rocketmq.store.timer.TimerMetrics;
@@ -76,25 +80,28 @@ private void syncTopicConfig() {
             try {
                 TopicConfigAndMappingSerializeWrapper topicWrapper =
                         this.brokerController.getBrokerOuterAPI().getAllTopicConfig(masterAddrBak);
-                if (!this.brokerController.getTopicConfigManager().getDataVersion()
-                        .equals(topicWrapper.getDataVersion())) {
+                TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager();
+                if (!topicConfigManager.getDataVersion().equals(topicWrapper.getDataVersion())) {
 
-                    this.brokerController.getTopicConfigManager().getDataVersion()
-                            .assignNewOne(topicWrapper.getDataVersion());
+                    topicConfigManager.getDataVersion().assignNewOne(topicWrapper.getDataVersion());
 
                     ConcurrentMap<String, TopicConfig> newTopicConfigTable = topicWrapper.getTopicConfigTable();
+                    ConcurrentMap<String, TopicConfig> topicConfigTable = topicConfigManager.getTopicConfigTable();
+
                     //delete
-                    ConcurrentMap<String, TopicConfig> topicConfigTable = this.brokerController.getTopicConfigManager().getTopicConfigTable();
-                    for (Iterator<Map.Entry<String, TopicConfig>> it = topicConfigTable.entrySet().iterator(); it.hasNext(); ) {
-                        Map.Entry<String, TopicConfig> item = it.next();
-                        if (!newTopicConfigTable.containsKey(item.getKey())) {
-                            it.remove();
+                    Iterator<Map.Entry<String, TopicConfig>> iterator = topicConfigTable.entrySet().iterator();
+                    while (iterator.hasNext()) {
+                        Map.Entry<String, TopicConfig> entry = iterator.next();
+                        if (!newTopicConfigTable.containsKey(entry.getKey())) {
+                            iterator.remove();
                         }
+                        topicConfigManager.deleteTopicConfig(entry.getKey());
                     }
+
                     //update
-                    topicConfigTable.putAll(newTopicConfigTable);
+                    newTopicConfigTable.values().forEach(topicConfigManager::updateSingleTopicConfigWithoutPersist);
 
-                    this.brokerController.getTopicConfigManager().persist();
+                    topicConfigManager.persist();
                 }
                 if (topicWrapper.getTopicQueueMappingDetailMap() != null
                         && !topicWrapper.getMappingDataVersion().equals(this.brokerController.getTopicQueueMappingManager().getDataVersion())) {
@@ -104,12 +111,7 @@ private void syncTopicConfig() {
                     ConcurrentMap<String, TopicConfig> newTopicConfigTable = topicWrapper.getTopicConfigTable();
                     //delete
                     ConcurrentMap<String, TopicConfig> topicConfigTable = this.brokerController.getTopicConfigManager().getTopicConfigTable();
-                    for (Iterator<Map.Entry<String, TopicConfig>> it = topicConfigTable.entrySet().iterator(); it.hasNext(); ) {
-                        Map.Entry<String, TopicConfig> item = it.next();
-                        if (!newTopicConfigTable.containsKey(item.getKey())) {
-                            it.remove();
-                        }
-                    }
+                    topicConfigTable.entrySet().removeIf(item -> !newTopicConfigTable.containsKey(item.getKey()));
                     //update
                     topicConfigTable.putAll(newTopicConfigTable);
 
@@ -174,13 +176,25 @@ private void syncSubscriptionGroupConfig() {
 
                 if (!this.brokerController.getSubscriptionGroupManager().getDataVersion()
                         .equals(subscriptionWrapper.getDataVersion())) {
-                    SubscriptionGroupManager subscriptionGroupManager =
-                            this.brokerController.getSubscriptionGroupManager();
-                    subscriptionGroupManager.getDataVersion().assignNewOne(
-                            subscriptionWrapper.getDataVersion());
-                    subscriptionGroupManager.getSubscriptionGroupTable().clear();
-                    subscriptionGroupManager.getSubscriptionGroupTable().putAll(
-                            subscriptionWrapper.getSubscriptionGroupTable());
+                    SubscriptionGroupManager subscriptionGroupManager = this.brokerController.getSubscriptionGroupManager();
+                    subscriptionGroupManager.getDataVersion().assignNewOne(subscriptionWrapper.getDataVersion());
+
+                    ConcurrentMap<String, SubscriptionGroupConfig> curSubscriptionGroupTable =
+                            subscriptionGroupManager.getSubscriptionGroupTable();
+                    ConcurrentMap<String, SubscriptionGroupConfig> newSubscriptionGroupTable =
+                            subscriptionWrapper.getSubscriptionGroupTable();
+                    // delete
+                    Iterator<Map.Entry<String, SubscriptionGroupConfig>> iterator = curSubscriptionGroupTable.entrySet().iterator();
+                    while (iterator.hasNext()) {
+                        Map.Entry<String, SubscriptionGroupConfig> configEntry = iterator.next();
+                        if (!newSubscriptionGroupTable.containsKey(configEntry.getKey())) {
+                            iterator.remove();
+                        }
+                        subscriptionGroupManager.deleteSubscriptionGroupConfig(configEntry.getKey());
+                    }
+                    // update
+                    newSubscriptionGroupTable.values().forEach(subscriptionGroupManager::updateSubscriptionGroupConfigWithoutPersist);
+                    // persist
                     subscriptionGroupManager.persist();
                     LOGGER.info("Update slave Subscription Group from master, {}", masterAddrBak);
                 }
@@ -199,10 +213,16 @@ private void syncMessageRequestMode() {
 
                 MessageRequestModeManager messageRequestModeManager =
                         this.brokerController.getQueryAssignmentProcessor().getMessageRequestModeManager();
-                messageRequestModeManager.getMessageRequestModeMap().clear();
-                messageRequestModeManager.getMessageRequestModeMap().putAll(
-                        messageRequestModeSerializeWrapper.getMessageRequestModeMap()
-                );
+                ConcurrentHashMap<String, ConcurrentHashMap<String, SetMessageRequestModeRequestBody>> curMessageRequestModeMap =
+                        messageRequestModeManager.getMessageRequestModeMap();
+                ConcurrentHashMap<String, ConcurrentHashMap<String, SetMessageRequestModeRequestBody>> newMessageRequestModeMap =
+                        messageRequestModeSerializeWrapper.getMessageRequestModeMap();
+
+                // delete
+                curMessageRequestModeMap.entrySet().removeIf(e -> !newMessageRequestModeMap.containsKey(e.getKey()));
+                // update
+                curMessageRequestModeMap.putAll(newMessageRequestModeMap);
+                // persist
                 messageRequestModeManager.persist();
                 LOGGER.info("Update slave Message Request Mode from master, {}", masterAddrBak);
             } catch (Exception e) {
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java
index 018083811e8..69e59fd8e7f 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java
@@ -43,4 +43,13 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config)
         }
         super.updateSubscriptionGroupConfig(config);
     }
+
+    @Override
+    public boolean containsSubscriptionGroup(String group) {
+        if (MixAll.isLmq(group)) {
+            return true;
+        } else {
+            return super.containsSubscriptionGroup(group);
+        }
+    }
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java
deleted file mode 100644
index e9a81a8d686..00000000000
--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.broker.subscription;
-
-import java.io.File;
-
-import org.apache.rocketmq.broker.BrokerController;
-import org.apache.rocketmq.common.config.RocksDBConfigManager;
-import org.apache.rocketmq.common.utils.DataConverter;
-import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
-
-public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager {
-
-    public RocksDBSubscriptionGroupManager(BrokerController brokerController) {
-        super(brokerController, false);
-        this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs());
-    }
-
-    @Override
-    public boolean load() {
-        if (!this.rocksDBConfigManager.load(configFilePath(), this::decode0)) {
-            return false;
-        }
-        this.init();
-        return true;
-    }
-
-    @Override
-    public boolean stop() {
-        return this.rocksDBConfigManager.stop();
-    }
-
-    @Override
-    protected SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) {
-        String groupName = subscriptionGroupConfig.getGroupName();
-        SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig);
-
-        try {
-            byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8);
-            byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible);
-            this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes);
-        } catch (Exception e) {
-            log.error("kv put sub Failed, {}", subscriptionGroupConfig.toString());
-        }
-        return oldConfig;
-    }
-
-    @Override
-    protected SubscriptionGroupConfig putSubscriptionGroupConfigIfAbsent(SubscriptionGroupConfig subscriptionGroupConfig) {
-        String groupName = subscriptionGroupConfig.getGroupName();
-        SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.putIfAbsent(groupName, subscriptionGroupConfig);
-        if (oldConfig == null) {
-            try {
-                byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8);
-                byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible);
-                this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes);
-            } catch (Exception e) {
-                log.error("kv put sub Failed, {}", subscriptionGroupConfig.toString());
-            }
-        }
-        return oldConfig;
-    }
-
-    @Override
-    protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) {
-        SubscriptionGroupConfig subscriptionGroupConfig = this.subscriptionGroupTable.remove(groupName);
-        try {
-            this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.CHARSET_UTF8));
-        } catch (Exception e) {
-            log.error("kv delete sub Failed, {}", subscriptionGroupConfig.toString());
-        }
-        return subscriptionGroupConfig;
-    }
-
-    @Override
-    protected void decode0(byte[] key, byte[] body) {
-        String groupName = new String(key, DataConverter.CHARSET_UTF8);
-        SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(body, SubscriptionGroupConfig.class);
-
-        this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig);
-        log.info("load exist local sub, {}", subscriptionGroupConfig.toString());
-    }
-
-    @Override
-    public synchronized void persist() {
-        if (brokerController.getMessageStoreConfig().isRealTimePersistRocksDBConfig()) {
-            this.rocksDBConfigManager.flushWAL();
-        }
-    }
-
-    @Override
-    public String configFilePath() {
-        return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "subscriptionGroups" + File.separator;
-    }
-}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
index e63b9305868..f3e669fb3ea 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
@@ -19,6 +19,7 @@
 import com.google.common.collect.ImmutableMap;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
@@ -48,7 +49,7 @@ public class SubscriptionGroupManager extends ConfigManager {
     private ConcurrentMap<String, ConcurrentMap<String, Integer>> forbiddenTable =
         new ConcurrentHashMap<>(4);
 
-    private final DataVersion dataVersion = new DataVersion();
+    protected final DataVersion dataVersion = new DataVersion();
     protected transient BrokerController brokerController;
 
     public SubscriptionGroupManager() {
@@ -121,7 +122,7 @@ protected void init() {
         }
     }
 
-    protected SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) {
+    public SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) {
         return this.subscriptionGroupTable.put(subscriptionGroupConfig.getGroupName(), subscriptionGroupConfig);
     }
 
@@ -138,6 +139,11 @@ protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName
     }
 
     public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) {
+        updateSubscriptionGroupConfigWithoutPersist(config);
+        this.persist();
+    }
+
+    public void updateSubscriptionGroupConfigWithoutPersist(SubscriptionGroupConfig config) {
         Map<String, String> newAttributes = request(config);
         Map<String, String> currentAttributes = current(config.getGroupName());
 
@@ -156,9 +162,14 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config)
             log.info("create new subscription group, {}", config);
         }
 
-        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-        dataVersion.nextVersion(stateMachineVersion);
+        updateDataVersion();
+    }
 
+    public void updateSubscriptionGroupConfigList(List<SubscriptionGroupConfig> configList) {
+        if (null == configList || configList.isEmpty()) {
+            return;
+        }
+        configList.forEach(this::updateSubscriptionGroupConfigWithoutPersist);
         this.persist();
     }
 
@@ -214,7 +225,7 @@ public int getForbidden(String group, String topic) {
         return topicForbidden;
     }
 
-    private void updateForbiddenValue(String group, String topic, Integer forbidden) {
+    protected void updateForbiddenValue(String group, String topic, Integer forbidden) {
         if (forbidden == null || forbidden <= 0) {
             this.forbiddenTable.remove(group);
             log.info("clear group forbidden, {}@{} ", group, topic);
@@ -233,8 +244,7 @@ private void updateForbiddenValue(String group, String topic, Integer forbidden)
             log.info("set group forbidden, {}@{} old: {} new: {}", group, topic, 0, forbidden);
         }
 
-        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-        dataVersion.nextVersion(stateMachineVersion);
+        updateDataVersion();
 
         this.persist();
     }
@@ -243,15 +253,15 @@ public void disableConsume(final String groupName) {
         SubscriptionGroupConfig old = getSubscriptionGroupConfig(groupName);
         if (old != null) {
             old.setConsumeEnable(false);
-            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-            dataVersion.nextVersion(stateMachineVersion);
+            updateDataVersion();
         }
     }
 
     public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) {
         SubscriptionGroupConfig subscriptionGroupConfig = getSubscriptionGroupConfig(group);
         if (null == subscriptionGroupConfig) {
-            if (brokerController.getBrokerConfig().isAutoCreateSubscriptionGroup() || MixAll.isSysConsumerGroup(group)) {
+            if (brokerController.getBrokerConfig().isAutoCreateSubscriptionGroup()
+                    || MixAll.isSysConsumerGroupAndEnableCreate(group, brokerController.getBrokerConfig().isEnableCreateSysGroup())) {
                 if (group.length() > Validators.CHARACTER_MAX_LENGTH || TopicValidator.isTopicOrGroupIllegal(group)) {
                     return null;
                 }
@@ -261,8 +271,7 @@ public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) {
                 if (null == preConfig) {
                     log.info("auto create a subscription group, {}", subscriptionGroupConfig.toString());
                 }
-                long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-                dataVersion.nextVersion(stateMachineVersion);
+                updateDataVersion();
                 this.persist();
             }
         }
@@ -326,13 +335,32 @@ public DataVersion getDataVersion() {
         return dataVersion;
     }
 
+    public boolean loadDataVersion() {
+        String fileName = null;
+        try {
+            fileName = this.configFilePath();
+            String jsonString = MixAll.file2String(fileName);
+            if (jsonString != null) {
+                SubscriptionGroupManager obj = RemotingSerializable.fromJson(jsonString, SubscriptionGroupManager.class);
+                if (obj != null) {
+                    this.dataVersion.assignNewOne(obj.dataVersion);
+                    this.printLoadDataWhenFirstBoot(obj);
+                    log.info("load subGroup dataVersion success,{},{}", fileName, obj.dataVersion);
+                }
+            }
+            return true;
+        } catch (Exception e) {
+            log.error("load subGroup dataVersion failed" + fileName, e);
+            return false;
+        }
+    }
+
     public void deleteSubscriptionGroupConfig(final String groupName) {
         SubscriptionGroupConfig old = removeSubscriptionGroupConfig(groupName);
         this.forbiddenTable.remove(groupName);
         if (old != null) {
             log.info("delete subscription group OK, subscription group:{}", old);
-            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-            dataVersion.nextVersion(stateMachineVersion);
+            updateDataVersion();
             this.persist();
         } else {
             log.warn("delete subscription group failed, subscription groupName: {} not exist", groupName);
@@ -369,4 +397,14 @@ private Map<String, String> current(String groupName) {
             }
         }
     }
+
+    public void setDataVersion(DataVersion dataVersion) {
+        this.dataVersion.assignNewOne(dataVersion);
+    }
+
+    public void updateDataVersion() {
+        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+        dataVersion.nextVersion(stateMachineVersion);
+    }
+
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java
deleted file mode 100644
index fddecf2d92a..00000000000
--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.broker.topic;
-
-import java.io.File;
-
-import org.apache.rocketmq.broker.BrokerController;
-import org.apache.rocketmq.common.TopicConfig;
-import org.apache.rocketmq.common.config.RocksDBConfigManager;
-import org.apache.rocketmq.common.utils.DataConverter;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
-
-public class RocksDBTopicConfigManager extends TopicConfigManager {
-
-    public RocksDBTopicConfigManager(BrokerController brokerController) {
-        super(brokerController, false);
-        this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs());
-    }
-
-    @Override
-    public boolean load() {
-        if (!this.rocksDBConfigManager.load(configFilePath(), this::decode0)) {
-            return false;
-        }
-        this.init();
-        return true;
-    }
-
-    @Override
-    public boolean stop() {
-        return this.rocksDBConfigManager.stop();
-    }
-
-    @Override
-    protected void decode0(byte[] key, byte[] body) {
-        String topicName = new String(key, DataConverter.CHARSET_UTF8);
-        TopicConfig topicConfig = JSON.parseObject(body, TopicConfig.class);
-
-        this.topicConfigTable.put(topicName, topicConfig);
-        log.info("load exist local topic, {}", topicConfig.toString());
-    }
-
-    @Override
-    public String configFilePath() {
-        return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "topics" + File.separator;
-    }
-
-    @Override
-    protected TopicConfig putTopicConfig(TopicConfig topicConfig) {
-        String topicName = topicConfig.getTopicName();
-        TopicConfig oldTopicConfig = this.topicConfigTable.put(topicName, topicConfig);
-        try {
-            byte[] keyBytes = topicName.getBytes(DataConverter.CHARSET_UTF8);
-            byte[] valueBytes = JSON.toJSONBytes(topicConfig, SerializerFeature.BrowserCompatible);
-            this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes);
-        } catch (Exception e) {
-            log.error("kv put topic Failed, {}", topicConfig.toString(), e);
-        }
-        return oldTopicConfig;
-    }
-
-    @Override
-    protected TopicConfig removeTopicConfig(String topicName) {
-        TopicConfig topicConfig = this.topicConfigTable.remove(topicName);
-        try {
-            this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.CHARSET_UTF8));
-        } catch (Exception e) {
-            log.error("kv remove topic Failed, {}", topicConfig.toString());
-        }
-        return topicConfig;
-    }
-
-    @Override
-    public synchronized void persist() {
-        if (brokerController.getMessageStoreConfig().isRealTimePersistRocksDBConfig()) {
-            this.rocksDBConfigManager.flushWAL();
-        }
-    }
-}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
index 511d29e12ad..b20cafc1018 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
@@ -18,6 +18,7 @@
 
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -51,6 +52,10 @@
 import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;
 import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
 import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;
+import org.apache.rocketmq.store.timer.TimerMessageStore;
+import org.apache.rocketmq.tieredstore.TieredMessageStore;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
@@ -61,7 +66,7 @@ public class TopicConfigManager extends ConfigManager {
 
     private transient final Lock topicConfigTableLock = new ReentrantLock();
     protected ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<>(1024);
-    private DataVersion dataVersion = new DataVersion();
+    protected DataVersion dataVersion = new DataVersion();
     protected transient BrokerController brokerController;
 
     public TopicConfigManager() {
@@ -207,9 +212,20 @@ protected void init() {
             topicConfig.setWriteQueueNums(1);
             putTopicConfig(topicConfig);
         }
+
+        {
+            if (this.brokerController.getMessageStoreConfig().isTimerWheelEnable()) {
+                String topic = TimerMessageStore.TIMER_TOPIC;
+                TopicConfig topicConfig = new TopicConfig(topic);
+                TopicValidator.addSystemTopic(topic);
+                topicConfig.setReadQueueNums(1);
+                topicConfig.setWriteQueueNums(1);
+                this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
+            }
+        }
     }
 
-    protected TopicConfig putTopicConfig(TopicConfig topicConfig) {
+    public TopicConfig putTopicConfig(TopicConfig topicConfig) {
         return this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
     }
 
@@ -277,8 +293,7 @@ public TopicConfig createTopicInSendMessageMethod(final String topic, final Stri
 
                         putTopicConfig(topicConfig);
 
-                        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-                        dataVersion.nextVersion(stateMachineVersion);
+                        updateDataVersion();
 
                         createNew = true;
 
@@ -321,8 +336,7 @@ public TopicConfig createTopicIfAbsent(TopicConfig topicConfig, boolean register
                     }
                     log.info("Create new topic [{}] config:[{}]", topicConfig.getTopicName(), topicConfig);
                     putTopicConfig(topicConfig);
-                    long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-                    dataVersion.nextVersion(stateMachineVersion);
+                    updateDataVersion();
                     createNew = true;
                     this.persist();
                 } finally {
@@ -381,8 +395,7 @@ public TopicConfig createTopicInSendMessageBackMethod(
                     log.info("create new topic {}", topicConfig);
                     putTopicConfig(topicConfig);
                     createNew = true;
-                    long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-                    dataVersion.nextVersion(stateMachineVersion);
+                    updateDataVersion();
                     this.persist();
                 } finally {
                     this.topicConfigTableLock.unlock();
@@ -422,8 +435,7 @@ public TopicConfig createTopicOfTranCheckMaxTime(final int clientDefaultTopicQue
                     log.info("create new topic {}", topicConfig);
                     putTopicConfig(topicConfig);
                     createNew = true;
-                    long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-                    dataVersion.nextVersion(stateMachineVersion);
+                    updateDataVersion();
                     this.persist();
                 } finally {
                     this.topicConfigTableLock.unlock();
@@ -456,8 +468,7 @@ public void updateTopicUnitFlag(final String topic, final boolean unit) {
 
             putTopicConfig(topicConfig);
 
-            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-            dataVersion.nextVersion(stateMachineVersion);
+            updateDataVersion();
 
             this.persist();
             registerBrokerData(topicConfig);
@@ -479,21 +490,19 @@ public void updateTopicUnitSubFlag(final String topic, final boolean hasUnitSub)
 
             putTopicConfig(topicConfig);
 
-            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-            dataVersion.nextVersion(stateMachineVersion);
+            updateDataVersion();
 
             this.persist();
             registerBrokerData(topicConfig);
         }
     }
 
-    public void updateTopicConfig(final TopicConfig topicConfig) {
+    public void updateSingleTopicConfigWithoutPersist(final TopicConfig topicConfig) {
         checkNotNull(topicConfig, "topicConfig shouldn't be null");
 
         Map<String, String> newAttributes = request(topicConfig);
         Map<String, String> currentAttributes = current(topicConfig.getTopicName());
 
-
         Map<String, String> finalAttributes = AttributeUtil.alterCurrentAttributes(
             this.topicConfigTable.get(topicConfig.getTopicName()) == null,
             TopicAttributes.ALL,
@@ -501,6 +510,7 @@ public void updateTopicConfig(final TopicConfig topicConfig) {
             ImmutableMap.copyOf(newAttributes));
 
         topicConfig.setAttributes(finalAttributes);
+        updateTieredStoreTopicMetadata(topicConfig, newAttributes);
 
         TopicConfig old = putTopicConfig(topicConfig);
         if (old != null) {
@@ -509,12 +519,46 @@ public void updateTopicConfig(final TopicConfig topicConfig) {
             log.info("create new topic [{}]", topicConfig);
         }
 
-        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-        dataVersion.nextVersion(stateMachineVersion);
+        updateDataVersion();
+    }
 
+    public void updateTopicConfig(final TopicConfig topicConfig) {
+        updateSingleTopicConfigWithoutPersist(topicConfig);
         this.persist(topicConfig.getTopicName(), topicConfig);
     }
 
+    public void updateTopicConfigList(final List<TopicConfig> topicConfigList) {
+        topicConfigList.forEach(this::updateSingleTopicConfigWithoutPersist);
+        this.persist();
+    }
+
+    private synchronized void updateTieredStoreTopicMetadata(final TopicConfig topicConfig, Map<String, String> newAttributes) {
+        if (!(brokerController.getMessageStore() instanceof TieredMessageStore)) {
+            if (newAttributes.get(TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getName()) != null) {
+                throw new IllegalArgumentException("Update topic reserveTime not supported");
+            }
+            return;
+        }
+
+        String topic = topicConfig.getTopicName();
+        long reserveTime = TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getDefaultValue();
+        String attr = topicConfig.getAttributes().get(TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getName());
+        if (attr != null) {
+            reserveTime = Long.parseLong(attr);
+        }
+
+        log.info("Update tiered storage metadata, topic {}, reserveTime {}", topic, reserveTime);
+        TieredMessageStore tieredMessageStore = (TieredMessageStore) brokerController.getMessageStore();
+        MetadataStore metadataStore = tieredMessageStore.getMetadataStore();
+        TopicMetadata topicMetadata = metadataStore.getTopic(topic);
+        if (topicMetadata == null) {
+            metadataStore.addTopic(topic, reserveTime);
+        } else if (topicMetadata.getReserveTime() != reserveTime) {
+            topicMetadata.setReserveTime(reserveTime);
+            metadataStore.updateTopic(topicMetadata);
+        }
+    }
+
     public void updateOrderTopicConfig(final KVTable orderKVTableFromNs) {
 
         if (orderKVTableFromNs != null && orderKVTableFromNs.getTable() != null) {
@@ -529,25 +573,8 @@ public void updateOrderTopicConfig(final KVTable orderKVTableFromNs) {
                 }
             }
 
-            // We don't have a mandatory rule to maintain the validity of order conf in NameServer,
-            // so we may overwrite the order field mistakenly.
-            // To avoid the above case, we comment the below codes, please use mqadmin API to update
-            // the order filed.
-            /*for (Map.Entry<String, TopicConfig> entry : this.topicConfigTable.entrySet()) {
-                String topic = entry.getKey();
-                if (!orderTopics.contains(topic)) {
-                    TopicConfig topicConfig = entry.getValue();
-                    if (topicConfig.isOrder()) {
-                        topicConfig.setOrder(false);
-                        isChange = true;
-                        log.info("update order topic config, topic={}, order={}", topic, false);
-                    }
-                }
-            }*/
-
             if (isChange) {
-                long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-                dataVersion.nextVersion(stateMachineVersion);
+                updateDataVersion();
                 this.persist();
             }
         }
@@ -571,8 +598,7 @@ public void deleteTopicConfig(final String topic) {
         TopicConfig old = removeTopicConfig(topic);
         if (old != null) {
             log.info("delete topic config OK, topic: {}", old);
-            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
-            dataVersion.nextVersion(stateMachineVersion);
+            updateDataVersion();
             this.persist();
         } else {
             log.warn("delete topic config failed, topic: {} not exists", topic);
@@ -611,6 +637,26 @@ public String encode() {
         return encode(false);
     }
 
+    public boolean loadDataVersion() {
+        String fileName = null;
+        try {
+            fileName = this.configFilePath();
+            String jsonString = MixAll.file2String(fileName);
+            if (jsonString != null) {
+                TopicConfigSerializeWrapper topicConfigSerializeWrapper =
+                    TopicConfigSerializeWrapper.fromJson(jsonString, TopicConfigSerializeWrapper.class);
+                if (topicConfigSerializeWrapper != null) {
+                    this.dataVersion.assignNewOne(topicConfigSerializeWrapper.getDataVersion());
+                    log.info("load topic metadata dataVersion success {}, {}", fileName, topicConfigSerializeWrapper.getDataVersion());
+                }
+            }
+            return true;
+        } catch (Exception e) {
+            log.error("load topic metadata dataVersion failed" + fileName, e);
+            return false;
+        }
+    }
+
     @Override
     public String configFilePath() {
         return BrokerPathConfigHelper.getTopicConfigPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir());
@@ -687,5 +733,11 @@ public boolean containsTopic(String topic) {
         return topicConfigTable.containsKey(topic);
     }
 
+    public void updateDataVersion() {
+        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+        dataVersion.nextVersion(stateMachineVersion);
+    }
+
+
 
 }
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java
index b5a86d3d083..c8d49f416c7 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java
@@ -50,6 +50,7 @@ public AbstractTransactionalMessageCheckListener(BrokerController brokerControll
 
     public void sendCheckMessage(MessageExt msgExt) throws Exception {
         CheckTransactionStateRequestHeader checkTransactionStateRequestHeader = new CheckTransactionStateRequestHeader();
+        checkTransactionStateRequestHeader.setTopic(msgExt.getTopic());
         checkTransactionStateRequestHeader.setCommitLogOffset(msgExt.getCommitLogOffset());
         checkTransactionStateRequestHeader.setOffsetMsgId(msgExt.getMsgId());
         checkTransactionStateRequestHeader.setMsgId(msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java
index 9fdfd0a7101..017803c624c 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java
@@ -203,9 +203,9 @@ public void check(long transactionTimeout, int transactionCheckMax,
                         log.info("Queue={} process time reach max={}", messageQueue, MAX_PROCESS_TIME_LIMIT);
                         break;
                     }
-                    if (removeMap.containsKey(i)) {
+                    Long removedOpOffset;
+                    if ((removedOpOffset = removeMap.remove(i)) != null) {
                         log.debug("Half offset {} has been committed/rolled back", i);
-                        Long removedOpOffset = removeMap.remove(i);
                         opMsgMap.get(removedOpOffset).remove(i);
                         if (opMsgMap.get(removedOpOffset).size() == 0) {
                             opMsgMap.remove(removedOpOffset);
@@ -456,8 +456,8 @@ private boolean checkPrepareQueueOffset(HashMap<Long, Long> removeMap, List<Long
             if (-1 == prepareQueueOffset) {
                 return false;
             } else {
-                if (removeMap.containsKey(prepareQueueOffset)) {
-                    long tmpOpOffset = removeMap.remove(prepareQueueOffset);
+                Long tmpOpOffset;
+                if ((tmpOpOffset = removeMap.remove(prepareQueueOffset)) != null) {
                     doneOpOffset.add(tmpOpOffset);
                     log.info("removeMap contain prepareQueueOffset. real_topic={},uniqKey={},immunityTime={},offset={}",
                             msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC),
diff --git a/broker/src/main/resources/rmq.broker.logback.xml b/broker/src/main/resources/rmq.broker.logback.xml
index 32dc297360e..fd63ef174da 100644
--- a/broker/src/main/resources/rmq.broker.logback.xml
+++ b/broker/src/main/resources/rmq.broker.logback.xml
@@ -592,6 +592,39 @@
         <appender-ref ref="RocketmqBrokerMetricsSiftingAppender_inner"/>
     </appender>
 
+    <appender name="RocketmqAuthAuditSiftingAppender_inner" class="ch.qos.logback.classic.sift.SiftingAppender">
+        <discriminator>
+            <key>brokerContainerLogDir</key>
+            <defaultValue>${file.separator}</defaultValue>
+        </discriminator>
+        <sift>
+            <appender name="RocketmqAuthAuditAppender"
+                      class="ch.qos.logback.core.rolling.RollingFileAppender">
+                <file>
+                    ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}auth_audit.log
+                </file>
+                <append>true</append>
+                <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+                    <fileNamePattern>
+                        ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}auth_audit.%i.log.gz
+                    </fileNamePattern>
+                    <minIndex>1</minIndex>
+                    <maxIndex>3</maxIndex>
+                </rollingPolicy>
+                <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+                    <maxFileSize>512MB</maxFileSize>
+                </triggeringPolicy>
+                <encoder>
+                    <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+                    <charset class="java.nio.charset.Charset">UTF-8</charset>
+                </encoder>
+            </appender>
+        </sift>
+    </appender>
+    <appender name="RocketmqAuthAuditSiftingAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqAuthAuditSiftingAppender_inner"/>
+    </appender>
+
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
             <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
@@ -681,6 +714,10 @@
         <appender-ref ref="RocketmqBrokerMetricsSiftingAppender"/>
     </logger>
 
+    <logger name="RocketmqAuthAudit" additivity="false" level="INFO">
+        <appender-ref ref="RocketmqAuthAuditSiftingAppender"/>
+    </logger>
+
     <root level="INFO">
         <appender-ref ref="DefaultSiftingAppender"/>
     </root>
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java
index 6035a20acb2..3ce1fe3dbdf 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java
@@ -26,11 +26,18 @@
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.future.FutureTaskExt;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.RemotingServer;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;
+import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
 import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.remoting.netty.RequestTask;
+import org.apache.rocketmq.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -94,4 +101,43 @@ public void run() {
         TimeUnit.MILLISECONDS.sleep(headSlowTimeMills);
         assertThat(brokerController.headSlowTimeMills(queue)).isGreaterThanOrEqualTo(headSlowTimeMills);
     }
+
+    @Test
+    public void testCustomRemotingServer() throws CloneNotSupportedException {
+        final RemotingServer mockRemotingServer = new NettyRemotingServer(nettyServerConfig);
+        final String mockRemotingServerName = "MOCK_REMOTING_SERVER";
+
+        BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, new NettyClientConfig(), messageStoreConfig);
+        brokerController.setRemotingServerByName(mockRemotingServerName, mockRemotingServer);
+        brokerController.initializeRemotingServer();
+
+        final RPCHook rpcHook = new RPCHook() {
+            @Override
+            public void doBeforeRequest(String remoteAddr, RemotingCommand request) {
+
+            }
+
+            @Override
+            public void doAfterResponse(String remoteAddr, RemotingCommand request, RemotingCommand response) {
+
+            }
+        };
+        brokerController.registerServerRPCHook(rpcHook);
+
+        // setRequestPipelineTest
+        final RequestPipeline requestPipeline = (ctx, request) -> {
+
+        };
+        brokerController.setRequestPipeline(requestPipeline);
+
+        NettyRemotingAbstract tcpRemotingServer = (NettyRemotingAbstract) brokerController.getRemotingServer();
+        Assert.assertTrue(tcpRemotingServer.getRPCHook().contains(rpcHook));
+
+        NettyRemotingAbstract fastRemotingServer = (NettyRemotingAbstract) brokerController.getFastRemotingServer();
+        Assert.assertTrue(fastRemotingServer.getRPCHook().contains(rpcHook));
+
+        NettyRemotingAbstract mockRemotingServer1 = (NettyRemotingAbstract) brokerController.getRemotingServerByName(mockRemotingServerName);
+        Assert.assertTrue(mockRemotingServer1.getRPCHook().contains(rpcHook));
+        Assert.assertSame(mockRemotingServer, mockRemotingServer1);
+    }
 }
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java
index 2541e755e39..766fcdd1e28 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java
@@ -20,28 +20,43 @@
 import com.google.common.base.Predicate;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import io.netty.channel.DefaultChannelPromise;
+import io.netty.util.concurrent.DefaultEventExecutor;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.rocketmq.auth.config.AuthConfig;
 import org.apache.rocketmq.broker.out.BrokerOuterAPI;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.PullStatus;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.BrokerIdentity;
 import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce;
 import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
 import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
 import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.netty.ResponseFuture;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
 import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
 import org.apache.rocketmq.remoting.protocol.body.ClusterInfo;
 import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
+import org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.namesrv.QueryDataVersionResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader;
 import org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;
@@ -55,9 +70,12 @@
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mock;
 import org.mockito.Spy;
+import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
-import org.mockito.junit.MockitoJUnitRunner;
 import org.mockito.stubbing.Answer;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -68,9 +86,11 @@
 import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.ArgumentMatchers.isNull;
 import static org.mockito.AdditionalMatchers.or;
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.when;
 
-@RunWith(MockitoJUnitRunner.class)
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(NettyRemotingClient.class)
 public class BrokerOuterAPITest {
     @Mock
     private ChannelHandlerContext handlerContext;
@@ -93,7 +113,7 @@ public class BrokerOuterAPITest {
     private BrokerOuterAPI brokerOuterAPI;
 
     public void init() throws Exception {
-        brokerOuterAPI = new BrokerOuterAPI(new NettyClientConfig());
+        brokerOuterAPI = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());
         Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
         field.setAccessible(true);
         field.set(brokerOuterAPI, nettyRemotingClient);
@@ -116,6 +136,9 @@ public void test_needRegister_normal() throws Exception {
 
     @Test
     public void test_needRegister_timeout() throws Exception {
+        if (MixAll.isMac()) {
+            return;
+        }
         init();
         brokerOuterAPI.start();
 
@@ -131,7 +154,7 @@ public RemotingCommand answer(InvocationOnMock invocation) throws Throwable {
                 } else if (invocation.getArgument(0) == nameserver2) {
                     return buildResponse(Boolean.FALSE);
                 } else if (invocation.getArgument(0) == nameserver3) {
-                    TimeUnit.MILLISECONDS.sleep(timeOut + 20);
+                    TimeUnit.MILLISECONDS.sleep(timeOut + 100);  // Increase sleep time to force timeout
                     return buildResponse(Boolean.TRUE);
                 }
                 return buildResponse(Boolean.TRUE);
@@ -250,4 +273,154 @@ public void testLookupAddressByDomain() throws Exception {
         });
         Assert.assertTrue(result.get());
     }
+
+    @Test
+    public void testPullMessageFromSpecificBrokerAsync_createChannel_null() throws Exception {
+        NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));
+        PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(null);
+        BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());
+        Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
+        field.setAccessible(true);
+        field.set(api, mockClient);
+
+        Triple<PullResult, String, Boolean> rst = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join();
+        Assert.assertNull(rst.getLeft());
+        Assert.assertTrue(rst.getMiddle().contains("connect"));
+        Assert.assertTrue(rst.getRight()); // need retry
+    }
+
+    @Test
+    public void testPullMessageFromSpecificBrokerAsync_createChannel_future_notSuccess() throws Exception {
+        NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));
+        DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor()));
+        PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(promise);
+        BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());
+        Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
+        field.setAccessible(true);
+        field.set(api, mockClient);
+
+        promise.tryFailure(new Throwable());
+        Triple<PullResult, String, Boolean> rst
+                = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join();
+        Assert.assertNull(rst.getLeft());
+        Assert.assertTrue(rst.getMiddle().contains("connect"));
+        Assert.assertTrue(rst.getRight()); // need retry
+    }
+
+    // skip other future status test
+
+    @Test
+    public void testPullMessageFromSpecificBrokerAsync_timeout() throws Exception {
+        Channel channel = Mockito.mock(Channel.class);
+        when(channel.isActive()).thenReturn(true);
+        NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));
+        DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor()));
+        PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(promise);
+        when(promise.channel()).thenReturn(channel);
+        BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());
+        Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
+        field.setAccessible(true);
+        field.set(api, mockClient);
+
+        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
+        doReturn(future).when(mockClient).invokeImpl(any(Channel.class), any(RemotingCommand.class), anyLong());
+        promise.trySuccess(null);
+        future.completeExceptionally(new RemotingTimeoutException("wait response on the channel timeout"));
+        Triple<PullResult, String, Boolean> rst = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join();
+        Assert.assertNull(rst.getLeft());
+        Assert.assertTrue(rst.getMiddle().contains("timeout"));
+        Assert.assertTrue(rst.getRight()); // need retry
+    }
+
+    @Test
+    public void testPullMessageFromSpecificBrokerAsync_brokerReturn_pullStatusCode() throws Exception {
+        Channel channel = Mockito.mock(Channel.class);
+        when(channel.isActive()).thenReturn(true);
+        NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));
+        DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor()));
+        PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(promise);
+        when(promise.channel()).thenReturn(channel);
+        BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());
+        Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
+        field.setAccessible(true);
+        field.set(api, mockClient);
+
+        int[] respCodes = new int[] {ResponseCode.SUCCESS, ResponseCode.PULL_NOT_FOUND, ResponseCode.PULL_RETRY_IMMEDIATELY, ResponseCode.PULL_OFFSET_MOVED};
+        PullStatus[] respStatus = new PullStatus[] {PullStatus.FOUND, PullStatus.NO_NEW_MSG, PullStatus.NO_MATCHED_MSG, PullStatus.OFFSET_ILLEGAL};
+        for (int i = 0; i < respCodes.length; i++) {
+            CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
+            doReturn(future).when(mockClient).invokeImpl(any(Channel.class), any(RemotingCommand.class), anyLong());
+            RemotingCommand response = mockPullMessageResponse(respCodes[i]);
+            ResponseFuture responseFuture = new ResponseFuture(channel, 0, null, 1000,
+                    resp -> { }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));
+            responseFuture.setResponseCommand(response);
+            promise.trySuccess(null);
+            future.complete(responseFuture);
+
+            Triple<PullResult, String, Boolean> rst = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join();
+            Assert.assertEquals(respStatus[i], rst.getLeft().getPullStatus());
+            if (ResponseCode.SUCCESS == respCodes[i]) {
+                Assert.assertEquals(1, rst.getLeft().getMsgFoundList().size());
+            } else {
+                Assert.assertNull(rst.getLeft().getMsgFoundList());
+            }
+            Assert.assertEquals(respStatus[i].name(), rst.getMiddle());
+            Assert.assertFalse(rst.getRight()); // no retry
+        }
+    }
+
+    @Test
+    public void testPullMessageFromSpecificBrokerAsync_brokerReturn_allOtherResponseCode() throws Exception {
+        Channel channel = Mockito.mock(Channel.class);
+        when(channel.isActive()).thenReturn(true);
+        NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));
+        DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor()));
+        PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(promise);
+        when(promise.channel()).thenReturn(channel);
+        BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());
+        Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
+        field.setAccessible(true);
+        field.set(api, mockClient);
+
+        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
+        doReturn(future).when(mockClient).invokeImpl(any(Channel.class), any(RemotingCommand.class), anyLong());
+        // test one code here, skip others
+        RemotingCommand response = mockPullMessageResponse(ResponseCode.SUBSCRIPTION_NOT_EXIST);
+        ResponseFuture responseFuture = new ResponseFuture(channel, 0, null, 1000,
+                resp -> { }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));
+        responseFuture.setResponseCommand(response);
+        promise.trySuccess(null);
+        future.complete(responseFuture);
+
+        Triple<PullResult, String, Boolean> rst = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join();
+        Assert.assertNull(rst.getLeft());
+        Assert.assertTrue(rst.getMiddle().contains(ResponseCode.SUBSCRIPTION_NOT_EXIST + ""));
+        Assert.assertTrue(rst.getRight()); // need retry
+    }
+
+    private RemotingCommand mockPullMessageResponse(int responseCode) throws Exception {
+        RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class);
+        response.setCode(responseCode);
+        if (responseCode == ResponseCode.SUCCESS) {
+            MessageExt msg = new MessageExt();
+            msg.setBody("HW".getBytes());
+            msg.setTopic("topic");
+            msg.setBornHost(new InetSocketAddress("127.0.0.1", 9000));
+            msg.setStoreHost(new InetSocketAddress("127.0.0.1", 9000));
+            byte[] encode = MessageDecoder.encode(msg, false);
+            response.setBody(encode);
+        }
+        PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader();
+        responseHeader.setNextBeginOffset(0L);
+        responseHeader.setMaxOffset(0L);
+        responseHeader.setMinOffset(0L);
+        responseHeader.setOffsetDelta(0L);
+        responseHeader.setTopicSysFlag(0);
+        responseHeader.setGroupSysFlag(0);
+        responseHeader.setSuggestWhichBrokerId(0L);
+        responseHeader.setForbiddenType(0);
+        response.makeCustomHeaderToNet();
+        return response;
+    }
+
 }
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerPathConfigHelperTest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerPathConfigHelperTest.java
index 3b260540838..61a0891c9e1 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerPathConfigHelperTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerPathConfigHelperTest.java
@@ -25,7 +25,7 @@
 public class BrokerPathConfigHelperTest {
 
     @Test
-    public void testGetLmqConsumerOffsetPath() {
+    public void testGetPath() {
         String lmqConsumerOffsetPath = BrokerPathConfigHelper.getLmqConsumerOffsetPath("/home/admin/store".replace("/", File.separator));
         assertEquals("/home/admin/store/config/lmqConsumerOffset.json".replace("/", File.separator), lmqConsumerOffsetPath);
 
@@ -38,5 +38,25 @@ public void testGetLmqConsumerOffsetPath() {
         String subscriptionGroupPath = BrokerPathConfigHelper.getSubscriptionGroupPath("/home/admin/store".replace("/", File.separator));
         assertEquals("/home/admin/store/config/subscriptionGroup.json".replace("/", File.separator), subscriptionGroupPath);
 
+        String topicQueueMappingPath = BrokerPathConfigHelper.getTopicQueueMappingPath("/home/admin/store".replace("/", File.separator));
+        assertEquals("/home/admin/store/config/topicQueueMapping.json".replace("/", File.separator), topicQueueMappingPath);
+
+        String consumerOrderInfoPath = BrokerPathConfigHelper.getConsumerOrderInfoPath("/home/admin/store".replace("/", File.separator));
+        assertEquals("/home/admin/store/config/consumerOrderInfo.json".replace("/", File.separator), consumerOrderInfoPath);
+
+        String timercheckPath = BrokerPathConfigHelper.getTimerCheckPath("/home/admin/store".replace("/", File.separator));
+        assertEquals("/home/admin/store/config/timercheck".replace("/", File.separator), timercheckPath);
+
+        String timermetricsPath = BrokerPathConfigHelper.getTimerMetricsPath("/home/admin/store".replace("/", File.separator));
+        assertEquals("/home/admin/store/config/timermetrics".replace("/", File.separator), timermetricsPath);
+
+        String transactionMetricsPath = BrokerPathConfigHelper.getTransactionMetricsPath("/home/admin/store".replace("/", File.separator));
+        assertEquals("/home/admin/store/config/transactionMetrics".replace("/", File.separator), transactionMetricsPath);
+
+        String consumerFilterPath = BrokerPathConfigHelper.getConsumerFilterPath("/home/admin/store".replace("/", File.separator));
+        assertEquals("/home/admin/store/config/consumerFilter.json".replace("/", File.separator), consumerFilterPath);
+
+        String messageRequestModePath = BrokerPathConfigHelper.getMessageRequestModePath("/home/admin/store".replace("/", File.separator));
+        assertEquals("/home/admin/store/config/messageRequestMode.json".replace("/", File.separator), messageRequestModePath);
     }
-}
\ No newline at end of file
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java
index 8c909824348..a23ad20037c 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java
@@ -18,10 +18,7 @@
 package org.apache.rocketmq.broker.client;
 
 import io.netty.channel.Channel;
-import java.util.HashSet;
-import java.util.Set;
 import org.apache.rocketmq.broker.BrokerController;
-import org.apache.rocketmq.broker.client.net.Broker2Client;
 import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
@@ -30,13 +27,25 @@
 import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
 import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
-import org.assertj.core.api.Assertions;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.Mock;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
@@ -49,19 +58,10 @@ public class ConsumerManagerTest {
 
     private ConsumerManager consumerManager;
 
-    private DefaultConsumerIdsChangeListener defaultConsumerIdsChangeListener;
-
     @Mock
     private BrokerController brokerController;
 
-    @Mock
-    private ConsumerFilterManager consumerFilterManager;
-
-    private BrokerConfig brokerConfig = new BrokerConfig();
-
-    private Broker2Client broker2Client;
-
-    private BrokerStatsManager brokerStatsManager;
+    private final BrokerConfig brokerConfig = new BrokerConfig();
 
     private static final String GROUP = "DEFAULT_GROUP";
 
@@ -74,40 +74,38 @@ public class ConsumerManagerTest {
     @Before
     public void before() {
         clientChannelInfo = new ClientChannelInfo(channel, CLIENT_ID, LanguageCode.JAVA, VERSION);
-        defaultConsumerIdsChangeListener = new DefaultConsumerIdsChangeListener(brokerController);
-        brokerStatsManager = new BrokerStatsManager(brokerConfig);
-        consumerManager = new ConsumerManager(defaultConsumerIdsChangeListener, brokerStatsManager, brokerConfig);
-        broker2Client = new Broker2Client(brokerController);
+        DefaultConsumerIdsChangeListener defaultConsumerIdsChangeListener = new DefaultConsumerIdsChangeListener(brokerController);
+        BrokerStatsManager brokerStatsManager = new BrokerStatsManager(brokerConfig);
+        consumerManager = spy(new ConsumerManager(defaultConsumerIdsChangeListener, brokerStatsManager, brokerConfig));
+        ConsumerFilterManager consumerFilterManager = mock(ConsumerFilterManager.class);
         when(brokerController.getConsumerFilterManager()).thenReturn(consumerFilterManager);
-        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
-        when(brokerController.getBroker2Client()).thenReturn(broker2Client);
     }
 
     @Test
     public void compensateBasicConsumerInfoTest() {
         ConsumerGroupInfo consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true);
-        Assertions.assertThat(consumerGroupInfo).isNull();
+        assertThat(consumerGroupInfo).isNull();
 
         consumerManager.compensateBasicConsumerInfo(GROUP, ConsumeType.CONSUME_ACTIVELY, MessageModel.BROADCASTING);
         consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true);
-        Assertions.assertThat(consumerGroupInfo).isNotNull();
-        Assertions.assertThat(consumerGroupInfo.getConsumeType()).isEqualTo(ConsumeType.CONSUME_ACTIVELY);
-        Assertions.assertThat(consumerGroupInfo.getMessageModel()).isEqualTo(MessageModel.BROADCASTING);
+        assertThat(consumerGroupInfo).isNotNull();
+        assertThat(consumerGroupInfo.getConsumeType()).isEqualTo(ConsumeType.CONSUME_ACTIVELY);
+        assertThat(consumerGroupInfo.getMessageModel()).isEqualTo(MessageModel.BROADCASTING);
     }
 
     @Test
     public void compensateSubscribeDataTest() {
         ConsumerGroupInfo consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true);
-        Assertions.assertThat(consumerGroupInfo).isNull();
+        assertThat(consumerGroupInfo).isNull();
 
         consumerManager.compensateSubscribeData(GROUP, TOPIC, new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL));
         consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true);
-        Assertions.assertThat(consumerGroupInfo).isNotNull();
-        Assertions.assertThat(consumerGroupInfo.getSubscriptionTable().size()).isEqualTo(1);
+        assertThat(consumerGroupInfo).isNotNull();
+        assertThat(consumerGroupInfo.getSubscriptionTable().size()).isEqualTo(1);
         SubscriptionData subscriptionData = consumerGroupInfo.getSubscriptionTable().get(TOPIC);
-        Assertions.assertThat(subscriptionData).isNotNull();
-        Assertions.assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC);
-        Assertions.assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL);
+        assertThat(subscriptionData).isNotNull();
+        assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC);
+        assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL);
     }
 
     @Test
@@ -118,7 +116,8 @@ public void registerConsumerTest() {
         subList.add(subscriptionData);
         consumerManager.registerConsumer(GROUP, clientChannelInfo, ConsumeType.CONSUME_PASSIVELY,
             MessageModel.BROADCASTING, ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET, subList, true);
-        Assertions.assertThat(consumerManager.getConsumerTable().get(GROUP)).isNotNull();
+        verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any());
+        assertThat(consumerManager.getConsumerTable().get(GROUP)).isNotNull();
     }
 
     @Test
@@ -128,63 +127,65 @@ public void unregisterConsumerTest() {
 
         // unregister
         consumerManager.unregisterConsumer(GROUP, clientChannelInfo, true);
-        Assertions.assertThat(consumerManager.getConsumerTable().get(GROUP)).isNull();
+        verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any());
+        assertThat(consumerManager.getConsumerTable().get(GROUP)).isNull();
     }
 
     @Test
     public void findChannelTest() {
         register();
         final ClientChannelInfo consumerManagerChannel = consumerManager.findChannel(GROUP, CLIENT_ID);
-        Assertions.assertThat(consumerManagerChannel).isNotNull();
+        assertThat(consumerManagerChannel).isNotNull();
     }
 
     @Test
     public void findSubscriptionDataTest() {
         register();
         final SubscriptionData subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC);
-        Assertions.assertThat(subscriptionData).isNotNull();
+        assertThat(subscriptionData).isNotNull();
     }
 
     @Test
     public void findSubscriptionDataCountTest() {
         register();
         final int count = consumerManager.findSubscriptionDataCount(GROUP);
-        assert count > 0;
+        assertTrue(count > 0);
     }
 
     @Test
     public void findSubscriptionTest() {
         SubscriptionData subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, true);
-        Assertions.assertThat(subscriptionData).isNull();
+        assertThat(subscriptionData).isNull();
 
         consumerManager.compensateSubscribeData(GROUP, TOPIC, new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL));
         subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, true);
-        Assertions.assertThat(subscriptionData).isNotNull();
-        Assertions.assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC);
-        Assertions.assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL);
+        assertThat(subscriptionData).isNotNull();
+        assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC);
+        assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL);
 
         subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, false);
-        Assertions.assertThat(subscriptionData).isNull();
+        assertThat(subscriptionData).isNull();
     }
 
     @Test
     public void scanNotActiveChannelTest() {
         clientChannelInfo.setLastUpdateTimestamp(System.currentTimeMillis() - brokerConfig.getChannelExpiredTimeout() * 2);
         consumerManager.scanNotActiveChannel();
-        Assertions.assertThat(consumerManager.getConsumerTable().size()).isEqualTo(0);
+        assertThat(consumerManager.getConsumerTable().size()).isEqualTo(0);
     }
 
     @Test
     public void queryTopicConsumeByWhoTest() {
         register();
         final HashSet<String> consumeGroup = consumerManager.queryTopicConsumeByWho(TOPIC);
-        assert consumeGroup.size() > 0;
+        assertFalse(consumeGroup.isEmpty());
     }
 
     @Test
     public void doChannelCloseEventTest() {
         consumerManager.doChannelCloseEvent("127.0.0.1", channel);
-        assert consumerManager.getConsumerTable().size() == 0;
+        verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any());
+        assertEquals(0, consumerManager.getConsumerTable().size());
     }
 
     private void register() {
@@ -203,8 +204,8 @@ public void removeExpireConsumerGroupInfo() {
         consumerManager.compensateSubscribeData(GROUP, TOPIC, subscriptionData);
         consumerManager.compensateSubscribeData(GROUP, TOPIC + "_1", new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL));
         consumerManager.removeExpireConsumerGroupInfo();
-        Assertions.assertThat(consumerManager.getConsumerGroupInfo(GROUP, true)).isNotNull();
-        Assertions.assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC)).isNull();
-        Assertions.assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC + "_1")).isNotNull();
+        assertThat(consumerManager.getConsumerGroupInfo(GROUP, true)).isNotNull();
+        assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC)).isNull();
+        assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC + "_1")).isNotNull();
     }
 }
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java
index 3d6091e02fb..451b0e044c7 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java
@@ -22,6 +22,8 @@
 import java.util.Map;
 
 import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.remoting.protocol.LanguageCode;
 import org.junit.Before;
 import org.junit.Test;
@@ -36,6 +38,8 @@
 
 @RunWith(MockitoJUnitRunner.class)
 public class ProducerManagerTest {
+
+    private BrokerConfig brokerConfig;
     private ProducerManager producerManager;
     private String group = "FooBar";
     private ClientChannelInfo clientInfo;
@@ -45,7 +49,8 @@ public class ProducerManagerTest {
 
     @Before
     public void init() {
-        producerManager = new ProducerManager();
+        brokerConfig = new BrokerConfig();
+        producerManager = new ProducerManager(null, brokerConfig);
         clientInfo = new ClientChannelInfo(channel, "clientId", LanguageCode.JAVA, 0);
     }
 
@@ -140,10 +145,20 @@ public void doChannelCloseEvent() throws Exception {
     }
 
     @Test
-    public void testRegisterProducer() throws Exception {
+    public void testRegisterProducer() {
+        brokerConfig.setEnableRegisterProducer(false);
+        brokerConfig.setRejectTransactionMessage(true);
         producerManager.registerProducer(group, clientInfo);
         Map<Channel, ClientChannelInfo> channelMap = producerManager.getGroupChannelTable().get(group);
         Channel channel1 = producerManager.findChannel("clientId");
+        assertThat(channelMap).isNull();
+        assertThat(channel1).isNull();
+
+        brokerConfig.setEnableRegisterProducer(true);
+        brokerConfig.setRejectTransactionMessage(false);
+        producerManager.registerProducer(group, clientInfo);
+        channelMap = producerManager.getGroupChannelTable().get(group);
+        channel1 = producerManager.findChannel("clientId");
         assertThat(channelMap).isNotNull();
         assertThat(channel1).isNotNull();
         assertThat(channelMap.get(channel)).isEqualTo(clientInfo);
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java
new file mode 100644
index 00000000000..ccb489aead2
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.client.net;
+
+import io.netty.channel.Channel;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
+import org.apache.rocketmq.broker.client.ConsumerManager;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MQVersion;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.remoting.RemotingServer;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody;
+import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;
+import org.apache.rocketmq.store.MessageStore;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class Broker2ClientTest {
+    
+    @Mock
+    private BrokerController brokerController;
+    
+    @Mock
+    private RemotingServer remotingServer;
+    
+    @Mock
+    private ConsumerManager consumerManager;
+    
+    @Mock
+    private TopicConfigManager topicConfigManager;
+    
+    @Mock
+    private ConsumerOffsetManager consumerOffsetManager;
+    
+    @Mock
+    private Channel channel;
+    
+    @Mock
+    private ConsumerGroupInfo consumerGroupInfo;
+    
+    private Broker2Client broker2Client;
+    
+    private final String defaultTopic = "defaultTopic";
+    
+    private final String defaultBroker = "defaultBroker";
+    
+    private final String defaultGroup = "defaultGroup";
+    
+    private final long timestamp = System.currentTimeMillis();
+    
+    private final boolean isForce = true;
+    
+    @Before
+    public void init() {
+        broker2Client = new Broker2Client(brokerController);
+        when(brokerController.getRemotingServer()).thenReturn(remotingServer);
+        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
+        when(brokerController.getConsumerManager()).thenReturn(consumerManager);
+        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);
+        when(brokerController.getBrokerConfig()).thenReturn(mock(BrokerConfig.class));
+        when(brokerController.getMessageStore()).thenReturn(mock(MessageStore.class));
+        when(consumerManager.getConsumerGroupInfo(any())).thenReturn(consumerGroupInfo);
+    }
+    
+    @Test
+    public void testCheckProducerTransactionState() throws Exception {
+        CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader();
+        broker2Client.checkProducerTransactionState("group", channel, requestHeader, createMessageExt());
+        verify(remotingServer).invokeOneway(eq(channel), any(RemotingCommand.class), eq(10L));
+    }
+    
+    @Test
+    public void testCheckProducerTransactionStateException() throws Exception {
+        CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader();
+        MessageExt messageExt = createMessageExt();
+        doThrow(new RuntimeException("Test Exception"))
+                .when(remotingServer)
+                .invokeOneway(any(Channel.class),
+                        any(RemotingCommand.class),
+                        anyLong());
+        broker2Client.checkProducerTransactionState("group", channel, requestHeader, messageExt);
+        verify(brokerController.getRemotingServer()).invokeOneway(eq(channel), any(RemotingCommand.class), eq(10L));
+    }
+    
+    @Test
+    public void testResetOffsetNoTopicConfig() throws RemotingCommandException {
+        when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(null);
+        RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce);
+        assertEquals(ResponseCode.TOPIC_NOT_EXIST, response.getCode());
+    }
+    
+    @Test
+    public void testResetOffsetNoConsumerGroupInfo() throws RemotingCommandException {
+        TopicConfig topicConfig = mock(TopicConfig.class);
+        when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig);
+        when(topicConfig.getWriteQueueNums()).thenReturn(1);
+        when(consumerOffsetManager.queryOffset(defaultGroup, defaultTopic, 0)).thenReturn(0L);
+        RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce);
+        assertEquals(ResponseCode.CONSUMER_NOT_ONLINE, response.getCode());
+    }
+    
+    @Test
+    public void testResetOffset() throws RemotingCommandException {
+        TopicConfig topicConfig = mock(TopicConfig.class);
+        when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig);
+        when(topicConfig.getWriteQueueNums()).thenReturn(1);
+        when(brokerController.getConsumerOffsetManager().queryOffset(defaultGroup, defaultTopic, 0)).thenReturn(0L);
+        BrokerConfig brokerConfig = mock(BrokerConfig.class);
+        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
+        when(brokerConfig.getBrokerName()).thenReturn(defaultBroker);
+        ConsumerGroupInfo consumerGroupInfo = mock(ConsumerGroupInfo.class);
+        when(consumerManager.getConsumerGroupInfo(defaultGroup)).thenReturn(consumerGroupInfo);
+        RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce);
+        assertEquals(ResponseCode.CONSUMER_NOT_ONLINE, response.getCode());
+    }
+    
+    @Test
+    public void testGetConsumeStatusNoConsumerOnline() {
+        when(consumerGroupInfo.getChannelInfoTable()).thenReturn(new ConcurrentHashMap<>());
+        RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, "");
+        assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode());
+    }
+    
+    @Test
+    public void testGetConsumeStatusClientDoesNotSupportFeature() {
+        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, "defaultClientId", null, MQVersion.Version.V3_0_6.ordinal());
+        ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable = new ConcurrentHashMap<>();
+        channelInfoTable.put(channel, clientChannelInfo);
+        when(consumerGroupInfo.getChannelInfoTable()).thenReturn(channelInfoTable);
+        RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, "");
+        assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode());
+    }
+    
+    @Test
+    public void testGetConsumeStatus() throws Exception {
+        ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable = new ConcurrentHashMap<>();
+        ClientChannelInfo clientChannelInfo = mock(ClientChannelInfo.class);
+        when(clientChannelInfo.getVersion()).thenReturn(MQVersion.CURRENT_VERSION);
+        channelInfoTable.put(channel, clientChannelInfo);
+        when(consumerGroupInfo.getChannelInfoTable()).thenReturn(channelInfoTable);
+        RemotingCommand responseMock = mock(RemotingCommand.class);
+        when(responseMock.getCode()).thenReturn(ResponseCode.SUCCESS);
+        when(responseMock.getBody()).thenReturn("{\"consumerTable\":{}}".getBytes(StandardCharsets.UTF_8));
+        when(remotingServer.invokeSync(any(Channel.class), any(RemotingCommand.class), anyLong())).thenReturn(responseMock);
+        RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, "");
+        assertEquals(ResponseCode.SUCCESS, response.getCode());
+        GetConsumerStatusBody body = RemotingSerializable.decode(response.getBody(), GetConsumerStatusBody.class);
+        assertEquals(1, body.getConsumerTable().size());
+    }
+    
+    private MessageExt createMessageExt() {
+        MessageExt result = new MessageExt();
+        result.setBody("body".getBytes(StandardCharsets.UTF_8));
+        result.setTopic(defaultTopic);
+        result.setBrokerName(defaultBroker);
+        result.putUserProperty("key", "value");
+        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup);
+        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1");
+        result.setKeys("keys");
+        SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911);
+        SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911);
+        result.setStoreHost(storeHost);
+        result.setBornHost(bornHost);
+        return result;
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManagerTest.java
new file mode 100644
index 00000000000..e231d61b6a7
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManagerTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.client.rebalance;
+
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RebalanceLockManagerTest {
+    
+    @Mock
+    private RebalanceLockManager.LockEntry lockEntry;
+    
+    private final RebalanceLockManager rebalanceLockManager = new RebalanceLockManager();
+    
+    private final String defaultTopic = "defaultTopic";
+    
+    private final String defaultBroker = "defaultBroker";
+    
+    private final String defaultGroup = "defaultGroup";
+    
+    private final String defaultClientId = "defaultClientId";
+    
+    @Test
+    public void testIsLockAllExpiredGroupNotExist() {
+        assertTrue(rebalanceLockManager.isLockAllExpired(defaultGroup));
+    }
+    
+    @Test
+    public void testIsLockAllExpiredGroupExist() throws IllegalAccessException {
+        FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true);
+        when(lockEntry.isExpired()).thenReturn(false);
+        assertFalse(rebalanceLockManager.isLockAllExpired(defaultGroup));
+    }
+    
+    @Test
+    public void testIsLockAllExpiredGroupExistSomeExpired() throws IllegalAccessException {
+        FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true);
+        when(lockEntry.isExpired()).thenReturn(true).thenReturn(false);
+        assertFalse(rebalanceLockManager.isLockAllExpired(defaultGroup));
+    }
+    
+    @Test
+    public void testTryLockNotLocked() {
+        assertTrue(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId));
+    }
+    
+    @Test
+    public void testTryLockSameClient() throws IllegalAccessException {
+        when(lockEntry.isLocked(defaultClientId)).thenReturn(true);
+        FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true);
+        assertTrue(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId));
+    }
+    
+    @Test
+    public void testTryLockDifferentClient() throws Exception {
+        when(lockEntry.isLocked(defaultClientId)).thenReturn(false);
+        FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true);
+        assertFalse(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId));
+    }
+    
+    @Test
+    public void testTryLockButExpired() throws IllegalAccessException {
+        when(lockEntry.isExpired()).thenReturn(true);
+        FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true);
+        assertTrue(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId));
+    }
+    
+    @Test
+    public void testTryLockBatchAllLocked() {
+        Set<MessageQueue> mqs = createMessageQueue(2);
+        Set<MessageQueue> actual = rebalanceLockManager.tryLockBatch(defaultGroup, mqs, defaultClientId);
+        assertEquals(mqs, actual);
+    }
+    
+    @Test
+    public void testTryLockBatchNoneLocked() throws IllegalAccessException {
+        when(lockEntry.isLocked(defaultClientId)).thenReturn(false);
+        FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true);
+        Set<MessageQueue> actual = rebalanceLockManager.tryLockBatch(defaultGroup, createMessageQueue(2), defaultClientId);
+        assertTrue(actual.isEmpty());
+    }
+    
+    @Test
+    public void testTryLockBatchSomeLocked() throws IllegalAccessException {
+        Set<MessageQueue> mqs = new HashSet<>();
+        MessageQueue mq1 = new MessageQueue(defaultTopic, defaultBroker, 0);
+        MessageQueue mq2 = new MessageQueue(defaultTopic, defaultBroker, 1);
+        mqs.add(mq1);
+        mqs.add(mq2);
+        when(lockEntry.isLocked(defaultClientId)).thenReturn(true).thenReturn(false);
+        FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true);
+        Set<MessageQueue> actual = rebalanceLockManager.tryLockBatch(defaultGroup, mqs, defaultClientId);
+        Set<MessageQueue> expected = new HashSet<>();
+        expected.add(mq2);
+        assertEquals(expected, actual);
+    }
+    
+    @Test
+    public void testUnlockBatch() throws IllegalAccessException {
+        when(lockEntry.getClientId()).thenReturn(defaultClientId);
+        ConcurrentMap<String, ConcurrentHashMap<MessageQueue, RebalanceLockManager.LockEntry>> mqLockTable = createMQLockTable();
+        FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", mqLockTable, true);
+        rebalanceLockManager.unlockBatch(defaultGroup, createMessageQueue(1), defaultClientId);
+        assertEquals(1, mqLockTable.get(defaultGroup).values().size());
+    }
+    
+    @Test
+    public void testUnlockBatchByOtherClient() throws IllegalAccessException {
+        when(lockEntry.getClientId()).thenReturn("otherClientId");
+        ConcurrentMap<String, ConcurrentHashMap<MessageQueue, RebalanceLockManager.LockEntry>> mqLockTable = createMQLockTable();
+        FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", mqLockTable, true);
+        rebalanceLockManager.unlockBatch(defaultGroup, createMessageQueue(1), defaultClientId);
+        assertEquals(2, mqLockTable.get(defaultGroup).values().size());
+    }
+    
+    private MessageQueue createDefaultMessageQueue() {
+        return createMessageQueue(1).iterator().next();
+    }
+    
+    private Set<MessageQueue> createMessageQueue(final int count) {
+        Set<MessageQueue> result = new HashSet<>();
+        for (int i = 0; i < count; i++) {
+            result.add(new MessageQueue(defaultTopic, defaultBroker, i));
+        }
+        return result;
+    }
+    
+    private ConcurrentMap<String, ConcurrentHashMap<MessageQueue, RebalanceLockManager.LockEntry>> createMQLockTable() {
+        MessageQueue messageQueue1 = new MessageQueue(defaultTopic, defaultBroker, 0);
+        MessageQueue messageQueue2 = new MessageQueue(defaultTopic, defaultBroker, 1);
+        ConcurrentHashMap<MessageQueue, RebalanceLockManager.LockEntry> lockEntryMap = new ConcurrentHashMap<>();
+        lockEntryMap.put(messageQueue1, lockEntry);
+        lockEntryMap.put(messageQueue2, lockEntry);
+        ConcurrentMap<String, ConcurrentHashMap<MessageQueue, RebalanceLockManager.LockEntry>> result = new ConcurrentHashMap<>();
+        result.put(defaultGroup, lockEntryMap);
+        return result;
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/coldctr/ColdDataCgCtrServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/coldctr/ColdDataCgCtrServiceTest.java
new file mode 100644
index 00000000000..7ccb3422fe3
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/coldctr/ColdDataCgCtrServiceTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.coldctr;
+
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.coldctr.AccAndTimeStamp;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ColdDataCgCtrServiceTest {
+
+    @Mock
+    private BrokerController brokerController;
+
+    @Mock
+    private BrokerConfig brokerConfig;
+
+    private ColdDataCgCtrService coldDataCgCtrService;
+
+    @Before
+    public void init() throws IllegalAccessException {
+        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
+        coldDataCgCtrService = new ColdDataCgCtrService(brokerController);
+        FieldUtils.writeField(coldDataCgCtrService, "cgColdThresholdMapRuntime", createCgColdThresholdMapRuntime(), true);
+        FieldUtils.writeField(coldDataCgCtrService, "cgColdThresholdMapConfig", createCgColdThresholdMapConfig(), true);
+    }
+
+    @Test
+    public void testGetColdDataFlowCtrInfo() {
+        String actual = coldDataCgCtrService.getColdDataFlowCtrInfo();
+        assertTrue(actual.contains("\"globalAcc\":0"));
+        assertTrue(actual.contains("\"cgColdReadThreshold\":0"));
+        assertTrue(actual.contains("\"globalColdReadThreshold\":0"));
+        assertTrue(actual.contains("\"configTable\":{\"consumerGroup2\":2048}"));
+        assertTrue(actual.contains("\"runtimeTable\":{\"consumerGroup1\":{\"coldAcc\":1,\"createTimeMills\":1,\"lastColdReadTimeMills\":1}}"));
+    }
+
+    private Map<String, AccAndTimeStamp> createCgColdThresholdMapRuntime() {
+        Map<String, AccAndTimeStamp> result = new ConcurrentHashMap<>();
+        AccAndTimeStamp accAndTimeStamp = new AccAndTimeStamp(new AtomicLong(1L));
+        accAndTimeStamp.setCreateTimeMills(1L);
+        accAndTimeStamp.setLastColdReadTimeMills(1L);
+        result.put("consumerGroup1", accAndTimeStamp);
+        return result;
+    }
+
+    private ConcurrentHashMap<String, Long> createCgColdThresholdMapConfig() {
+        ConcurrentHashMap<String, Long> result = new ConcurrentHashMap<>();
+        result.put("consumerGroup2", 2048L);
+        return result;
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2Test.java b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2Test.java
new file mode 100644
index 00000000000..132bd5c1a56
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2Test.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.config.v2;
+
+import java.io.File;
+import java.io.IOException;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ConsumerOffsetManagerV2Test {
+
+    private ConfigStorage configStorage;
+
+    private ConsumerOffsetManagerV2 consumerOffsetManagerV2;
+
+    @Mock
+    private BrokerController controller;
+
+    private MessageStoreConfig messageStoreConfig;
+
+    @Rule
+    public TemporaryFolder tf = new TemporaryFolder();
+
+    @After
+    public void cleanUp() {
+        if (null != configStorage) {
+            configStorage.shutdown();
+        }
+    }
+
+    @Before
+    public void setUp() throws IOException {
+        BrokerConfig brokerConfig = new BrokerConfig();
+        Mockito.doReturn(brokerConfig).when(controller).getBrokerConfig();
+
+        File configStoreDir = tf.newFolder();
+        messageStoreConfig = new MessageStoreConfig();
+        messageStoreConfig.setStorePathRootDir(configStoreDir.getAbsolutePath());
+        configStorage = new ConfigStorage(messageStoreConfig);
+        configStorage.start();
+        consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage);
+    }
+
+    /**
+     * Verify consumer offset can survive restarts
+     */
+    @Test
+    public void testCommitOffset_Standard() {
+        Assert.assertTrue(consumerOffsetManagerV2.load());
+
+        String clientHost = "localhost";
+        String topic = "T1";
+        String group = "G0";
+        int queueId = 1;
+        long queueOffset = 100;
+        consumerOffsetManagerV2.commitOffset(clientHost, group, topic, queueId, queueOffset);
+        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId));
+
+        configStorage.shutdown();
+        consumerOffsetManagerV2.getOffsetTable().clear();
+        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId));
+
+        configStorage = new ConfigStorage(messageStoreConfig);
+        configStorage.start();
+        consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage);
+        consumerOffsetManagerV2.load();
+        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId));
+    }
+
+    /**
+     * Verify commit offset can survive config store restart
+     */
+    @Test
+    public void testCommitOffset_LMQ() {
+        Assert.assertTrue(consumerOffsetManagerV2.load());
+
+        String clientHost = "localhost";
+        String topic = MixAll.LMQ_PREFIX + "T1";
+        String group = "G0";
+        int queueId = 1;
+        long queueOffset = 100;
+        consumerOffsetManagerV2.commitOffset(clientHost, group, topic, queueId, queueOffset);
+        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId));
+
+        configStorage.shutdown();
+
+        configStorage = new ConfigStorage(messageStoreConfig);
+        configStorage.start();
+        consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage);
+        consumerOffsetManagerV2.load();
+        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId));
+    }
+
+
+    /**
+     * Verify commit offset can survive config store restart
+     */
+    @Test
+    public void testCommitPullOffset_LMQ() {
+        Assert.assertTrue(consumerOffsetManagerV2.load());
+
+        String clientHost = "localhost";
+        String topic = MixAll.LMQ_PREFIX + "T1";
+        String group = "G0";
+        int queueId = 1;
+        long queueOffset = 100;
+        consumerOffsetManagerV2.commitPullOffset(clientHost, group, topic, queueId, queueOffset);
+        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryPullOffset(group, topic, queueId));
+
+        configStorage.shutdown();
+
+        configStorage = new ConfigStorage(messageStoreConfig);
+        configStorage.start();
+        consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage);
+        consumerOffsetManagerV2.load();
+        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryPullOffset(group, topic, queueId));
+    }
+
+    /**
+     * Verify commit offset can survive config store restart
+     */
+    @Test
+    public void testRemoveByTopicAtGroup() {
+        Assert.assertTrue(consumerOffsetManagerV2.load());
+
+        String clientHost = "localhost";
+        String topic = MixAll.LMQ_PREFIX + "T1";
+        String topic2 = MixAll.LMQ_PREFIX + "T2";
+        String group = "G0";
+        int queueId = 1;
+        long queueOffset = 100;
+        consumerOffsetManagerV2.commitOffset(clientHost, group, topic, queueId, queueOffset);
+        consumerOffsetManagerV2.commitOffset(clientHost, group, topic2, queueId, queueOffset);
+        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId));
+        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic2, queueId));
+
+        consumerOffsetManagerV2.removeConsumerOffset(topic + ConsumerOffsetManager.TOPIC_GROUP_SEPARATOR + group);
+        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId));
+        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic2, queueId));
+
+        configStorage.shutdown();
+
+        configStorage = new ConfigStorage(messageStoreConfig);
+        configStorage.start();
+        consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage);
+        consumerOffsetManagerV2.load();
+        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId));
+        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic2, queueId));
+    }
+
+    /**
+     * Verify commit offset can survive config store restart
+     */
+    @Test
+    public void testRemoveByGroup() {
+        Assert.assertTrue(consumerOffsetManagerV2.load());
+
+        String clientHost = "localhost";
+        String topic = MixAll.LMQ_PREFIX + "T1";
+        String topic2 = MixAll.LMQ_PREFIX + "T2";
+        String group = "G0";
+        int queueId = 1;
+        long queueOffset = 100;
+        consumerOffsetManagerV2.commitOffset(clientHost, group, topic, queueId, queueOffset);
+        consumerOffsetManagerV2.commitOffset(clientHost, group, topic2, queueId, queueOffset);
+        Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId));
+        consumerOffsetManagerV2.removeOffset(group);
+        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId));
+        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic2, queueId));
+
+        configStorage.shutdown();
+
+        configStorage = new ConfigStorage(messageStoreConfig);
+        configStorage.start();
+        consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage);
+        consumerOffsetManagerV2.load();
+        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId));
+        Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic2, queueId));
+    }
+
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2Test.java b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2Test.java
new file mode 100644
index 00000000000..4ff8a81e60a
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2Test.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.config.v2;
+
+import java.io.File;
+import java.io.IOException;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.remoting.protocol.subscription.GroupRetryPolicy;
+import org.apache.rocketmq.remoting.protocol.subscription.GroupRetryPolicyType;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class SubscriptionGroupManagerV2Test {
+
+    private MessageStoreConfig messageStoreConfig;
+
+    private ConfigStorage configStorage;
+
+    private SubscriptionGroupManagerV2 subscriptionGroupManagerV2;
+
+    @Mock
+    private BrokerController controller;
+
+    @Mock
+    private MessageStore messageStore;
+
+    @Rule
+    public TemporaryFolder tf = new TemporaryFolder();
+
+    @After
+    public void cleanUp() {
+        if (null != configStorage) {
+            configStorage.shutdown();
+        }
+    }
+
+    @Before
+    public void setUp() throws IOException {
+        BrokerConfig brokerConfig = new BrokerConfig();
+        brokerConfig.setAutoCreateSubscriptionGroup(false);
+        Mockito.doReturn(brokerConfig).when(controller).getBrokerConfig();
+
+        Mockito.doReturn(messageStore).when(controller).getMessageStore();
+        Mockito.doReturn(1L).when(messageStore).getStateMachineVersion();
+
+        File configStoreDir = tf.newFolder();
+        messageStoreConfig = new MessageStoreConfig();
+        messageStoreConfig.setStorePathRootDir(configStoreDir.getAbsolutePath());
+        configStorage = new ConfigStorage(messageStoreConfig);
+        configStorage.start();
+        subscriptionGroupManagerV2 = new SubscriptionGroupManagerV2(controller, configStorage);
+    }
+
+
+    @Test
+    public void testUpdateSubscriptionGroupConfig() {
+        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
+        subscriptionGroupConfig.setGroupName("G1");
+        subscriptionGroupConfig.setConsumeEnable(true);
+        subscriptionGroupConfig.setRetryMaxTimes(16);
+        subscriptionGroupConfig.setGroupSysFlag(1);
+        GroupRetryPolicy retryPolicy = new GroupRetryPolicy();
+        retryPolicy.setType(GroupRetryPolicyType.EXPONENTIAL);
+        subscriptionGroupConfig.setGroupRetryPolicy(retryPolicy);
+        subscriptionGroupConfig.setBrokerId(1);
+        subscriptionGroupConfig.setConsumeBroadcastEnable(true);
+        subscriptionGroupConfig.setConsumeMessageOrderly(true);
+        subscriptionGroupConfig.setConsumeTimeoutMinute(30);
+        subscriptionGroupConfig.setConsumeFromMinEnable(true);
+        subscriptionGroupConfig.setWhichBrokerWhenConsumeSlowly(1);
+        subscriptionGroupConfig.setNotifyConsumerIdsChangedEnable(true);
+        subscriptionGroupManagerV2.updateSubscriptionGroupConfig(subscriptionGroupConfig);
+
+        SubscriptionGroupConfig found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName());
+        Assert.assertEquals(subscriptionGroupConfig, found);
+
+        subscriptionGroupManagerV2.getSubscriptionGroupTable().clear();
+        configStorage.shutdown();
+
+        configStorage = new ConfigStorage(messageStoreConfig);
+        configStorage.start();
+        subscriptionGroupManagerV2 = new SubscriptionGroupManagerV2(controller, configStorage);
+        subscriptionGroupManagerV2.load();
+        found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName());
+        Assert.assertEquals(subscriptionGroupConfig, found);
+    }
+
+
+    @Test
+    public void testDeleteSubscriptionGroupConfig() {
+        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
+        subscriptionGroupConfig.setGroupName("G1");
+        subscriptionGroupConfig.setConsumeEnable(true);
+        subscriptionGroupConfig.setRetryMaxTimes(16);
+        subscriptionGroupConfig.setGroupSysFlag(1);
+        GroupRetryPolicy retryPolicy = new GroupRetryPolicy();
+        retryPolicy.setType(GroupRetryPolicyType.EXPONENTIAL);
+        subscriptionGroupConfig.setGroupRetryPolicy(retryPolicy);
+        subscriptionGroupConfig.setBrokerId(1);
+        subscriptionGroupConfig.setConsumeBroadcastEnable(true);
+        subscriptionGroupConfig.setConsumeMessageOrderly(true);
+        subscriptionGroupConfig.setConsumeTimeoutMinute(30);
+        subscriptionGroupConfig.setConsumeFromMinEnable(true);
+        subscriptionGroupConfig.setWhichBrokerWhenConsumeSlowly(1);
+        subscriptionGroupConfig.setNotifyConsumerIdsChangedEnable(true);
+        subscriptionGroupManagerV2.updateSubscriptionGroupConfig(subscriptionGroupConfig);
+
+        SubscriptionGroupConfig found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName());
+        Assert.assertEquals(subscriptionGroupConfig, found);
+        subscriptionGroupManagerV2.removeSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName());
+
+        found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName());
+        Assert.assertNull(found);
+
+        configStorage.shutdown();
+
+        configStorage = new ConfigStorage(messageStoreConfig);
+        configStorage.start();
+
+        subscriptionGroupManagerV2 = new SubscriptionGroupManagerV2(controller, configStorage);
+        subscriptionGroupManagerV2.load();
+        found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName());
+        Assert.assertNull(found);
+    }
+
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2Test.java b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2Test.java
new file mode 100644
index 00000000000..731a1f538fb
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2Test.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.config.v2;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class TopicConfigManagerV2Test {
+
+    private MessageStoreConfig messageStoreConfig;
+
+    private ConfigStorage configStorage;
+
+    @Mock
+    private BrokerController controller;
+
+    @Mock
+    private MessageStore messageStore;
+
+    @Rule
+    public TemporaryFolder tf = new TemporaryFolder();
+
+    @After
+    public void cleanUp() {
+        if (null != configStorage) {
+            configStorage.shutdown();
+        }
+    }
+
+    @Before
+    public void setUp() throws IOException {
+        BrokerConfig brokerConfig = new BrokerConfig();
+        Mockito.doReturn(brokerConfig).when(controller).getBrokerConfig();
+
+        messageStoreConfig = new MessageStoreConfig();
+        Mockito.doReturn(messageStoreConfig).when(controller).getMessageStoreConfig();
+        Mockito.doReturn(messageStore).when(controller).getMessageStore();
+
+        File configStoreDir = tf.newFolder();
+        messageStoreConfig.setStorePathRootDir(configStoreDir.getAbsolutePath());
+
+        configStorage = new ConfigStorage(messageStoreConfig);
+        configStorage.start();
+    }
+
+    @Test
+    public void testUpdateTopicConfig() {
+        TopicConfigManagerV2 topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage);
+        topicConfigManagerV2.load();
+
+        TopicConfig topicConfig = new TopicConfig();
+        String topicName = "T1";
+        topicConfig.setTopicName(topicName);
+        topicConfig.setPerm(6);
+        topicConfig.setReadQueueNums(8);
+        topicConfig.setWriteQueueNums(4);
+        topicConfig.setOrder(true);
+        topicConfig.setTopicSysFlag(4);
+        topicConfigManagerV2.updateTopicConfig(topicConfig);
+
+        Assert.assertTrue(configStorage.shutdown());
+
+        topicConfigManagerV2.getTopicConfigTable().clear();
+
+        configStorage = new ConfigStorage(messageStoreConfig);
+        Assert.assertTrue(configStorage.start());
+        topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage);
+        Assert.assertTrue(topicConfigManagerV2.load());
+
+        TopicConfig loaded = topicConfigManagerV2.selectTopicConfig(topicName);
+        Assert.assertNotNull(loaded);
+        Assert.assertEquals(topicName, loaded.getTopicName());
+        Assert.assertEquals(6, loaded.getPerm());
+        Assert.assertEquals(8, loaded.getReadQueueNums());
+        Assert.assertEquals(4, loaded.getWriteQueueNums());
+        Assert.assertTrue(loaded.isOrder());
+        Assert.assertEquals(4, loaded.getTopicSysFlag());
+
+        Assert.assertTrue(topicConfigManagerV2.containsTopic(topicName));
+    }
+
+    @Test
+    public void testRemoveTopicConfig() {
+        TopicConfig topicConfig = new TopicConfig();
+        String topicName = "T1";
+        topicConfig.setTopicName(topicName);
+        topicConfig.setPerm(6);
+        topicConfig.setReadQueueNums(8);
+        topicConfig.setWriteQueueNums(4);
+        topicConfig.setOrder(true);
+        topicConfig.setTopicSysFlag(4);
+        TopicConfigManagerV2 topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage);
+        topicConfigManagerV2.updateTopicConfig(topicConfig);
+        topicConfigManagerV2.removeTopicConfig(topicName);
+        Assert.assertFalse(topicConfigManagerV2.containsTopic(topicName));
+        Assert.assertTrue(configStorage.shutdown());
+
+        configStorage = new ConfigStorage(messageStoreConfig);
+        Assert.assertTrue(configStorage.start());
+        topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage);
+        Assert.assertTrue(topicConfigManagerV2.load());
+        Assert.assertFalse(topicConfigManagerV2.containsTopic(topicName));
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java b/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java
index d01a6f76f5e..39ec0d8d94f 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.rocketmq.broker.controller;
 
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.out.BrokerOuterAPI;
 import org.apache.rocketmq.broker.slave.SlaveSynchronize;
@@ -36,29 +37,31 @@
 import org.apache.rocketmq.store.ha.autoswitch.BrokerMetadata;
 import org.apache.rocketmq.store.ha.autoswitch.TempBrokerMetadata;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mockito.Mock;
 import org.mockito.Mockito;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
+import org.mockito.junit.MockitoJUnitRunner;
 
 import java.io.File;
 import java.time.Duration;
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.UUID;
 
 import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.when;
 
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(ReplicasManager.class)
+@RunWith(MockitoJUnitRunner.class)
 public class ReplicasManagerRegisterTest {
 
     public static final String STORE_BASE_PATH = System.getProperty("java.io.tmpdir") + File.separator + "ReplicasManagerRegisterTest";
@@ -74,7 +77,14 @@ public class ReplicasManagerRegisterTest {
     public static final String CONTROLLER_ADDR = "127.0.0.1:8888";
 
     public static final BrokerConfig BROKER_CONFIG;
-    private final HashSet<Long> syncStateSet = new HashSet<>(Arrays.asList(1L));
+    
+    private final HashSet<Long> syncStateSet = new HashSet<>(Collections.singletonList(1L));
+    
+    @Mock
+    private BrokerMetadata brokerMetadata;
+    
+    @Mock
+    private TempBrokerMetadata tempBrokerMetadata;
 
     static {
         BROKER_CONFIG = new BrokerConfig();
@@ -133,18 +143,19 @@ public void testBrokerRegisterSuccess() throws Exception {
         when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L));
         when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader());
         when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(),  syncStateSet));
-        when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet));
+        when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong()))
+                .thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet));
 
         ReplicasManager replicasManager0 = new ReplicasManager(mockedBrokerController);
         replicasManager0.start();
         await().atMost(Duration.ofMillis(1000)).until(() ->
             replicasManager0.getState() == ReplicasManager.State.RUNNING
         );
-        Assert.assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState());
-        Assert.assertEquals(1L, replicasManager0.getBrokerControllerId().longValue());
+        assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState());
+        assertEquals(1L, replicasManager0.getBrokerControllerId().longValue());
         checkMetadataFile(replicasManager0.getBrokerMetadata(), 1L);
-        Assert.assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded());
-        Assert.assertFalse(replicasManager0.getTempBrokerMetadata().fileExists());
+        assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded());
+        assertFalse(replicasManager0.getTempBrokerMetadata().fileExists());
         replicasManager0.shutdown();
     }
 
@@ -160,18 +171,18 @@ public void testBrokerRegisterSuccessAndRestartWithChangedBrokerConfig() throws
         await().atMost(Duration.ofMillis(1000)).until(() ->
                 replicasManager0.getState() == ReplicasManager.State.RUNNING
         );
-        Assert.assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState());
-        Assert.assertEquals(1L, replicasManager0.getBrokerControllerId().longValue());
+        assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState());
+        assertEquals(1L, replicasManager0.getBrokerControllerId().longValue());
         checkMetadataFile(replicasManager0.getBrokerMetadata(), 1L);
-        Assert.assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded());
-        Assert.assertFalse(replicasManager0.getTempBrokerMetadata().fileExists());
+        assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded());
+        assertFalse(replicasManager0.getTempBrokerMetadata().fileExists());
         replicasManager0.shutdown();
 
         // change broker name in broker config
         mockedBrokerController.getBrokerConfig().setBrokerName(BROKER_NAME + "1");
         ReplicasManager replicasManagerRestart = new ReplicasManager(mockedBrokerController);
         replicasManagerRestart.start();
-        Assert.assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState());
+        assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState());
         mockedBrokerController.getBrokerConfig().setBrokerName(BROKER_NAME);
         replicasManagerRestart.shutdown();
 
@@ -179,7 +190,7 @@ public void testBrokerRegisterSuccessAndRestartWithChangedBrokerConfig() throws
         mockedBrokerController.getBrokerConfig().setBrokerClusterName(CLUSTER_NAME + "1");
         replicasManagerRestart = new ReplicasManager(mockedBrokerController);
         replicasManagerRestart.start();
-        Assert.assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState());
+        assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState());
         mockedBrokerController.getBrokerConfig().setBrokerClusterName(CLUSTER_NAME);
         replicasManagerRestart.shutdown();
     }
@@ -190,32 +201,29 @@ public void testRegisterFailedAtGetNextBrokerId() throws Exception {
         when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenThrow(new RuntimeException());
 
         replicasManager.start();
-
-        Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState());
-        Assert.assertEquals(ReplicasManager.RegisterState.INITIAL, replicasManager.getRegisterState());
-        Assert.assertFalse(replicasManager.getTempBrokerMetadata().fileExists());
-        Assert.assertFalse(replicasManager.getBrokerMetadata().fileExists());
-        Assert.assertNull(replicasManager.getBrokerControllerId());
+        
+        assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState());
+        assertEquals(ReplicasManager.RegisterState.INITIAL, replicasManager.getRegisterState());
+        assertFalse(replicasManager.getTempBrokerMetadata().fileExists());
+        assertFalse(replicasManager.getBrokerMetadata().fileExists());
+        assertNull(replicasManager.getBrokerControllerId());
         replicasManager.shutdown();
     }
 
     @Test
     public void testRegisterFailedAtCreateTempFile() throws Exception {
-        ReplicasManager replicasManager = new ReplicasManager(mockedBrokerController);
+        ReplicasManager spyReplicasManager = new ReplicasManager(mockedBrokerController);
         when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L));
-        when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader());
-        when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(),  syncStateSet));
-        when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet));
-        ReplicasManager spyReplicasManager = PowerMockito.spy(replicasManager);
-        PowerMockito.doReturn(false).when(spyReplicasManager, "createTempMetadataFile", anyLong());
+        FieldUtils.writeDeclaredField(spyReplicasManager, "tempBrokerMetadata", tempBrokerMetadata, true);
+        doThrow(new RuntimeException("Test exception")).when(tempBrokerMetadata).updateAndPersist(any(), any(), anyLong(), any());
 
         spyReplicasManager.start();
-
-        Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState());
-        Assert.assertEquals(ReplicasManager.RegisterState.INITIAL, spyReplicasManager.getRegisterState());
-        Assert.assertFalse(spyReplicasManager.getTempBrokerMetadata().fileExists());
-        Assert.assertFalse(spyReplicasManager.getBrokerMetadata().fileExists());
-        Assert.assertNull(spyReplicasManager.getBrokerControllerId());
+        
+        assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState());
+        assertEquals(ReplicasManager.RegisterState.INITIAL, spyReplicasManager.getRegisterState());
+        assertFalse(spyReplicasManager.getTempBrokerMetadata().fileExists());
+        assertFalse(spyReplicasManager.getBrokerMetadata().fileExists());
+        assertNull(spyReplicasManager.getBrokerControllerId());
         spyReplicasManager.shutdown();
     }
 
@@ -224,61 +232,57 @@ public void testRegisterFailedAtApplyBrokerIdFailed() throws Exception {
         ReplicasManager replicasManager = new ReplicasManager(mockedBrokerController);
         when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L));
         when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenThrow(new RuntimeException());
-        when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(),  syncStateSet));
-        when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet));
 
         replicasManager.start();
-
-        Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState());
-        Assert.assertNotEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState());
-        Assert.assertNotEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager.getRegisterState());
+        
+        assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState());
+        assertNotEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState());
+        assertNotEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager.getRegisterState());
 
         replicasManager.shutdown();
-
-        Assert.assertFalse(replicasManager.getBrokerMetadata().fileExists());
-        Assert.assertNull(replicasManager.getBrokerControllerId());
+        
+        assertFalse(replicasManager.getBrokerMetadata().fileExists());
+        assertNull(replicasManager.getBrokerControllerId());
     }
 
     @Test
     public void testRegisterFailedAtCreateMetadataFileAndDeleteTemp() throws Exception {
-        ReplicasManager replicasManager = new ReplicasManager(mockedBrokerController);
+        ReplicasManager spyReplicasManager = new ReplicasManager(mockedBrokerController);
         when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L));
         when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader());
         when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(),  syncStateSet));
         when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet));
-
-        ReplicasManager spyReplicasManager = PowerMockito.spy(replicasManager);
-        PowerMockito.doReturn(false).when(spyReplicasManager, "createMetadataFileAndDeleteTemp");
+        
+        FieldUtils.writeDeclaredField(spyReplicasManager, "brokerMetadata", brokerMetadata, true);
+        doThrow(new RuntimeException("Test exception")).when(brokerMetadata).updateAndPersist(any(), any(), anyLong());
 
         spyReplicasManager.start();
-
-        Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState());
-        Assert.assertEquals(ReplicasManager.RegisterState.CREATE_TEMP_METADATA_FILE_DONE, spyReplicasManager.getRegisterState());
+        
+        assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState());
+        assertEquals(ReplicasManager.RegisterState.CREATE_TEMP_METADATA_FILE_DONE, spyReplicasManager.getRegisterState());
         TempBrokerMetadata tempBrokerMetadata = spyReplicasManager.getTempBrokerMetadata();
-        Assert.assertTrue(tempBrokerMetadata.fileExists());
-        Assert.assertTrue(tempBrokerMetadata.isLoaded());
-        Assert.assertFalse(spyReplicasManager.getBrokerMetadata().fileExists());
-        Assert.assertNull(spyReplicasManager.getBrokerControllerId());
+        assertTrue(tempBrokerMetadata.fileExists());
+        assertTrue(tempBrokerMetadata.isLoaded());
+        assertFalse(spyReplicasManager.getBrokerMetadata().fileExists());
+        assertNull(spyReplicasManager.getBrokerControllerId());
 
         spyReplicasManager.shutdown();
 
         // restart, we expect that this replicasManager still keep the tempMetadata and still try to finish its registering
         ReplicasManager replicasManagerNew = new ReplicasManager(mockedBrokerController);
-        // because apply brokerId: 1 has succeeded, so now next broker id is 2
-        when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 2L));
 
         replicasManagerNew.start();
-
-        Assert.assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState());
-        Assert.assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState());
+        
+        assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState());
+        assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState());
         // tempMetadata has been cleared
-        Assert.assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists());
-        Assert.assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded());
+        assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists());
+        assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded());
         // metadata has been persisted
-        Assert.assertTrue(replicasManagerNew.getBrokerMetadata().fileExists());
-        Assert.assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded());
-        Assert.assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue());
-        Assert.assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue());
+        assertTrue(replicasManagerNew.getBrokerMetadata().fileExists());
+        assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded());
+        assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue());
+        assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue());
         replicasManagerNew.shutdown();
     }
 
@@ -291,62 +295,57 @@ public void testRegisterFailedAtRegisterSuccess() throws Exception {
         when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet));
 
         replicasManager.start();
-
-        Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState());
-        Assert.assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState());
+        
+        assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState());
+        assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState());
         TempBrokerMetadata tempBrokerMetadata = replicasManager.getTempBrokerMetadata();
         // temp metadata has been cleared
-        Assert.assertFalse(tempBrokerMetadata.fileExists());
-        Assert.assertFalse(tempBrokerMetadata.isLoaded());
+        assertFalse(tempBrokerMetadata.fileExists());
+        assertFalse(tempBrokerMetadata.isLoaded());
         // metadata has been persisted
-        Assert.assertTrue(replicasManager.getBrokerMetadata().fileExists());
-        Assert.assertTrue(replicasManager.getBrokerMetadata().isLoaded());
-        Assert.assertEquals(1L, replicasManager.getBrokerMetadata().getBrokerId().longValue());
-        Assert.assertEquals(1L, replicasManager.getBrokerControllerId().longValue());
+        assertTrue(replicasManager.getBrokerMetadata().fileExists());
+        assertTrue(replicasManager.getBrokerMetadata().isLoaded());
+        assertEquals(1L, replicasManager.getBrokerMetadata().getBrokerId().longValue());
+        assertEquals(1L, replicasManager.getBrokerControllerId().longValue());
 
         replicasManager.shutdown();
 
         Mockito.reset(mockedBrokerOuterAPI);
-        when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet));
+        when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong()))
+                .thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet));
         when(mockedBrokerOuterAPI.getControllerMetaData(any())).thenReturn(
                 new GetMetaDataResponseHeader("default-group", "dledger-a", CONTROLLER_ADDR, true, CONTROLLER_ADDR));
         when(mockedBrokerOuterAPI.checkAddressReachable(any())).thenReturn(true);
 
         // restart, we expect that this replicasManager still keep the metadata and still try to finish its registering
         ReplicasManager replicasManagerNew = new ReplicasManager(mockedBrokerController);
-        // because apply brokerId: 1 has succeeded, so now next broker id is 2
-        when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 2L));
-        // because apply brokerId: 1 has succeeded, so next request which try to apply brokerId: 1 will be failed
-        when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), eq(1L), any(), any())).thenThrow(new RuntimeException());
         when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(),  syncStateSet));
         replicasManagerNew.start();
-
-        Assert.assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState());
-        Assert.assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState());
+        
+        assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState());
+        assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState());
         // tempMetadata has been cleared
-        Assert.assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists());
-        Assert.assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded());
+        assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists());
+        assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded());
         // metadata has been persisted
-        Assert.assertTrue(replicasManagerNew.getBrokerMetadata().fileExists());
-        Assert.assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded());
-        Assert.assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue());
-        Assert.assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue());
+        assertTrue(replicasManagerNew.getBrokerMetadata().fileExists());
+        assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded());
+        assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue());
+        assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue());
         replicasManagerNew.shutdown();
     }
 
 
     private void checkMetadataFile(BrokerMetadata brokerMetadata0 ,Long brokerId) throws Exception {
-        Assert.assertEquals(brokerId, brokerMetadata0.getBrokerId());
-        Assert.assertTrue(brokerMetadata0.fileExists());
+        assertEquals(brokerId, brokerMetadata0.getBrokerId());
+        assertTrue(brokerMetadata0.fileExists());
         BrokerMetadata brokerMetadata = new BrokerMetadata(brokerMetadata0.getFilePath());
         brokerMetadata.readFromFile();
-        Assert.assertEquals(brokerMetadata0, brokerMetadata);
+        assertEquals(brokerMetadata0, brokerMetadata);
     }
 
     @After
     public void clear() {
         UtilAll.deleteFile(new File(STORE_BASE_PATH));
     }
-
-
 }
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerTest.java
index c863f7ac96c..9f17f2bd593 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerTest.java
@@ -17,12 +17,15 @@
 
 package org.apache.rocketmq.broker.controller;
 
+import com.google.common.collect.Lists;
 import java.io.File;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
-
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.out.BrokerOuterAPI;
 import org.apache.rocketmq.broker.slave.SlaveSynchronize;
@@ -31,11 +34,11 @@
 import org.apache.rocketmq.common.Pair;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.remoting.protocol.body.SyncStateSet;
+import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdResponseHeader;
-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerResponseHeader;
 import org.apache.rocketmq.store.DefaultMessageStore;
 import org.apache.rocketmq.store.RunningFlags;
@@ -52,6 +55,7 @@
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
@@ -189,11 +193,11 @@ public void changeBrokerRoleTest() {
         syncStateSetA.add(BROKER_ID_2);
         // not equal to localAddress
         Assertions.assertThatCode(() -> replicasManager.changeBrokerRole(BROKER_ID_2, NEW_MASTER_ADDRESS, NEW_MASTER_EPOCH, OLD_MASTER_EPOCH, syncStateSetB))
-                .doesNotThrowAnyException();
+            .doesNotThrowAnyException();
 
         // equal to localAddress
         Assertions.assertThatCode(() -> replicasManager.changeBrokerRole(BROKER_ID_1, OLD_MASTER_ADDRESS, NEW_MASTER_EPOCH, OLD_MASTER_EPOCH, syncStateSetA))
-                .doesNotThrowAnyException();
+            .doesNotThrowAnyException();
     }
 
     @Test
@@ -206,6 +210,28 @@ public void changeToMasterTest() {
     @Test
     public void changeToSlaveTest() {
         Assertions.assertThatCode(() -> replicasManager.changeToSlave(NEW_MASTER_ADDRESS, NEW_MASTER_EPOCH, BROKER_ID_2))
-                .doesNotThrowAnyException();
+            .doesNotThrowAnyException();
     }
+
+    @Test
+    public void testUpdateControllerAddr() throws Exception {
+        final String controllerAddr = "192.168.1.1";
+        brokerConfig.setFetchControllerAddrByDnsLookup(true);
+        when(brokerOuterAPI.dnsLookupAddressByDomain(anyString())).thenReturn(Lists.newArrayList(controllerAddr));
+        Method method = ReplicasManager.class.getDeclaredMethod("updateControllerAddr");
+        method.setAccessible(true);
+        method.invoke(replicasManager);
+
+        List<String> addresses = replicasManager.getControllerAddresses();
+        Assertions.assertThat(addresses).contains(controllerAddr);
+
+        // Simulating dns resolution exceptions
+        when(brokerOuterAPI.dnsLookupAddressByDomain(anyString())).thenReturn(new ArrayList<>());
+
+        method.invoke(replicasManager);
+        addresses = replicasManager.getControllerAddresses();
+        Assertions.assertThat(addresses).contains(controllerAddr);
+
+    }
+
 }
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java
index d7bd753d776..27fc37dbec8 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java
@@ -17,27 +17,39 @@
 
 package org.apache.rocketmq.broker.failover;
 
+import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Triple;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.out.BrokerOuterAPI;
 import org.apache.rocketmq.broker.topic.TopicRouteInfoManager;
 import org.apache.rocketmq.client.consumer.PullResult;
 import org.apache.rocketmq.client.consumer.PullStatus;
+import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
 import org.apache.rocketmq.client.producer.DefaultMQProducer;
 import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.store.DefaultMessageStore;
 import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
 import org.apache.rocketmq.store.logfile.DefaultMappedFile;
 import org.apache.rocketmq.store.logfile.MappedFile;
+import org.apache.rocketmq.tieredstore.TieredMessageStore;
 import org.assertj.core.api.Assertions;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -49,8 +61,10 @@
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.verify;
 
 @RunWith(MockitoJUnitRunner.class)
 public class EscapeBridgeTest {
@@ -68,11 +82,20 @@ public class EscapeBridgeTest {
     @Mock
     private DefaultMessageStore defaultMessageStore;
 
+    @Mock
+    private TieredMessageStore tieredMessageStore;
+
     private GetMessageResult getMessageResult;
 
     @Mock
     private DefaultMQProducer defaultMQProducer;
 
+    @Mock
+    private TopicRouteInfoManager topicRouteInfoManager;
+
+    @Mock
+    private BrokerOuterAPI brokerOuterAPI;
+
     private static final String BROKER_NAME = "broker_a";
 
     private static final String TEST_TOPIC = "TEST_TOPIC";
@@ -92,14 +115,10 @@ public void before() throws Exception {
         when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);
         when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(CompletableFuture.completedFuture(getMessageResult));
 
-        TopicRouteInfoManager topicRouteInfoManager = mock(TopicRouteInfoManager.class);
         when(brokerController.getTopicRouteInfoManager()).thenReturn(topicRouteInfoManager);
         when(topicRouteInfoManager.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn("");
 
-        BrokerOuterAPI brokerOuterAPI = mock(BrokerOuterAPI.class);
         when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);
-        when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong()))
-            .thenReturn(CompletableFuture.completedFuture(new PullResult(PullStatus.FOUND, -1, -1, -1, new ArrayList<>())));
 
         brokerConfig.setEnableSlaveActingMaster(true);
         brokerConfig.setEnableRemoteEscape(true);
@@ -179,6 +198,75 @@ public void getMessageAsyncTest() {
         Assertions.assertThatCode(() -> escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false)).doesNotThrowAnyException();
     }
 
+    @Test
+    public void getMessageAsyncTest_localStore_getMessageAsync_null() {
+        when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(defaultMessageStore);
+        when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any()))
+                .thenReturn(CompletableFuture.completedFuture(null));
+        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join();
+        Assert.assertNull(rst.getLeft());
+        Assert.assertEquals("getMessageResult is null", rst.getMiddle());
+        Assert.assertFalse(rst.getRight()); // no retry
+    }
+
+    @Test
+    public void getMessageAsyncTest_localStore_decodeNothing_DefaultMessageStore() throws Exception {
+        when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(defaultMessageStore);
+        for (GetMessageStatus status : GetMessageStatus.values()) {
+            GetMessageResult getMessageResult = mockGetMessageResult(0, TEST_TOPIC, null);
+            getMessageResult.setStatus(status);
+            when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any()))
+                    .thenReturn(CompletableFuture.completedFuture(getMessageResult));
+            Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join();
+            Assert.assertNull(rst.getLeft());
+            Assert.assertEquals("Can not get msg", rst.getMiddle());
+            Assert.assertFalse(rst.getRight()); // DefaultMessageStore, no retry
+        }
+    }
+
+    @Test
+    public void getMessageAsyncTest_localStore_decodeNothing_TieredMessageStore() throws Exception {
+        when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(tieredMessageStore);
+        for (GetMessageStatus status : GetMessageStatus.values()) {
+            GetMessageResult getMessageResult = new GetMessageResult();
+            getMessageResult.setStatus(status);
+            when(tieredMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any()))
+                    .thenReturn(CompletableFuture.completedFuture(getMessageResult));
+            Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join();
+            Assert.assertNull(rst.getLeft());
+            Assert.assertEquals("Can not get msg", rst.getMiddle());
+            if (GetMessageStatus.OFFSET_FOUND_NULL.equals(status)) {
+                Assert.assertTrue(rst.getRight()); // TieredMessageStore returns OFFSET_FOUND_NULL, need retry
+            } else {
+                Assert.assertFalse(rst.getRight()); // other status, like DefaultMessageStore, no retry
+            }
+        }
+    }
+
+    @Test
+    public void getMessageAsyncTest_localStore_message_found() throws Exception {
+        when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(defaultMessageStore);
+        when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any()))
+                .thenReturn(CompletableFuture.completedFuture(mockGetMessageResult(2, TEST_TOPIC, "HW".getBytes())));
+        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join();
+        Assert.assertNotNull(rst.getLeft());
+        Assert.assertEquals(0, rst.getLeft().getQueueOffset());
+        Assert.assertTrue(Arrays.equals("HW".getBytes(), rst.getLeft().getBody()));
+        Assert.assertFalse(rst.getRight());
+    }
+
+    @Test
+    public void getMessageAsyncTest_remoteStore_addressNotFound() throws Exception {
+        when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(null);
+
+        // just test address not found, since we have complete tests of getMessageFromRemoteAsync()
+        when(topicRouteInfoManager.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(null);
+        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join();
+        Assert.assertNull(rst.getLeft());
+        Assert.assertEquals("brokerAddress not found", rst.getMiddle());
+        Assert.assertTrue(rst.getRight()); // need retry
+    }
+
     @Test
     public void getMessageFromRemoteTest() {
         Assertions.assertThatCode(() -> escapeBridge.getMessageFromRemote(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME)).doesNotThrowAnyException();
@@ -189,6 +277,54 @@ public void getMessageFromRemoteAsyncTest() {
         Assertions.assertThatCode(() -> escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME)).doesNotThrowAnyException();
     }
 
+    @Test
+    public void getMessageFromRemoteAsyncTest_exception_caught() throws Exception {
+        when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong()))
+                .thenThrow(new RemotingException("mock remoting exception"));
+        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join();
+        Assert.assertNull(rst.getLeft());
+        Assert.assertEquals("Get message from remote failed", rst.getMiddle());
+        Assert.assertTrue(rst.getRight()); // need retry
+    }
+
+    @Test
+    public void getMessageFromRemoteAsyncTest_brokerAddressNotFound() throws Exception {
+        when(topicRouteInfoManager.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(null);
+        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join();
+        Assert.assertNull(rst.getLeft());
+        Assert.assertEquals("brokerAddress not found", rst.getMiddle());
+        Assert.assertTrue(rst.getRight()); // need retry
+    }
+
+    @Test
+    public void getMessageFromRemoteAsyncTest_message_found() throws Exception {
+        PullResult pullResult = new PullResult(PullStatus.FOUND, 1, 1, 1, Arrays.asList(new MessageExt()));
+        when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong()))
+                .thenReturn(CompletableFuture.completedFuture(Triple.of(pullResult, "", false))); // right value is ignored
+        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join();
+        Assert.assertNotNull(rst.getLeft());
+        Assert.assertTrue(StringUtils.isEmpty(rst.getMiddle()));
+        Assert.assertFalse(rst.getRight()); // no retry
+    }
+
+    @Test
+    public void getMessageFromRemoteAsyncTest_message_notFound() throws Exception {
+        PullResult pullResult = new PullResult(PullStatus.NO_MATCHED_MSG, 1, 1, 1, null);
+        when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong()))
+                .thenReturn(CompletableFuture.completedFuture(Triple.of(pullResult, "no msg", false)));
+        Triple<MessageExt, String, Boolean> rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join();
+        Assert.assertNull(rst.getLeft());
+        Assert.assertEquals("no msg", rst.getMiddle());
+        Assert.assertFalse(rst.getRight()); // no retry
+
+        when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong()))
+                .thenReturn(CompletableFuture.completedFuture(Triple.of(null, "other resp code", true)));
+        rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join();
+        Assert.assertNull(rst.getLeft());
+        Assert.assertEquals("other resp code", rst.getMiddle());
+        Assert.assertTrue(rst.getRight()); // need retry
+    }
+
     @Test
     public void decodeMsgListTest() {
         ByteBuffer byteBuffer = ByteBuffer.allocate(10);
@@ -199,4 +335,98 @@ public void decodeMsgListTest() {
         Assertions.assertThatCode(() -> escapeBridge.decodeMsgList(getMessageResult, false)).doesNotThrowAnyException();
     }
 
+    @Test
+    public void decodeMsgListTest_messageNotNull() throws Exception {
+        MessageExt msg = new MessageExt();
+        msg.setBody("HW".getBytes());
+        msg.setTopic("topic");
+        msg.setBornHost(new InetSocketAddress("127.0.0.1", 9000));
+        msg.setStoreHost(new InetSocketAddress("127.0.0.1", 9000));
+        ByteBuffer byteBuffer = ByteBuffer.wrap(MessageDecoder.encode(msg, false));
+        SelectMappedBufferResult result = new SelectMappedBufferResult(0, byteBuffer, 10, new DefaultMappedFile());
+
+
+        getMessageResult.addMessage(result);
+        getMessageResult.getMessageQueueOffset().add(0L);
+        List<MessageExt> list = escapeBridge.decodeMsgList(getMessageResult, false); // skip deCompressBody test
+        Assert.assertEquals(1, list.size());
+        Assert.assertTrue(Arrays.equals(msg.getBody(), list.get(0).getBody()));
+    }
+
+    @Test
+    public void testPutMessageToRemoteBroker_noSpecificBrokerName_hasRemoteBroker() throws Exception {
+        MessageExtBrokerInner message = new MessageExtBrokerInner();
+        message.setTopic(TEST_TOPIC);
+        String anotherBrokerName = "broker_b";
+        TopicPublishInfo publishInfo = mockTopicPublishInfo(BROKER_NAME, anotherBrokerName);
+        when(topicRouteInfoManager.tryToFindTopicPublishInfo(anyString())).thenReturn(publishInfo);
+        when(topicRouteInfoManager.findBrokerAddressInPublish(anotherBrokerName)).thenReturn("127.0.0.1");
+        escapeBridge.putMessageToRemoteBroker(message, null);
+        verify(brokerOuterAPI).sendMessageToSpecificBroker(eq("127.0.0.1"), eq(anotherBrokerName), any(MessageExtBrokerInner.class), anyString(), anyLong());
+    }
+
+    @Test
+    public void testPutMessageToRemoteBroker_noSpecificBrokerName_noRemoteBroker() throws Exception {
+        MessageExtBrokerInner message = new MessageExtBrokerInner();
+        message.setTopic(TEST_TOPIC);
+        TopicPublishInfo publishInfo = mockTopicPublishInfo(BROKER_NAME);
+        when(topicRouteInfoManager.tryToFindTopicPublishInfo(anyString())).thenReturn(publishInfo);
+        escapeBridge.putMessageToRemoteBroker(message, null);
+        verify(topicRouteInfoManager, times(0)).findBrokerAddressInPublish(anyString());
+    }
+
+    @Test
+    public void testPutMessageToRemoteBroker_specificBrokerName_equals() throws Exception {
+        escapeBridge.putMessageToRemoteBroker(new MessageExtBrokerInner(), BROKER_NAME);
+        verify(topicRouteInfoManager, times(0)).tryToFindTopicPublishInfo(anyString());
+    }
+
+    @Test
+    public void testPutMessageToRemoteBroker_specificBrokerName_addressNotFound() throws Exception {
+        MessageExtBrokerInner message = new MessageExtBrokerInner();
+        message.setTopic(TEST_TOPIC);
+        TopicPublishInfo publishInfo = mockTopicPublishInfo(BROKER_NAME);
+        when(topicRouteInfoManager.tryToFindTopicPublishInfo(anyString())).thenReturn(publishInfo);
+        escapeBridge.putMessageToRemoteBroker(message, "whatever");
+        verify(topicRouteInfoManager).findBrokerAddressInPublish(eq("whatever"));
+        verify(brokerOuterAPI, times(0)).sendMessageToSpecificBroker(anyString(), anyString(), any(MessageExtBrokerInner.class), anyString(), anyLong());
+    }
+
+    @Test
+    public void testPutMessageToRemoteBroker_specificBrokerName_addressFound() throws Exception {
+        MessageExtBrokerInner message = new MessageExtBrokerInner();
+        message.setTopic(TEST_TOPIC);
+        String anotherBrokerName = "broker_b";
+        TopicPublishInfo publishInfo = mockTopicPublishInfo(BROKER_NAME, anotherBrokerName);
+        when(topicRouteInfoManager.tryToFindTopicPublishInfo(anyString())).thenReturn(publishInfo);
+        when(topicRouteInfoManager.findBrokerAddressInPublish(anotherBrokerName)).thenReturn("127.0.0.1");
+        escapeBridge.putMessageToRemoteBroker(message, anotherBrokerName);
+        verify(brokerOuterAPI).sendMessageToSpecificBroker(eq("127.0.0.1"), eq(anotherBrokerName), any(MessageExtBrokerInner.class), anyString(), anyLong());
+    }
+
+    private GetMessageResult mockGetMessageResult(int count, String topic, byte[] body) throws Exception {
+        GetMessageResult result = new GetMessageResult();
+        for (int i = 0; i < count; i++) {
+            MessageExt msg = new MessageExt();
+            msg.setBody(body);
+            msg.setTopic(topic);
+            msg.setBornHost(new InetSocketAddress("127.0.0.1", 9000));
+            msg.setStoreHost(new InetSocketAddress("127.0.0.1", 9000));
+            ByteBuffer byteBuffer = ByteBuffer.wrap(MessageDecoder.encode(msg, false));
+            SelectMappedBufferResult bufferResult = new SelectMappedBufferResult(0, byteBuffer, body.length, new DefaultMappedFile());
+
+            result.addMessage(bufferResult);
+            result.getMessageQueueOffset().add(i + 0L);
+        }
+        return result;
+    }
+
+    private TopicPublishInfo mockTopicPublishInfo(String... brokerNames) {
+        TopicPublishInfo topicPublishInfo = new TopicPublishInfo();
+        for (String brokerName : brokerNames) {
+            topicPublishInfo.getMessageQueueList().add(new MessageQueue(TEST_TOPIC, brokerName, 0));
+        }
+        return topicPublishInfo;
+    }
+
 }
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java
index 31b547cf1be..2216a1d50c1 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java
@@ -19,16 +19,46 @@
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.future.FutureTaskExt;
 import org.apache.rocketmq.remoting.netty.RequestTask;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.MessageStore;
+import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class BrokerFastFailureTest {
+
+    private BrokerController brokerController;
+
+    private final BrokerConfig brokerConfig = new BrokerConfig();
+
+    private MessageStore messageStore;
+
+    @Before
+    public void setUp() {
+        brokerController = Mockito.mock(BrokerController.class);
+        messageStore = Mockito.mock(DefaultMessageStore.class);
+        BlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();
+        Mockito.when(brokerController.getSendThreadPoolQueue()).thenReturn(queue);
+        Mockito.when(brokerController.getPullThreadPoolQueue()).thenReturn(queue);
+        Mockito.when(brokerController.getLitePullThreadPoolQueue()).thenReturn(queue);
+        Mockito.when(brokerController.getHeartbeatThreadPoolQueue()).thenReturn(queue);
+        Mockito.when(brokerController.getEndTransactionThreadPoolQueue()).thenReturn(queue);
+        Mockito.when(brokerController.getAdminBrokerThreadPoolQueue()).thenReturn(queue);
+        Mockito.when(brokerController.getAckThreadPoolQueue()).thenReturn(queue);
+        Mockito.when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
+        Mockito.when(messageStore.isOSPageCacheBusy()).thenReturn(false);
+        Mockito.when(brokerController.getMessageStore()).thenReturn(messageStore);
+    }
+
     @Test
     public void testCleanExpiredRequestInQueue() throws Exception {
-        BrokerFastFailure brokerFastFailure = new BrokerFastFailure(null);
+        BrokerFastFailure brokerFastFailure = new BrokerFastFailure(brokerController);
 
         BlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();
         brokerFastFailure.cleanExpiredRequestInQueue(queue, 1);
@@ -63,4 +93,40 @@ public void run() {
         assertThat(((FutureTaskExt) queue.peek()).getRunnable()).isEqualTo(requestTask);
     }
 
+    @Test
+    public void testCleanExpiredCustomRequestInQueue() throws Exception {
+        BrokerFastFailure brokerFastFailure = new BrokerFastFailure(brokerController);
+        brokerFastFailure.start();
+        brokerConfig.setWaitTimeMillsInAckQueue(10);
+        BlockingQueue<Runnable> customThreadPoolQueue = new LinkedBlockingQueue<>();
+        brokerFastFailure.addCleanExpiredRequestQueue(customThreadPoolQueue, () -> brokerConfig.getWaitTimeMillsInAckQueue());
+
+        Runnable runnable = new Runnable() {
+            @Override
+            public void run() {
+
+            }
+        };
+        RequestTask requestTask = new RequestTask(runnable, null, null);
+        customThreadPoolQueue.add(new FutureTaskExt<>(requestTask, null));
+
+        Thread.sleep(2000);
+
+        assertThat(customThreadPoolQueue.size()).isEqualTo(0);
+        assertThat(requestTask.isStopRun()).isEqualTo(true);
+
+        brokerConfig.setWaitTimeMillsInAckQueue(10000);
+
+        RequestTask requestTask2 = new RequestTask(runnable, null, null);
+        customThreadPoolQueue.add(new FutureTaskExt<>(requestTask2, null));
+
+        Thread.sleep(1000);
+
+        assertThat(customThreadPoolQueue.size()).isEqualTo(1);
+        assertThat(((FutureTaskExt) customThreadPoolQueue.peek()).getRunnable()).isEqualTo(requestTask2);
+
+        brokerFastFailure.shutdown();
+
+    }
+
 }
\ No newline at end of file
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLongPollingServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLongPollingServiceTest.java
new file mode 100644
index 00000000000..003bf09842a
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLongPollingServiceTest.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.longpolling;
+
+import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.store.MessageFilter;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ExecutorService;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class PopLongPollingServiceTest {
+    
+    @Mock
+    private BrokerController brokerController;
+
+    @Mock
+    private NettyRequestProcessor processor;
+
+    @Mock
+    private ChannelHandlerContext ctx;
+
+    @Mock
+    private ExecutorService pullMessageExecutor;
+
+    private PopLongPollingService popLongPollingService;
+
+    private final String defaultTopic = "defaultTopic";
+
+    @Before
+    public void init() {
+        BrokerConfig brokerConfig = new BrokerConfig();
+        brokerConfig.setPopPollingMapSize(100);
+        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
+        popLongPollingService = spy(new PopLongPollingService(brokerController, processor, true));
+    }
+
+    @Test
+    public void testNotifyMessageArrivingWithRetryTopic() {
+        int queueId = 0;
+        doNothing().when(popLongPollingService).notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, -1L, null, 0L, null, null);
+        popLongPollingService.notifyMessageArrivingWithRetryTopic(defaultTopic, queueId);
+        verify(popLongPollingService, times(1)).notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, -1L, null, 0L, null, null);
+    }
+
+    @Test
+    public void testNotifyMessageArriving() {
+        int queueId = 0;
+        Long tagsCode = 123L;
+        long offset = 123L;
+        long msgStoreTime = System.currentTimeMillis();
+        byte[] filterBitMap = new byte[] {0x01};
+        Map<String, String> properties = new ConcurrentHashMap<>();
+        doNothing().when(popLongPollingService).notifyMessageArriving(defaultTopic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);
+        popLongPollingService.notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);
+        verify(popLongPollingService).notifyMessageArriving(defaultTopic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);
+    }
+
+    @Test
+    public void testNotifyMessageArrivingValidRequest() throws Exception {
+        String cid = "CID_1";
+        int queueId = 0;
+        ConcurrentLinkedHashMap<String, ConcurrentHashMap<String, Byte>> topicCidMap = new ConcurrentLinkedHashMap.Builder<String, ConcurrentHashMap<String, Byte>>()
+            .maximumWeightedCapacity(10).build();
+        ConcurrentHashMap<String, Byte> cids = new ConcurrentHashMap<>();
+        cids.put(cid, (byte) 1);
+        topicCidMap.put(defaultTopic, cids);
+        popLongPollingService = new PopLongPollingService(brokerController, processor, true);
+        ConcurrentLinkedHashMap<String, ConcurrentSkipListSet<PopRequest>> pollingMap =
+            new ConcurrentLinkedHashMap.Builder<String, ConcurrentSkipListSet<PopRequest>>().maximumWeightedCapacity(this.brokerController.getBrokerConfig().getPopPollingMapSize()).build();
+        Channel channel = mock(Channel.class);
+        when(channel.isActive()).thenReturn(true);
+        PopRequest popRequest = mock(PopRequest.class);
+        MessageFilter messageFilter = mock(MessageFilter.class);
+        SubscriptionData subscriptionData = mock(SubscriptionData.class);
+        when(popRequest.getMessageFilter()).thenReturn(messageFilter);
+        when(popRequest.getSubscriptionData()).thenReturn(subscriptionData);
+        when(popRequest.getChannel()).thenReturn(channel);
+        String pollingKey = KeyBuilder.buildPollingKey(defaultTopic, cid, queueId);
+        ConcurrentSkipListSet popRequests = mock(ConcurrentSkipListSet.class);
+        when(popRequests.pollLast()).thenReturn(popRequest);
+        pollingMap.put(pollingKey, popRequests);
+        FieldUtils.writeDeclaredField(popLongPollingService, "topicCidMap", topicCidMap, true);
+        FieldUtils.writeDeclaredField(popLongPollingService, "pollingMap", pollingMap, true);
+        boolean actual = popLongPollingService.notifyMessageArriving(defaultTopic, queueId, cid, null, 0, null, null);
+        assertFalse(actual);
+    }
+
+    @Test
+    public void testWakeUpNullRequest() {
+        assertFalse(popLongPollingService.wakeUp(null));
+    }
+
+    @Test
+    public void testWakeUpIncompleteRequest() {
+        PopRequest request = mock(PopRequest.class);
+        when(request.complete()).thenReturn(false);
+        assertFalse(popLongPollingService.wakeUp(request));
+    }
+
+    @Test
+    public void testWakeUpInactiveChannel() {
+        PopRequest request = mock(PopRequest.class);
+        when(request.complete()).thenReturn(true);
+        when(request.getCtx()).thenReturn(ctx);
+        Channel channel = mock(Channel.class);
+        when(ctx.channel()).thenReturn(channel);
+        when(channel.isActive()).thenReturn(true);
+        when(brokerController.getPullMessageExecutor()).thenReturn(pullMessageExecutor);
+        assertTrue(popLongPollingService.wakeUp(request));
+    }
+
+    @Test
+    public void testWakeUpValidRequestWithException() throws Exception {
+        PopRequest request = mock(PopRequest.class);
+        when(request.complete()).thenReturn(true);
+        when(request.getCtx()).thenReturn(ctx);
+        Channel channel = mock(Channel.class);
+        when(ctx.channel()).thenReturn(channel);
+        when(request.getChannel()).thenReturn(channel);
+        when(channel.isActive()).thenReturn(true);
+        when(brokerController.getPullMessageExecutor()).thenReturn(pullMessageExecutor);
+        when(processor.processRequest(any(), any())).thenThrow(new RuntimeException("Test Exception"));
+        assertTrue(popLongPollingService.wakeUp(request));
+        ArgumentCaptor<Runnable> captor = ArgumentCaptor.forClass(Runnable.class);
+        verify(pullMessageExecutor).submit(captor.capture());
+        captor.getValue().run();
+        verify(processor).processRequest(any(), any());
+    }
+
+    @Test
+    public void testPollingNotPolling() {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        RemotingCommand remotingCommand = mock(RemotingCommand.class);
+        PollingHeader requestHeader = mock(PollingHeader.class);
+        SubscriptionData subscriptionData = mock(SubscriptionData.class);
+        MessageFilter messageFilter = mock(MessageFilter.class);
+        when(requestHeader.getPollTime()).thenReturn(0L);
+        PollingResult result = popLongPollingService.polling(ctx, remotingCommand, requestHeader, subscriptionData, messageFilter);
+        assertEquals(PollingResult.NOT_POLLING, result);
+    }
+
+    @Test
+    public void testPollingServicePollingTimeout() throws IllegalAccessException {
+        String cid = "CID_1";
+        popLongPollingService = new PopLongPollingService(brokerController, processor, true);
+        popLongPollingService.shutdown();
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        RemotingCommand remotingCommand = mock(RemotingCommand.class);
+        PollingHeader requestHeader = mock(PollingHeader.class);
+        SubscriptionData subscriptionData = mock(SubscriptionData.class);
+        MessageFilter messageFilter = mock(MessageFilter.class);
+        when(requestHeader.getPollTime()).thenReturn(1000L);
+        when(requestHeader.getTopic()).thenReturn(defaultTopic);
+        when(requestHeader.getConsumerGroup()).thenReturn("defaultGroup");
+        ConcurrentLinkedHashMap<String, ConcurrentHashMap<String, Byte>> topicCidMap = new ConcurrentLinkedHashMap.Builder<String, ConcurrentHashMap<String, Byte>>()
+            .maximumWeightedCapacity(10).build();
+        ConcurrentHashMap<String, Byte> cids = new ConcurrentHashMap<>();
+        cids.put(cid, (byte) 1);
+        topicCidMap.put(defaultTopic, cids);
+        FieldUtils.writeDeclaredField(popLongPollingService, "topicCidMap", topicCidMap, true);
+        PollingResult result = popLongPollingService.polling(ctx, remotingCommand, requestHeader, subscriptionData, messageFilter);
+        assertEquals(PollingResult.POLLING_TIMEOUT, result);
+    }
+
+    @Test
+    public void testPollingPollingSuc() {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        RemotingCommand remotingCommand = mock(RemotingCommand.class);
+        PollingHeader requestHeader = mock(PollingHeader.class);
+        SubscriptionData subscriptionData = mock(SubscriptionData.class);
+        MessageFilter messageFilter = mock(MessageFilter.class);
+        when(requestHeader.getPollTime()).thenReturn(1000L);
+        when(requestHeader.getBornTime()).thenReturn(System.currentTimeMillis());
+        when(requestHeader.getTopic()).thenReturn("topic");
+        when(requestHeader.getConsumerGroup()).thenReturn("cid");
+        when(requestHeader.getQueueId()).thenReturn(0);
+        PollingResult result = popLongPollingService.polling(ctx, remotingCommand, requestHeader, subscriptionData, messageFilter);
+        assertEquals(PollingResult.POLLING_SUC, result);
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManagerTest.java
index 11f7ae8215a..9264eb4b56b 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManagerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManagerTest.java
@@ -20,8 +20,25 @@
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.common.Attributes;
 import io.opentelemetry.api.common.AttributesBuilder;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.attribute.TopicMessageType;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.metrics.MetricsExporterType;
+import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.junit.Test;
 
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class BrokerMetricsManagerTest {
@@ -29,7 +46,7 @@ public class BrokerMetricsManagerTest {
     @Test
     public void testNewAttributesBuilder() {
         Attributes attributes = BrokerMetricsManager.newAttributesBuilder().put("a", "b")
-            .build();
+                .build();
         assertThat(attributes.get(AttributeKey.stringKey("a"))).isEqualTo("b");
     }
 
@@ -37,6 +54,7 @@ public void testNewAttributesBuilder() {
     public void testCustomizedAttributesBuilder() {
         BrokerMetricsManager.attributesBuilderSupplier = () -> new AttributesBuilder() {
             private AttributesBuilder attributesBuilder = Attributes.builder();
+
             @Override
             public Attributes build() {
                 return attributesBuilder.put("customized", "value").build();
@@ -61,8 +79,263 @@ public AttributesBuilder putAll(Attributes attributes) {
             }
         };
         Attributes attributes = BrokerMetricsManager.newAttributesBuilder().put("a", "b")
-            .build();
+                .build();
         assertThat(attributes.get(AttributeKey.stringKey("a"))).isEqualTo("b");
         assertThat(attributes.get(AttributeKey.stringKey("customized"))).isEqualTo("value");
     }
+
+
+    @Test
+    public void testIsRetryOrDlqTopicWithRetryTopic() {
+        String topic = MixAll.RETRY_GROUP_TOPIC_PREFIX + "TestTopic";
+        boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic);
+        assertThat(result).isTrue();
+    }
+
+    @Test
+    public void testIsRetryOrDlqTopicWithDlqTopic() {
+        String topic = MixAll.DLQ_GROUP_TOPIC_PREFIX + "TestTopic";
+        boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic);
+        assertThat(result).isTrue();
+    }
+
+    @Test
+    public void testIsRetryOrDlqTopicWithNonRetryOrDlqTopic() {
+        String topic = "NormalTopic";
+        boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic);
+        assertThat(result).isFalse();
+    }
+
+    @Test
+    public void testIsRetryOrDlqTopicWithEmptyTopic() {
+        String topic = "";
+        boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic);
+        assertThat(result).isFalse();
+    }
+
+    @Test
+    public void testIsRetryOrDlqTopicWithNullTopic() {
+        String topic = null;
+        boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic);
+        assertThat(result).isFalse();
+    }
+
+    @Test
+    public void testIsSystemGroup_SystemGroup_ReturnsTrue() {
+        String group = "FooGroup";
+        String systemGroup = MixAll.CID_RMQ_SYS_PREFIX + group;
+        boolean result = BrokerMetricsManager.isSystemGroup(systemGroup);
+        assertThat(result).isTrue();
+    }
+
+    @Test
+    public void testIsSystemGroup_NonSystemGroup_ReturnsFalse() {
+        String group = "FooGroup";
+        boolean result = BrokerMetricsManager.isSystemGroup(group);
+        assertThat(result).isFalse();
+    }
+
+    @Test
+    public void testIsSystemGroup_EmptyGroup_ReturnsFalse() {
+        String group = "";
+        boolean result = BrokerMetricsManager.isSystemGroup(group);
+        assertThat(result).isFalse();
+    }
+
+    @Test
+    public void testIsSystemGroup_NullGroup_ReturnsFalse() {
+        String group = null;
+        boolean result = BrokerMetricsManager.isSystemGroup(group);
+        assertThat(result).isFalse();
+    }
+
+    @Test
+    public void testIsSystem_SystemTopicOrSystemGroup_ReturnsTrue() {
+        String topic = "FooTopic";
+        String group = "FooGroup";
+        String systemTopic = TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC;
+        String systemGroup = MixAll.CID_RMQ_SYS_PREFIX + group;
+
+        boolean resultTopic = BrokerMetricsManager.isSystem(systemTopic, group);
+        assertThat(resultTopic).isTrue();
+
+        boolean resultGroup = BrokerMetricsManager.isSystem(topic, systemGroup);
+        assertThat(resultGroup).isTrue();
+    }
+
+    @Test
+    public void testIsSystem_NonSystemTopicAndGroup_ReturnsFalse() {
+        String topic = "FooTopic";
+        String group = "FooGroup";
+        boolean result = BrokerMetricsManager.isSystem(topic, group);
+        assertThat(result).isFalse();
+    }
+
+    @Test
+    public void testIsSystem_EmptyTopicAndGroup_ReturnsFalse() {
+        String topic = "";
+        String group = "";
+        boolean result = BrokerMetricsManager.isSystem(topic, group);
+        assertThat(result).isFalse();
+    }
+
+    @Test
+    public void testGetMessageTypeAsNormal() {
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        requestHeader.setProperties("");
+
+        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);
+        assertThat(TopicMessageType.NORMAL).isEqualTo(result);
+    }
+
+    @Test
+    public void testGetMessageTypeAsTransaction() {
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+
+        Map<String, String> map = new HashMap<>();
+        map.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, "true");
+        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));
+
+        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);
+        assertThat(TopicMessageType.TRANSACTION).isEqualTo(result);
+    }
+
+    @Test
+    public void testGetMessageTypeAsFifo() {
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        Map<String, String> map = new HashMap<>();
+        map.put(MessageConst.PROPERTY_SHARDING_KEY, "shardingKey");
+        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));
+
+        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);
+        assertThat(TopicMessageType.FIFO).isEqualTo(result);
+    }
+
+    @Test
+    public void testGetMessageTypeAsDelayLevel() {
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        Map<String, String> map = new HashMap<>();
+        map.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, "1");
+        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));
+
+        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);
+        assertThat(TopicMessageType.DELAY).isEqualTo(result);
+    }
+
+    @Test
+    public void testGetMessageTypeAsDeliverMS() {
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        Map<String, String> map = new HashMap<>();
+        map.put(MessageConst.PROPERTY_TIMER_DELIVER_MS, "10");
+        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));
+
+        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);
+        assertThat(TopicMessageType.DELAY).isEqualTo(result);
+    }
+
+    @Test
+    public void testGetMessageTypeAsDelaySEC() {
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        Map<String, String> map = new HashMap<>();
+        map.put(MessageConst.PROPERTY_TIMER_DELAY_SEC, "1");
+        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));
+
+        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);
+        assertThat(TopicMessageType.DELAY).isEqualTo(result);
+    }
+
+    @Test
+    public void testGetMessageTypeAsDelayMS() {
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        Map<String, String> map = new HashMap<>();
+        map.put(MessageConst.PROPERTY_TIMER_DELAY_MS, "10");
+        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));
+
+        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);
+        assertThat(TopicMessageType.DELAY).isEqualTo(result);
+    }
+
+    @Test
+    public void testGetMessageTypeWithUnknownProperty() {
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        Map<String, String> map = new HashMap<>();
+        map.put("unknownProperty", "unknownValue");
+        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));
+
+        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);
+        assertThat(TopicMessageType.NORMAL).isEqualTo(result);
+    }
+
+    @Test
+    public void testGetMessageTypeWithMultipleProperties() {
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        Map<String, String> map = new HashMap<>();
+        map.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, "1");
+        map.put(MessageConst.PROPERTY_SHARDING_KEY, "shardingKey");
+        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));
+
+        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);
+        assertThat(TopicMessageType.FIFO).isEqualTo(result);
+    }
+
+    @Test
+    public void testGetMessageTypeWithTransactionFlagButOtherPropertiesPresent() {
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        Map<String, String> map = new HashMap<>();
+        map.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, "true");
+        map.put(MessageConst.PROPERTY_SHARDING_KEY, "shardingKey");
+        requestHeader.setProperties(MessageDecoder.messageProperties2String(map));
+
+        TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader);
+        assertThat(TopicMessageType.TRANSACTION).isEqualTo(result);
+    }
+
+    @Test
+    public void testGetMessageTypeWithEmptyProperties() {
+        TopicMessageType result = BrokerMetricsManager.getMessageType(new SendMessageRequestHeader());
+        assertThat(TopicMessageType.NORMAL).isEqualTo(result);
+    }
+
+    @Test
+    public void testCreateMetricsManager() {
+        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+        String storePathRootDir = System.getProperty("java.io.tmpdir") + File.separator + "store-"
+                + UUID.randomUUID();
+        messageStoreConfig.setStorePathRootDir(storePathRootDir);
+        BrokerConfig brokerConfig = new BrokerConfig();
+
+        NettyServerConfig nettyServerConfig = new NettyServerConfig();
+        nettyServerConfig.setListenPort(0);
+
+        BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig,
+                new NettyClientConfig(), messageStoreConfig);
+
+        BrokerMetricsManager metricsManager = new BrokerMetricsManager(brokerController);
+
+        assertThat(metricsManager.getBrokerMeter()).isNull();
+    }
+
+    @Test
+    public void testCreateMetricsManagerLogType() throws CloneNotSupportedException {
+        BrokerConfig brokerConfig = new BrokerConfig();
+        brokerConfig.setMetricsExporterType(MetricsExporterType.LOG);
+        brokerConfig.setMetricsLabel("label1:value1;label2:value2");
+        brokerConfig.setMetricsOtelCardinalityLimit(1);
+
+        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+        String storePathRootDir = System.getProperty("java.io.tmpdir") + File.separator + "store-"
+                + UUID.randomUUID();
+        messageStoreConfig.setStorePathRootDir(storePathRootDir);
+
+        NettyServerConfig nettyServerConfig = new NettyServerConfig();
+        nettyServerConfig.setListenPort(0);
+
+        BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig,
+                new NettyClientConfig(), messageStoreConfig);
+        brokerController.initialize();
+
+        BrokerMetricsManager metricsManager = new BrokerMetricsManager(brokerController);
+
+        assertThat(metricsManager.getBrokerMeter()).isNotNull();
+    }
 }
\ No newline at end of file
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java
index 9dc00f9d6b1..ad5af92646e 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java
@@ -25,6 +25,7 @@
 import org.apache.rocketmq.broker.client.ConsumerManager;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -51,7 +52,7 @@ public class BroadcastOffsetManagerTest {
     private BroadcastOffsetManager broadcastOffsetManager;
 
     @Before
-    public void before() {
+    public void before() throws ConsumeQueueException {
         brokerConfig.setEnableBroadcastOffsetStore(true);
         brokerConfig.setBroadcastOffsetExpireSecond(1);
         brokerConfig.setBroadcastOffsetExpireMaxSecond(5);
@@ -84,7 +85,7 @@ public void before() {
     }
 
     @Test
-    public void testBroadcastOffsetSwitch() {
+    public void testBroadcastOffsetSwitch() throws ConsumeQueueException {
         // client1 connect to broker
         onlineClientIdSet.add("client1");
         long offset = broadcastOffsetManager.queryInitOffset("group", "topic", 0, "client1", 0, false);
@@ -160,4 +161,4 @@ public void testBroadcastOffsetExpire() {
             return broadcastOffsetManager.offsetStoreMap.isEmpty();
         });
     }
-}
\ No newline at end of file
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManagerTest.java
index 7bd289a6f11..d980090a23f 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManagerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManagerTest.java
@@ -18,6 +18,7 @@
 package org.apache.rocketmq.broker.offset;
 
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.junit.Assert;
 import org.junit.Before;
@@ -27,6 +28,7 @@
 import java.util.concurrent.ConcurrentMap;
 import org.mockito.Mockito;
 
+import static org.apache.rocketmq.broker.offset.ConsumerOffsetManager.TOPIC_GROUP_SEPARATOR;
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class ConsumerOffsetManagerTest {
@@ -65,6 +67,22 @@ public void cleanOffsetByTopic_Exist() {
         assertThat(!consumerOffsetManager.getOffsetTable().containsKey(KEY)).isTrue();
     }
 
+    @Test
+    public void removeOffsetByGroupTest() {
+        String topic = "TopicName";
+        String group = "GroupName";
+        Mockito.when(brokerController.getBrokerConfig()).thenReturn(new BrokerConfig());
+        consumerOffsetManager.commitOffset("Commit", group, topic, 0, 100);
+        consumerOffsetManager.assignResetOffset(topic, group, 0, 100);
+        consumerOffsetManager.commitPullOffset("Pull", group, topic, 0, 100);
+        consumerOffsetManager.removeOffset(group);
+        Assert.assertFalse(consumerOffsetManager.getOffsetTable().containsKey(topic + TOPIC_GROUP_SEPARATOR + group));
+
+        consumerOffsetManager.commitPullOffset("Pull", group, topic, 0, 100);
+        consumerOffsetManager.clearPullOffset(group, topic);
+        Assert.assertEquals(-1L, consumerOffsetManager.queryPullOffset(group, topic, 0));
+    }
+
     @Test
     public void testOffsetPersistInMemory() {
         ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable = consumerOffsetManager.getOffsetTable();
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java
index 93689efa586..1fdf454d5e0 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java
@@ -22,6 +22,7 @@
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.processor.PopMessageProcessor;
 import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.assertj.core.util.Lists;
 import org.junit.Before;
 import org.junit.Test;
@@ -50,7 +51,7 @@ public class ConsumerOrderInfoManagerLockFreeNotifyTest {
     private final BrokerController brokerController = mock(BrokerController.class);
 
     @Before
-    public void before() {
+    public void before() throws ConsumeQueueException {
         notified = new AtomicBoolean(false);
         brokerConfig.setEnableNotifyAfterPopOrderLockRelease(true);
         when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
@@ -175,4 +176,4 @@ public void testRecover() {
         await().atLeast(Duration.ofSeconds(2)).atMost(Duration.ofSeconds(4)).until(notified::get);
         assertTrue(consumerOrderInfoManager.getConsumerOrderInfoLockManager().getTimeoutMap().isEmpty());
     }
-}
\ No newline at end of file
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerTest.java
index 25b418c9344..4414eda54e9 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerTest.java
@@ -21,7 +21,6 @@
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
@@ -29,7 +28,6 @@
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
-import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
 import org.assertj.core.util.Lists;
 import org.junit.Before;
 import org.junit.Test;
@@ -384,9 +382,7 @@ public void testAutoCleanAndEncode() {
 
         SubscriptionGroupManager subscriptionGroupManager = mock(SubscriptionGroupManager.class);
         when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);
-        ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupConfigConcurrentMap = new ConcurrentHashMap<>();
-        subscriptionGroupConfigConcurrentMap.put(GROUP, new SubscriptionGroupConfig());
-        when(subscriptionGroupManager.getSubscriptionGroupTable()).thenReturn(subscriptionGroupConfigConcurrentMap);
+        when(subscriptionGroupManager.containsSubscriptionGroup(GROUP)).thenReturn(true);
 
         TopicConfig topicConfig = new TopicConfig(TOPIC);
         when(topicConfigManager.selectTopicConfig(eq(TOPIC))).thenReturn(topicConfig);
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManagerTest.java
index 58b690c9a34..5a7a2c38acb 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManagerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManagerTest.java
@@ -21,6 +21,7 @@
 import java.util.concurrent.ConcurrentMap;
 
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.junit.After;
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java
new file mode 100644
index 00000000000..aa5003fc103
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.offset;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.when;
+
+public class RocksDBLmqConsumerOffsetManagerTest {
+    private static final String LMQ_GROUP = MixAll.LMQ_PREFIX + "FooBarGroup";
+    private static final String NON_LMQ_GROUP = "nonLmqGroup";
+
+    private static final String LMQ_TOPIC = MixAll.LMQ_PREFIX + "FooBarTopic";
+    private static final String NON_LMQ_TOPIC = "FooBarTopic";
+    private static final int QUEUE_ID = 0;
+    private static final long OFFSET = 12345;
+
+    private BrokerController brokerController;
+
+    private RocksDBConsumerOffsetManager offsetManager;
+
+    @Before
+    public void setUp() {
+        brokerController = Mockito.mock(BrokerController.class);
+        when(brokerController.getMessageStoreConfig()).thenReturn(Mockito.mock(MessageStoreConfig.class));
+        when(brokerController.getBrokerConfig()).thenReturn(new BrokerConfig());
+        offsetManager = new RocksDBConsumerOffsetManager(brokerController);
+    }
+
+
+    @Test
+    public void testQueryOffsetForNonLmq() {
+        long actualOffset = offsetManager.queryOffset(NON_LMQ_GROUP, NON_LMQ_TOPIC, QUEUE_ID);
+        // Verify
+        assertEquals("Offset should not be null.", -1, actualOffset);
+    }
+
+
+    @Test
+    public void testQueryOffsetForLmqGroupWithExistingOffset() {
+        offsetManager.commitOffset("127.0.0.1",LMQ_GROUP, LMQ_TOPIC, QUEUE_ID, OFFSET);
+
+        // Act
+        Map<Integer, Long> actualOffsets = offsetManager.queryOffset(LMQ_GROUP, LMQ_TOPIC);
+
+        // Assert
+        assertNotNull(actualOffsets);
+        assertEquals(1, actualOffsets.size());
+        assertEquals(OFFSET, (long) actualOffsets.get(0));
+    }
+
+    @Test
+    public void testQueryOffsetForLmqGroupWithoutExistingOffset() {
+        // Act
+        Map<Integer, Long> actualOffsets = offsetManager.queryOffset(LMQ_GROUP, "nonExistingTopic");
+        // Assert
+        assertNull(actualOffsets);
+    }
+
+    @Test
+    public void testQueryOffsetForNonLmqGroup() {
+        // Arrange
+        Map<Integer, Long> mockOffsets = new HashMap<>();
+        mockOffsets.put(QUEUE_ID, OFFSET);
+
+        offsetManager.commitOffset("clientHost", NON_LMQ_GROUP, NON_LMQ_TOPIC, QUEUE_ID, OFFSET);
+
+        // Act
+        Map<Integer, Long> actualOffsets = offsetManager.queryOffset(NON_LMQ_GROUP, NON_LMQ_TOPIC);
+
+        // Assert
+        assertNotNull(actualOffsets);
+        assertEquals("Offsets should match the mocked return value for non-LMQ groups", mockOffsets, actualOffsets);
+    }
+
+    @Test
+    public void testCommitOffsetForLmq() {
+        // Execute
+        offsetManager.commitOffset("clientHost", LMQ_GROUP, LMQ_TOPIC, QUEUE_ID, OFFSET);
+        // Verify
+        Long expectedOffset = offsetManager.getOffsetTable().get(getLMQKey()).get(QUEUE_ID);
+        assertEquals("Offset should be updated correctly.", OFFSET, expectedOffset.longValue());
+    }
+
+    private String getLMQKey() {
+        return LMQ_TOPIC + "@" + LMQ_GROUP;
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapperTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapperTest.java
new file mode 100644
index 00000000000..c01e63f31f7
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapperTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.offset;
+
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.rocketmq.broker.config.v1.RocksDBOffsetSerializeWrapper;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class RocksDBOffsetSerializeWrapperTest {
+
+    private RocksDBOffsetSerializeWrapper wrapper;
+
+    @Before
+    public void setUp() {
+        wrapper = new RocksDBOffsetSerializeWrapper();
+    }
+
+    @Test
+    public void testGetOffsetTable_ShouldReturnConcurrentHashMap() {
+        ConcurrentMap<Integer, Long> offsetTable = wrapper.getOffsetTable();
+        assertNotNull("The offsetTable should not be null", offsetTable);
+    }
+
+    @Test
+    public void testSetOffsetTable_ShouldSetTheOffsetTableCorrectly() {
+        ConcurrentMap<Integer, Long> newOffsetTable = new ConcurrentHashMap<>();
+        wrapper.setOffsetTable(newOffsetTable);
+        ConcurrentMap<Integer, Long> offsetTable = wrapper.getOffsetTable();
+        assertEquals("The offsetTable should be the same as the one set", newOffsetTable, offsetTable);
+    }
+
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java
new file mode 100644
index 00000000000..6a805b04340
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.offset;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.collections.MapUtils;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.Pair;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.DispatchRequest;
+import org.apache.rocketmq.store.RocksDBMessageStore;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
+import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface;
+import org.apache.rocketmq.store.queue.CqUnit;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.awaitility.Awaitility;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.rocksdb.RocksDBException;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RocksdbTransferOffsetAndCqTest {
+
+    private final String basePath = Paths.get(System.getProperty("user.home"),
+        "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString();
+
+    private final String topic = "topic";
+    private final String group = "group";
+    private final String clientHost = "clientHost";
+    private final int queueId = 1;
+
+    private RocksDBConsumerOffsetManager rocksdbConsumerOffsetManager;
+
+    private ConsumerOffsetManager consumerOffsetManager;
+
+    private DefaultMessageStore defaultMessageStore;
+
+    @Mock
+    private BrokerController brokerController;
+
+    @Before
+    public void init() throws IOException {
+        if (notToBeExecuted()) {
+            return;
+        }
+        BrokerConfig brokerConfig = new BrokerConfig();
+        brokerConfig.setConsumerOffsetUpdateVersionStep(10);
+        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+        messageStoreConfig.setStorePathRootDir(basePath);
+        messageStoreConfig.setRocksdbCQDoubleWriteEnable(true);
+        Mockito.lenient().when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
+        Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
+
+        defaultMessageStore = new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager("aaa", true), null,
+            brokerConfig, new ConcurrentHashMap<String, TopicConfig>());
+        defaultMessageStore.enableRocksdbCQWrite();
+        defaultMessageStore.loadCheckPoint();
+
+        consumerOffsetManager = new ConsumerOffsetManager(brokerController);
+        consumerOffsetManager.load();
+
+        rocksdbConsumerOffsetManager = new RocksDBConsumerOffsetManager(brokerController);
+    }
+
+    @Test
+    public void testTransferOffset() {
+        if (notToBeExecuted()) {
+            return;
+        }
+
+        for (int i = 0; i < 200; i++) {
+            consumerOffsetManager.commitOffset(clientHost, group, topic, queueId, i);
+        }
+
+        ConcurrentMap<String, ConcurrentMap<Integer, Long>> offsetTable = consumerOffsetManager.getOffsetTable();
+        ConcurrentMap<Integer, Long> map = offsetTable.get(topic + "@" + group);
+        Assert.assertTrue(MapUtils.isNotEmpty(map));
+
+        Long offset = map.get(queueId);
+        Assert.assertEquals(199L, (long) offset);
+
+        long offsetDataVersion = consumerOffsetManager.getDataVersion().getCounter().get();
+        Assert.assertEquals(20L, offsetDataVersion);
+
+        consumerOffsetManager.persist();
+
+        boolean loadResult = rocksdbConsumerOffsetManager.load();
+        Assert.assertTrue(loadResult);
+
+        ConcurrentMap<String, ConcurrentMap<Integer, Long>> rocksdbOffsetTable = rocksdbConsumerOffsetManager.getOffsetTable();
+
+        ConcurrentMap<Integer, Long> rocksdbMap = rocksdbOffsetTable.get(topic + "@" + group);
+        Assert.assertTrue(MapUtils.isNotEmpty(rocksdbMap));
+
+        Long aLong1 = rocksdbMap.get(queueId);
+        Assert.assertEquals(199L, (long) aLong1);
+
+        long rocksdbOffset = rocksdbConsumerOffsetManager.getDataVersion().getCounter().get();
+        Assert.assertEquals(21L, rocksdbOffset);
+    }
+
+    @Test
+    public void testRocksdbCqWrite() throws RocksDBException {
+        if (notToBeExecuted()) {
+            return;
+        }
+        RocksDBMessageStore kvStore = defaultMessageStore.getRocksDBMessageStore();
+        ConsumeQueueStoreInterface store = kvStore.getConsumeQueueStore();
+        store.start();
+        ConsumeQueueInterface rocksdbCq = defaultMessageStore.getRocksDBMessageStore().findConsumeQueue(topic, queueId);
+        ConsumeQueueInterface fileCq = defaultMessageStore.findConsumeQueue(topic, queueId);
+        for (int i = 0; i < 200; i++) {
+            DispatchRequest request = new DispatchRequest(topic, queueId, i, 200, 0, System.currentTimeMillis(), i, "", "", 0, 0, new HashMap<>());
+            fileCq.putMessagePositionInfoWrapper(request);
+            store.putMessagePositionInfoWrapper(request);
+        }
+        Awaitility.await()
+            .pollInterval(100, TimeUnit.MILLISECONDS)
+            .atMost(3, TimeUnit.SECONDS)
+            .until(() -> rocksdbCq.getMaxOffsetInQueue() == 200);
+        Pair<CqUnit, Long> unit = rocksdbCq.getCqUnitAndStoreTime(100);
+        Pair<CqUnit, Long> unit1 = fileCq.getCqUnitAndStoreTime(100);
+        Assert.assertEquals(unit.getObject1().getPos(), unit1.getObject1().getPos());
+    }
+
+    /**
+     * No need to skip macOS platform.
+     * @return true if some platform is NOT a good fit for this test case.
+     */
+    private boolean notToBeExecuted() {
+        return false;
+    }
+
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerCacheTest.java b/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerCacheTest.java
new file mode 100644
index 00000000000..3f6e893a527
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerCacheTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import java.util.Collections;
+import java.util.Queue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.awaitility.Awaitility;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.mockito.ArgumentMatchers.any;
+
+public class PopConsumerCacheTest {
+
+    private final String attemptId = "attemptId";
+    private final String topicId = "TopicTest";
+    private final String groupId = "GroupTest";
+    private final int queueId = 2;
+
+    @Test
+    public void consumerRecordsTest() {
+        BrokerConfig brokerConfig = new BrokerConfig();
+        brokerConfig.setPopConsumerKVServiceLog(true);
+        PopConsumerCache.ConsumerRecords consumerRecords =
+            new PopConsumerCache.ConsumerRecords(brokerConfig, groupId, topicId, queueId);
+        Assert.assertNotNull(consumerRecords.toString());
+
+        for (int i = 0; i < 5; i++) {
+            consumerRecords.write(new PopConsumerRecord(i, groupId, topicId, queueId, 0,
+                20000, 100 + i, attemptId));
+        }
+        Assert.assertEquals(100, consumerRecords.getMinOffsetInBuffer());
+        Assert.assertEquals(5, consumerRecords.getInFlightRecordCount());
+
+        for (int i = 0; i < 2; i++) {
+            consumerRecords.delete(new PopConsumerRecord(i, groupId, topicId, queueId, 0,
+                20000, 100 + i, attemptId));
+        }
+        Assert.assertEquals(102, consumerRecords.getMinOffsetInBuffer());
+        Assert.assertEquals(3, consumerRecords.getInFlightRecordCount());
+
+        long bufferTimeout = brokerConfig.getPopCkStayBufferTime();
+        Assert.assertEquals(1, consumerRecords.removeExpiredRecords(bufferTimeout + 2).size());
+        Assert.assertNull(consumerRecords.removeExpiredRecords(bufferTimeout + 2));
+        Assert.assertEquals(2, consumerRecords.removeExpiredRecords(bufferTimeout + 4).size());
+        Assert.assertNull(consumerRecords.removeExpiredRecords(bufferTimeout + 4));
+    }
+
+    @Test
+    public void consumerOffsetTest() throws IllegalAccessException {
+        BrokerController brokerController = Mockito.mock(BrokerController.class);
+        PopConsumerKVStore consumerKVStore = Mockito.mock(PopConsumerRocksdbStore.class);
+        PopConsumerLockService consumerLockService = Mockito.mock(PopConsumerLockService.class);
+        ConsumerOffsetManager consumerOffsetManager = Mockito.mock(ConsumerOffsetManager.class);
+        Mockito.when(brokerController.getBrokerConfig()).thenReturn(new BrokerConfig());
+        Mockito.when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);
+        Mockito.when(consumerLockService.tryLock(groupId, topicId)).thenReturn(true);
+
+        PopConsumerCache consumerCache =
+            new PopConsumerCache(brokerController, consumerKVStore, consumerLockService, null);
+        consumerCache.commitOffset("CommitOffsetTest", groupId, topicId, queueId, 100L);
+        consumerCache.removeRecords(groupId, topicId, queueId);
+
+        AtomicInteger estimateCacheSize = (AtomicInteger) FieldUtils.readField(
+            consumerCache, "estimateCacheSize", true);
+        estimateCacheSize.set(2);
+        consumerCache.start();
+        Awaitility.await().until(() -> estimateCacheSize.get() == 0);
+        consumerCache.shutdown();
+    }
+
+    @Test
+    public void consumerCacheTest() {
+        BrokerController brokerController = Mockito.mock(BrokerController.class);
+        PopConsumerKVStore consumerKVStore = Mockito.mock(PopConsumerRocksdbStore.class);
+        PopConsumerLockService consumerLockService = Mockito.mock(PopConsumerLockService.class);
+        Mockito.when(brokerController.getBrokerConfig()).thenReturn(new BrokerConfig());
+
+        PopConsumerCache consumerCache =
+            new PopConsumerCache(brokerController, consumerKVStore, consumerLockService, null);
+        Assert.assertEquals(-1L, consumerCache.getMinOffsetInCache(groupId, topicId, queueId));
+        Assert.assertEquals(0, consumerCache.getPopInFlightMessageCount(groupId, topicId, queueId));
+        Assert.assertEquals(0, consumerCache.getCacheKeySize());
+
+        // write
+        for (int i = 0; i < 3; i++) {
+            PopConsumerRecord record = new PopConsumerRecord(2L, groupId, topicId, queueId,
+                0, 20000, 100 + i, attemptId);
+            Assert.assertEquals(consumerCache.getKey(record), consumerCache.getKey(groupId, topicId, queueId));
+            consumerCache.writeRecords(Collections.singletonList(record));
+        }
+        Assert.assertEquals(100, consumerCache.getMinOffsetInCache(groupId, topicId, queueId));
+        Assert.assertEquals(3, consumerCache.getPopInFlightMessageCount(groupId, topicId, queueId));
+        Assert.assertEquals(1, consumerCache.getCacheKeySize());
+        Assert.assertEquals(3, consumerCache.getCacheSize());
+        Assert.assertFalse(consumerCache.isCacheFull());
+
+        // delete
+        PopConsumerRecord record = new PopConsumerRecord(2L, groupId, topicId, queueId,
+            0, 20000, 100, attemptId);
+        Assert.assertEquals(0, consumerCache.deleteRecords(Collections.singletonList(record)).size());
+        Assert.assertEquals(101, consumerCache.getMinOffsetInCache(groupId, topicId, queueId));
+        Assert.assertEquals(2, consumerCache.getPopInFlightMessageCount(groupId, topicId, queueId));
+        Assert.assertEquals(2, consumerCache.getCacheSize());
+
+        record = new PopConsumerRecord(2L, groupId, topicId, queueId,
+            0, 20000, 104, attemptId);
+        Assert.assertEquals(1, consumerCache.deleteRecords(Collections.singletonList(record)).size());
+        Assert.assertEquals(101, consumerCache.getMinOffsetInCache(groupId, topicId, queueId));
+        Assert.assertEquals(2, consumerCache.getPopInFlightMessageCount(groupId, topicId, queueId));
+
+        // clean expired records
+        Queue<PopConsumerRecord> consumerRecordList = new LinkedBlockingQueue<>();
+        consumerCache.cleanupRecords(consumerRecordList::add);
+        Assert.assertEquals(2, consumerRecordList.size());
+
+        // clean all
+        Mockito.when(consumerLockService.isLockTimeout(any(), any())).thenReturn(true);
+        consumerRecordList.clear();
+        consumerCache.cleanupRecords(consumerRecordList::add);
+        Assert.assertEquals(0, consumerRecordList.size());
+    }
+}
\ No newline at end of file
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerContextTest.java b/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerContextTest.java
new file mode 100644
index 00000000000..6f009423f9d
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerContextTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import org.apache.rocketmq.common.constant.ConsumeInitMode;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class PopConsumerContextTest {
+
+    @Test
+    public void consumerContextTest() {
+        long popTime = System.currentTimeMillis();
+        PopConsumerContext context = new PopConsumerContext("127.0.0.1:6789",
+            popTime, 20_000, "GroupId", true, ConsumeInitMode.MIN, "attemptId");
+
+        Assert.assertFalse(context.isFound());
+        Assert.assertEquals("127.0.0.1:6789", context.getClientHost());
+        Assert.assertEquals(popTime, context.getPopTime());
+        Assert.assertEquals(20_000, context.getInvisibleTime());
+        Assert.assertEquals("GroupId", context.getGroupId());
+        Assert.assertTrue(context.isFifo());
+        Assert.assertEquals("attemptId", context.getAttemptId());
+        Assert.assertEquals(0, context.getRestCount());
+
+        GetMessageResult getMessageResult = new GetMessageResult();
+        getMessageResult.setStatus(GetMessageStatus.FOUND);
+        getMessageResult.setMinOffset(10L);
+        getMessageResult.setMaxOffset(20L);
+        getMessageResult.setNextBeginOffset(15L);
+        getMessageResult.addMessage(Mockito.mock(SelectMappedBufferResult.class), 10);
+        getMessageResult.addMessage(Mockito.mock(SelectMappedBufferResult.class), 12);
+        getMessageResult.addMessage(Mockito.mock(SelectMappedBufferResult.class), 13);
+
+        context.addGetMessageResult(getMessageResult,
+            "TopicId", 3, PopConsumerRecord.RetryType.NORMAL_TOPIC, 1);
+
+        Assert.assertEquals(3, context.getMessageCount());
+        Assert.assertEquals(
+            getMessageResult.getMaxOffset() - getMessageResult.getNextBeginOffset(), context.getRestCount());
+
+        // check header
+        Assert.assertNotNull(context.toString());
+        Assert.assertEquals("0 3 1", context.getStartOffsetInfo());
+        Assert.assertEquals("0 3 10,12,13", context.getMsgOffsetInfo());
+        Assert.assertNotNull(context.getOrderCountInfoBuilder());
+        Assert.assertEquals("", context.getOrderCountInfo());
+
+        Assert.assertEquals(1, context.getGetMessageResultList().size());
+        Assert.assertEquals(3, context.getPopConsumerRecordList().size());
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerLockServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerLockServiceTest.java
new file mode 100644
index 00000000000..b5af2f31798
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerLockServiceTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import java.lang.reflect.Field;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.common.PopAckConstants;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class PopConsumerLockServiceTest {
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void consumerLockTest() throws NoSuchFieldException, IllegalAccessException {
+        String groupId = "groupId";
+        String topicId = "topicId";
+
+        PopConsumerLockService lockService =
+            new PopConsumerLockService(TimeUnit.MINUTES.toMillis(2));
+
+        Assert.assertTrue(lockService.tryLock(groupId, topicId));
+        Assert.assertFalse(lockService.tryLock(groupId, topicId));
+        lockService.unlock(groupId, topicId);
+
+        Assert.assertTrue(lockService.tryLock(groupId, topicId));
+        Assert.assertFalse(lockService.tryLock(groupId, topicId));
+        Assert.assertFalse(lockService.isLockTimeout(groupId, topicId));
+        lockService.removeTimeout();
+
+        // set expired
+        Field field = PopConsumerLockService.class.getDeclaredField("lockTable");
+        field.setAccessible(true);
+        Map<String, PopConsumerLockService.TimedLock> table =
+            (Map<String, PopConsumerLockService.TimedLock>) field.get(lockService);
+
+        Field lockTime = PopConsumerLockService.TimedLock.class.getDeclaredField("lockTime");
+        lockTime.setAccessible(true);
+        lockTime.set(table.get(groupId + PopAckConstants.SPLIT + topicId),
+            System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(3));
+        lockService.removeTimeout();
+
+        Assert.assertEquals(0, table.size());
+    }
+}
\ No newline at end of file
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerRecordTest.java b/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerRecordTest.java
new file mode 100644
index 00000000000..24a79b33f31
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerRecordTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import java.util.UUID;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class PopConsumerRecordTest {
+
+    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+
+    @Test
+    public void retryCodeTest() {
+        Assert.assertEquals("NORMAL_TOPIC code should be 0",
+            0, PopConsumerRecord.RetryType.NORMAL_TOPIC.getCode());
+        Assert.assertEquals("RETRY_TOPIC code should be 1",
+            1, PopConsumerRecord.RetryType.RETRY_TOPIC_V1.getCode());
+        Assert.assertEquals("RETRY_TOPIC_V2 code should be 2",
+            2, PopConsumerRecord.RetryType.RETRY_TOPIC_V2.getCode());
+    }
+
+    @Test
+    public void deliveryRecordSerializeTest() {
+        PopConsumerRecord consumerRecord = new PopConsumerRecord();
+        consumerRecord.setPopTime(System.currentTimeMillis());
+        consumerRecord.setGroupId("GroupId");
+        consumerRecord.setTopicId("TopicId");
+        consumerRecord.setQueueId(3);
+        consumerRecord.setRetryFlag(PopConsumerRecord.RetryType.RETRY_TOPIC_V1.getCode());
+        consumerRecord.setInvisibleTime(20);
+        consumerRecord.setOffset(100);
+        consumerRecord.setAttemptTimes(2);
+        consumerRecord.setAttemptId(UUID.randomUUID().toString().toUpperCase());
+
+        Assert.assertTrue(consumerRecord.isRetry());
+        Assert.assertEquals(consumerRecord.getPopTime() + consumerRecord.getInvisibleTime(),
+            consumerRecord.getVisibilityTimeout());
+        Assert.assertEquals(8 + "GroupId".length() + 1 + "TopicId".length() + 1 + 4 + 1 + 8,
+            consumerRecord.getKeyBytes().length);
+        log.info("ConsumerRecord={}", consumerRecord.toString());
+
+        PopConsumerRecord decodeRecord = PopConsumerRecord.decode(consumerRecord.getValueBytes());
+        PopConsumerRecord consumerRecord2 = new PopConsumerRecord(consumerRecord.getPopTime(),
+            consumerRecord.getGroupId(), consumerRecord.getTopicId(), consumerRecord.getQueueId(),
+            consumerRecord.getRetryFlag(), consumerRecord.getInvisibleTime(),
+            consumerRecord.getOffset(), consumerRecord.getAttemptId());
+        Assert.assertEquals(decodeRecord.getPopTime(), consumerRecord2.getPopTime());
+        Assert.assertEquals(decodeRecord.getGroupId(), consumerRecord2.getGroupId());
+        Assert.assertEquals(decodeRecord.getTopicId(), consumerRecord2.getTopicId());
+        Assert.assertEquals(decodeRecord.getQueueId(), consumerRecord2.getQueueId());
+        Assert.assertEquals(decodeRecord.getRetryFlag(), consumerRecord2.getRetryFlag());
+        Assert.assertEquals(decodeRecord.getInvisibleTime(), consumerRecord2.getInvisibleTime());
+        Assert.assertEquals(decodeRecord.getOffset(), consumerRecord2.getOffset());
+        Assert.assertEquals(0, consumerRecord2.getAttemptTimes());
+        Assert.assertEquals(decodeRecord.getAttemptId(), consumerRecord2.getAttemptId());
+    }
+}
\ No newline at end of file
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerRocksdbStoreTest.java b/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerRocksdbStoreTest.java
new file mode 100644
index 00000000000..3c2b190d1cd
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerRocksdbStoreTest.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import com.google.common.base.Stopwatch;
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.rocketmq.common.config.AbstractRocksDBStorage;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PopConsumerRocksdbStoreTest {
+
+    private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+    private static final String CONSUMER_STORE_PATH = "consumer_rocksdb";
+
+    public static String getRandomStorePath() {
+        return Paths.get(System.getProperty("user.home"), "store_test", CONSUMER_STORE_PATH,
+            UUID.randomUUID().toString().replace("-", "").toUpperCase().substring(0, 16)).toString();
+    }
+
+    public static void deleteStoreDirectory(String storePath) {
+        try {
+            FileUtils.deleteDirectory(new File(storePath));
+        } catch (IOException e) {
+            log.error("Delete store directory failed, filePath: {}", storePath, e);
+        }
+    }
+
+    public static PopConsumerRecord getConsumerRecord() {
+        return new PopConsumerRecord(1L, "GroupTest", "TopicTest", 2,
+            PopConsumerRecord.RetryType.NORMAL_TOPIC.getCode(), TimeUnit.SECONDS.toMillis(20), 100L, "AttemptId");
+    }
+
+    @Test
+    public void rocksdbStoreWriteDeleteTest() {
+        String filePath = getRandomStorePath();
+        PopConsumerKVStore consumerStore = new PopConsumerRocksdbStore(filePath);
+        Assert.assertEquals(filePath, consumerStore.getFilePath());
+
+        consumerStore.start();
+        consumerStore.writeRecords(IntStream.range(0, 3).boxed()
+            .flatMap(i ->
+                IntStream.range(0, 5).mapToObj(j -> {
+                    PopConsumerRecord consumerRecord = getConsumerRecord();
+                    consumerRecord.setPopTime(j);
+                    consumerRecord.setQueueId(i);
+                    consumerRecord.setOffset(100L + j);
+                    return consumerRecord;
+                })
+            )
+            .collect(Collectors.toList()));
+        consumerStore.deleteRecords(IntStream.range(0, 2).boxed()
+            .flatMap(i ->
+                IntStream.range(0, 5).mapToObj(j -> {
+                    PopConsumerRecord consumerRecord = getConsumerRecord();
+                    consumerRecord.setPopTime(j);
+                    consumerRecord.setQueueId(i);
+                    consumerRecord.setOffset(100L + j);
+                    return consumerRecord;
+                })
+            )
+            .collect(Collectors.toList()));
+
+        List<PopConsumerRecord> consumerRecords =
+            consumerStore.scanExpiredRecords(0, 20002, 2);
+        Assert.assertEquals(2, consumerRecords.size());
+        consumerStore.deleteRecords(consumerRecords);
+
+        consumerRecords = consumerStore.scanExpiredRecords(0, 20003, 2);
+        Assert.assertEquals(1, consumerRecords.size());
+        consumerStore.deleteRecords(consumerRecords);
+
+        consumerRecords = consumerStore.scanExpiredRecords(0, 20005, 3);
+        Assert.assertEquals(2, consumerRecords.size());
+
+        consumerStore.shutdown();
+        deleteStoreDirectory(filePath);
+    }
+
+    private long getDirectorySizeRecursive(File directory) {
+        long size = 0;
+        File[] files = directory.listFiles();
+        if (files != null) {
+            for (File file : files) {
+                if (file.isFile()) {
+                    size += file.length();
+                } else if (file.isDirectory()) {
+                    size += getDirectorySizeRecursive(file);
+                }
+            }
+        }
+        return size;
+    }
+
+    @Test
+    @Ignore
+    @SuppressWarnings("ConstantValue")
+    public void tombstoneDeletionTest() throws IllegalAccessException, NoSuchFieldException {
+        PopConsumerRocksdbStore rocksdbStore = new PopConsumerRocksdbStore(getRandomStorePath());
+        rocksdbStore.start();
+
+        int iterCount = 1000 * 1000;
+        boolean useSeekFirstDelete = false;
+        Field dbField = AbstractRocksDBStorage.class.getDeclaredField("db");
+        dbField.setAccessible(true);
+        RocksDB rocksDB = (RocksDB) dbField.get(rocksdbStore);
+
+        long currentTime = 0L;
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        for (int i = 0; i < iterCount; i++) {
+            List<PopConsumerRecord> records = new ArrayList<>();
+            for (int j = 0; j < 1000; j++) {
+                PopConsumerRecord record = getConsumerRecord();
+                record.setPopTime((long) i * iterCount + j);
+                record.setGroupId("GroupTest");
+                record.setTopicId("TopicTest");
+                record.setQueueId(i % 10);
+                record.setRetryFlag(0);
+                record.setInvisibleTime(TimeUnit.SECONDS.toMillis(30));
+                record.setOffset(i);
+                records.add(record);
+            }
+            rocksdbStore.writeRecords(records);
+
+            long start = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+            List<PopConsumerRecord> deleteList = new ArrayList<>();
+            if (useSeekFirstDelete) {
+                try (RocksIterator iterator = rocksDB.newIterator(rocksdbStore.columnFamilyHandle)) {
+                    iterator.seekToFirst();
+                    if (i % 10 == 0) {
+                        long fileSize = getDirectorySizeRecursive(new File(rocksdbStore.getFilePath()));
+                        log.info("DirectorySize={}, Cost={}ms",
+                            MessageStoreUtil.toHumanReadable(fileSize), stopwatch.elapsed(TimeUnit.MILLISECONDS) - start);
+                    }
+                    while (iterator.isValid() && deleteList.size() < 1024) {
+                        deleteList.add(PopConsumerRecord.decode(iterator.value()));
+                        iterator.next();
+                    }
+                }
+            } else {
+                long upper = System.currentTimeMillis();
+                deleteList = rocksdbStore.scanExpiredRecords(currentTime, upper, 800);
+                if (!deleteList.isEmpty()) {
+                    currentTime = deleteList.get(deleteList.size() - 1).getVisibilityTimeout();
+                }
+                long scanCost = stopwatch.elapsed(TimeUnit.MILLISECONDS) - start;
+                if (i % 100 == 0) {
+                    long fileSize = getDirectorySizeRecursive(new File(rocksdbStore.getFilePath()));
+                    long seekTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+                    try (RocksIterator iterator = rocksDB.newIterator(rocksdbStore.columnFamilyHandle)) {
+                        iterator.seekToFirst();
+                    }
+                    log.info("DirectorySize={}, Cost={}ms, SeekFirstCost={}ms", MessageStoreUtil.toHumanReadable(fileSize),
+                        scanCost, stopwatch.elapsed(TimeUnit.MILLISECONDS) - seekTime);
+                }
+            }
+            rocksdbStore.deleteRecords(deleteList);
+        }
+        rocksdbStore.shutdown();
+        deleteStoreDirectory(rocksdbStore.getFilePath());
+    }
+}
\ No newline at end of file
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerServiceTest.java
new file mode 100644
index 00000000000..9c23a8625eb
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/pop/PopConsumerServiceTest.java
@@ -0,0 +1,454 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.pop;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.failover.EscapeBridge;
+import org.apache.rocketmq.broker.longpolling.PopLongPollingService;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.broker.offset.ConsumerOrderInfoManager;
+import org.apache.rocketmq.broker.processor.PopMessageProcessor;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.ConsumeInitMode;
+import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.AppendMessageResult;
+import org.apache.rocketmq.store.AppendMessageStatus;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+
+public class PopConsumerServiceTest {
+
+    private final String clientHost = "127.0.0.1:8888";
+    private final String groupId = "groupId";
+    private final String topicId = "topicId";
+    private final int queueId = 2;
+    private final String attemptId = UUID.randomUUID().toString().toUpperCase();
+    private final String filePath = PopConsumerRocksdbStoreTest.getRandomStorePath();
+
+    private BrokerController brokerController;
+    private PopConsumerService consumerService;
+
+    @Before
+    public void init() throws IOException {
+        BrokerConfig brokerConfig = new BrokerConfig();
+        brokerConfig.setEnablePopLog(true);
+        brokerConfig.setEnablePopBufferMerge(true);
+        brokerConfig.setEnablePopMessageThreshold(true);
+        brokerConfig.setPopInflightMessageThreshold(100);
+        brokerConfig.setPopConsumerKVServiceLog(true);
+        brokerConfig.setEnableRetryTopicV2(true);
+        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+        messageStoreConfig.setStorePathRootDir(filePath);
+
+        TopicConfigManager topicConfigManager = Mockito.mock(TopicConfigManager.class);
+        ConsumerOffsetManager consumerOffsetManager = Mockito.mock(ConsumerOffsetManager.class);
+        PopMessageProcessor popMessageProcessor = Mockito.mock(PopMessageProcessor.class);
+        PopLongPollingService popLongPollingService = Mockito.mock(PopLongPollingService.class);
+        ConsumerOrderInfoManager consumerOrderInfoManager = Mockito.mock(ConsumerOrderInfoManager.class);
+
+        brokerController = Mockito.mock(BrokerController.class);
+        Mockito.when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
+        Mockito.when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
+        Mockito.when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
+        Mockito.when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);
+        Mockito.when(brokerController.getPopMessageProcessor()).thenReturn(popMessageProcessor);
+        Mockito.when(popMessageProcessor.getPopLongPollingService()).thenReturn(popLongPollingService);
+        Mockito.when(brokerController.getConsumerOrderInfoManager()).thenReturn(consumerOrderInfoManager);
+
+        consumerService = new PopConsumerService(brokerController);
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        FileUtils.deleteDirectory(new File(filePath));
+    }
+
+    public PopConsumerRecord getConsumerTestRecord() {
+        PopConsumerRecord popConsumerRecord = new PopConsumerRecord();
+        popConsumerRecord.setPopTime(System.currentTimeMillis());
+        popConsumerRecord.setGroupId(groupId);
+        popConsumerRecord.setTopicId(topicId);
+        popConsumerRecord.setQueueId(queueId);
+        popConsumerRecord.setRetryFlag(PopConsumerRecord.RetryType.NORMAL_TOPIC.getCode());
+        popConsumerRecord.setAttemptTimes(0);
+        popConsumerRecord.setInvisibleTime(TimeUnit.SECONDS.toMillis(20));
+        popConsumerRecord.setAttemptId(UUID.randomUUID().toString().toUpperCase());
+        return popConsumerRecord;
+    }
+
+    @Test
+    public void isPopShouldStopTest() throws IllegalAccessException {
+        Assert.assertFalse(consumerService.isPopShouldStop(groupId, topicId, queueId));
+        PopConsumerCache consumerCache = (PopConsumerCache) FieldUtils.readField(
+            consumerService, "popConsumerCache", true);
+        for (int i = 0; i < 100; i++) {
+            PopConsumerRecord record = getConsumerTestRecord();
+            record.setOffset(i);
+            consumerCache.writeRecords(Collections.singletonList(record));
+        }
+        Assert.assertTrue(consumerService.isPopShouldStop(groupId, topicId, queueId));
+    }
+
+    @Test
+    public void pendingFilterCountTest() throws ConsumeQueueException {
+        MessageStore messageStore = Mockito.mock(MessageStore.class);
+        Mockito.when(messageStore.getMaxOffsetInQueue(topicId, queueId)).thenReturn(100L);
+        Mockito.when(brokerController.getMessageStore()).thenReturn(messageStore);
+        ConsumerOffsetManager consumerOffsetManager = brokerController.getConsumerOffsetManager();
+        Mockito.when(consumerOffsetManager.queryOffset(groupId, topicId, queueId)).thenReturn(20L);
+        Assert.assertEquals(consumerService.getPendingFilterCount(groupId, topicId, queueId), 80L);
+    }
+
+    private MessageExt getMessageExt() {
+        MessageExt messageExt = new MessageExt();
+        messageExt.setTopic(topicId);
+        messageExt.setQueueId(queueId);
+        messageExt.setBody(new byte[128]);
+        messageExt.setBornHost(new InetSocketAddress("127.0.0.1", 8080));
+        messageExt.setStoreHost(new InetSocketAddress("127.0.0.1", 8080));
+        messageExt.putUserProperty("Key", "Value");
+        return messageExt;
+    }
+
+    @Test
+    public void recodeRetryMessageTest() throws Exception {
+        GetMessageResult getMessageResult = new GetMessageResult();
+        getMessageResult.setStatus(GetMessageStatus.FOUND);
+
+        // result is empty
+        SelectMappedBufferResult bufferResult = new SelectMappedBufferResult(
+            0, ByteBuffer.allocate(10), 10, null);
+        getMessageResult.addMessage(bufferResult);
+        getMessageResult.getMessageMapedList().clear();
+        GetMessageResult result = consumerService.recodeRetryMessage(
+            getMessageResult, topicId, 0, 100, 200);
+        Assert.assertEquals(0, result.getMessageMapedList().size());
+
+        ByteBuffer buffer = ByteBuffer.wrap(
+            MessageDecoder.encode(getMessageExt(), false));
+        getMessageResult = new GetMessageResult();
+        getMessageResult.setStatus(GetMessageStatus.FOUND);
+        getMessageResult.addMessage(new SelectMappedBufferResult(
+            0, buffer, buffer.remaining(), null));
+        result = consumerService.recodeRetryMessage(
+            getMessageResult, topicId, 0, 100, 200);
+        Assert.assertNotNull(result);
+        Assert.assertEquals(1, result.getMessageMapedList().size());
+    }
+
+    @Test
+    public void addGetMessageResultTest() {
+        PopConsumerContext context = new PopConsumerContext(
+            clientHost, System.currentTimeMillis(), 20000, groupId, false, ConsumeInitMode.MIN, attemptId);
+        GetMessageResult result = new GetMessageResult();
+        result.setStatus(GetMessageStatus.FOUND);
+        result.getMessageQueueOffset().add(100L);
+        consumerService.handleGetMessageResult(
+            context, result, topicId, queueId, PopConsumerRecord.RetryType.NORMAL_TOPIC, 100);
+        Assert.assertEquals(1, context.getGetMessageResultList().size());
+    }
+
+    @Test
+    public void getMessageAsyncTest() throws Exception {
+        MessageStore messageStore = Mockito.mock(MessageStore.class);
+        Mockito.when(brokerController.getMessageStore()).thenReturn(messageStore);
+        Mockito.when(messageStore.getMessageAsync(groupId, topicId, queueId, 0, 10, null))
+            .thenReturn(CompletableFuture.completedFuture(null));
+        GetMessageResult getMessageResult = consumerService.getMessageAsync(
+            "127.0.0.1:8888", groupId, topicId, queueId, 0, 10, null).join();
+        Assert.assertNull(getMessageResult);
+
+        // success when first get message
+        GetMessageResult firstGetMessageResult = new GetMessageResult();
+        firstGetMessageResult.setStatus(GetMessageStatus.FOUND);
+        Mockito.when(messageStore.getMessageAsync(groupId, topicId, queueId, 0, 10, null))
+            .thenReturn(CompletableFuture.completedFuture(firstGetMessageResult));
+        getMessageResult = consumerService.getMessageAsync(
+            "127.0.0.1:8888", groupId, topicId, queueId, 0, 10, null).join();
+        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
+
+        // reset offset from server
+        firstGetMessageResult.setStatus(GetMessageStatus.OFFSET_FOUND_NULL);
+        firstGetMessageResult.setNextBeginOffset(25);
+        GetMessageResult resetGetMessageResult = new GetMessageResult();
+        resetGetMessageResult.setStatus(GetMessageStatus.FOUND);
+        Mockito.when(messageStore.getMessageAsync(groupId, topicId, queueId, 25, 10, null))
+            .thenReturn(CompletableFuture.completedFuture(resetGetMessageResult));
+        getMessageResult = consumerService.getMessageAsync(
+            "127.0.0.1:8888", groupId, topicId, queueId, 0, 10, null).join();
+        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
+
+        // fifo block
+        PopConsumerContext context = new PopConsumerContext(
+            clientHost, System.currentTimeMillis(), 20000, groupId, false, ConsumeInitMode.MIN, attemptId);
+        consumerService.setFifoBlocked(context, groupId, topicId, queueId, Collections.singletonList(100L));
+        Mockito.when(brokerController.getConsumerOrderInfoManager()
+            .checkBlock(anyString(), anyString(), anyString(), anyInt(), anyLong())).thenReturn(true);
+        Assert.assertTrue(consumerService.isFifoBlocked(context, groupId, topicId, queueId));
+
+        // get message async normal
+        CompletableFuture<PopConsumerContext> future = CompletableFuture.completedFuture(context);
+        Assert.assertEquals(0L, consumerService.getMessageAsync(future, clientHost, groupId, topicId, queueId,
+            10, null, PopConsumerRecord.RetryType.NORMAL_TOPIC).join().getRestCount());
+
+        // get message result full, no need get again
+        for (int i = 0; i < 10; i++) {
+            ByteBuffer buffer = ByteBuffer.wrap(MessageDecoder.encode(getMessageExt(), false));
+            getMessageResult.addMessage(new SelectMappedBufferResult(
+                0, buffer, buffer.remaining(), null), i);
+        }
+        context.addGetMessageResult(getMessageResult, topicId, queueId, PopConsumerRecord.RetryType.NORMAL_TOPIC, 0);
+
+        Mockito.when(brokerController.getMessageStore().getMaxOffsetInQueue(topicId, queueId)).thenReturn(100L);
+        Mockito.when(brokerController.getConsumerOffsetManager().queryOffset(groupId, topicId, queueId)).thenReturn(0L);
+        Assert.assertEquals(100L, consumerService.getMessageAsync(future, clientHost, groupId, topicId, queueId,
+            10, null, PopConsumerRecord.RetryType.NORMAL_TOPIC).join().getRestCount());
+
+        // fifo block test
+        context = new PopConsumerContext(
+            clientHost, System.currentTimeMillis(), 20000, groupId, true, ConsumeInitMode.MIN, attemptId);
+        future = CompletableFuture.completedFuture(context);
+        Assert.assertEquals(0L, consumerService.getMessageAsync(future, clientHost, groupId, topicId, queueId,
+            10, null, PopConsumerRecord.RetryType.NORMAL_TOPIC).join().getRestCount());
+    }
+
+    @Test
+    public void popAsyncTest() {
+        PopConsumerService consumerServiceSpy = Mockito.spy(consumerService);
+        TopicConfigManager topicConfigManager = Mockito.mock(TopicConfigManager.class);
+        Mockito.when(topicConfigManager.selectTopicConfig(topicId)).thenReturn(new TopicConfig(
+            topicId, 2, 2, PermName.PERM_READ | PermName.PERM_WRITE, 0));
+        Mockito.when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
+
+        String[] retryTopic = new String[] {
+            KeyBuilder.buildPopRetryTopicV1(topicId, groupId),
+            KeyBuilder.buildPopRetryTopicV2(topicId, groupId)
+        };
+
+        for (String retry : retryTopic) {
+            GetMessageResult getMessageResult = new GetMessageResult();
+            getMessageResult.setStatus(GetMessageStatus.NO_MATCHED_MESSAGE);
+            getMessageResult.setMinOffset(0L);
+            getMessageResult.setMaxOffset(1L);
+            getMessageResult.setNextBeginOffset(1L);
+            Mockito.doReturn(CompletableFuture.completedFuture(getMessageResult))
+                .when(consumerServiceSpy).getMessageAsync(clientHost, groupId, retry, 0, 0, 10, null);
+            Mockito.doReturn(CompletableFuture.completedFuture(getMessageResult))
+                .when(consumerServiceSpy).getMessageAsync(clientHost, groupId, retry, 0, 0, 8, null);
+        }
+
+        for (int i = -1; i < 2; i++) {
+            GetMessageResult getMessageResult = new GetMessageResult();
+            getMessageResult.setStatus(GetMessageStatus.FOUND);
+            getMessageResult.setMinOffset(0L);
+            getMessageResult.setMaxOffset(1L);
+            getMessageResult.setNextBeginOffset(1L);
+            getMessageResult.addMessage(Mockito.mock(SelectMappedBufferResult.class), 1L);
+
+            Mockito.doReturn(CompletableFuture.completedFuture(getMessageResult))
+                .when(consumerServiceSpy).getMessageAsync(clientHost, groupId, topicId, i, 0, 8, null);
+            Mockito.doReturn(CompletableFuture.completedFuture(getMessageResult))
+                .when(consumerServiceSpy).getMessageAsync(clientHost, groupId, topicId, i, 0, 9, null);
+            Mockito.doReturn(CompletableFuture.completedFuture(getMessageResult))
+                .when(consumerServiceSpy).getMessageAsync(clientHost, groupId, topicId, i, 0, 10, null);
+        }
+
+        // pop broker
+        consumerServiceSpy.popAsync(clientHost, System.currentTimeMillis(),
+            20000, groupId, topicId, -1, 10, false, attemptId, ConsumeInitMode.MIN, null).join();
+    }
+
+    @Test
+    public void ackAsyncTest() {
+        long current = System.currentTimeMillis();
+        consumerService.getPopConsumerStore().start();
+        consumerService.ackAsync(
+            current, 10, groupId, topicId, queueId, 100).join();
+        consumerService.changeInvisibilityDuration(current, 10,
+            current + 100, 10, groupId, topicId, queueId, 100);
+        consumerService.shutdown();
+    }
+
+    @Test
+    public void reviveRetryTest() {
+        Mockito.when(brokerController.getTopicConfigManager().selectTopicConfig(topicId)).thenReturn(null);
+        Mockito.when(brokerController.getConsumerOffsetManager().queryOffset(groupId, topicId, 0)).thenReturn(-1L);
+
+        consumerService.createRetryTopicIfNeeded(groupId, topicId);
+        consumerService.clearCache(groupId, topicId, queueId);
+        MessageExt messageExt = new MessageExt();
+        messageExt.setBody("body".getBytes());
+        messageExt.setBornTimestamp(System.currentTimeMillis());
+        messageExt.setFlag(0);
+        messageExt.setSysFlag(0);
+        messageExt.setReconsumeTimes(1);
+        messageExt.putUserProperty("key", "value");
+
+        PopConsumerRecord record = new PopConsumerRecord();
+        record.setTopicId("topic");
+        record.setGroupId("group");
+        Mockito.when(brokerController.getBrokerStatsManager()).thenReturn(Mockito.mock(BrokerStatsManager.class));
+        Mockito.when(brokerController.getEscapeBridge()).thenReturn(Mockito.mock(EscapeBridge.class));
+        Mockito.when(brokerController.getEscapeBridge().putMessageToSpecificQueue(any(MessageExtBrokerInner.class)))
+            .thenReturn(new PutMessageResult(
+                PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
+
+        PopConsumerService consumerServiceSpy = Mockito.spy(consumerService);
+        Mockito.doNothing().when(consumerServiceSpy).createRetryTopicIfNeeded(any(), any());
+        Assert.assertTrue(consumerServiceSpy.reviveRetry(record, messageExt));
+
+        // write message error
+        Mockito.when(brokerController.getEscapeBridge().putMessageToSpecificQueue(any(MessageExtBrokerInner.class)))
+            .thenReturn(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR,
+                new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));
+        Assert.assertFalse(consumerServiceSpy.reviveRetry(record, messageExt));
+
+        // revive backoff
+        consumerService.getPopConsumerStore().start();
+        List<PopConsumerRecord> consumerRecordList = IntStream.range(0, 3)
+            .mapToObj(i -> {
+                PopConsumerRecord temp = new PopConsumerRecord();
+                temp.setPopTime(0);
+                temp.setInvisibleTime(20 * 1000);
+                temp.setTopicId("topic");
+                temp.setGroupId("group");
+                temp.setQueueId(2);
+                temp.setOffset(i);
+                return temp;
+            })
+            .collect(Collectors.toList());
+        consumerService.getPopConsumerStore().writeRecords(consumerRecordList);
+
+        Mockito.doReturn(CompletableFuture.completedFuture(null))
+            .when(consumerServiceSpy).getMessageAsync(any(PopConsumerRecord.class));
+        consumerServiceSpy.revive(new AtomicLong(20 * 1000), 1);
+
+        Mockito.doReturn(CompletableFuture.completedFuture(
+                Triple.of(null, "GetMessageResult is null", false)))
+            .when(consumerServiceSpy).getMessageAsync(any(PopConsumerRecord.class));
+        consumerServiceSpy.revive(new AtomicLong(20 * 1000), 1);
+
+        Mockito.doReturn(CompletableFuture.completedFuture(
+                Triple.of(Mockito.mock(MessageExt.class), null, false)))
+            .when(consumerServiceSpy).getMessageAsync(any(PopConsumerRecord.class));
+        consumerServiceSpy.revive(new AtomicLong(20 * 1000), 1);
+        consumerService.shutdown();
+    }
+
+    @Test
+    public void reviveBackoffRetryTest() {
+        Mockito.when(brokerController.getEscapeBridge()).thenReturn(Mockito.mock(EscapeBridge.class));
+        PopConsumerService consumerServiceSpy = Mockito.spy(consumerService);
+
+        consumerService.getPopConsumerStore().start();
+
+        long popTime = 1000000000L;
+        long invisibleTime = 60 * 1000L;
+        PopConsumerRecord record = new PopConsumerRecord();
+        record.setPopTime(popTime);
+        record.setInvisibleTime(invisibleTime);
+        record.setTopicId("topic");
+        record.setGroupId("group");
+        record.setQueueId(0);
+        record.setOffset(0);
+        consumerService.getPopConsumerStore().writeRecords(Collections.singletonList(record));
+
+        Mockito.doReturn(CompletableFuture.completedFuture(Triple.of(Mockito.mock(MessageExt.class), "", false)))
+            .when(consumerServiceSpy).getMessageAsync(any(PopConsumerRecord.class));
+        Mockito.when(brokerController.getEscapeBridge().putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenReturn(
+            new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))
+        );
+
+        long visibleTimestamp = popTime + invisibleTime;
+
+        // revive fails
+        Assert.assertEquals(1, consumerServiceSpy.revive(new AtomicLong(visibleTimestamp), 1));
+        // should be invisible now
+        Assert.assertEquals(0, consumerService.getPopConsumerStore().scanExpiredRecords(0, visibleTimestamp, 1).size());
+        // will be visible again in 10 seconds
+        Assert.assertEquals(1, consumerService.getPopConsumerStore().scanExpiredRecords(visibleTimestamp, System.currentTimeMillis() + visibleTimestamp + 10 * 1000, 1).size());
+
+        consumerService.shutdown();
+    }
+
+    @Test
+    public void transferToFsStoreTest() {
+        Assert.assertNotNull(consumerService.getServiceName());
+        List<PopConsumerRecord> consumerRecordList = IntStream.range(0, 3)
+            .mapToObj(i -> {
+                PopConsumerRecord temp = new PopConsumerRecord();
+                temp.setPopTime(0);
+                temp.setInvisibleTime(20 * 1000);
+                temp.setTopicId("topic");
+                temp.setGroupId("group");
+                temp.setQueueId(2);
+                temp.setOffset(i);
+                return temp;
+            })
+            .collect(Collectors.toList());
+
+        Mockito.when(brokerController.getPopMessageProcessor().buildCkMsg(any(), anyInt()))
+            .thenReturn(new MessageExtBrokerInner());
+        Mockito.when(brokerController.getMessageStore()).thenReturn(Mockito.mock(MessageStore.class));
+        Mockito.when(brokerController.getMessageStore().asyncPutMessage(any()))
+            .thenReturn(CompletableFuture.completedFuture(
+                new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));
+
+        consumerService.start();
+        consumerService.getPopConsumerStore().writeRecords(consumerRecordList);
+        consumerService.transferToFsStore();
+        consumerService.shutdown();
+    }
+}
\ No newline at end of file
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java
index c0afb46c330..757b01b63ff 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java
@@ -47,6 +47,7 @@
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -93,7 +94,7 @@ public class AckMessageProcessorTest {
     private static final long MAX_OFFSET_IN_QUEUE = 999;
 
     @Before
-    public void init() throws IllegalAccessException, NoSuchFieldException {
+    public void init() throws IllegalAccessException, NoSuchFieldException, ConsumeQueueException {
         clientInfo = new ClientChannelInfo(channel, "127.0.0.1", LanguageCode.JAVA, 0);
         brokerController.setMessageStore(messageStore);
         Field field = BrokerController.class.getDeclaredField("broker2Client");
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
index e4fcc690d11..a6bcca954d7 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
@@ -20,33 +20,36 @@
 import com.google.common.collect.Sets;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.LongAdder;
+import org.apache.rocketmq.auth.authentication.enums.UserType;
+import org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authorization.enums.Decision;
+import org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.authorization.model.Environment;
+import org.apache.rocketmq.auth.authorization.model.Resource;
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
 import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
 import org.apache.rocketmq.broker.client.ConsumerManager;
+import org.apache.rocketmq.broker.client.net.Broker2Client;
 import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
 import org.apache.rocketmq.broker.schedule.ScheduleMessageService;
-import org.apache.rocketmq.broker.subscription.RocksDBSubscriptionGroupManager;
-import org.apache.rocketmq.broker.topic.RocksDBTopicConfigManager;
+import org.apache.rocketmq.broker.config.v1.RocksDBSubscriptionGroupManager;
+import org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager;
 import org.apache.rocketmq.broker.topic.TopicConfigManager;
 import org.apache.rocketmq.common.BoundaryType;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.MQVersion;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.TopicFilterType;
 import org.apache.rocketmq.common.TopicQueueId;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.attribute.AttributeParser;
+import org.apache.rocketmq.common.constant.FIleReadaheadMode;
 import org.apache.rocketmq.common.constant.PermName;
 import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
 import org.apache.rocketmq.common.message.MessageAccessor;
@@ -54,31 +57,65 @@
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
+import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
 import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
 import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.body.AclInfo;
+import org.apache.rocketmq.remoting.protocol.body.CreateTopicListRequestBody;
+import org.apache.rocketmq.remoting.protocol.body.GroupList;
+import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
 import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;
+import org.apache.rocketmq.remoting.protocol.body.QueryCorrectionOffsetBody;
 import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;
+import org.apache.rocketmq.remoting.protocol.body.UserInfo;
+import org.apache.rocketmq.remoting.protocol.header.CreateAclRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.CreateUserRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.DeleteAclRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.DeleteUserRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ExchangeHAInfoResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetAclRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetAllTopicConfigResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetMinOffsetRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetTopicConfigRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetUserRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ListAclsRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ListUsersRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.NotifyMinBrokerIdChangeRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryCorrectionOffsetHeader;
+import org.apache.rocketmq.remoting.protocol.header.QuerySubscriptionByConsumerRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ResetMasterFlushOffsetHeader;
+import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ResumeCheckHalfMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SearchOffsetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateAclRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateUserRequestHeader;
 import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
 import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
 import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.store.CommitLog;
 import org.apache.rocketmq.store.DefaultMessageStore;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.logfile.DefaultMappedFile;
 import org.apache.rocketmq.store.stats.BrokerStats;
+import org.apache.rocketmq.store.timer.TimerCheckpoint;
+import org.apache.rocketmq.store.timer.TimerMessageStore;
+import org.apache.rocketmq.store.timer.TimerMetrics;
+import org.apache.rocketmq.store.util.LibC;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -87,12 +124,33 @@
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.LongAdder;
+
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anySet;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -110,9 +168,8 @@ public class AdminBrokerProcessorTest {
     private Channel channel;
 
     @Spy
-    private BrokerController
-        brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(),
-            new MessageStoreConfig());
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(),
+        new MessageStoreConfig(), null);
 
     @Mock
     private MessageStore messageStore;
@@ -140,10 +197,37 @@ public class AdminBrokerProcessorTest {
     private DefaultMessageStore defaultMessageStore;
     @Mock
     private ScheduleMessageService scheduleMessageService;
+    @Mock
+    private AuthenticationMetadataManager authenticationMetadataManager;
+    @Mock
+    private AuthorizationMetadataManager authorizationMetadataManager;
+
+    @Mock
+    private TimerMessageStore timerMessageStore;
+
+    @Mock
+    private TimerMetrics timerMetrics;
+
+    @Mock
+    private MessageStoreConfig messageStoreConfig;
+
+    @Mock
+    private CommitLog commitLog;
+
+    @Mock
+    private Broker2Client broker2Client;
+
+    @Mock
+    private ClientChannelInfo clientChannelInfo;
 
     @Before
     public void init() throws Exception {
         brokerController.setMessageStore(messageStore);
+        brokerController.setAuthenticationMetadataManager(authenticationMetadataManager);
+        brokerController.setAuthorizationMetadataManager(authorizationMetadataManager);
+        Field field = BrokerController.class.getDeclaredField("broker2Client");
+        field.setAccessible(true);
+        field.set(brokerController, broker2Client);
 
         //doReturn(sendMessageProcessor).when(brokerController).getSendMessageProcessor();
 
@@ -167,6 +251,8 @@ public void init() throws Exception {
 
         brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig(topic));
         brokerController.getMessageStoreConfig().setTimerWheelEnable(false);
+        when(this.brokerController.getMessageStore().getTimerMessageStore()).thenReturn(timerMessageStore);
+        when(this.timerMessageStore.getTimerMetrics()).thenReturn(timerMetrics);
     }
 
     @After
@@ -233,7 +319,7 @@ public void testUpdateAndCreateTopic() throws Exception {
         for (String topic : systemTopicSet) {
             RemotingCommand request = buildCreateTopicRequest(topic);
             RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
-            assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+            assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);
             assertThat(response.getRemark()).isEqualTo("The topic[" + topic + "] is conflict with system topic.");
         }
 
@@ -241,12 +327,64 @@ public void testUpdateAndCreateTopic() throws Exception {
         String topic = "";
         RemotingCommand request = buildCreateTopicRequest(topic);
         RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
-        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);
 
         topic = "TEST_CREATE_TOPIC";
         request = buildCreateTopicRequest(topic);
         response = adminBrokerProcessor.processRequest(handlerContext, request);
         assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        // test deny MIXED topic type
+        brokerController.getBrokerConfig().setEnableMixedMessageType(false);
+        topic = "TEST_MIXED_TYPE";
+        Map<String, String> attributes = new HashMap<>();
+        attributes.put("+message.type", "MIXED");
+        request = buildCreateTopicRequest(topic, attributes);
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);
+        // test allow MIXED topic type
+        brokerController.getBrokerConfig().setEnableMixedMessageType(true);
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testUpdateAndCreateTopicList() throws RemotingCommandException {
+        List<String> systemTopicList = new ArrayList<>(systemTopicSet);
+        RemotingCommand request = buildCreateTopicListRequest(systemTopicList);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);
+        assertThat(response.getRemark()).isEqualTo("The topic[" + systemTopicList.get(0) + "] is conflict with system topic.");
+
+        List<String> inValidTopicList = new ArrayList<>();
+        inValidTopicList.add("");
+        request = buildCreateTopicListRequest(inValidTopicList);
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);
+
+        List<String> topicList = new ArrayList<>();
+        topicList.add("TEST_CREATE_TOPIC");
+        topicList.add("TEST_CREATE_TOPIC1");
+        request = buildCreateTopicListRequest(topicList);
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+        //test no changes
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        // test deny MIXED topic type
+        brokerController.getBrokerConfig().setEnableMixedMessageType(false);
+        topicList.add("TEST_MIXED_TYPE");
+        topicList.add("TEST_MIXED_TYPE1");
+        Map<String, String> attributes = new HashMap<>();
+        attributes.put("+message.type", "MIXED");
+        request = buildCreateTopicListRequest(topicList, attributes);
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);
+        // test allow MIXED topic type
+        brokerController.getBrokerConfig().setEnableMixedMessageType(true);
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
     }
 
     @Test
@@ -264,7 +402,7 @@ public void testDeleteTopic() throws Exception {
         for (String topic : systemTopicSet) {
             RemotingCommand request = buildDeleteTopicRequest(topic);
             RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
-            assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+            assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);
             assertThat(response.getRemark()).isEqualTo("The topic[" + topic + "] is conflict with system topic.");
         }
 
@@ -634,19 +772,619 @@ public void testGetTopicConfig() throws Exception {
         }
     }
 
+    @Test
+    public void testCreateUser() throws RemotingCommandException {
+        when(authenticationMetadataManager.createUser(any(User.class)))
+            .thenReturn(CompletableFuture.completedFuture(null));
+
+        CreateUserRequestHeader createUserRequestHeader = new CreateUserRequestHeader();
+        createUserRequestHeader.setUsername("abc");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_CREATE_USER, createUserRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        UserInfo userInfo = UserInfo.of("abc", "123", UserType.NORMAL.getName());
+        request.setBody(JSON.toJSONBytes(userInfo));
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        when(authenticationMetadataManager.isSuperUser(eq("rocketmq"))).thenReturn(CompletableFuture.completedFuture(true));
+        createUserRequestHeader = new CreateUserRequestHeader();
+        createUserRequestHeader.setUsername("super");
+        request = RemotingCommand.createRequestCommand(RequestCode.AUTH_CREATE_USER, createUserRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        userInfo = UserInfo.of("super", "123", UserType.SUPER.getName());
+        request.setBody(JSON.toJSONBytes(userInfo));
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        when(authenticationMetadataManager.isSuperUser(eq("rocketmq"))).thenReturn(CompletableFuture.completedFuture(false));
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+    }
+
+    @Test
+    public void testUpdateUser() throws RemotingCommandException {
+        when(authenticationMetadataManager.updateUser(any(User.class)))
+            .thenReturn(CompletableFuture.completedFuture(null));
+        when(authenticationMetadataManager.getUser(eq("abc"))).thenReturn(CompletableFuture.completedFuture(User.of("abc", "123", UserType.NORMAL)));
+        when(authenticationMetadataManager.getUser(eq("super"))).thenReturn(CompletableFuture.completedFuture(User.of("super", "123", UserType.SUPER)));
+
+        UpdateUserRequestHeader updateUserRequestHeader = new UpdateUserRequestHeader();
+        updateUserRequestHeader.setUsername("abc");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_UPDATE_USER, updateUserRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        UserInfo userInfo = UserInfo.of("abc", "123", UserType.NORMAL.getName());
+        request.setBody(JSON.toJSONBytes(userInfo));
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        when(authenticationMetadataManager.isSuperUser(eq("rocketmq"))).thenReturn(CompletableFuture.completedFuture(true));
+        updateUserRequestHeader = new UpdateUserRequestHeader();
+        updateUserRequestHeader.setUsername("super");
+        request = RemotingCommand.createRequestCommand(RequestCode.AUTH_UPDATE_USER, updateUserRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        userInfo = UserInfo.of("super", "123", UserType.SUPER.getName());
+        request.setBody(JSON.toJSONBytes(userInfo));
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        when(authenticationMetadataManager.isSuperUser(eq("rocketmq"))).thenReturn(CompletableFuture.completedFuture(false));
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+    }
+
+    @Test
+    public void testDeleteUser() throws RemotingCommandException {
+        when(authenticationMetadataManager.deleteUser(any(String.class)))
+            .thenReturn(CompletableFuture.completedFuture(null));
+        when(authenticationMetadataManager.getUser(eq("abc"))).thenReturn(CompletableFuture.completedFuture(User.of("abc", "123", UserType.NORMAL)));
+        when(authenticationMetadataManager.getUser(eq("super"))).thenReturn(CompletableFuture.completedFuture(User.of("super", "123", UserType.SUPER)));
+
+        DeleteUserRequestHeader deleteUserRequestHeader = new DeleteUserRequestHeader();
+        deleteUserRequestHeader.setUsername("abc");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_DELETE_USER, deleteUserRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        when(authenticationMetadataManager.isSuperUser(eq("rocketmq"))).thenReturn(CompletableFuture.completedFuture(true));
+        deleteUserRequestHeader = new DeleteUserRequestHeader();
+        deleteUserRequestHeader.setUsername("super");
+        request = RemotingCommand.createRequestCommand(RequestCode.AUTH_DELETE_USER, deleteUserRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        when(authenticationMetadataManager.isSuperUser(eq("rocketmq"))).thenReturn(CompletableFuture.completedFuture(false));
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);
+    }
+
+    @Test
+    public void testGetUser() throws RemotingCommandException {
+        when(authenticationMetadataManager.getUser(eq("abc"))).thenReturn(CompletableFuture.completedFuture(User.of("abc", "123", UserType.NORMAL)));
+
+        GetUserRequestHeader getUserRequestHeader = new GetUserRequestHeader();
+        getUserRequestHeader.setUsername("abc");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_GET_USER, getUserRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+        UserInfo userInfo = JSON.parseObject(new String(response.getBody()), UserInfo.class);
+        assertThat(userInfo.getUsername()).isEqualTo("abc");
+        assertThat(userInfo.getPassword()).isEqualTo("123");
+        assertThat(userInfo.getUserType()).isEqualTo("Normal");
+    }
+
+    @Test
+    public void testListUser() throws RemotingCommandException {
+        when(authenticationMetadataManager.listUser(eq("abc"))).thenReturn(CompletableFuture.completedFuture(Arrays.asList(User.of("abc", "123", UserType.NORMAL))));
+
+        ListUsersRequestHeader listUserRequestHeader = new ListUsersRequestHeader();
+        listUserRequestHeader.setFilter("abc");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_LIST_USER, listUserRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+        List<UserInfo> userInfo = JSON.parseArray(new String(response.getBody()), UserInfo.class);
+        assertThat(userInfo.get(0).getUsername()).isEqualTo("abc");
+        assertThat(userInfo.get(0).getPassword()).isEqualTo("123");
+        assertThat(userInfo.get(0).getUserType()).isEqualTo("Normal");
+    }
+
+    @Test
+    public void testCreateAcl() throws RemotingCommandException {
+        when(authorizationMetadataManager.createAcl(any(Acl.class)))
+            .thenReturn(CompletableFuture.completedFuture(null));
+
+        CreateAclRequestHeader createAclRequestHeader = new CreateAclRequestHeader();
+        createAclRequestHeader.setSubject("User:abc");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_CREATE_ACL, createAclRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        AclInfo aclInfo = AclInfo.of("User:abc", Arrays.asList("Topic:*"), Arrays.asList("PUB"), Arrays.asList("192.168.0.1"), "Grant");
+        request.setBody(JSON.toJSONBytes(aclInfo));
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testUpdateAcl() throws RemotingCommandException {
+        when(authorizationMetadataManager.updateAcl(any(Acl.class)))
+            .thenReturn(CompletableFuture.completedFuture(null));
+
+        UpdateAclRequestHeader updateAclRequestHeader = new UpdateAclRequestHeader();
+        updateAclRequestHeader.setSubject("User:abc");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_UPDATE_ACL, updateAclRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        AclInfo aclInfo = AclInfo.of("User:abc", Arrays.asList("Topic:*"), Arrays.asList("PUB"), Arrays.asList("192.168.0.1"), "Grant");
+        request.setBody(JSON.toJSONBytes(aclInfo));
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testDeleteAcl() throws RemotingCommandException {
+        when(authorizationMetadataManager.deleteAcl(any(), any(), any()))
+            .thenReturn(CompletableFuture.completedFuture(null));
+
+        DeleteAclRequestHeader deleteAclRequestHeader = new DeleteAclRequestHeader();
+        deleteAclRequestHeader.setSubject("User:abc");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_DELETE_ACL, deleteAclRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testGetAcl() throws RemotingCommandException {
+        Acl aclInfo = Acl.of(User.of("abc"), Arrays.asList(Resource.of("Topic:*")), Arrays.asList(Action.PUB), Environment.of("192.168.0.1"), Decision.ALLOW);
+        when(authorizationMetadataManager.getAcl(any(Subject.class))).thenReturn(CompletableFuture.completedFuture(aclInfo));
+
+        GetAclRequestHeader getAclRequestHeader = new GetAclRequestHeader();
+        getAclRequestHeader.setSubject("User:abc");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_GET_ACL, getAclRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+        AclInfo aclInfoData = JSON.parseObject(new String(response.getBody()), AclInfo.class);
+        assertThat(aclInfoData.getSubject()).isEqualTo("User:abc");
+        assertThat(aclInfoData.getPolicies().get(0).getEntries().get(0).getResource()).isEqualTo("Topic:*");
+        assertThat(aclInfoData.getPolicies().get(0).getEntries().get(0).getActions()).containsAll(Arrays.asList(Action.PUB.getName()));
+        assertThat(aclInfoData.getPolicies().get(0).getEntries().get(0).getSourceIps()).containsAll(Arrays.asList("192.168.0.1"));
+        assertThat(aclInfoData.getPolicies().get(0).getEntries().get(0).getDecision()).isEqualTo("Allow");
+    }
+
+    @Test
+    public void testListAcl() throws RemotingCommandException {
+        Acl aclInfo = Acl.of(User.of("abc"), Arrays.asList(Resource.of("Topic:*")), Arrays.asList(Action.PUB), Environment.of("192.168.0.1"), Decision.ALLOW);
+        when(authorizationMetadataManager.listAcl(any(), any())).thenReturn(CompletableFuture.completedFuture(Arrays.asList(aclInfo)));
+
+        ListAclsRequestHeader listAclRequestHeader = new ListAclsRequestHeader();
+        listAclRequestHeader.setSubjectFilter("User:abc");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_LIST_ACL, listAclRequestHeader);
+        request.setVersion(441);
+        request.addExtField("AccessKey", "rocketmq");
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+        List<AclInfo> aclInfoData = JSON.parseArray(new String(response.getBody()), AclInfo.class);
+        assertThat(aclInfoData.get(0).getSubject()).isEqualTo("User:abc");
+        assertThat(aclInfoData.get(0).getPolicies().get(0).getEntries().get(0).getResource()).isEqualTo("Topic:*");
+        assertThat(aclInfoData.get(0).getPolicies().get(0).getEntries().get(0).getActions()).containsAll(Arrays.asList(Action.PUB.getName()));
+        assertThat(aclInfoData.get(0).getPolicies().get(0).getEntries().get(0).getSourceIps()).containsAll(Arrays.asList("192.168.0.1"));
+        assertThat(aclInfoData.get(0).getPolicies().get(0).getEntries().get(0).getDecision()).isEqualTo("Allow");
+    }
+
+    @Test
+    public void testGetTimeCheckPoint() throws RemotingCommandException {
+        when(this.brokerController.getTimerCheckpoint()).thenReturn(null);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TIMER_CHECK_POINT, null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+        assertThat(response.getRemark()).isEqualTo("The checkpoint is null");
+
+        when(this.brokerController.getTimerCheckpoint()).thenReturn(new TimerCheckpoint());
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+    @Test
+    public void testGetTimeMetrics() throws RemotingCommandException, IOException {
+        when(this.brokerController.getMessageStore().getTimerMessageStore()).thenReturn(null);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TIMER_METRICS, null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+
+        when(this.brokerController.getMessageStore().getTimerMessageStore()).thenReturn(timerMessageStore);
+        when(this.timerMessageStore.getTimerMetrics()).thenReturn(timerMetrics);
+        when(this.timerMetrics.encode()).thenReturn(new TimerMetrics.TimerMetricsSerializeWrapper().toJson(false));
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testUpdateColdDataFlowCtrGroupConfig() throws RemotingCommandException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_COLD_DATA_FLOW_CTR_CONFIG, null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        request.setBody("consumerGroup1=1".getBytes());
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        request.setBody("".getBytes());
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testRemoveColdDataFlowCtrGroupConfig() throws RemotingCommandException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REMOVE_COLD_DATA_FLOW_CTR_CONFIG, null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        request.setBody("consumerGroup1".getBytes());
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testGetColdDataFlowCtrInfo() throws RemotingCommandException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_COLD_DATA_FLOW_CTR_INFO, null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testSetCommitLogReadAheadMode() throws RemotingCommandException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SET_COMMITLOG_READ_MODE, null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+
+        HashMap<String, String> extfields = new HashMap<>();
+        extfields.put(FIleReadaheadMode.READ_AHEAD_MODE, String.valueOf(LibC.MADV_DONTNEED));
+        request.setExtFields(extfields);
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);
+
+        extfields.clear();
+        extfields.put(FIleReadaheadMode.READ_AHEAD_MODE, String.valueOf(LibC.MADV_NORMAL));
+        request.setExtFields(extfields);
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        this.brokerController.setMessageStore(defaultMessageStore);
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+
+        when(this.defaultMessageStore.getMessageStoreConfig()).thenReturn(messageStoreConfig);
+        when(this.defaultMessageStore.getCommitLog()).thenReturn(commitLog);
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testGetUnknownCmdResponse() throws RemotingCommandException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(10000, null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.REQUEST_CODE_NOT_SUPPORTED);
+    }
+
+    @Test
+    public void testGetAllMessageRequestMode() throws RemotingCommandException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_MESSAGE_REQUEST_MODE, null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testResetOffset() throws RemotingCommandException {
+        ResetOffsetRequestHeader requestHeader =
+                createRequestHeader("topic","group",-1,false,-1,-1);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader);
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);
+
+        this.brokerController.getTopicConfigManager().getTopicConfigTable().put("topic", new TopicConfig("topic"));
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
+
+        this.brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().put("group", new SubscriptionGroupConfig());
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        requestHeader.setQueueId(0);
+        request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader);
+        request.makeCustomHeaderToNet();
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        requestHeader.setOffset(2L);
+        request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader);
+        request.makeCustomHeaderToNet();
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+    }
+
+    @Test
+    public void testGetConsumerStatus() throws RemotingCommandException {
+        GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader();
+        requestHeader.setGroup("group");
+        requestHeader.setTopic("topic");
+        requestHeader.setClientAddr("");
+        RemotingCommand request = RemotingCommand
+                .createRequestCommand(RequestCode.INVOKE_BROKER_TO_GET_CONSUMER_STATUS, requestHeader);
+        RemotingCommand responseCommand = RemotingCommand.createResponseCommand(null);
+        responseCommand.setCode(ResponseCode.SUCCESS);
+        when(broker2Client.getConsumeStatus(anyString(),anyString(),anyString())).thenReturn(responseCommand);
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testQueryTopicConsumeByWho() throws RemotingCommandException {
+        QueryTopicConsumeByWhoRequestHeader requestHeader = new QueryTopicConsumeByWhoRequestHeader();
+        requestHeader.setTopic("topic");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_CONSUME_BY_WHO, requestHeader);
+        request.makeCustomHeaderToNet();
+        HashSet<String> groups = new HashSet<>();
+        groups.add("group");
+        when(brokerController.getConsumerManager()).thenReturn(consumerManager);
+        when(consumerManager.queryTopicConsumeByWho(anyString())).thenReturn(groups);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+        assertThat(RemotingSerializable.decode(response.getBody(), GroupList.class)
+                .getGroupList().contains("group"))
+                .isEqualTo(groups.contains("group"));
+    }
+
+    @Test
+    public void testQueryTopicByConsumer() throws RemotingCommandException {
+        QueryTopicsByConsumerRequestHeader requestHeader = new QueryTopicsByConsumerRequestHeader();
+        requestHeader.setGroup("group");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPICS_BY_CONSUMER, requestHeader);
+        request.makeCustomHeaderToNet();
+        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testQuerySubscriptionByConsumer() throws RemotingCommandException {
+        QuerySubscriptionByConsumerRequestHeader requestHeader = new QuerySubscriptionByConsumerRequestHeader();
+        requestHeader.setGroup("group");
+        requestHeader.setTopic("topic");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_SUBSCRIPTION_BY_CONSUMER, requestHeader);
+        request.makeCustomHeaderToNet();
+        when(brokerController.getConsumerManager()).thenReturn(consumerManager);
+        when(consumerManager.findSubscriptionData(anyString(),anyString())).thenReturn(null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testGetSystemTopicListFromBroker() throws RemotingCommandException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_BROKER, null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testCleanExpiredConsumeQueue() throws RemotingCommandException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLEAN_EXPIRED_CONSUMEQUEUE, null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testDeleteExpiredCommitLog() throws RemotingCommandException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_EXPIRED_COMMITLOG, null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testCleanUnusedTopic() throws RemotingCommandException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLEAN_UNUSED_TOPIC, null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testGetConsumerRunningInfo() throws RemotingCommandException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException {
+        when(brokerController.getConsumerManager()).thenReturn(consumerManager);
+        when(consumerManager.findChannel(anyString(),anyString())).thenReturn(null);
+        GetConsumerRunningInfoRequestHeader requestHeader = new GetConsumerRunningInfoRequestHeader();
+        requestHeader.setClientId("client");
+        requestHeader.setConsumerGroup("group");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, requestHeader);
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+
+        when(consumerManager.findChannel(anyString(),anyString())).thenReturn(clientChannelInfo);
+        when(clientChannelInfo.getVersion()).thenReturn(MQVersion.Version.V3_0_0_SNAPSHOT.ordinal());
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+
+        when(clientChannelInfo.getVersion()).thenReturn(MQVersion.Version.V5_2_3.ordinal());
+        when(brokerController.getBroker2Client()).thenReturn(broker2Client);
+        when(clientChannelInfo.getChannel()).thenReturn(channel);
+        RemotingCommand responseCommand = RemotingCommand.createResponseCommand(null);
+        responseCommand.setCode(ResponseCode.SUCCESS);
+        when(broker2Client.callClient(any(Channel.class),any(RemotingCommand.class))).thenReturn(responseCommand);
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        when(broker2Client.callClient(any(Channel.class),any(RemotingCommand.class))).thenThrow(new RemotingTimeoutException("timeout"));
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.CONSUME_MSG_TIMEOUT);
+    }
+
+    @Test
+    public void testQueryCorrectionOffset() throws RemotingCommandException {
+        Map<Integer, Long> correctionOffsetMap = new HashMap<>();
+        correctionOffsetMap.put(0, 100L);
+        correctionOffsetMap.put(1, 200L);
+        Map<Integer, Long> compareOffsetMap = new HashMap<>();
+        compareOffsetMap.put(0, 80L);
+        compareOffsetMap.put(1, 300L);
+        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);
+        when(consumerOffsetManager.queryMinOffsetInAllGroup(anyString(),anyString())).thenReturn(correctionOffsetMap);
+        when(consumerOffsetManager.queryOffset(anyString(),anyString())).thenReturn(compareOffsetMap);
+        QueryCorrectionOffsetHeader queryCorrectionOffsetHeader = new QueryCorrectionOffsetHeader();
+        queryCorrectionOffsetHeader.setTopic("topic");
+        queryCorrectionOffsetHeader.setCompareGroup("group");
+        queryCorrectionOffsetHeader.setFilterGroups("");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CORRECTION_OFFSET, queryCorrectionOffsetHeader);
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+        QueryCorrectionOffsetBody body = RemotingSerializable.decode(response.getBody(), QueryCorrectionOffsetBody.class);
+        Map<Integer, Long> correctionOffsets = body.getCorrectionOffsets();
+        assertThat(correctionOffsets.get(0)).isEqualTo(Long.MAX_VALUE);
+        assertThat(correctionOffsets.get(1)).isEqualTo(200L);
+    }
+
+    @Test
+    public void testNotifyMinBrokerIdChange() throws RemotingCommandException {
+        NotifyMinBrokerIdChangeRequestHeader requestHeader = new NotifyMinBrokerIdChangeRequestHeader();
+        requestHeader.setMinBrokerId(1L);
+        requestHeader.setMinBrokerAddr("127.0.0.1:10912");
+        requestHeader.setOfflineBrokerAddr("127.0.0.1:10911");
+        requestHeader.setHaBrokerAddr("");
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFY_MIN_BROKER_ID_CHANGE, requestHeader);
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testUpdateBrokerHaInfo() throws RemotingCommandException {
+        ExchangeHAInfoResponseHeader requestHeader = new ExchangeHAInfoResponseHeader();
+        requestHeader.setMasterAddress("127.0.0.1:10911");
+        requestHeader.setMasterFlushOffset(0L);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader);
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        when(brokerController.getMessageStore()).thenReturn(messageStore);
+        requestHeader.setMasterHaAddress("127.0.0.1:10912");
+        request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader);
+        request.makeCustomHeaderToNet();
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        when(messageStore.getMasterFlushedOffset()).thenReturn(0L);
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testGetBrokerHaStatus() throws RemotingCommandException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_HA_STATUS,null);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+
+        when(brokerController.getMessageStore()).thenReturn(messageStore);
+        when(messageStore.getHARuntimeInfo()).thenReturn(new HARuntimeInfo());
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testResetMasterFlushOffset() throws RemotingCommandException {
+        ResetMasterFlushOffsetHeader requestHeader = new ResetMasterFlushOffsetHeader();
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RESET_MASTER_FLUSH_OFFSET,requestHeader);
+        RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        requestHeader.setMasterFlushOffset(0L);
+        request.makeCustomHeaderToNet();
+        response = adminBrokerProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    private ResetOffsetRequestHeader createRequestHeader(String topic,String group,long timestamp,boolean force,long offset,int queueId) {
+        ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader();
+        requestHeader.setTopic(topic);
+        requestHeader.setGroup(group);
+        requestHeader.setTimestamp(timestamp);
+        requestHeader.setForce(force);
+        requestHeader.setOffset(offset);
+        requestHeader.setQueueId(queueId);
+        return requestHeader;
+    }
+
     private RemotingCommand buildCreateTopicRequest(String topic) {
+        return buildCreateTopicRequest(topic, null);
+    }
+
+    private RemotingCommand buildCreateTopicRequest(String topic, Map<String, String> attributes) {
         CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader();
         requestHeader.setTopic(topic);
         requestHeader.setTopicFilterType(TopicFilterType.SINGLE_TAG.name());
         requestHeader.setReadQueueNums(8);
         requestHeader.setWriteQueueNums(8);
         requestHeader.setPerm(PermName.PERM_READ | PermName.PERM_WRITE);
-
+        if (attributes != null) {
+            requestHeader.setAttributes(AttributeParser.parseToString(attributes));
+        }
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, requestHeader);
         request.makeCustomHeaderToNet();
         return request;
     }
 
+    private RemotingCommand buildCreateTopicListRequest(List<String> topicList) {
+        return buildCreateTopicListRequest(topicList, null);
+    }
+
+    private RemotingCommand buildCreateTopicListRequest(List<String> topicList, Map<String, String> attributes) {
+        List<TopicConfig> topicConfigList = new ArrayList<>();
+        for (String topic:topicList) {
+            TopicConfig topicConfig = new TopicConfig(topic);
+            topicConfig.setReadQueueNums(8);
+            topicConfig.setWriteQueueNums(8);
+            topicConfig.setTopicFilterType(TopicFilterType.SINGLE_TAG);
+            topicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE);
+            topicConfig.setTopicSysFlag(0);
+            topicConfig.setOrder(false);
+            if (attributes != null) {
+                topicConfig.setAttributes(new HashMap<>(attributes));
+            }
+            topicConfigList.add(topicConfig);
+        }
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC_LIST, null);
+        CreateTopicListRequestBody createTopicListRequestBody = new CreateTopicListRequestBody(topicConfigList);
+        request.setBody(createTopicListRequestBody.encode());
+        return request;
+    }
+
     private RemotingCommand buildDeleteTopicRequest(String topic) {
         DeleteTopicRequestHeader requestHeader = new DeleteTopicRequestHeader();
         requestHeader.setTopic(topic);
@@ -675,6 +1413,7 @@ private SelectMappedBufferResult createSelectMappedBufferResult() {
 
     private ResumeCheckHalfMessageRequestHeader createResumeCheckHalfMessageRequestHeader() {
         ResumeCheckHalfMessageRequestHeader header = new ResumeCheckHalfMessageRequestHeader();
+        header.setTopic("topic");
         header.setMsgId("C0A803CA00002A9F0000000000031367");
         return header;
     }
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java
index e51e110a7a8..e15d51b4a87 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java
@@ -19,6 +19,7 @@
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
 import java.lang.reflect.Field;
+import java.util.concurrent.CompletableFuture;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.client.ClientChannelInfo;
 import org.apache.rocketmq.broker.client.net.Broker2Client;
@@ -28,8 +29,6 @@
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
-import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
-import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
 import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.remoting.protocol.LanguageCode;
@@ -45,6 +44,7 @@
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -55,6 +55,8 @@
 import static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -62,7 +64,7 @@
 public class ChangeInvisibleTimeProcessorTest {
     private ChangeInvisibleTimeProcessor changeInvisibleTimeProcessor;
     @Spy
-    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig(), null);
     @Mock
     private ChannelHandlerContext handlerContext;
     @Mock
@@ -107,8 +109,9 @@ public void init() throws IllegalAccessException, NoSuchFieldException {
     }
 
     @Test
-    public void testProcessRequest_Success() throws RemotingCommandException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException {
-        when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
+    public void testProcessRequest_Success() throws RemotingCommandException, ConsumeQueueException {
+        when(messageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(2L);
+        when(escapeBridge.asyncPutMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));
         int queueId = 0;
         long queueOffset = 0;
         long popTime = System.currentTimeMillis() - 1_000;
@@ -132,4 +135,31 @@ public void testProcessRequest_Success() throws RemotingCommandException, Interr
         assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);
         assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque());
     }
+
+    @Test
+    public void testProcessRequest_NoMessage() throws RemotingCommandException, ConsumeQueueException {
+        when(messageStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(2L);
+        int queueId = 0;
+        long queueOffset = 2;
+        long popTime = System.currentTimeMillis() - 1_000;
+        long invisibleTime = 30_000;
+        int reviveQid = 0;
+        String brokerName = "test_broker";
+        String extraInfo = ExtraInfoUtil.buildExtraInfo(queueOffset, popTime, invisibleTime, reviveQid,
+            topic, brokerName, queueId) + MessageConst.KEY_SEPARATOR + queueOffset;
+
+        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();
+        requestHeader.setTopic(topic);
+        requestHeader.setQueueId(queueId);
+        requestHeader.setOffset(queueOffset);
+        requestHeader.setConsumerGroup(group);
+        requestHeader.setExtraInfo(extraInfo);
+        requestHeader.setInvisibleTime(invisibleTime);
+
+        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader);
+        request.makeCustomHeaderToNet();
+        RemotingCommand responseToReturn = changeInvisibleTimeProcessor.processRequest(handlerContext, request);
+        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_MESSAGE);
+        assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque());
+    }
 }
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessorTest.java
index dd7584b5276..6b3c2578af3 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessorTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessorTest.java
@@ -16,18 +16,36 @@
  */
 package org.apache.rocketmq.broker.processor;
 
+import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.broker.out.BrokerOuterAPI;
+import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
 import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.broker.topic.TopicQueueMappingManager;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.TopicConfig;
-import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
 import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
-import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetResponseHeader;
+import org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem;
+import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingContext;
+import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.remoting.rpc.RpcClient;
+import org.apache.rocketmq.remoting.rpc.RpcException;
+import org.apache.rocketmq.remoting.rpc.RpcResponse;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.junit.Before;
@@ -37,7 +55,17 @@
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
@@ -49,42 +77,202 @@ public class ConsumerManageProcessorTest {
     private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
     @Mock
     private MessageStore messageStore;
+    @Mock
+    private Channel channel;
+    @Mock
+    private ConsumerOffsetManager consumerOffsetManager;
+    @Mock
+    private BrokerOuterAPI brokerOuterAPI;
+    @Mock
+    private RpcClient rpcClient;
+    @Mock
+    private Future<RpcResponse> responseFuture;
+    @Mock
+    private TopicQueueMappingContext mappingContext;
 
     private String topic = "FooBar";
     private String group = "FooBarGroup";
 
     @Before
-    public void init() {
+    public void init() throws RpcException {
         brokerController.setMessageStore(messageStore);
         TopicConfigManager topicConfigManager = new TopicConfigManager(brokerController);
         topicConfigManager.getTopicConfigTable().put(topic, new TopicConfig(topic));
         when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
+        SubscriptionGroupManager subscriptionGroupManager = new SubscriptionGroupManager(brokerController);
+        subscriptionGroupManager.getSubscriptionGroupTable().put(group, new SubscriptionGroupConfig());
+        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);
         consumerManageProcessor = new ConsumerManageProcessor(brokerController);
+        when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);
+        when(brokerOuterAPI.getRpcClient()).thenReturn(rpcClient);
+        when(rpcClient.invoke(any(),anyLong())).thenReturn(responseFuture);
+        TopicQueueMappingDetail topicQueueMappingDetail = new TopicQueueMappingDetail();
+        topicQueueMappingDetail.setBname("BrokerA");
+        when(mappingContext.getMappingDetail()).thenReturn(topicQueueMappingDetail);
     }
 
     @Test
     public void testUpdateConsumerOffset_InvalidTopic() throws Exception {
-        RemotingCommand request = createConsumerManageCommand(RequestCode.UPDATE_CONSUMER_OFFSET);
-        request.addExtField("topic", "InvalidTopic");
+        RemotingCommand request = buildUpdateConsumerOffsetRequest(group, "InvalidTopic", 0, 0);
         RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request);
         assertThat(response).isNotNull();
         assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);
     }
 
-    private RemotingCommand createConsumerManageCommand(int requestCode) {
-        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
-        requestHeader.setProducerGroup(group);
+    @Test
+    public void testUpdateConsumerOffset_GroupNotExist() throws Exception {
+        RemotingCommand request = buildUpdateConsumerOffsetRequest("NotExistGroup", topic, 0, 0);
+        RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
+    }
+
+    @Test
+    public void testUpdateConsumerOffset() throws RemotingCommandException {
+        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);
+        when(consumerOffsetManager.hasOffsetReset(anyString(),anyString(),anyInt())).thenReturn(true);
+        RemotingCommand request = buildUpdateConsumerOffsetRequest(group, topic, 0, 0);
+        RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        when(consumerOffsetManager.hasOffsetReset(anyString(),anyString(),anyInt())).thenReturn(false);
+        response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testGetConsumerListByGroup() throws RemotingCommandException {
+        GetConsumerListByGroupRequestHeader requestHeader = new GetConsumerListByGroupRequestHeader();
+        requestHeader.setConsumerGroup(group);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, requestHeader);
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+
+        brokerController.getConsumerManager().getConsumerTable().put(group,new ConsumerGroupInfo(group));
+        response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
+
+        ConsumerGroupInfo consumerGroupInfo =
+                this.brokerController.getConsumerManager().getConsumerGroupInfo(
+                        requestHeader.getConsumerGroup());
+        consumerGroupInfo.getChannelInfoTable().put(channel,new ClientChannelInfo(channel));
+        response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testQueryConsumerOffset() throws RemotingCommandException, ExecutionException, InterruptedException {
+        RemotingCommand request = buildQueryConsumerOffsetRequest(group, topic, 0, true);
+        RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.QUERY_NOT_FOUND);
+
+        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);
+        when(consumerOffsetManager.queryOffset(anyString(),anyString(),anyInt())).thenReturn(0L);
+        response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        when(consumerOffsetManager.queryOffset(anyString(),anyString(),anyInt())).thenReturn(-1L);
+        when(messageStore.getMinOffsetInQueue(anyString(),anyInt())).thenReturn(-1L);
+        when(messageStore.checkInMemByConsumeOffset(anyString(),anyInt(),anyLong(),anyInt())).thenReturn(true);
+        response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        TopicQueueMappingManager topicQueueMappingManager = mock(TopicQueueMappingManager.class);
+        when(brokerController.getTopicQueueMappingManager()).thenReturn(topicQueueMappingManager);
+        when(topicQueueMappingManager.buildTopicQueueMappingContext(any(QueryConsumerOffsetRequestHeader.class))).thenReturn(mappingContext);
+        response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.NOT_LEADER_FOR_QUEUE);
+
+        List<LogicQueueMappingItem> items = new ArrayList<>();
+        LogicQueueMappingItem item1 = createLogicQueueMappingItem("BrokerC", 0, 0L, 0L);
+        items.add(item1);
+        when(mappingContext.getMappingItemList()).thenReturn(items);
+        when(mappingContext.getLeaderItem()).thenReturn(item1);
+        when(mappingContext.getCurrentItem()).thenReturn(item1);
+        when(mappingContext.isLeader()).thenReturn(true);
+        response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        LogicQueueMappingItem item2 = createLogicQueueMappingItem("BrokerA", 0, 0L, 0L);
+        items.add(item2);
+        QueryConsumerOffsetResponseHeader queryConsumerOffsetResponseHeader = new QueryConsumerOffsetResponseHeader();
+        queryConsumerOffsetResponseHeader.setOffset(0L);
+        RpcResponse rpcResponse = new RpcResponse(ResponseCode.SUCCESS,queryConsumerOffsetResponseHeader,null);
+        when(responseFuture.get()).thenReturn(rpcResponse);
+        response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+
+        queryConsumerOffsetResponseHeader.setOffset(-1L);
+        rpcResponse = new RpcResponse(ResponseCode.SUCCESS,queryConsumerOffsetResponseHeader,null);
+        when(responseFuture.get()).thenReturn(rpcResponse);
+        response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.QUERY_NOT_FOUND);
+    }
+
+    @Test
+    public void testRewriteRequestForStaticTopic() throws RpcException, ExecutionException, InterruptedException {
+        UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader();
+        requestHeader.setConsumerGroup(group);
+        requestHeader.setTopic(topic);
+        requestHeader.setQueueId(0);
+        requestHeader.setCommitOffset(0L);
+
+        RemotingCommand response = consumerManageProcessor.rewriteRequestForStaticTopic(requestHeader, mappingContext);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.NOT_LEADER_FOR_QUEUE);
+
+        List<LogicQueueMappingItem> items = new ArrayList<>();
+        LogicQueueMappingItem item = createLogicQueueMappingItem("BrokerC", 0, 0L, 0L);
+        items.add(item);
+        when(mappingContext.getMappingItemList()).thenReturn(items);
+        when(mappingContext.isLeader()).thenReturn(true);
+        RpcResponse rpcResponse = new RpcResponse(ResponseCode.SUCCESS,new UpdateConsumerOffsetResponseHeader(),null);
+        when(responseFuture.get()).thenReturn(rpcResponse);
+        response = consumerManageProcessor.rewriteRequestForStaticTopic(requestHeader, mappingContext);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    public RemotingCommand buildQueryConsumerOffsetRequest(String group, String topic, int queueId,boolean setZeroIfNotFound) {
+        QueryConsumerOffsetRequestHeader requestHeader = new QueryConsumerOffsetRequestHeader();
+        requestHeader.setConsumerGroup(group);
+        requestHeader.setTopic(topic);
+        requestHeader.setQueueId(queueId);
+        requestHeader.setSetZeroIfNotFound(setZeroIfNotFound);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader);
+        request.makeCustomHeaderToNet();
+        return request;
+    }
+
+    public LogicQueueMappingItem createLogicQueueMappingItem(String brokerName, int queueId, long startOffset, long logicOffset) {
+        LogicQueueMappingItem item = new LogicQueueMappingItem();
+        item.setBname(brokerName);
+        item.setQueueId(queueId);
+        item.setStartOffset(startOffset);
+        item.setLogicOffset(logicOffset);
+        return item;
+    }
+
+    private RemotingCommand buildUpdateConsumerOffsetRequest(String group, String topic, int queueId, long offset) {
+        UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader();
+        requestHeader.setConsumerGroup(group);
         requestHeader.setTopic(topic);
-        requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC);
-        requestHeader.setDefaultTopicQueueNums(3);
-        requestHeader.setQueueId(1);
-        requestHeader.setSysFlag(0);
-        requestHeader.setBornTimestamp(System.currentTimeMillis());
-        requestHeader.setFlag(124);
-        requestHeader.setReconsumeTimes(0);
-
-        RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader);
-        request.setBody(new byte[] {'a'});
+        requestHeader.setQueueId(queueId);
+        requestHeader.setCommitOffset(offset);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, requestHeader);
         request.makeCustomHeaderToNet();
         return request;
     }
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java
index a364a1bbeeb..e4360f147b0 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java
@@ -26,6 +26,7 @@
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.common.stats.Stats;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
@@ -56,6 +57,8 @@
 @RunWith(MockitoJUnitRunner.class)
 public class EndTransactionProcessorTest {
 
+    private static final String TOPIC = "trans_topic_test";
+
     private EndTransactionProcessor endTransactionProcessor;
 
     @Mock
@@ -64,7 +67,7 @@ public class EndTransactionProcessorTest {
     @Spy
     private BrokerController
         brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(),
-            new MessageStoreConfig());
+            new MessageStoreConfig(), null);
 
     @Mock
     private MessageStore messageStore;
@@ -95,20 +98,26 @@ private OperationResult createResponse(int status) {
     public void testProcessRequest() throws RemotingCommandException {
         when(transactionMsgService.commitMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS));
         when(messageStore.putMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
+                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, createAppendMessageResult(AppendMessageStatus.PUT_OK)));
         RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_COMMIT_TYPE, false);
         RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request);
         assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+        assertThat(brokerController.getBrokerStatsManager().getStatsItem(Stats.BROKER_PUT_NUMS, brokerController.getBrokerConfig().getBrokerClusterName()).getValue().sum()).isEqualTo(1);
+        assertThat(brokerController.getBrokerStatsManager().getStatsItem(Stats.TOPIC_PUT_NUMS, TOPIC).getValue().sum()).isEqualTo(1L);
+        assertThat(brokerController.getBrokerStatsManager().getStatsItem(Stats.TOPIC_PUT_SIZE, TOPIC).getValue().sum()).isEqualTo(1L);
     }
 
     @Test
     public void testProcessRequest_CheckMessage() throws RemotingCommandException {
         when(transactionMsgService.commitMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS));
         when(messageStore.putMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
+                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, createAppendMessageResult(AppendMessageStatus.PUT_OK)));
         RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_COMMIT_TYPE, true);
         RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request);
         assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+        assertThat(brokerController.getBrokerStatsManager().getStatsItem(Stats.BROKER_PUT_NUMS, brokerController.getBrokerConfig().getBrokerClusterName()).getValue().sum()).isEqualTo(1);
+        assertThat(brokerController.getBrokerStatsManager().getStatsItem(Stats.TOPIC_PUT_NUMS, TOPIC).getValue().sum()).isEqualTo(1L);
+        assertThat(brokerController.getBrokerStatsManager().getStatsItem(Stats.TOPIC_PUT_SIZE, TOPIC).getValue().sum()).isEqualTo(1L);
     }
 
     @Test
@@ -148,6 +157,7 @@ private MessageExt createDefaultMessageExt() {
         messageExt.setQueueId(0);
         messageExt.setCommitLogOffset(123456789L);
         messageExt.setQueueOffset(1234);
+        MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_TOPIC, TOPIC);
         MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_QUEUE_ID, "0");
         MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_TRANSACTION_PREPARED, "true");
         MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_PRODUCER_GROUP, "testTransactionGroup");
@@ -156,6 +166,7 @@ private MessageExt createDefaultMessageExt() {
 
     private EndTransactionRequestHeader createEndTransactionRequestHeader(int status, boolean isCheckMsg) {
         EndTransactionRequestHeader header = new EndTransactionRequestHeader();
+        header.setTopic("topic");
         header.setCommitLogOffset(123456789L);
         header.setFromTransactionCheck(isCheckMsg);
         header.setCommitOrRollback(status);
@@ -195,4 +206,12 @@ private MessageExt createRejectMessageExt() {
         MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS, "60");
         return messageExt;
     }
+
+    private AppendMessageResult createAppendMessageResult(AppendMessageStatus status) {
+        AppendMessageResult result = new AppendMessageResult(status);
+        result.setMsgId("12345678");
+        result.setMsgNum(1);
+        result.setWroteBytes(1);
+        return result;
+    }
 }
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PeekMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PeekMessageProcessorTest.java
new file mode 100644
index 00000000000..9baf2a6ebb3
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PeekMessageProcessorTest.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.PeekMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class PeekMessageProcessorTest {
+
+    private PeekMessageProcessor peekMessageProcessor;
+
+    @Spy
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
+
+    @Mock
+    private ChannelHandlerContext handlerContext;
+
+    @Mock
+    private MessageStore messageStore;
+
+    @Mock
+    private SubscriptionGroupManager subscriptionGroupManager;
+
+    @Mock
+    private ConsumerOffsetManager consumerOffsetManager;
+
+    @Mock
+    private SubscriptionGroupConfig subscriptionGroupConfig;
+
+    @Mock
+    private Channel channel;
+
+    private TopicConfigManager topicConfigManager;
+
+    @Before
+    public void init() {
+        peekMessageProcessor = new PeekMessageProcessor(brokerController);
+        when(brokerController.getMessageStore()).thenReturn(messageStore);
+        topicConfigManager = new TopicConfigManager(brokerController);
+        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
+        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);
+        when(subscriptionGroupManager.findSubscriptionGroupConfig(anyString())).thenReturn(subscriptionGroupConfig);
+        when(subscriptionGroupConfig.isConsumeEnable()).thenReturn(true);
+        topicConfigManager.getTopicConfigTable().put("topic", new TopicConfig("topic"));
+        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);
+        when(consumerOffsetManager.queryOffset(anyString(), anyString(), anyInt())).thenReturn(-1L);
+        when(messageStore.getMinOffsetInQueue(anyString(),anyInt())).thenReturn(0L);
+        when(handlerContext.channel()).thenReturn(channel);
+        when(channel.remoteAddress()).thenReturn(new InetSocketAddress("127.0.0.1", 12345));
+    }
+
+    @Test
+    public void testProcessRequest() throws RemotingCommandException {
+        RemotingCommand request = createPeekMessageRequest("group","topic",0);
+        GetMessageResult getMessageResult = new GetMessageResult();
+        getMessageResult.setStatus(GetMessageStatus.FOUND);
+        ByteBuffer bb = ByteBuffer.allocate(64);
+        bb.putLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION, System.currentTimeMillis());
+        SelectMappedBufferResult mappedBufferResult1 = new SelectMappedBufferResult(0, bb, 64, null);
+        for (int i = 0; i < 10;i++) {
+            getMessageResult.addMessage(mappedBufferResult1);
+        }
+        when(messageStore.getMessage(anyString(),anyString(),anyInt(),anyLong(),anyInt(),any())).thenReturn(getMessageResult);
+        RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testProcessRequest_NoPermission() throws RemotingCommandException {
+        this.brokerController.getBrokerConfig().setBrokerPermission(PermName.PERM_WRITE);
+        RemotingCommand request = createPeekMessageRequest("group","topic",0);
+        RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);
+        this.brokerController.getBrokerConfig().setBrokerPermission(PermName.PERM_WRITE | PermName.PERM_READ);
+
+        topicConfigManager.getTopicConfigTable().get("topic").setPerm(PermName.PERM_WRITE);
+        response = peekMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);
+        topicConfigManager.getTopicConfigTable().get("topic").setPerm(PermName.PERM_WRITE | PermName.PERM_READ);
+
+        when(subscriptionGroupConfig.isConsumeEnable()).thenReturn(false);
+        response = peekMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);
+    }
+
+    @Test
+    public void testProcessRequest_TopicNotExist() throws RemotingCommandException {
+        RemotingCommand request = createPeekMessageRequest("group1","topic1",0);
+        RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);
+    }
+
+    @Test
+    public void testProcessRequest_SubscriptionGroupNotExist() throws RemotingCommandException {
+        when(subscriptionGroupManager.findSubscriptionGroupConfig(anyString())).thenReturn(null);
+        RemotingCommand request = createPeekMessageRequest("group","topic",0);
+        RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
+    }
+
+    @Test
+    public void testProcessRequest_QueueIdError() throws RemotingCommandException {
+        RemotingCommand request = createPeekMessageRequest("group","topic",17);
+        RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER);
+    }
+
+    private RemotingCommand createPeekMessageRequest(String group,String topic,int queueId) {
+        PeekMessageRequestHeader peekMessageRequestHeader = new PeekMessageRequestHeader();
+        peekMessageRequestHeader.setConsumerGroup(group);
+        peekMessageRequestHeader.setTopic(topic);
+        peekMessageRequestHeader.setMaxMsgNums(10);
+        peekMessageRequestHeader.setQueueId(queueId);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PEEK_MESSAGE, peekMessageRequestHeader);
+        request.makeCustomHeaderToNet();
+        return request;
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java
index d8c8fa1034e..fdb0690e5dc 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java
@@ -23,6 +23,7 @@
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.client.ClientChannelInfo;
 import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.KeyBuilder;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.constant.ConsumeInitMode;
 import org.apache.rocketmq.common.message.MessageDecoder;
@@ -39,7 +40,9 @@
 import org.apache.rocketmq.store.GetMessageStatus;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.logfile.DefaultMappedFile;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -53,6 +56,7 @@
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
@@ -151,14 +155,70 @@ public void testProcessRequest_whenTimerWheelIsFalse() throws RemotingCommandExc
         assertThat(response.getRemark()).contains("pop message is forbidden because timerWheelEnable is false");
     }
 
+    @Test
+    public void testGetInitOffset_retryTopic() throws RemotingCommandException {
+        when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());
+        String newGroup = group + "-" + System.currentTimeMillis();
+        String retryTopic = KeyBuilder.buildPopRetryTopic(topic, newGroup);
+        long minOffset = 100L;
+        when(messageStore.getMinOffsetInQueue(retryTopic, 0)).thenReturn(minOffset);
+        brokerController.getTopicConfigManager().getTopicConfigTable().put(retryTopic, new TopicConfig(retryTopic, 1, 1));
+        GetMessageResult getMessageResult = createGetMessageResult(0);
+        when(messageStore.getMessageAsync(eq(newGroup), anyString(), anyInt(), anyLong(), anyInt(), any()))
+                .thenReturn(CompletableFuture.completedFuture(getMessageResult));
+
+        long offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, retryTopic, 0);
+        Assert.assertEquals(-1, offset);
+
+        RemotingCommand request = createPopMsgCommand(newGroup, topic, 0, ConsumeInitMode.MAX);
+        popMessageProcessor.processRequest(handlerContext, request);
+        offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, retryTopic, 0);
+        Assert.assertEquals(minOffset, offset);
+
+        when(messageStore.getMinOffsetInQueue(retryTopic, 0)).thenReturn(minOffset * 2);
+        popMessageProcessor.processRequest(handlerContext, request);
+        offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, retryTopic, 0);
+        Assert.assertEquals(minOffset, offset); // will not entry getInitOffset() again
+        messageStore.getMinOffsetInQueue(retryTopic, 0); // prevent UnnecessaryStubbingException
+    }
+
+    @Test
+    public void testGetInitOffset_normalTopic() throws RemotingCommandException, ConsumeQueueException {
+        long maxOffset = 999L;
+        when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());
+        when(messageStore.getMaxOffsetInQueue(topic, 0)).thenReturn(maxOffset);
+        String newGroup = group + "-" + System.currentTimeMillis();
+        GetMessageResult getMessageResult = createGetMessageResult(0);
+        when(messageStore.getMessageAsync(eq(newGroup), anyString(), anyInt(), anyLong(), anyInt(), any()))
+                .thenReturn(CompletableFuture.completedFuture(getMessageResult));
+
+        long offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, topic, 0);
+        Assert.assertEquals(-1, offset);
+
+        RemotingCommand request = createPopMsgCommand(newGroup, topic, 0, ConsumeInitMode.MAX);
+        popMessageProcessor.processRequest(handlerContext, request);
+        offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, topic, 0);
+        Assert.assertEquals(maxOffset - 1, offset); // checkInMem return false
+
+        when(messageStore.getMaxOffsetInQueue(topic, 0)).thenReturn(maxOffset * 2);
+        popMessageProcessor.processRequest(handlerContext, request);
+        offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, topic, 0);
+        Assert.assertEquals(maxOffset - 1, offset); // will not entry getInitOffset() again
+        messageStore.getMaxOffsetInQueue(topic, 0); // prevent UnnecessaryStubbingException
+    }
+
     private RemotingCommand createPopMsgCommand() {
+        return createPopMsgCommand(group, topic, -1, ConsumeInitMode.MAX);
+    }
+
+    private RemotingCommand createPopMsgCommand(String group, String topic, int queueId, int initMode) {
         PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();
         requestHeader.setConsumerGroup(group);
         requestHeader.setMaxMsgNums(30);
-        requestHeader.setQueueId(-1);
+        requestHeader.setQueueId(queueId);
         requestHeader.setTopic(topic);
         requestHeader.setInvisibleTime(10_000);
-        requestHeader.setInitMode(ConsumeInitMode.MAX);
+        requestHeader.setInitMode(initMode);
         requestHeader.setOrder(false);
         requestHeader.setPollTime(15_000);
         requestHeader.setBornTime(System.currentTimeMillis());
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java
index 78b76264fef..3010e836101 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java
@@ -20,12 +20,17 @@
 import java.net.SocketAddress;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.commons.lang3.tuple.Triple;
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.failover.EscapeBridge;
 import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
 import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
 import org.apache.rocketmq.broker.topic.TopicConfigManager;
 import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.KeyBuilder;
 import org.apache.rocketmq.common.PopAckConstants;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.message.MessageConst;
@@ -36,9 +41,14 @@
 import org.apache.rocketmq.common.utils.NetworkUtil;
 import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
 import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.AppendMessageResult;
+import org.apache.rocketmq.store.AppendMessageStatus;
 import org.apache.rocketmq.store.pop.AckMsg;
 import org.apache.rocketmq.store.pop.PopCheckPoint;
 import org.apache.rocketmq.store.timer.TimerMessageStore;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -50,19 +60,25 @@
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.times;
 
 @RunWith(MockitoJUnitRunner.Silent.class)
 public class PopReviveServiceTest {
 
-    private static final String REVIVE_TOPIC = PopAckConstants.REVIVE_TOPIC + "test";
+    private static final String CLUSTER_NAME = "test";
+    private static final String REVIVE_TOPIC = PopAckConstants.buildClusterReviveTopic(CLUSTER_NAME);
     private static final int REVIVE_QUEUE_ID = 0;
     private static final String GROUP = "group";
     private static final String TOPIC = "topic";
     private static final SocketAddress STORE_HOST = NetworkUtil.string2SocketAddress("127.0.0.1:8080");
+    private static final Long INVISIBLE_TIME = 1000L;
 
     @Mock
     private MessageStore messageStore;
@@ -76,6 +92,9 @@ public class PopReviveServiceTest {
     private SubscriptionGroupManager subscriptionGroupManager;
     @Mock
     private BrokerController brokerController;
+    @Mock
+    private EscapeBridge escapeBridge;
+    private PopMessageProcessor popMessageProcessor;
 
     private BrokerConfig brokerConfig;
     private PopReviveService popReviveService;
@@ -83,12 +102,13 @@ public class PopReviveServiceTest {
     @Before
     public void before() {
         brokerConfig = new BrokerConfig();
-
+        brokerConfig.setBrokerClusterName(CLUSTER_NAME);
         when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
         when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);
         when(brokerController.getMessageStore()).thenReturn(messageStore);
         when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
         when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);
+        when(brokerController.getEscapeBridge()).thenReturn(escapeBridge);
         when(messageStore.getTimerMessageStore()).thenReturn(timerMessageStore);
         when(timerMessageStore.getDequeueBehind()).thenReturn(0L);
         when(timerMessageStore.getEnqueueBehind()).thenReturn(0L);
@@ -96,6 +116,9 @@ public void before() {
         when(topicConfigManager.selectTopicConfig(anyString())).thenReturn(new TopicConfig());
         when(subscriptionGroupManager.findSubscriptionGroupConfig(anyString())).thenReturn(new SubscriptionGroupConfig());
 
+        popMessageProcessor = new PopMessageProcessor(brokerController); // a real one, not mock
+        when(brokerController.getPopMessageProcessor()).thenReturn(popMessageProcessor);
+
         popReviveService = spy(new PopReviveService(brokerController, REVIVE_TOPIC, REVIVE_QUEUE_ID));
         popReviveService.setShouldRunPopRevive(true);
     }
@@ -204,6 +227,184 @@ public void testSkipLongWaiteAckWithSameAck() throws Throwable {
         assertEquals(maxReviveOffset, commitOffsetCaptor.getValue().longValue());
     }
 
+    @Test
+    public void testReviveMsgFromCk_messageFound_writeRetryOK() throws Throwable {
+        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);
+        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();
+        reviveObj.map.put("", ck);
+        reviveObj.endTime = System.currentTimeMillis();
+        StringBuilder actualRetryTopic = new StringBuilder();
+
+        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))
+                .thenReturn(CompletableFuture.completedFuture(Triple.of(new MessageExt(), "", false)));
+        when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> {
+            MessageExtBrokerInner msg = invocation.getArgument(0);
+            actualRetryTopic.append(msg.getTopic());
+            return new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK));
+        });
+
+        popReviveService.mergeAndRevive(reviveObj);
+        Assert.assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, GROUP, false), actualRetryTopic.toString());
+        verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry
+        verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK
+    }
+
+    @Test
+    public void testReviveMsgFromCk_messageFound_writeRetryFailed_rewriteCK() throws Throwable {
+        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);
+        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();
+        reviveObj.map.put("", ck);
+        reviveObj.endTime = System.currentTimeMillis();
+        StringBuilder actualRetryTopic = new StringBuilder();
+        StringBuilder actualReviveTopic = new StringBuilder();
+        AtomicLong actualInvisibleTime = new AtomicLong(0L);
+
+        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))
+                .thenReturn(CompletableFuture.completedFuture(Triple.of(new MessageExt(), "", false)));
+        when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> {
+            MessageExtBrokerInner msg = invocation.getArgument(0);
+            actualRetryTopic.append(msg.getTopic());
+            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED));
+        });
+        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> {
+            MessageExtBrokerInner msg = invocation.getArgument(0);
+            actualReviveTopic.append(msg.getTopic());
+            PopCheckPoint rewriteCK = JSON.parseObject(msg.getBody(), PopCheckPoint.class);
+            actualInvisibleTime.set(rewriteCK.getReviveTime());
+            return new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK));
+        });
+
+        popReviveService.mergeAndRevive(reviveObj);
+        Assert.assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, GROUP, false), actualRetryTopic.toString());
+        Assert.assertEquals(REVIVE_TOPIC, actualReviveTopic.toString());
+        Assert.assertEquals(INVISIBLE_TIME + 10 * 1000L, actualInvisibleTime.get()); // first interval is 10s
+        verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry
+        verify(messageStore, times(1)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK
+    }
+
+    @Test
+    public void testReviveMsgFromCk_messageFound_writeRetryFailed_rewriteCK_end() throws Throwable {
+        brokerConfig.setSkipWhenCKRePutReachMaxTimes(true);
+        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);
+        ck.setRePutTimes("17");
+        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();
+        reviveObj.map.put("", ck);
+        reviveObj.endTime = System.currentTimeMillis();
+        StringBuilder actualRetryTopic = new StringBuilder();
+
+        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))
+                .thenReturn(CompletableFuture.completedFuture(Triple.of(new MessageExt(), "", false)));
+        when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> {
+            MessageExtBrokerInner msg = invocation.getArgument(0);
+            actualRetryTopic.append(msg.getTopic());
+            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED));
+        });
+
+        popReviveService.mergeAndRevive(reviveObj);
+        Assert.assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, GROUP, false), actualRetryTopic.toString());
+        verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry
+        verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK
+    }
+
+    @Test
+    public void testReviveMsgFromCk_messageFound_writeRetryFailed_rewriteCK_noEnd() throws Throwable {
+        brokerConfig.setSkipWhenCKRePutReachMaxTimes(false);
+        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);
+        ck.setRePutTimes(Byte.MAX_VALUE + "");
+        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();
+        reviveObj.map.put("", ck);
+        reviveObj.endTime = System.currentTimeMillis();
+        StringBuilder actualRetryTopic = new StringBuilder();
+
+        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))
+                .thenReturn(CompletableFuture.completedFuture(Triple.of(new MessageExt(), "", false)));
+        when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> {
+            MessageExtBrokerInner msg = invocation.getArgument(0);
+            actualRetryTopic.append(msg.getTopic());
+            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED));
+        });
+
+        popReviveService.mergeAndRevive(reviveObj);
+        Assert.assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, GROUP, false), actualRetryTopic.toString());
+        verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry
+        verify(messageStore, times(1)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK
+    }
+
+    @Test
+    public void testReviveMsgFromCk_messageNotFound_noRetry() throws Throwable {
+        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);
+        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();
+        reviveObj.map.put("", ck);
+        reviveObj.endTime = System.currentTimeMillis();
+
+        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))
+                .thenReturn(CompletableFuture.completedFuture(Triple.of(null, "", false)));
+
+        popReviveService.mergeAndRevive(reviveObj);
+        verify(escapeBridge, times(0)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry
+        verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK
+    }
+
+    @Test
+    public void testReviveMsgFromCk_messageNotFound_needRetry() throws Throwable {
+        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);
+        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();
+        reviveObj.map.put("", ck);
+        reviveObj.endTime = System.currentTimeMillis();
+        StringBuilder actualReviveTopic = new StringBuilder();
+        AtomicLong actualInvisibleTime = new AtomicLong(0L);
+
+        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))
+                .thenReturn(CompletableFuture.completedFuture(Triple.of(null, "", true)));
+        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> {
+            MessageExtBrokerInner msg = invocation.getArgument(0);
+            actualReviveTopic.append(msg.getTopic());
+            PopCheckPoint rewriteCK = JSON.parseObject(msg.getBody(), PopCheckPoint.class);
+            actualInvisibleTime.set(rewriteCK.getReviveTime());
+            return new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK));
+        });
+
+        popReviveService.mergeAndRevive(reviveObj);
+        Assert.assertEquals(REVIVE_TOPIC, actualReviveTopic.toString());
+        Assert.assertEquals(INVISIBLE_TIME + 10 * 1000L, actualInvisibleTime.get()); // first interval is 10s
+        verify(escapeBridge, times(0)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry
+        verify(messageStore, times(1)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK
+    }
+
+    @Test
+    public void testReviveMsgFromCk_messageNotFound_needRetry_end() throws Throwable {
+        brokerConfig.setSkipWhenCKRePutReachMaxTimes(true);
+        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);
+        ck.setRePutTimes("17");
+        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();
+        reviveObj.map.put("", ck);
+        reviveObj.endTime = System.currentTimeMillis();
+
+        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))
+                .thenReturn(CompletableFuture.completedFuture(Triple.of(null, "", true)));
+
+        popReviveService.mergeAndRevive(reviveObj);
+        verify(escapeBridge, times(0)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry
+        verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK
+    }
+
+    @Test
+    public void testReviveMsgFromCk_messageNotFound_needRetry_noEnd() throws Throwable {
+        brokerConfig.setSkipWhenCKRePutReachMaxTimes(false);
+        PopCheckPoint ck = buildPopCheckPoint(0, 0, 0);
+        ck.setRePutTimes(Byte.MAX_VALUE + "");
+        PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj();
+        reviveObj.map.put("", ck);
+        reviveObj.endTime = System.currentTimeMillis();
+
+        when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean()))
+                .thenReturn(CompletableFuture.completedFuture(Triple.of(null, "", true)));
+
+        popReviveService.mergeAndRevive(reviveObj);
+        verify(escapeBridge, times(0)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry
+        verify(messageStore, times(1)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK
+    }
+
     public static PopCheckPoint buildPopCheckPoint(long startOffset, long popTime, long reviveOffset) {
         PopCheckPoint ck = new PopCheckPoint();
         ck.setStartOffset(startOffset);
@@ -214,7 +415,8 @@ public static PopCheckPoint buildPopCheckPoint(long startOffset, long popTime, l
         ck.setNum((byte) 1);
         ck.setBitMap(0);
         ck.setReviveOffset(reviveOffset);
-        ck.setInvisibleTime(1000);
+        ck.setInvisibleTime(INVISIBLE_TIME);
+        ck.setBrokerName("broker-a");
         return ck;
     }
 
@@ -226,6 +428,7 @@ public static AckMsg buildAckMsg(long offset, long popTime) {
         ackMsg.setTopic(TOPIC);
         ackMsg.setQueueId(0);
         ackMsg.setPopTime(popTime);
+        ackMsg.setBrokerName("broker-a");
 
         return ackMsg;
     }
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java
index e91c1a09617..67ff74897ef 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java
@@ -19,8 +19,10 @@
 import com.google.common.collect.ImmutableSet;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
+import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.client.ClientChannelInfo;
 import org.apache.rocketmq.broker.topic.TopicRouteInfoManager;
@@ -126,6 +128,24 @@ public void testSetMessageRequestMode_RetryTopic() throws Exception {
         assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);
     }
 
+    @Test
+    public void testDoLoadBalance() throws Exception {
+        Method method = queryAssignmentProcessor.getClass()
+            .getDeclaredMethod("doLoadBalance", String.class, String.class, String.class, MessageModel.class,
+                String.class, SetMessageRequestModeRequestBody.class, ChannelHandlerContext.class);
+        method.setAccessible(true);
+
+        Set<MessageQueue> mqs1 = (Set<MessageQueue>) method.invoke(
+            queryAssignmentProcessor, MixAll.LMQ_PREFIX + topic, group, "127.0.0.1", MessageModel.CLUSTERING,
+            new AllocateMessageQueueAveragely().getName(), new SetMessageRequestModeRequestBody(), handlerContext);
+        Set<MessageQueue> mqs2 = (Set<MessageQueue>) method.invoke(
+            queryAssignmentProcessor, MixAll.LMQ_PREFIX + topic, group, "127.0.0.2", MessageModel.CLUSTERING,
+            new AllocateMessageQueueAveragely().getName(), new SetMessageRequestModeRequestBody(), handlerContext);
+
+        assertThat(mqs1).hasSize(1);
+        assertThat(mqs2).isEmpty();
+    }
+
     @Test
     public void testAllocate4Pop() {
         testAllocate4Pop(new AllocateMessageQueueAveragely());
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryMessageProcessorTest.java
new file mode 100644
index 00000000000..0fd54df7d8a
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryMessageProcessorTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.QueryMessageResult;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.HashMap;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class QueryMessageProcessorTest {
+    private QueryMessageProcessor queryMessageProcessor;
+    @Spy
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
+
+    @Mock
+    private MessageStore messageStore;
+
+    @Mock
+    private ChannelHandlerContext handlerContext;
+
+    @Mock
+    private Channel channel;
+
+    @Mock
+    private ChannelFuture channelFuture;
+
+    @Before
+    public void init() {
+        when(handlerContext.channel()).thenReturn(channel);
+        queryMessageProcessor = new QueryMessageProcessor(brokerController);
+        when(brokerController.getMessageStore()).thenReturn(messageStore);
+        when(channel.writeAndFlush(any())).thenReturn(channelFuture);
+    }
+
+    @Test
+    public void testQueryMessage() throws RemotingCommandException {
+        QueryMessageResult result = new QueryMessageResult();
+        result.setIndexLastUpdateTimestamp(100);
+        result.setIndexLastUpdatePhyoffset(0);
+        result.addMessage(new SelectMappedBufferResult(0, null, 0, null));
+
+        when(messageStore.queryMessage(anyString(),anyString(),anyInt(),anyLong(),anyLong())).thenReturn(result);
+        RemotingCommand request = createQueryMessageRequest("topic", "msgKey", 1, 100, 200,"false");
+        request.makeCustomHeaderToNet();
+        RemotingCommand response = queryMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertEquals(response.getCode(), ResponseCode.QUERY_NOT_FOUND);
+
+        result.addMessage(new SelectMappedBufferResult(0, null, 1, null));
+        when(messageStore.queryMessage(anyString(),anyString(),anyInt(),anyLong(),anyLong())).thenReturn(result);
+        response = queryMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertNull(response);
+    }
+
+    @Test
+    public void testViewMessageById() throws RemotingCommandException {
+        ViewMessageRequestHeader viewMessageRequestHeader = new ViewMessageRequestHeader();
+        viewMessageRequestHeader.setTopic("topic");
+        viewMessageRequestHeader.setOffset(0L);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.VIEW_MESSAGE_BY_ID, viewMessageRequestHeader);
+        request.makeCustomHeaderToNet();
+        request.setCode(RequestCode.VIEW_MESSAGE_BY_ID);
+
+        when(messageStore.selectOneMessageByOffset(anyLong())).thenReturn(null);
+        RemotingCommand response = queryMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertEquals(response.getCode(), ResponseCode.SYSTEM_ERROR);
+
+        when(messageStore.selectOneMessageByOffset(anyLong())).thenReturn(new SelectMappedBufferResult(0, null, 0, null));
+        response = queryMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertNull(response);
+    }
+
+    private RemotingCommand createQueryMessageRequest(String topic, String key, int maxNum, long beginTimestamp, long endTimestamp,String flag) {
+        QueryMessageRequestHeader requestHeader = new QueryMessageRequestHeader();
+        requestHeader.setTopic(topic);
+        requestHeader.setKey(key);
+        requestHeader.setMaxNum(maxNum);
+        requestHeader.setBeginTimestamp(beginTimestamp);
+        requestHeader.setEndTimestamp(endTimestamp);
+
+        HashMap<String, String> extFields = new HashMap<>();
+        extFields.put(MixAll.UNIQUE_MSG_QUERY_FLAG, flag);
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, requestHeader);
+        request.setExtFields(extFields);
+        return request;
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/RecallMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/RecallMessageProcessorTest.java
new file mode 100644
index 00000000000..d28eb2f1dff
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/RecallMessageProcessorTest.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;
+import org.apache.rocketmq.store.AppendMessageResult;
+import org.apache.rocketmq.store.AppendMessageStatus;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.config.BrokerRole;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.verify;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RecallMessageProcessorTest {
+    private static final String TOPIC = "topic";
+    private static final String BROKER_NAME = "brokerName";
+
+    private RecallMessageProcessor recallMessageProcessor;
+    @Mock
+    private BrokerConfig brokerConfig;
+    @Mock
+    private BrokerController brokerController;
+    @Mock
+    private ChannelHandlerContext handlerContext;
+    @Mock
+    private MessageStoreConfig messageStoreConfig;
+    @Mock
+    private TopicConfigManager topicConfigManager;
+    @Mock
+    private MessageStore messageStore;
+    @Mock
+    private BrokerStatsManager brokerStatsManager;
+    @Mock
+    private Channel channel;
+
+    @Before
+    public void init() throws IllegalAccessException, NoSuchFieldException {
+        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
+        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
+        when(brokerController.getMessageStore()).thenReturn(messageStore);
+        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
+        when(brokerConfig.getBrokerName()).thenReturn(BROKER_NAME);
+        when(brokerConfig.isRecallMessageEnable()).thenReturn(true);
+        when(brokerController.getBrokerStatsManager()).thenReturn(brokerStatsManager);
+        when(handlerContext.channel()).thenReturn(channel);
+        recallMessageProcessor = new RecallMessageProcessor(brokerController);
+    }
+
+    @Test
+    public void testBuildMessage() {
+        String timestampStr = String.valueOf(System.currentTimeMillis());
+        String id = "id";
+        RecallMessageHandle.HandleV1 handle = new RecallMessageHandle.HandleV1(TOPIC, "brokerName", timestampStr, id);
+        MessageExtBrokerInner msg =
+            recallMessageProcessor.buildMessage(handlerContext, new RecallMessageRequestHeader(), handle);
+
+        Assert.assertEquals(TOPIC, msg.getTopic());
+        Map<String, String> properties = MessageDecoder.string2messageProperties(msg.getPropertiesString());
+        Assert.assertEquals(timestampStr, properties.get(MessageConst.PROPERTY_TIMER_DELIVER_MS));
+        Assert.assertEquals(id, properties.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
+        Assert.assertEquals(TOPIC + "+" + id, properties.get(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY));
+    }
+
+    @Test
+    public void testHandlePutMessageResult() {
+        MessageExt message = new MessageExt();
+        MessageAccessor.putProperty(message, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "id");
+        RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class);
+        recallMessageProcessor.handlePutMessageResult(null, null, response, message, handlerContext, 0L);
+        Assert.assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode());
+
+        List<PutMessageStatus> okStatus = Arrays.asList(PutMessageStatus.PUT_OK, PutMessageStatus.FLUSH_DISK_TIMEOUT,
+            PutMessageStatus.FLUSH_SLAVE_TIMEOUT, PutMessageStatus.SLAVE_NOT_AVAILABLE);
+
+        for (PutMessageStatus status : PutMessageStatus.values()) {
+            PutMessageResult putMessageResult =
+                new PutMessageResult(status, new AppendMessageResult(AppendMessageStatus.PUT_OK));
+            recallMessageProcessor.handlePutMessageResult(putMessageResult, null, response, message, handlerContext, 0L);
+            if (okStatus.contains(status)) {
+                Assert.assertEquals(ResponseCode.SUCCESS, response.getCode());
+                RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();
+                Assert.assertEquals("id", responseHeader.getMsgId());
+            } else {
+                Assert.assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode());
+            }
+        }
+    }
+
+    @Test
+    public void testProcessRequest_notEnable() throws RemotingCommandException {
+        when(brokerConfig.isRecallMessageEnable()).thenReturn(false);
+        RemotingCommand request = mockRequest(0, TOPIC, TOPIC, "id", BROKER_NAME);
+        RemotingCommand response = recallMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertEquals(ResponseCode.NO_PERMISSION, response.getCode());
+    }
+
+    @Test
+    public void testProcessRequest_invalidStatus() throws RemotingCommandException {
+        RemotingCommand request = mockRequest(0, TOPIC, TOPIC, "id", BROKER_NAME);
+        RemotingCommand response;
+
+        // role slave
+        when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE);
+        response = recallMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertEquals(ResponseCode.SLAVE_NOT_AVAILABLE, response.getCode());
+
+        // not reach startTimestamp
+        when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SYNC_MASTER);
+        when(messageStore.now()).thenReturn(0L);
+        when(brokerConfig.getStartAcceptSendRequestTimeStamp()).thenReturn(System.currentTimeMillis());
+        response = recallMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertEquals(ResponseCode.SERVICE_NOT_AVAILABLE, response.getCode());
+    }
+
+    @Test
+    public void testProcessRequest_notWriteable() throws RemotingCommandException {
+        when(brokerConfig.getBrokerPermission()).thenReturn(4);
+        when(brokerConfig.isAllowRecallWhenBrokerNotWriteable()).thenReturn(false);
+        RemotingCommand request = mockRequest(0, TOPIC, TOPIC, "id", BROKER_NAME);
+        RemotingCommand response = recallMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertEquals(ResponseCode.SERVICE_NOT_AVAILABLE, response.getCode());
+    }
+
+    @Test
+    public void testProcessRequest_topicNotFound_or_notMatch() throws RemotingCommandException {
+        when(brokerConfig.getBrokerPermission()).thenReturn(6);
+        RemotingCommand request;
+        RemotingCommand response;
+
+        // not found
+        request = mockRequest(0, TOPIC, TOPIC, "id", BROKER_NAME);
+        response = recallMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertEquals(ResponseCode.TOPIC_NOT_EXIST, response.getCode());
+
+        // not match
+        when(topicConfigManager.selectTopicConfig(TOPIC)).thenReturn(new TopicConfig(TOPIC));
+        request = mockRequest(0, TOPIC, "anotherTopic", "id", BROKER_NAME);
+        response = recallMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode());
+    }
+
+    @Test
+    public void testProcessRequest_brokerNameNotMatch() throws RemotingCommandException {
+        when(brokerConfig.getBrokerPermission()).thenReturn(6);
+        when(topicConfigManager.selectTopicConfig(TOPIC)).thenReturn(new TopicConfig(TOPIC));
+
+        RemotingCommand request = mockRequest(0, TOPIC, "anotherTopic", "id", BROKER_NAME + "_other");
+        RemotingCommand response = recallMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode());
+    }
+
+    @Test
+    public void testProcessRequest_timestampInvalid() throws RemotingCommandException {
+        when(brokerConfig.getBrokerPermission()).thenReturn(6);
+        when(topicConfigManager.selectTopicConfig(TOPIC)).thenReturn(new TopicConfig(TOPIC));
+        RemotingCommand request;
+        RemotingCommand response;
+
+        // past timestamp
+        request = mockRequest(0, TOPIC, TOPIC, "id", BROKER_NAME);
+        response = recallMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode());
+
+        // timestamp overflow
+        when(messageStoreConfig.getTimerMaxDelaySec()).thenReturn(86400);
+        request = mockRequest(System.currentTimeMillis() + 86400 * 2 * 1000, TOPIC, TOPIC, "id", BROKER_NAME);
+        response = recallMessageProcessor.processRequest(handlerContext, request);
+        Assert.assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode());
+    }
+
+    @Test
+    public void testProcessRequest_success() throws RemotingCommandException {
+        when(brokerConfig.getBrokerPermission()).thenReturn(6);
+        when(topicConfigManager.selectTopicConfig(TOPIC)).thenReturn(new TopicConfig(TOPIC));
+        when(messageStoreConfig.getTimerMaxDelaySec()).thenReturn(86400);
+        when(messageStore.putMessage(any())).thenReturn(
+            new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
+
+        String msgId = "msgId";
+        RemotingCommand request = mockRequest(System.currentTimeMillis() + 90 * 1000, TOPIC, TOPIC, msgId, BROKER_NAME);
+        RemotingCommand response = recallMessageProcessor.processRequest(handlerContext, request);
+        RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();
+        Assert.assertEquals(ResponseCode.SUCCESS, response.getCode());
+        Assert.assertEquals(msgId, responseHeader.getMsgId());
+        verify(messageStore, times(1)).putMessage(any());
+    }
+
+    private RemotingCommand mockRequest(long timestamp, String requestTopic, String handleTopic,
+        String msgId, String brokerName) {
+        String handle =
+            RecallMessageHandle.HandleV1.buildHandle(handleTopic, brokerName, String.valueOf(timestamp), msgId);
+        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();
+        requestHeader.setProducerGroup("group");
+        requestHeader.setTopic(requestTopic);
+        requestHeader.setRecallHandle(handle);
+        requestHeader.setBrokerName(brokerName);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader);
+        request.makeCustomHeaderToNet();
+        return request;
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java
index e046c888438..9da6a96ec99 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java
@@ -25,6 +25,7 @@
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
+import org.apache.commons.codec.DecoderException;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.AbortProcessException;
 import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;
@@ -36,10 +37,13 @@
 import org.apache.rocketmq.broker.transaction.TransactionalMessageService;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageClientIDSetter;
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -50,12 +54,14 @@
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
 import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;
 import org.apache.rocketmq.store.AppendMessageResult;
 import org.apache.rocketmq.store.AppendMessageStatus;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -69,6 +75,8 @@
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.times;
 
 @RunWith(MockitoJUnitRunner.class)
 public class SendMessageProcessorTest {
@@ -78,6 +86,8 @@ public class SendMessageProcessorTest {
     @Mock
     private Channel channel;
     @Spy
+    private BrokerConfig brokerConfig;
+    @Spy
     private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(),
         new NettyClientConfig(), new MessageStoreConfig());
     @Mock
@@ -98,6 +108,7 @@ public void init() {
         when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);
         when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
         when(brokerController.getPutMessageFutureExecutor()).thenReturn(Executors.newSingleThreadExecutor());
+        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
         when(messageStore.now()).thenReturn(System.currentTimeMillis());
         when(channel.remoteAddress()).thenReturn(new InetSocketAddress(1024));
         when(handlerContext.channel()).thenReturn(channel);
@@ -174,7 +185,7 @@ public void testProcessRequest_FlushSlaveTimeout() throws Exception {
     public void testProcessRequest_PageCacheBusy() throws Exception {
         when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).
             thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
-        assertPutResult(ResponseCode.SYSTEM_ERROR);
+        assertPutResult(ResponseCode.SYSTEM_BUSY);
     }
 
     @Test
@@ -299,6 +310,60 @@ public void consumeMessageAfter(ConsumeMessageContext context) {
         assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
     }
 
+    @Test
+    public void testAttachRecallHandle_skip() {
+        MessageExt message = new MessageExt();
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_BATCH_MESSAGE, null);
+        sendMessageProcessor.attachRecallHandle(request, message, new SendMessageResponseHeader());
+
+        request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, null);
+        sendMessageProcessor.attachRecallHandle(request, message, new SendMessageResponseHeader());
+
+        request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, null);
+        sendMessageProcessor.attachRecallHandle(request, message, new SendMessageResponseHeader());
+
+        request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, null);
+        sendMessageProcessor.attachRecallHandle(request, message, new SendMessageResponseHeader());
+
+        verify(brokerConfig, times(0)).getBrokerName();
+    }
+
+    @Test
+    public void testAttachRecallHandle_doAttach() throws DecoderException {
+        int[] precisionSet = {100, 200, 500, 1000};
+        SendMessageResponseHeader responseHeader = new SendMessageResponseHeader();
+        String id = MessageClientIDSetter.createUniqID();
+        long timestamp = System.currentTimeMillis();
+
+        for (int precisionMs : precisionSet) {
+            long deliverMs = floor(timestamp, precisionMs);
+            MessageExt message = new MessageExt();
+            MessageAccessor.putProperty(message, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, id);
+            MessageAccessor.putProperty(message, MessageConst.PROPERTY_TIMER_OUT_MS, String.valueOf(deliverMs));
+            MessageAccessor.putProperty(message, MessageConst.PROPERTY_REAL_TOPIC, topic);
+
+            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, null);
+            sendMessageProcessor.attachRecallHandle(request, message, responseHeader);
+            Assert.assertNotNull(responseHeader.getRecallHandle());
+            RecallMessageHandle.HandleV1 v1 =
+                (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(responseHeader.getRecallHandle());
+            Assert.assertEquals(id, v1.getMessageId());
+            Assert.assertEquals(topic, v1.getTopic());
+            Assert.assertEquals(deliverMs + 1, Long.parseLong(v1.getTimestampStr()));
+            Assert.assertEquals(deliverMs, floor(Long.valueOf(v1.getTimestampStr()), precisionMs));
+        }
+    }
+
+    private long floor(long deliverMs, int precisionMs) {
+        assert precisionMs > 0;
+        if (deliverMs % precisionMs == 0) {
+            deliverMs -= precisionMs;
+        } else {
+            deliverMs = deliverMs / precisionMs * precisionMs;
+        }
+        return deliverMs;
+    }
+
     private RemotingCommand createSendTransactionMsgCommand(int requestCode) {
         SendMessageRequestHeader header = createSendMsgRequestHeader();
         int sysFlag = header.getSysFlag();
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/slave/SlaveSynchronizeAtomicTest.java b/broker/src/test/java/org/apache/rocketmq/broker/slave/SlaveSynchronizeAtomicTest.java
new file mode 100644
index 00000000000..75db22e7e77
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/slave/SlaveSynchronizeAtomicTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.slave;
+
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.loadbalance.MessageRequestModeManager;
+import org.apache.rocketmq.broker.out.BrokerOuterAPI;
+import org.apache.rocketmq.broker.processor.QueryAssignmentProcessor;
+import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.remoting.exception.RemotingConnectException;
+import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
+import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.apache.rocketmq.remoting.protocol.body.MessageRequestModeSerializeWrapper;
+import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class SlaveSynchronizeAtomicTest {
+    @Spy
+    private BrokerController brokerController =
+            new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(),
+                    new MessageStoreConfig());
+
+    private SlaveSynchronize slaveSynchronize;
+
+    @Mock
+    private BrokerOuterAPI brokerOuterAPI;
+
+    @Mock
+    private TopicConfigManager topicConfigManager;
+
+
+    @Mock
+    private SubscriptionGroupManager subscriptionGroupManager;
+
+    @Mock
+    private QueryAssignmentProcessor queryAssignmentProcessor;
+
+    @Mock
+    private MessageRequestModeManager messageRequestModeManager;
+
+
+    private static final String BROKER_ADDR = "127.0.0.1:10911";
+    private final SubscriptionGroupWrapper subscriptionGroupWrapper = createSubscriptionGroupWrapper();
+    private final MessageRequestModeSerializeWrapper requestModeSerializeWrapper = createMessageRequestModeWrapper();
+    private final DataVersion dataVersion = new DataVersion();
+
+    @Before
+    public void init() {
+        for (int i = 0; i < 100000; i++) {
+            subscriptionGroupWrapper.getSubscriptionGroupTable().put("group" + i, new SubscriptionGroupConfig());
+        }
+        for (int i = 0; i < 100000; i++) {
+            requestModeSerializeWrapper.getMessageRequestModeMap().put("topic" + i, new ConcurrentHashMap<>());
+        }
+        when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);
+        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
+        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);
+        when(subscriptionGroupManager.getDataVersion()).thenReturn(dataVersion);
+        when(subscriptionGroupManager.getSubscriptionGroupTable()).thenReturn(
+                subscriptionGroupWrapper.getSubscriptionGroupTable());
+        slaveSynchronize = new SlaveSynchronize(brokerController);
+        slaveSynchronize.setMasterAddr(BROKER_ADDR);
+    }
+
+    private SubscriptionGroupWrapper createSubscriptionGroupWrapper() {
+        SubscriptionGroupWrapper wrapper = new SubscriptionGroupWrapper();
+        wrapper.setSubscriptionGroupTable(new ConcurrentHashMap<>());
+        DataVersion dataVersion = new DataVersion();
+        dataVersion.setStateVersion(1L);
+        wrapper.setDataVersion(dataVersion);
+        return wrapper;
+    }
+
+    private MessageRequestModeSerializeWrapper createMessageRequestModeWrapper() {
+        MessageRequestModeSerializeWrapper wrapper = new MessageRequestModeSerializeWrapper();
+        wrapper.setMessageRequestModeMap(new ConcurrentHashMap<>());
+        return wrapper;
+    }
+
+    @Test
+    public void testSyncAtomically()
+            throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException,
+            InterruptedException {
+        when(brokerOuterAPI.getAllSubscriptionGroupConfig(anyString())).thenReturn(subscriptionGroupWrapper);
+        when(brokerOuterAPI.getAllMessageRequestMode(anyString())).thenReturn(requestModeSerializeWrapper);
+
+        CountDownLatch countDownLatch = new CountDownLatch(1);
+        new Thread(() -> {
+            while (countDownLatch.getCount() > 0) {
+                dataVersion.nextVersion();
+                try {
+                    slaveSynchronize.syncAll();
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }).start();
+
+        for (int i = 0; i < 10000000; i++) {
+            Assert.assertTrue(subscriptionGroupWrapper.getSubscriptionGroupTable()
+                    .containsKey("group" + ThreadLocalRandom.current().nextInt(0, 100000)));
+            Assert.assertTrue(requestModeSerializeWrapper.getMessageRequestModeMap()
+                    .containsKey("topic" + ThreadLocalRandom.current().nextInt(0, 100000)));
+        }
+        countDownLatch.countDown();
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/slave/SlaveSynchronizeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/slave/SlaveSynchronizeTest.java
new file mode 100644
index 00000000000..95db733d0d1
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/slave/SlaveSynchronizeTest.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.slave;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.loadbalance.MessageRequestModeManager;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.broker.out.BrokerOuterAPI;
+import org.apache.rocketmq.broker.processor.QueryAssignmentProcessor;
+import org.apache.rocketmq.broker.schedule.ScheduleMessageService;
+import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.remoting.exception.RemotingConnectException;
+import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
+import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.apache.rocketmq.remoting.protocol.body.ConsumerOffsetSerializeWrapper;
+import org.apache.rocketmq.remoting.protocol.body.MessageRequestModeSerializeWrapper;
+import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;
+import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.timer.TimerCheckpoint;
+import org.apache.rocketmq.store.timer.TimerMessageStore;
+import org.apache.rocketmq.store.timer.TimerMetrics;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.UnsupportedEncodingException;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class SlaveSynchronizeTest {
+    @Spy
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
+
+    private SlaveSynchronize slaveSynchronize;
+
+    @Mock
+    private BrokerOuterAPI brokerOuterAPI;
+
+    @Mock
+    private TopicConfigManager topicConfigManager;
+
+    @Mock
+    private ConsumerOffsetManager consumerOffsetManager;
+
+    @Mock
+    private MessageStoreConfig messageStoreConfig;
+
+    @Mock
+    private MessageStore messageStore;
+
+    @Mock
+    private ScheduleMessageService scheduleMessageService;
+
+    @Mock
+    private SubscriptionGroupManager subscriptionGroupManager;
+
+    @Mock
+    private QueryAssignmentProcessor queryAssignmentProcessor;
+
+    @Mock
+    private MessageRequestModeManager messageRequestModeManager;
+
+    @Mock
+    private TimerMessageStore timerMessageStore;
+
+    @Mock
+    private TimerMetrics timerMetrics;
+
+    @Mock
+    private TimerCheckpoint timerCheckpoint;
+
+    private static final String BROKER_ADDR = "127.0.0.1:10911";
+
+    @Before
+    public void init() {
+        when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);
+        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
+        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
+        when(brokerController.getScheduleMessageService()).thenReturn(scheduleMessageService);
+        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);
+        when(brokerController.getQueryAssignmentProcessor()).thenReturn(queryAssignmentProcessor);
+        when(brokerController.getMessageStore()).thenReturn(messageStore);
+        when(brokerController.getTimerMessageStore()).thenReturn(timerMessageStore);
+        when(brokerController.getTimerCheckpoint()).thenReturn(timerCheckpoint);
+        when(topicConfigManager.getDataVersion()).thenReturn(new DataVersion());
+        when(topicConfigManager.getTopicConfigTable()).thenReturn(new ConcurrentHashMap<>());
+        when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);
+        when(consumerOffsetManager.getOffsetTable()).thenReturn(new ConcurrentHashMap<>());
+        when(consumerOffsetManager.getDataVersion()).thenReturn(new DataVersion());
+        when(subscriptionGroupManager.getDataVersion()).thenReturn(new DataVersion());
+        when(subscriptionGroupManager.getSubscriptionGroupTable()).thenReturn(new ConcurrentHashMap<>());
+        when(queryAssignmentProcessor.getMessageRequestModeManager()).thenReturn(messageRequestModeManager);
+        when(messageRequestModeManager.getMessageRequestModeMap()).thenReturn(new ConcurrentHashMap<>());
+        when(messageStoreConfig.isTimerWheelEnable()).thenReturn(true);
+        when(messageStore.getTimerMessageStore()).thenReturn(timerMessageStore);
+        when(timerMessageStore.isShouldRunningDequeue()).thenReturn(false);
+        when(timerMessageStore.getTimerMetrics()).thenReturn(timerMetrics);
+        when(timerMetrics.getDataVersion()).thenReturn(new DataVersion());
+        when(timerCheckpoint.getDataVersion()).thenReturn(new DataVersion());
+        slaveSynchronize = new SlaveSynchronize(brokerController);
+        slaveSynchronize.setMasterAddr(BROKER_ADDR);
+    }
+
+    @Test
+    public void testSyncAll() throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException, UnsupportedEncodingException {
+        TopicConfig newTopicConfig = new TopicConfig("NewTopic");
+        when(brokerOuterAPI.getAllTopicConfig(anyString())).thenReturn(createTopicConfigWrapper(newTopicConfig));
+        when(brokerOuterAPI.getAllConsumerOffset(anyString())).thenReturn(createConsumerOffsetWrapper());
+        when(brokerOuterAPI.getAllDelayOffset(anyString())).thenReturn("");
+        when(brokerOuterAPI.getAllSubscriptionGroupConfig(anyString())).thenReturn(createSubscriptionGroupWrapper());
+        when(brokerOuterAPI.getAllMessageRequestMode(anyString())).thenReturn(createMessageRequestModeWrapper());
+        when(brokerOuterAPI.getTimerMetrics(anyString())).thenReturn(createTimerMetricsWrapper());
+        slaveSynchronize.syncAll();
+        Assert.assertEquals(1, this.brokerController.getTopicConfigManager().getDataVersion().getStateVersion());
+        Assert.assertEquals(1, this.brokerController.getTopicQueueMappingManager().getDataVersion().getStateVersion());
+        Assert.assertEquals(1, consumerOffsetManager.getDataVersion().getStateVersion());
+        Assert.assertEquals(1, subscriptionGroupManager.getDataVersion().getStateVersion());
+        Assert.assertEquals(1, timerMetrics.getDataVersion().getStateVersion());
+    }
+
+    @Test
+    public void testGetMasterAddr() {
+        Assert.assertEquals(BROKER_ADDR, slaveSynchronize.getMasterAddr());
+    }
+
+    @Test
+    public void testSyncTimerCheckPoint() throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        when(brokerOuterAPI.getTimerCheckPoint(anyString())).thenReturn(timerCheckpoint);
+        slaveSynchronize.syncTimerCheckPoint();
+        Assert.assertEquals(0, timerCheckpoint.getDataVersion().getStateVersion());
+    }
+
+    private TopicConfigAndMappingSerializeWrapper createTopicConfigWrapper(TopicConfig topicConfig) {
+        TopicConfigAndMappingSerializeWrapper wrapper = new TopicConfigAndMappingSerializeWrapper();
+        wrapper.setTopicConfigTable(new ConcurrentHashMap<>());
+        wrapper.getTopicConfigTable().put(topicConfig.getTopicName(), topicConfig);
+        DataVersion dataVersion = new DataVersion();
+        dataVersion.setStateVersion(1L);
+        wrapper.setDataVersion(dataVersion);
+        wrapper.setMappingDataVersion(dataVersion);
+        return wrapper;
+    }
+
+    private ConsumerOffsetSerializeWrapper createConsumerOffsetWrapper() {
+        ConsumerOffsetSerializeWrapper wrapper = new ConsumerOffsetSerializeWrapper();
+        wrapper.setOffsetTable(new ConcurrentHashMap<>());
+        DataVersion dataVersion = new DataVersion();
+        dataVersion.setStateVersion(1L);
+        wrapper.setDataVersion(dataVersion);
+        return wrapper;
+    }
+
+    private SubscriptionGroupWrapper createSubscriptionGroupWrapper() {
+        SubscriptionGroupWrapper wrapper = new SubscriptionGroupWrapper();
+        wrapper.setSubscriptionGroupTable(new ConcurrentHashMap<>());
+        DataVersion dataVersion = new DataVersion();
+        dataVersion.setStateVersion(1L);
+        wrapper.setDataVersion(dataVersion);
+        return wrapper;
+    }
+
+    private MessageRequestModeSerializeWrapper createMessageRequestModeWrapper() {
+        MessageRequestModeSerializeWrapper wrapper = new MessageRequestModeSerializeWrapper();
+        wrapper.setMessageRequestModeMap(new ConcurrentHashMap<>());
+        return wrapper;
+    }
+
+    private TimerMetrics.TimerMetricsSerializeWrapper createTimerMetricsWrapper() {
+        TimerMetrics.TimerMetricsSerializeWrapper wrapper = new TimerMetrics.TimerMetricsSerializeWrapper();
+        wrapper.setTimingCount(new ConcurrentHashMap<>());
+        DataVersion dataVersion = new DataVersion();
+        dataVersion.setStateVersion(1L);
+        wrapper.setDataVersion(dataVersion);
+        return wrapper;
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java b/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java
new file mode 100644
index 00000000000..c75fe0d6a03
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java
@@ -0,0 +1,340 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.subscription;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.config.v1.RocksDBSubscriptionGroupManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RocksdbGroupConfigTransferTest {
+    private final String basePath = Paths.get(System.getProperty("user.home"),
+            "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString();
+
+    private RocksDBSubscriptionGroupManager rocksDBSubscriptionGroupManager;
+
+    private SubscriptionGroupManager jsonSubscriptionGroupManager;
+    @Mock
+    private BrokerController brokerController;
+
+    @Mock
+    private DefaultMessageStore defaultMessageStore;
+
+    @Before
+    public void init() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        BrokerConfig brokerConfig = new BrokerConfig();
+        Mockito.lenient().when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
+        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+        messageStoreConfig.setStorePathRootDir(basePath);
+        Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
+        Mockito.lenient().when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);
+        when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L);
+    }
+
+    @After
+    public void destroy() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        Path pathToBeDeleted = Paths.get(basePath);
+
+        try {
+            Files.walk(pathToBeDeleted)
+                    .sorted(Comparator.reverseOrder())
+                    .forEach(path -> {
+                        try {
+                            Files.delete(path);
+                        } catch (IOException e) {
+                            // ignore
+                        }
+                    });
+        } catch (IOException e) {
+            // ignore
+        }
+        if (rocksDBSubscriptionGroupManager != null) {
+            rocksDBSubscriptionGroupManager.stop();
+        }
+    }
+
+
+    public void initRocksDBSubscriptionGroupManager() {
+        if (rocksDBSubscriptionGroupManager == null) {
+            rocksDBSubscriptionGroupManager = new RocksDBSubscriptionGroupManager(brokerController);
+            rocksDBSubscriptionGroupManager.load();
+        }
+    }
+
+    public void initJsonSubscriptionGroupManager() {
+        if (jsonSubscriptionGroupManager == null) {
+            jsonSubscriptionGroupManager = new SubscriptionGroupManager(brokerController);
+            jsonSubscriptionGroupManager.load();
+        }
+    }
+
+    @Test
+    public void theFirstTimeLoadJsonSubscriptionGroupManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        initJsonSubscriptionGroupManager();
+        DataVersion dataVersion = jsonSubscriptionGroupManager.getDataVersion();
+        Assert.assertNotNull(dataVersion);
+        Assert.assertEquals(0L, dataVersion.getCounter().get());
+        Assert.assertEquals(0L, dataVersion.getStateVersion());
+        Assert.assertNotEquals(0, jsonSubscriptionGroupManager.getSubscriptionGroupTable().size());
+    }
+
+    @Test
+    public void theFirstTimeLoadRocksDBSubscriptionGroupManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        initRocksDBSubscriptionGroupManager();
+        DataVersion dataVersion = rocksDBSubscriptionGroupManager.getDataVersion();
+        Assert.assertNotNull(dataVersion);
+        Assert.assertEquals(0L, dataVersion.getCounter().get());
+        Assert.assertEquals(0L, dataVersion.getStateVersion());
+        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size());
+    }
+
+
+    @Test
+    public void addGroupLoadJsonSubscriptionGroupManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        initJsonSubscriptionGroupManager();
+        int beforeSize = jsonSubscriptionGroupManager.getSubscriptionGroupTable().size();
+        String groupName = "testAddGroupConfig-" + System.currentTimeMillis();
+
+        Map<String, String> attributes = new HashMap<>();
+
+        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
+        subscriptionGroupConfig.setGroupName(groupName);
+        subscriptionGroupConfig.setAttributes(attributes);
+        DataVersion beforeDataVersion = jsonSubscriptionGroupManager.getDataVersion();
+        long beforeDataVersionCounter = beforeDataVersion.getCounter().get();
+        long beforeTimestamp = beforeDataVersion.getTimestamp();
+
+        jsonSubscriptionGroupManager.updateSubscriptionGroupConfig(subscriptionGroupConfig);
+
+        int afterSize = jsonSubscriptionGroupManager.getSubscriptionGroupTable().size();
+        DataVersion afterDataVersion = jsonSubscriptionGroupManager.getDataVersion();
+        long afterDataVersionCounter = afterDataVersion.getCounter().get();
+        long afterTimestamp = afterDataVersion.getTimestamp();
+
+        Assert.assertEquals(0, beforeDataVersionCounter);
+        Assert.assertEquals(1, afterDataVersionCounter);
+        Assert.assertEquals(1, afterSize - beforeSize);
+        Assert.assertTrue(afterTimestamp >= beforeTimestamp);
+    }
+
+    @Test
+    public void addForbiddenGroupLoadJsonSubscriptionGroupManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        initJsonSubscriptionGroupManager();
+        int beforeSize = jsonSubscriptionGroupManager.getForbiddenTable().size();
+        String groupName = "testAddGroupConfig-" + System.currentTimeMillis();
+
+        Map<String, String> attributes = new HashMap<>();
+
+        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
+        subscriptionGroupConfig.setGroupName(groupName);
+        subscriptionGroupConfig.setAttributes(attributes);
+        DataVersion beforeDataVersion = jsonSubscriptionGroupManager.getDataVersion();
+        long beforeDataVersionCounter = beforeDataVersion.getCounter().get();
+        long beforeTimestamp = beforeDataVersion.getTimestamp();
+
+        jsonSubscriptionGroupManager.setForbidden(groupName, "topic", 0);
+        int afterSize = jsonSubscriptionGroupManager.getForbiddenTable().size();
+        DataVersion afterDataVersion = jsonSubscriptionGroupManager.getDataVersion();
+        long afterDataVersionCounter = afterDataVersion.getCounter().get();
+        long afterTimestamp = afterDataVersion.getTimestamp();
+
+        Assert.assertEquals(1, afterDataVersionCounter - beforeDataVersionCounter);
+        Assert.assertEquals(1, afterSize - beforeSize);
+        Assert.assertTrue(afterTimestamp >= beforeTimestamp);
+    }
+
+    @Test
+    public void addGroupLoadRocksdbSubscriptionGroupManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        initRocksDBSubscriptionGroupManager();
+        int beforeSize = rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size();
+        String groupName = "testAddGroupConfig-" + System.currentTimeMillis();
+
+        Map<String, String> attributes = new HashMap<>();
+
+        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
+        subscriptionGroupConfig.setGroupName(groupName);
+        subscriptionGroupConfig.setAttributes(attributes);
+        DataVersion beforeDataVersion = rocksDBSubscriptionGroupManager.getDataVersion();
+        long beforeDataVersionCounter = beforeDataVersion.getCounter().get();
+        long beforeTimestamp = beforeDataVersion.getTimestamp();
+
+        rocksDBSubscriptionGroupManager.updateSubscriptionGroupConfig(subscriptionGroupConfig);
+        int afterSize = rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size();
+        DataVersion afterDataVersion = rocksDBSubscriptionGroupManager.getDataVersion();
+        long afterDataVersionCounter = afterDataVersion.getCounter().get();
+        long afterTimestamp = afterDataVersion.getTimestamp();
+        Assert.assertEquals(1, afterDataVersionCounter);
+        Assert.assertEquals(0, beforeDataVersionCounter);
+        Assert.assertEquals(1, afterSize - beforeSize);
+        Assert.assertTrue(afterTimestamp >= beforeTimestamp);
+    }
+
+    @Test
+    public void addForbiddenLoadRocksdbSubscriptionGroupManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        initRocksDBSubscriptionGroupManager();
+        int beforeSize = rocksDBSubscriptionGroupManager.getForbiddenTable().size();
+        String groupName = "testAddGroupConfig-" + System.currentTimeMillis();
+
+        Map<String, String> attributes = new HashMap<>();
+
+        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
+        subscriptionGroupConfig.setGroupName(groupName);
+        subscriptionGroupConfig.setAttributes(attributes);
+        DataVersion beforeDataVersion = rocksDBSubscriptionGroupManager.getDataVersion();
+        long beforeDataVersionCounter = beforeDataVersion.getCounter().get();
+        long beforeTimestamp = beforeDataVersion.getTimestamp();
+
+        rocksDBSubscriptionGroupManager.updateForbidden(groupName, "topic", 0, true);
+
+        int afterSize = rocksDBSubscriptionGroupManager.getForbiddenTable().size();
+        DataVersion afterDataVersion = rocksDBSubscriptionGroupManager.getDataVersion();
+        long afterDataVersionCounter = afterDataVersion.getCounter().get();
+        long afterTimestamp = afterDataVersion.getTimestamp();
+        Assert.assertEquals(1, afterDataVersionCounter - beforeDataVersionCounter);
+        Assert.assertEquals(1, afterSize - beforeSize);
+        Assert.assertTrue(afterTimestamp >= beforeTimestamp);
+        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size());
+    }
+
+    @Test
+    public void theSecondTimeLoadJsonSubscriptionGroupManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        addGroupLoadJsonSubscriptionGroupManager();
+        jsonSubscriptionGroupManager.stop();
+        rocksDBSubscriptionGroupManager = null;
+        addForbiddenGroupLoadJsonSubscriptionGroupManager();
+        jsonSubscriptionGroupManager.stop();
+        rocksDBSubscriptionGroupManager = null;
+        jsonSubscriptionGroupManager = new SubscriptionGroupManager(brokerController);
+        jsonSubscriptionGroupManager.load();
+        DataVersion dataVersion = jsonSubscriptionGroupManager.getDataVersion();
+        Assert.assertNotNull(dataVersion);
+        Assert.assertEquals(2L, dataVersion.getCounter().get());
+        Assert.assertEquals(0L, dataVersion.getStateVersion());
+        Assert.assertNotEquals(0, jsonSubscriptionGroupManager.getSubscriptionGroupTable().size());
+        Assert.assertNotEquals(0, jsonSubscriptionGroupManager.getForbiddenTable().size());
+        Assert.assertNotEquals(0, jsonSubscriptionGroupManager.getSubscriptionGroupTable().size());
+    }
+
+    @Test
+    public void theSecondTimeLoadRocksdbTopicConfigManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        addGroupLoadRocksdbSubscriptionGroupManager();
+        rocksDBSubscriptionGroupManager.stop();
+        rocksDBSubscriptionGroupManager = null;
+        addForbiddenLoadRocksdbSubscriptionGroupManager();
+        rocksDBSubscriptionGroupManager.stop();
+        rocksDBSubscriptionGroupManager = null;
+        rocksDBSubscriptionGroupManager = new RocksDBSubscriptionGroupManager(brokerController);
+        rocksDBSubscriptionGroupManager.load();
+        DataVersion dataVersion = rocksDBSubscriptionGroupManager.getDataVersion();
+        Assert.assertNotNull(dataVersion);
+        Assert.assertEquals(2L, dataVersion.getCounter().get());
+        Assert.assertEquals(0L, dataVersion.getStateVersion());
+        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size());
+        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getForbiddenTable().size());
+        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size());
+    }
+
+
+    @Test
+    public void jsonUpgradeToRocksdb() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        addGroupLoadJsonSubscriptionGroupManager();
+        addForbiddenGroupLoadJsonSubscriptionGroupManager();
+        initRocksDBSubscriptionGroupManager();
+        DataVersion dataVersion = rocksDBSubscriptionGroupManager.getDataVersion();
+        Assert.assertNotNull(dataVersion);
+        Assert.assertEquals(3L, dataVersion.getCounter().get());
+        Assert.assertEquals(0L, dataVersion.getStateVersion());
+        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getForbiddenTable().size());
+        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size());
+        Assert.assertEquals(rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size(), jsonSubscriptionGroupManager.getSubscriptionGroupTable().size());
+        Assert.assertEquals(rocksDBSubscriptionGroupManager.getForbiddenTable().size(), jsonSubscriptionGroupManager.getForbiddenTable().size());
+
+        rocksDBSubscriptionGroupManager.stop();
+        rocksDBSubscriptionGroupManager = new RocksDBSubscriptionGroupManager(brokerController);
+        rocksDBSubscriptionGroupManager.load();
+        dataVersion = rocksDBSubscriptionGroupManager.getDataVersion();
+        Assert.assertEquals(3L, dataVersion.getCounter().get());
+        Assert.assertEquals(0L, dataVersion.getStateVersion());
+        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getForbiddenTable().size());
+        Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size());
+        Assert.assertEquals(rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size(), jsonSubscriptionGroupManager.getSubscriptionGroupTable().size());
+        Assert.assertEquals(rocksDBSubscriptionGroupManager.getForbiddenTable().size(), jsonSubscriptionGroupManager.getForbiddenTable().size());
+    }
+
+    private boolean notToBeExecuted() {
+        return MixAll.isMac();
+    }
+
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java
index 3c829437cf1..3384d479c6e 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java
@@ -18,7 +18,12 @@
 package org.apache.rocketmq.broker.subscription;
 
 import com.google.common.collect.ImmutableMap;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.SubscriptionGroupAttributes;
@@ -30,36 +35,46 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.junit.MockitoJUnitRunner;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
 
 @RunWith(MockitoJUnitRunner.class)
 public class SubscriptionGroupManagerTest {
     private String group = "group";
+
+    private final String basePath = Paths.get(System.getProperty("user.home"),
+            "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString();
     @Mock
     private BrokerController brokerControllerMock;
     private SubscriptionGroupManager subscriptionGroupManager;
 
     @Before
     public void before() {
+        if (notToBeExecuted()) {
+            return;
+        }
         SubscriptionGroupAttributes.ALL.put("test", new BooleanAttribute(
             "test",
             false,
             false
         ));
         subscriptionGroupManager = spy(new SubscriptionGroupManager(brokerControllerMock));
-        when(brokerControllerMock.getMessageStore()).thenReturn(null);
-        doNothing().when(subscriptionGroupManager).persist();
+        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+        messageStoreConfig.setStorePathRootDir(basePath);
+        Mockito.lenient().when(brokerControllerMock.getMessageStoreConfig()).thenReturn(messageStoreConfig);
     }
 
     @After
     public void destroy() {
-        if (MixAll.isMac()) {
+        if (notToBeExecuted()) {
             return;
         }
         if (subscriptionGroupManager != null) {
@@ -69,18 +84,18 @@ public void destroy() {
 
     @Test
     public void testUpdateAndCreateSubscriptionGroupInRocksdb() {
-        if (MixAll.isMac()) {
+        if (notToBeExecuted()) {
             return;
         }
-        when(brokerControllerMock.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());
-        subscriptionGroupManager = spy(new RocksDBSubscriptionGroupManager(brokerControllerMock));
-        subscriptionGroupManager.load();
         group += System.currentTimeMillis();
         updateSubscriptionGroupConfig();
     }
 
     @Test
     public void updateSubscriptionGroupConfig() {
+        if (notToBeExecuted()) {
+            return;
+        }
         SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
         subscriptionGroupConfig.setGroupName(group);
         Map<String, String> attr = ImmutableMap.of("+test", "true");
@@ -99,4 +114,56 @@ public void updateSubscriptionGroupConfig() {
         assertThatThrownBy(() -> subscriptionGroupManager.updateSubscriptionGroupConfig(subscriptionGroupConfig1))
             .isInstanceOf(RuntimeException.class).hasMessage("attempt to update an unchangeable attribute. key: test");
     }
+
+    private boolean notToBeExecuted() {
+        return MixAll.isMac();
+    }
+    @Test
+    public void testUpdateSubscriptionGroupConfigList_NullConfigList() {
+        if (notToBeExecuted()) {
+            return;
+        }
+
+        subscriptionGroupManager.updateSubscriptionGroupConfigList(null);
+        // Verifying that persist() is not called
+        verify(subscriptionGroupManager, never()).persist();
+    }
+
+    @Test
+    public void testUpdateSubscriptionGroupConfigList_EmptyConfigList() {
+        if (notToBeExecuted()) {
+            return;
+        }
+
+        subscriptionGroupManager.updateSubscriptionGroupConfigList(Collections.emptyList());
+        // Verifying that persist() is not called
+        verify(subscriptionGroupManager, never()).persist();
+    }
+
+    @Test
+    public void testUpdateSubscriptionGroupConfigList_ValidConfigList() {
+        if (notToBeExecuted()) {
+            return;
+        }
+
+        final List<SubscriptionGroupConfig> configList = new LinkedList<>();
+        final List<String> groupNames = new LinkedList<>();
+        for (int i = 0; i < 10; i++) {
+            SubscriptionGroupConfig config = new SubscriptionGroupConfig();
+            String groupName = String.format("group-%d", i);
+            config.setGroupName(groupName);
+            configList.add(config);
+            groupNames.add(groupName);
+        }
+
+        subscriptionGroupManager.updateSubscriptionGroupConfigList(configList);
+
+        // Verifying that persist() is called once
+        verify(subscriptionGroupManager, times(1)).persist();
+
+        groupNames.forEach(groupName ->
+            assertThat(subscriptionGroupManager.getSubscriptionGroupTable().get(groupName)).isNotNull());
+
+    }
+
 }
\ No newline at end of file
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java
index ed71a3313a8..fa3ef95f55f 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java
@@ -16,11 +16,15 @@
  */
 package org.apache.rocketmq.broker.topic;
 
+import java.nio.file.Paths;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.UUID;
+
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.TopicAttributes;
@@ -39,6 +43,7 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.junit.MockitoJUnitRunner;
 
 import static com.google.common.collect.Sets.newHashSet;
@@ -47,6 +52,10 @@
 
 @RunWith(MockitoJUnitRunner.class)
 public class RocksdbTopicConfigManagerTest {
+
+    private final String basePath = Paths.get(System.getProperty("user.home"),
+            "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString();
+
     private RocksDBTopicConfigManager topicConfigManager;
     @Mock
     private BrokerController brokerController;
@@ -62,9 +71,10 @@ public void init() {
         BrokerConfig brokerConfig = new BrokerConfig();
         when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
         MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+        messageStoreConfig.setStorePathRootDir(basePath);
         when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
-        when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);
-        when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L);
+        Mockito.lenient().when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);
+        Mockito.lenient().when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L);
         topicConfigManager = new RocksDBTopicConfigManager(brokerController);
         topicConfigManager.load();
     }
@@ -197,7 +207,6 @@ public void testNormalAddKeyOnCreating() {
         TopicConfig existingTopicConfig = topicConfigManager.getTopicConfigTable().get(topic);
         Assert.assertEquals("enum-2", existingTopicConfig.getAttributes().get("enum.key"));
         Assert.assertEquals("16", existingTopicConfig.getAttributes().get("long.range.key"));
-        //        assert file
     }
 
     @Test
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java
new file mode 100644
index 00000000000..e925ed4bd8a
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.topic;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RocksdbTopicConfigTransferTest {
+
+    private final String basePath = Paths.get(System.getProperty("user.home"),
+            "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString();
+
+    private RocksDBTopicConfigManager rocksdbTopicConfigManager;
+
+    private TopicConfigManager jsonTopicConfigManager;
+    @Mock
+    private BrokerController brokerController;
+
+    @Mock
+    private DefaultMessageStore defaultMessageStore;
+
+    @Before
+    public void init() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        BrokerConfig brokerConfig = new BrokerConfig();
+        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
+        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+        messageStoreConfig.setStorePathRootDir(basePath);
+        Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
+        when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);
+        when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L);
+    }
+
+    @After
+    public void destroy() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        Path pathToBeDeleted = Paths.get(basePath);
+        try {
+            Files.walk(pathToBeDeleted)
+                    .sorted(Comparator.reverseOrder())
+                    .forEach(path -> {
+                        try {
+                            Files.delete(path);
+                        } catch (IOException e) {
+                            // ignore
+                        }
+                    });
+        } catch (IOException e) {
+            // ignore
+        }
+        if (rocksdbTopicConfigManager != null) {
+            rocksdbTopicConfigManager.stop();
+        }
+    }
+
+    public void initRocksdbTopicConfigManager() {
+        if (rocksdbTopicConfigManager == null) {
+            rocksdbTopicConfigManager = new RocksDBTopicConfigManager(brokerController);
+            rocksdbTopicConfigManager.load();
+        }
+    }
+
+    public void initJsonTopicConfigManager() {
+        if (jsonTopicConfigManager == null) {
+            jsonTopicConfigManager = new TopicConfigManager(brokerController);
+            jsonTopicConfigManager.load();
+        }
+    }
+
+    @Test
+    public void theFirstTimeLoadJsonTopicConfigManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        initJsonTopicConfigManager();
+        DataVersion dataVersion = jsonTopicConfigManager.getDataVersion();
+        Assert.assertNotNull(dataVersion);
+        Assert.assertEquals(0L, dataVersion.getCounter().get());
+        Assert.assertEquals(0L, dataVersion.getStateVersion());
+        Assert.assertNotEquals(0, jsonTopicConfigManager.getTopicConfigTable().size());
+    }
+
+    @Test
+    public void theFirstTimeLoadRocksdbTopicConfigManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        initRocksdbTopicConfigManager();
+        DataVersion dataVersion = rocksdbTopicConfigManager.getDataVersion();
+        Assert.assertNotNull(dataVersion);
+        Assert.assertEquals(0L, dataVersion.getCounter().get());
+        Assert.assertEquals(0L, dataVersion.getStateVersion());
+        Assert.assertNotEquals(0, rocksdbTopicConfigManager.getTopicConfigTable().size());
+    }
+
+
+    @Test
+    public void addTopicLoadJsonTopicConfigManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        initJsonTopicConfigManager();
+        String topicName = "testAddTopicConfig-" + System.currentTimeMillis();
+
+        Map<String, String> attributes = new HashMap<>();
+
+        TopicConfig topicConfig = new TopicConfig();
+        topicConfig.setTopicName(topicName);
+        topicConfig.setAttributes(attributes);
+        DataVersion beforeDataVersion = jsonTopicConfigManager.getDataVersion();
+        long beforeDataVersionCounter = beforeDataVersion.getCounter().get();
+        long beforeTimestamp = beforeDataVersion.getTimestamp();
+
+        jsonTopicConfigManager.updateTopicConfig(topicConfig);
+
+        DataVersion afterDataVersion = jsonTopicConfigManager.getDataVersion();
+        long afterDataVersionCounter = afterDataVersion.getCounter().get();
+        long afterTimestamp = afterDataVersion.getTimestamp();
+
+        Assert.assertEquals(0, beforeDataVersionCounter);
+        Assert.assertEquals(1, afterDataVersionCounter);
+        Assert.assertTrue(afterTimestamp >= beforeTimestamp);
+    }
+
+    @Test
+    public void addTopicLoadRocksdbTopicConfigManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        initRocksdbTopicConfigManager();
+        String topicName = "testAddTopicConfig-" + System.currentTimeMillis();
+
+        Map<String, String> attributes = new HashMap<>();
+
+        TopicConfig topicConfig = new TopicConfig();
+        topicConfig.setTopicName(topicName);
+        topicConfig.setAttributes(attributes);
+        DataVersion beforeDataVersion = rocksdbTopicConfigManager.getDataVersion();
+        long beforeDataVersionCounter = beforeDataVersion.getCounter().get();
+        long beforeTimestamp = beforeDataVersion.getTimestamp();
+
+        rocksdbTopicConfigManager.updateTopicConfig(topicConfig);
+
+        DataVersion afterDataVersion = rocksdbTopicConfigManager.getDataVersion();
+        long afterDataVersionCounter = afterDataVersion.getCounter().get();
+        long afterTimestamp = afterDataVersion.getTimestamp();
+        Assert.assertEquals(0, beforeDataVersionCounter);
+        Assert.assertEquals(1, afterDataVersionCounter);
+        Assert.assertTrue(afterTimestamp >= beforeTimestamp);
+    }
+
+    @Test
+    public void theSecondTimeLoadJsonTopicConfigManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        addTopicLoadJsonTopicConfigManager();
+        jsonTopicConfigManager.stop();
+        jsonTopicConfigManager = new TopicConfigManager(brokerController);
+        jsonTopicConfigManager.load();
+        DataVersion dataVersion = jsonTopicConfigManager.getDataVersion();
+        Assert.assertNotNull(dataVersion);
+        Assert.assertEquals(1L, dataVersion.getCounter().get());
+        Assert.assertEquals(0L, dataVersion.getStateVersion());
+        Assert.assertNotEquals(0, jsonTopicConfigManager.getTopicConfigTable().size());
+    }
+
+    @Test
+    public void theSecondTimeLoadRocksdbTopicConfigManager() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        addTopicLoadRocksdbTopicConfigManager();
+        rocksdbTopicConfigManager.stop();
+        rocksdbTopicConfigManager = null;
+        rocksdbTopicConfigManager = new RocksDBTopicConfigManager(brokerController);
+        rocksdbTopicConfigManager.load();
+        DataVersion dataVersion = rocksdbTopicConfigManager.getDataVersion();
+        Assert.assertNotNull(dataVersion);
+        Assert.assertEquals(1L, dataVersion.getCounter().get());
+        Assert.assertEquals(0L, dataVersion.getStateVersion());
+        Assert.assertNotEquals(0, rocksdbTopicConfigManager.getTopicConfigTable().size());
+    }
+
+    @Test
+    public void jsonUpgradeToRocksdb() {
+        if (notToBeExecuted()) {
+            return;
+        }
+        addTopicLoadJsonTopicConfigManager();
+        initRocksdbTopicConfigManager();
+        DataVersion dataVersion = rocksdbTopicConfigManager.getDataVersion();
+        Assert.assertNotNull(dataVersion);
+        Assert.assertEquals(2L, dataVersion.getCounter().get());
+        Assert.assertEquals(0L, dataVersion.getStateVersion());
+        Assert.assertNotEquals(0, rocksdbTopicConfigManager.getTopicConfigTable().size());
+        Assert.assertEquals(rocksdbTopicConfigManager.getTopicConfigTable().size(), jsonTopicConfigManager.getTopicConfigTable().size());
+
+        rocksdbTopicConfigManager.stop();
+        rocksdbTopicConfigManager = new RocksDBTopicConfigManager(brokerController);
+        rocksdbTopicConfigManager.load();
+        dataVersion = rocksdbTopicConfigManager.getDataVersion();
+        Assert.assertEquals(2L, dataVersion.getCounter().get());
+        Assert.assertEquals(0L, dataVersion.getStateVersion());
+        Assert.assertNotEquals(0, rocksdbTopicConfigManager.getTopicConfigTable().size());
+        Assert.assertEquals(rocksdbTopicConfigManager.getTopicConfigTable().size(), rocksdbTopicConfigManager.getTopicConfigTable().size());
+    }
+
+
+    private boolean notToBeExecuted() {
+        return MixAll.isMac();
+    }
+
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java
index 6052a79d413..3fd1d14c3a2 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java
@@ -16,10 +16,13 @@
  */
 package org.apache.rocketmq.broker.topic;
 
+import java.nio.file.Paths;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.UUID;
+
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.TopicAttributes;
@@ -37,6 +40,7 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.junit.MockitoJUnitRunner;
 
 import static com.google.common.collect.Sets.newHashSet;
@@ -45,6 +49,9 @@
 
 @RunWith(MockitoJUnitRunner.class)
 public class TopicConfigManagerTest {
+
+    private final String basePath = Paths.get(System.getProperty("user.home"),
+            "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString();
     private TopicConfigManager topicConfigManager;
     @Mock
     private BrokerController brokerController;
@@ -57,8 +64,9 @@ public void init() {
         BrokerConfig brokerConfig = new BrokerConfig();
         when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
         MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+        messageStoreConfig.setStorePathRootDir(basePath);
         when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
-        when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);
+        Mockito.lenient().when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);
         when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L);
         topicConfigManager = new TopicConfigManager(brokerController);
     }
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicQueueMappingCleanServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicQueueMappingCleanServiceTest.java
new file mode 100644
index 00000000000..c7079c5248f
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicQueueMappingCleanServiceTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.topic;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.out.BrokerOuterAPI;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
+import org.apache.rocketmq.remoting.protocol.admin.TopicOffset;
+import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem;
+import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;
+import org.apache.rocketmq.remoting.rpc.RpcClient;
+import org.apache.rocketmq.remoting.rpc.RpcRequest;
+import org.apache.rocketmq.remoting.rpc.RpcResponse;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TopicQueueMappingCleanServiceTest {
+    
+    @Mock
+    private BrokerController brokerController;
+    
+    @Mock
+    private TopicQueueMappingManager topicQueueMappingManager;
+    
+    @Mock
+    private RpcClient rpcClient;
+    
+    @Mock
+    private MessageStoreConfig messageStoreConfig;
+    
+    @Mock
+    private BrokerConfig brokerConfig;
+    
+    @Mock
+    private BrokerOuterAPI brokerOuterAPI;
+    
+    private TopicQueueMappingCleanService topicQueueMappingCleanService;
+    
+    private final String defaultTopic = "defaultTopic";
+    
+    private final String defaultBroker = "defaultBroker";
+    
+    private final String deleteWhen = "00;01;02;03;04;05;06;07;08;09;10;11;12;13;14;15;16;17;18;19;20;21;22;23";
+    
+    @Before
+    public void init() {
+        when(brokerOuterAPI.getRpcClient()).thenReturn(rpcClient);
+        when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);
+        when(brokerController.getTopicQueueMappingManager()).thenReturn(topicQueueMappingManager);
+        when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
+        when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
+        topicQueueMappingCleanService = new TopicQueueMappingCleanService(brokerController);
+    }
+    
+    @Test
+    public void testCleanItemExpiredNoChange() throws Exception {
+        when(messageStoreConfig.getDeleteWhen()).thenReturn("04");
+        topicQueueMappingCleanService.cleanItemExpired();
+        verify(topicQueueMappingManager, never()).updateTopicQueueMapping(any(), anyBoolean(), anyBoolean(), anyBoolean());
+    }
+    
+    @Test
+    public void testCleanItemExpiredWithChange() throws Exception {
+        when(messageStoreConfig.getDeleteWhen()).thenReturn(deleteWhen);
+        TopicQueueMappingDetail mappingDetail = new TopicQueueMappingDetail(defaultTopic, 2, defaultBroker, 1);
+        mappingDetail.getHostedQueues().put(0,
+                Arrays.asList(new LogicQueueMappingItem(0, 0, defaultBroker, 0, 0, 100, 0, 0),
+                        new LogicQueueMappingItem(0, 1, defaultBroker, 1, 100, 200, 0, 0)));
+        when(topicQueueMappingManager.getTopicQueueMappingTable()).thenReturn(new ConcurrentHashMap<>(Collections.singletonMap(defaultTopic, mappingDetail)));
+        when(brokerConfig.getBrokerName()).thenReturn(defaultBroker);
+        TopicStatsTable topicStatsTable = mock(TopicStatsTable.class);
+        Map<MessageQueue, TopicOffset> offsetTable = new ConcurrentHashMap<>();
+        TopicOffset topicOffset = new TopicOffset();
+        topicOffset.setMinOffset(0);
+        topicOffset.setMaxOffset(0);
+        MessageQueue messageQueue = new MessageQueue(defaultTopic, defaultBroker, 0);
+        offsetTable.put(messageQueue, topicOffset);
+        when(topicStatsTable.getOffsetTable()).thenReturn(offsetTable);
+        when(rpcClient.invoke(any(RpcRequest.class), anyLong())).thenReturn(CompletableFuture.completedFuture(new RpcResponse(0, null, topicStatsTable)));
+        DataVersion dataVersion = mock(DataVersion.class);
+        when(topicQueueMappingManager.getDataVersion()).thenReturn(dataVersion);
+        topicQueueMappingCleanService.cleanItemExpired();
+        verify(topicQueueMappingManager, times(1)).updateTopicQueueMapping(any(), anyBoolean(), anyBoolean(), anyBoolean());
+    }
+    
+    @Test
+    public void testCleanItemListMoreThanSecondGen() throws Exception {
+        when(brokerConfig.getBrokerName()).thenReturn(defaultBroker);
+        when(messageStoreConfig.getDeleteWhen()).thenReturn(deleteWhen);
+        TopicQueueMappingDetail mappingDetail = new TopicQueueMappingDetail(defaultTopic, 1, defaultBroker, 1);
+        mappingDetail.setHostedQueues(new ConcurrentHashMap<>());
+        LogicQueueMappingItem logicQueueMappingItem = mock(LogicQueueMappingItem.class);
+        when(logicQueueMappingItem.getBname()).thenReturn("broker");
+        mappingDetail.getHostedQueues().put(0, Collections.singletonList(logicQueueMappingItem));
+        ConcurrentMap<String, TopicQueueMappingDetail> topicQueueMappingTable = new ConcurrentHashMap<>();
+        topicQueueMappingTable.put(defaultBroker, mappingDetail);
+        when(topicQueueMappingManager.getTopicQueueMappingTable()).thenReturn(topicQueueMappingTable);
+        TopicRouteData topicRouteData = new TopicRouteData();
+        when(brokerOuterAPI.getTopicRouteInfoFromNameServer(any(), anyLong())).thenReturn(topicRouteData);
+        topicQueueMappingCleanService.cleanItemListMoreThanSecondGen();
+        verify(brokerOuterAPI, times(1)).getTopicRouteInfoFromNameServer(any(), anyLong());
+    }
+    
+    @Test
+    public void testCleanItemListMoreThanSecondGenNoChange() throws Exception {
+        when(messageStoreConfig.getDeleteWhen()).thenReturn("04");
+        topicQueueMappingCleanService.cleanItemListMoreThanSecondGen();
+        verify(brokerOuterAPI, never()).getTopicRouteInfoFromNameServer(anyString(), anyLong());
+        verify(rpcClient, never()).invoke(any(RpcRequest.class), anyLong());
+    }
+    
+    @Test
+    public void testCleanItemListMoreThanSecondGenException() throws Exception {
+        when(brokerConfig.getBrokerName()).thenReturn(defaultBroker);
+        when(messageStoreConfig.getDeleteWhen()).thenReturn(deleteWhen);
+        TopicQueueMappingDetail mappingDetail = new TopicQueueMappingDetail(defaultTopic, 1, defaultBroker, 1);
+        mappingDetail.setHostedQueues(new ConcurrentHashMap<>());
+        LogicQueueMappingItem logicQueueMappingItem = mock(LogicQueueMappingItem.class);
+        when(logicQueueMappingItem.getBname()).thenReturn("broker");
+        mappingDetail.getHostedQueues().put(0, Collections.singletonList(logicQueueMappingItem));
+        ConcurrentMap<String, TopicQueueMappingDetail> topicQueueMappingTable = new ConcurrentHashMap<>();
+        topicQueueMappingTable.put(defaultBroker, mappingDetail);
+        when(topicQueueMappingManager.getTopicQueueMappingTable()).thenReturn(topicQueueMappingTable);
+        when(brokerOuterAPI.getTopicRouteInfoFromNameServer(any(), anyLong())).thenThrow(new RemotingException("Test exception"));
+        topicQueueMappingCleanService.cleanItemListMoreThanSecondGen();
+        verify(brokerOuterAPI, times(1)).getTopicRouteInfoFromNameServer(any(), anyLong());
+    }
+}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java
index 2a63774555e..b92c07dd478 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java
@@ -72,7 +72,7 @@ public class TransactionalMessageServiceImplTest {
 
     @Spy
     private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(),
-        new NettyClientConfig(), new MessageStoreConfig());
+        new NettyClientConfig(), new MessageStoreConfig(), null);
 
     @Mock
     private AbstractTransactionalMessageCheckListener listener;
@@ -237,6 +237,7 @@ private Set<MessageQueue> createMessageQueueSet(String topic) {
 
     private EndTransactionRequestHeader createEndTransactionRequestHeader(int status) {
         EndTransactionRequestHeader header = new EndTransactionRequestHeader();
+        header.setTopic("topic");
         header.setCommitLogOffset(123456789L);
         header.setCommitOrRollback(status);
         header.setMsgId("12345678");
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java b/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java
index 53fba00fa9e..40b12ab1c21 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/util/ServiceProviderTest.java
@@ -17,7 +17,6 @@
 
 package org.apache.rocketmq.broker.util;
 
-import org.apache.rocketmq.acl.AccessValidator;
 import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;
 import org.apache.rocketmq.broker.transaction.TransactionalMessageService;
 import org.apache.rocketmq.common.utils.ServiceProvider;
@@ -25,8 +24,6 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-import java.util.List;
-
 public class ServiceProviderTest {
 
     @Test
@@ -41,10 +38,4 @@ public void loadAbstractTransactionListenerTest() {
                 AbstractTransactionalMessageCheckListener.class);
         assertThat(listener).isNotNull();
     }
-    
-    @Test
-    public void loadAccessValidatorTest() {
-        List<AccessValidator> accessValidators = ServiceProvider.load(AccessValidator.class);
-        assertThat(accessValidators).isNotNull();
-    }
 }
diff --git a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator b/broker/src/test/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator
deleted file mode 100644
index 1abc92e0162..00000000000
--- a/broker/src/test/resources/META-INF/service/org.apache.rocketmq.acl.AccessValidator
+++ /dev/null
@@ -1 +0,0 @@
-org.apache.rocketmq.acl.plain.PlainAccessValidator
\ No newline at end of file
diff --git a/broker/src/test/resources/rmq.logback-test.xml b/broker/src/test/resources/rmq.logback-test.xml
index 8695d52d57c..7a2ff0bc933 100644
--- a/broker/src/test/resources/rmq.logback-test.xml
+++ b/broker/src/test/resources/rmq.logback-test.xml
@@ -19,9 +19,7 @@
 
     <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
         <layout class="ch.qos.logback.classic.PatternLayout">
-            <Pattern>
-                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n
-            </Pattern>
+            <pattern>${CONSOLE_LOG_PATTERN}</pattern>
         </layout>
     </appender>
 
@@ -29,7 +27,10 @@
         <appender-ref ref="CONSOLE"/>
     </logger>
 
-    <root level="error">
+    <property name="CONSOLE_LOG_PATTERN"
+              value="%d{yyyy-MM-dd HH:mm:ss.SSS,GMT+8} ${LOG_LEVEL_PATTERN:-%5p} [%20.20logger{39}] [%30.30t] %m%n"/>
+
+    <root level="info">
         <appender-ref ref="CONSOLE"/>
     </root>
 
diff --git a/client/BUILD.bazel b/client/BUILD.bazel
index 46e29452b95..e6edebe6bec 100644
--- a/client/BUILD.bazel
+++ b/client/BUILD.bazel
@@ -28,12 +28,15 @@ java_library(
         "@maven//:com_github_luben_zstd_jni",
         "@maven//:org_lz4_lz4_java",
         "@maven//:com_alibaba_fastjson",
+        "@maven//:com_alibaba_fastjson2_fastjson2",
         "@maven//:io_netty_netty_all",
         "@maven//:io_opentracing_opentracing_api",
         "@maven//:commons_collections_commons_collections",
         "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
         "@maven//:io_github_aliyunmq_rocketmq_logback_classic",
         "@maven//:com_google_guava_guava",
+        "@maven//:commons_codec_commons_codec",
+        "@maven//:org_yaml_snakeyaml",
     ],
 )
 
@@ -50,9 +53,11 @@ java_library(
         "@maven//:io_netty_netty_all",   
         "@maven//:io_opentracing_opentracing_api",        
         "@maven//:io_opentracing_opentracing_mock",   
-        "@maven//:org_awaitility_awaitility",         
+        "@maven//:org_awaitility_awaitility",
+        "@maven//:org_mockito_mockito_junit_jupiter",
+        "@maven//:com_alibaba_fastjson2_fastjson2",
     ],
-    resources = glob(["src/test/resources/certs/*.pem"]) + glob(["src/test/resources/certs/*.key"])
+    resources = glob(["src/test/resources/certs/*.pem"]) + glob(["src/test/resources/certs/*.key"]) + glob(["src/test/resources/**/*.yml"])
 )
 
 GenTestRules(
diff --git a/client/pom.xml b/client/pom.xml
index d6fb3889b7a..663f9b73955 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -54,10 +54,6 @@
             <groupId>io.opentracing</groupId>
             <artifactId>opentracing-mock</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-        </dependency>
         <dependency>
             <groupId>io.github.aliyunmq</groupId>
             <artifactId>rocketmq-slf4j-api</artifactId>
@@ -66,5 +62,9 @@
             <groupId>io.github.aliyunmq</groupId>
             <artifactId>rocketmq-logback-classic</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+        </dependency>
     </dependencies>
 </project>
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclClientRPCHook.java b/client/src/main/java/org/apache/rocketmq/acl/common/AclClientRPCHook.java
similarity index 83%
rename from acl/src/main/java/org/apache/rocketmq/acl/common/AclClientRPCHook.java
rename to client/src/main/java/org/apache/rocketmq/acl/common/AclClientRPCHook.java
index 294db4f069a..9fbcc9fe5fb 100644
--- a/acl/src/main/java/org/apache/rocketmq/acl/common/AclClientRPCHook.java
+++ b/client/src/main/java/org/apache/rocketmq/acl/common/AclClientRPCHook.java
@@ -16,15 +16,12 @@
  */
 package org.apache.rocketmq.acl.common;
 
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
-import static org.apache.rocketmq.acl.common.SessionCredentials.ACCESS_KEY;
-import static org.apache.rocketmq.acl.common.SessionCredentials.SECURITY_TOKEN;
-import static org.apache.rocketmq.acl.common.SessionCredentials.SIGNATURE;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
 
 public class AclClientRPCHook implements RPCHook {
     private final SessionCredentials sessionCredentials;
@@ -36,14 +33,14 @@ public AclClientRPCHook(SessionCredentials sessionCredentials) {
     @Override
     public void doBeforeRequest(String remoteAddr, RemotingCommand request) {
         // Add AccessKey and SecurityToken into signature calculating.
-        request.addExtField(ACCESS_KEY, sessionCredentials.getAccessKey());
+        request.addExtField(SessionCredentials.ACCESS_KEY, sessionCredentials.getAccessKey());
         // The SecurityToken value is unnecessary,user can choose this one.
         if (sessionCredentials.getSecurityToken() != null) {
-            request.addExtField(SECURITY_TOKEN, sessionCredentials.getSecurityToken());
+            request.addExtField(SessionCredentials.SECURITY_TOKEN, sessionCredentials.getSecurityToken());
         }
         byte[] total = AclUtils.combineRequestContent(request, parseRequestContent(request));
         String signature = AclUtils.calSignature(total, sessionCredentials.getSecretKey());
-        request.addExtField(SIGNATURE, signature);
+        request.addExtField(SessionCredentials.SIGNATURE, signature);
     }
 
     @Override
diff --git a/client/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java b/client/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java
new file mode 100644
index 00000000000..228e0e27b31
--- /dev/null
+++ b/client/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.acl.common;
+
+public class AclConstants {
+
+    public static final String CONFIG_ACCESS_KEY = "accessKey";
+
+    public static final String CONFIG_SECRET_KEY = "secretKey";
+
+    public static final String PUB = "PUB";
+
+    public static final String SUB = "SUB";
+
+    public static final String DENY = "DENY";
+
+    public static final String PUB_SUB = "PUB|SUB";
+
+    public static final String SUB_PUB = "SUB|PUB";
+}
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclException.java b/client/src/main/java/org/apache/rocketmq/acl/common/AclException.java
similarity index 100%
rename from acl/src/main/java/org/apache/rocketmq/acl/common/AclException.java
rename to client/src/main/java/org/apache/rocketmq/acl/common/AclException.java
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclSigner.java b/client/src/main/java/org/apache/rocketmq/acl/common/AclSigner.java
similarity index 99%
rename from acl/src/main/java/org/apache/rocketmq/acl/common/AclSigner.java
rename to client/src/main/java/org/apache/rocketmq/acl/common/AclSigner.java
index b4baa897225..a113ec12d24 100644
--- a/acl/src/main/java/org/apache/rocketmq/acl/common/AclSigner.java
+++ b/client/src/main/java/org/apache/rocketmq/acl/common/AclSigner.java
@@ -16,15 +16,16 @@
  */
 package org.apache.rocketmq.acl.common;
 
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import javax.crypto.Mac;
-import javax.crypto.spec.SecretKeySpec;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
 public class AclSigner {
     public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
     public static final SigningAlgorithm DEFAULT_ALGORITHM = SigningAlgorithm.HmacSHA1;
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java b/client/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java
similarity index 92%
rename from acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java
rename to client/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java
index 65f04f54339..4fbcd0ca2a3 100644
--- a/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java
+++ b/client/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java
@@ -16,14 +16,7 @@
  */
 package org.apache.rocketmq.acl.common;
 
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.util.Map;
-import java.util.SortedMap;
-
-import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson2.JSONObject;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
@@ -32,6 +25,12 @@
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.yaml.snakeyaml.Yaml;
 
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.util.Map;
+import java.util.SortedMap;
+
 import static org.apache.rocketmq.acl.common.SessionCredentials.CHARSET;
 
 public class AclUtils {
@@ -40,7 +39,7 @@ public class AclUtils {
 
     public static byte[] combineRequestContent(RemotingCommand request, SortedMap<String, String> fieldsMap) {
         try {
-            StringBuilder sb = new StringBuilder("");
+            StringBuilder sb = new StringBuilder();
             for (Map.Entry<String, String> entry : fieldsMap.entrySet()) {
                 if (!SessionCredentials.SIGNATURE.equals(entry.getKey())) {
                     sb.append(entry.getValue());
@@ -63,20 +62,19 @@ public static byte[] combineBytes(byte[] b1, byte[] b2) {
     }
 
     public static String calSignature(byte[] data, String secretKey) {
-        String signature = AclSigner.calSignature(data, secretKey);
-        return signature;
+        return AclSigner.calSignature(data, secretKey);
     }
 
     public static void IPv6AddressCheck(String netAddress) {
         if (isAsterisk(netAddress) || isMinus(netAddress)) {
             int asterisk = netAddress.indexOf("*");
             int minus = netAddress.indexOf("-");
-//            '*' must be the end of netAddress if it exists
+            // '*' must be the end of netAddress if it exists
             if (asterisk > -1 && asterisk != netAddress.length() - 1) {
                 throw new AclException(String.format("NetAddress examine scope Exception netAddress is %s", netAddress));
             }
 
-//            format like "2::ac5:78:1-200:*" or "2::ac5:78:1-200" is legal
+            // format like "2::ac5:78:1-200:*" or "2::ac5:78:1-200" is legal
             if (minus > -1) {
                 if (asterisk == -1) {
                     if (minus <= netAddress.lastIndexOf(":")) {
@@ -128,7 +126,7 @@ public static String[] getAddresses(String netAddress, String partialAddress) {
     }
 
     public static boolean isScope(String netAddress, int index) {
-//        IPv6 Address
+        // IPv6 Address
         if (isColon(netAddress)) {
             netAddress = expandIP(netAddress, 8);
             String[] strArray = StringUtils.split(netAddress, ":");
@@ -249,18 +247,6 @@ public static <T> T getYamlDataObject(InputStream fis, Class<T> clazz) {
         }
     }
 
-    public static boolean writeDataObject(String path, Object dataMap) {
-        Yaml yaml = new Yaml();
-        try (PrintWriter pw = new PrintWriter(path, "UTF-8")) {
-            String dumpAsMap = yaml.dumpAsMap(dataMap);
-            pw.print(dumpAsMap);
-            pw.flush();
-        } catch (Exception e) {
-            throw new AclException(e.getMessage(), e);
-        }
-        return true;
-    }
-
     public static RPCHook getAclRPCHook(String fileName) {
         JSONObject yamlDataObject;
         try {
diff --git a/client/src/main/java/org/apache/rocketmq/acl/common/Permission.java b/client/src/main/java/org/apache/rocketmq/acl/common/Permission.java
new file mode 100644
index 00000000000..3d7ac814f79
--- /dev/null
+++ b/client/src/main/java/org/apache/rocketmq/acl/common/Permission.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.acl.common;
+
+public class Permission {
+
+    public static final byte DENY = 1;
+    public static final byte ANY = 1 << 1;
+    public static final byte PUB = 1 << 2;
+    public static final byte SUB = 1 << 3;
+
+    public static byte parsePermFromString(String permString) {
+        if (permString == null) {
+            return Permission.DENY;
+        }
+        switch (permString.trim()) {
+            case AclConstants.PUB:
+                return Permission.PUB;
+            case AclConstants.SUB:
+                return Permission.SUB;
+            case AclConstants.PUB_SUB:
+            case AclConstants.SUB_PUB:
+                return Permission.PUB | Permission.SUB;
+            case AclConstants.DENY:
+                return Permission.DENY;
+            default:
+                return Permission.DENY;
+        }
+    }
+}
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/SessionCredentials.java b/client/src/main/java/org/apache/rocketmq/acl/common/SessionCredentials.java
similarity index 99%
rename from acl/src/main/java/org/apache/rocketmq/acl/common/SessionCredentials.java
rename to client/src/main/java/org/apache/rocketmq/acl/common/SessionCredentials.java
index dfc06d4f3a4..95af5943936 100644
--- a/acl/src/main/java/org/apache/rocketmq/acl/common/SessionCredentials.java
+++ b/client/src/main/java/org/apache/rocketmq/acl/common/SessionCredentials.java
@@ -16,12 +16,13 @@
  */
 package org.apache.rocketmq.acl.common;
 
+import org.apache.rocketmq.common.MixAll;
+
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.Properties;
-import org.apache.rocketmq.common.MixAll;
 
 public class SessionCredentials {
     public static final Charset CHARSET = StandardCharsets.UTF_8;
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/SigningAlgorithm.java b/client/src/main/java/org/apache/rocketmq/acl/common/SigningAlgorithm.java
similarity index 100%
rename from acl/src/main/java/org/apache/rocketmq/acl/common/SigningAlgorithm.java
rename to client/src/main/java/org/apache/rocketmq/acl/common/SigningAlgorithm.java
diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
index 8a7beffc704..696b073b373 100644
--- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
+++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
@@ -20,6 +20,7 @@
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.message.MessageQueue;
@@ -64,6 +65,8 @@ public class ClientConfig {
      */
     private int persistConsumerOffsetInterval = 1000 * 5;
     private long pullTimeDelayMillsWhenException = 1000;
+
+    private int traceMsgBatchNum = 10;
     private boolean unitMode = false;
     private String unitName;
     private boolean decodeReadBody = Boolean.parseBoolean(System.getProperty(DECODE_READ_BODY, "true"));
@@ -98,6 +101,16 @@ public class ClientConfig {
 
     private boolean enableHeartbeatChannelEventListener = true;
 
+    /**
+     * The switch for message trace
+     */
+    protected boolean enableTrace = false;
+
+    /**
+     * The name value of message trace topic. If not set, the default trace topic name will be used.
+     */
+    protected String traceTopic;
+
     public String buildMQClientId() {
         StringBuilder sb = new StringBuilder();
         sb.append(this.getClientIP());
@@ -117,6 +130,14 @@ public String buildMQClientId() {
         return sb.toString();
     }
 
+    public int getTraceMsgBatchNum() {
+        return traceMsgBatchNum;
+    }
+
+    public void setTraceMsgBatchNum(int traceMsgBatchNum) {
+        this.traceMsgBatchNum = traceMsgBatchNum;
+    }
+
     public String getClientIP() {
         return clientIP;
     }
@@ -215,6 +236,8 @@ public void resetClientConfig(final ClientConfig cc) {
         this.detectInterval = cc.detectInterval;
         this.detectTimeout = cc.detectTimeout;
         this.namespaceV2 = cc.namespaceV2;
+        this.enableTrace = cc.enableTrace;
+        this.traceTopic = cc.traceTopic;
     }
 
     public ClientConfig cloneClientConfig() {
@@ -245,6 +268,8 @@ public ClientConfig cloneClientConfig() {
         cc.detectInterval = detectInterval;
         cc.detectTimeout = detectTimeout;
         cc.namespaceV2 = namespaceV2;
+        cc.enableTrace = enableTrace;
+        cc.traceTopic = traceTopic;
         return cc;
     }
 
@@ -474,6 +499,22 @@ public void setUseHeartbeatV2(boolean useHeartbeatV2) {
         this.useHeartbeatV2 = useHeartbeatV2;
     }
 
+    public boolean isEnableTrace() {
+        return enableTrace;
+    }
+
+    public void setEnableTrace(boolean enableTrace) {
+        this.enableTrace = enableTrace;
+    }
+
+    public String getTraceTopic() {
+        return traceTopic;
+    }
+
+    public void setTraceTopic(String traceTopic) {
+        this.traceTopic = traceTopic;
+    }
+
     @Override
     public String toString() {
         return "ClientConfig{" +
@@ -505,6 +546,8 @@ public String toString() {
             ", sendLatencyEnable=" + sendLatencyEnable +
             ", startDetectorEnable=" + startDetectorEnable +
             ", enableHeartbeatChannelEventListener=" + enableHeartbeatChannelEventListener +
+            ", enableTrace=" + enableTrace +
+            ", traceTopic='" + traceTopic + '\'' +
             '}';
     }
 }
diff --git a/client/src/main/java/org/apache/rocketmq/client/MQAdmin.java b/client/src/main/java/org/apache/rocketmq/client/MQAdmin.java
index c2e936be43f..4864adda1c8 100644
--- a/client/src/main/java/org/apache/rocketmq/client/MQAdmin.java
+++ b/client/src/main/java/org/apache/rocketmq/client/MQAdmin.java
@@ -83,15 +83,6 @@ void createTopic(String key, String newTopic, int queueNum, int topicSysFlag, Ma
      */
     long earliestMsgStoreTime(final MessageQueue mq) throws MQClientException;
 
-    /**
-     * Query message according to message id
-     *
-     * @param offsetMsgId message id
-     * @return message
-     */
-    MessageExt viewMessage(final String offsetMsgId) throws RemotingException, MQBrokerException,
-        InterruptedException, MQClientException;
-
     /**
      * Query messages
      *
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java
index c193c6a42e4..20857f14e08 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java
@@ -169,15 +169,7 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon
      */
     private TraceDispatcher traceDispatcher = null;
 
-    /**
-     * The flag for message trace
-     */
-    private boolean enableMsgTrace = false;
-
-    /**
-     * The name value of message trace topic.If you don't config,you can use the default trace topic name.
-     */
-    private String customizedTraceTopic;
+    private RPCHook rpcHook;
 
     /**
      * Default constructor.
@@ -208,10 +200,11 @@ public DefaultLitePullConsumer(RPCHook rpcHook) {
      * Constructor specifying consumer group, RPC hook
      *
      * @param consumerGroup Consumer group.
-     * @param rpcHook RPC hook to execute before each remoting command.
+     * @param rpcHook       RPC hook to execute before each remoting command.
      */
     public DefaultLitePullConsumer(final String consumerGroup, RPCHook rpcHook) {
         this.consumerGroup = consumerGroup;
+        this.rpcHook = rpcHook;
         this.enableStreamRequestType = true;
         defaultLitePullConsumerImpl = new DefaultLitePullConsumerImpl(this, rpcHook);
     }
@@ -220,12 +213,13 @@ public DefaultLitePullConsumer(final String consumerGroup, RPCHook rpcHook) {
      * Constructor specifying namespace, consumer group and RPC hook.
      *
      * @param consumerGroup Consumer group.
-     * @param rpcHook RPC hook to execute before each remoting command.
+     * @param rpcHook       RPC hook to execute before each remoting command.
      */
     @Deprecated
     public DefaultLitePullConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook) {
         this.namespace = namespace;
         this.consumerGroup = consumerGroup;
+        this.rpcHook = rpcHook;
         this.enableStreamRequestType = true;
         defaultLitePullConsumerImpl = new DefaultLitePullConsumerImpl(this, rpcHook);
     }
@@ -276,6 +270,7 @@ public void subscribe(String topic, MessageSelector messageSelector) throws MQCl
     public void unsubscribe(String topic) {
         this.defaultLitePullConsumerImpl.unsubscribe(withNamespace(topic));
     }
+
     @Override
     public void assign(Collection<MessageQueue> messageQueues) {
         defaultLitePullConsumerImpl.assign(queuesWithNamespace(messageQueues));
@@ -344,7 +339,8 @@ public void commit() {
         this.defaultLitePullConsumerImpl.commitAll();
     }
 
-    @Override public void commit(Map<MessageQueue, Long> offsetMap, boolean persist) {
+    @Override
+    public void commit(Map<MessageQueue, Long> offsetMap, boolean persist) {
         this.defaultLitePullConsumerImpl.commit(offsetMap, persist);
     }
 
@@ -367,11 +363,11 @@ public Set<MessageQueue> assignment() throws MQClientException {
      * @param messageQueueListener
      */
     @Override
-    public void subscribe(String topic, String subExpression, MessageQueueListener messageQueueListener) throws MQClientException {
+    public void subscribe(String topic, String subExpression,
+        MessageQueueListener messageQueueListener) throws MQClientException {
         this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), subExpression, messageQueueListener);
     }
 
-
     @Override
     public void commit(final Set<MessageQueue> messageQueues, boolean persist) {
         this.defaultLitePullConsumerImpl.commit(messageQueues, persist);
@@ -592,15 +588,12 @@ public TraceDispatcher getTraceDispatcher() {
         return traceDispatcher;
     }
 
-    public void setCustomizedTraceTopic(String customizedTraceTopic) {
-        this.customizedTraceTopic = customizedTraceTopic;
-    }
-
     private void setTraceDispatcher() {
-        if (isEnableMsgTrace()) {
+        if (enableTrace) {
             try {
-                AsyncTraceDispatcher traceDispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, customizedTraceTopic, null);
+                AsyncTraceDispatcher traceDispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, getTraceMsgBatchNum(), traceTopic, rpcHook);
                 traceDispatcher.getTraceProducer().setUseTLS(this.isUseTLS());
+                traceDispatcher.setNamespaceV2(namespaceV2);
                 this.traceDispatcher = traceDispatcher;
                 this.defaultLitePullConsumerImpl.registerConsumeMessageHook(
                     new ConsumeMessageTraceHookImpl(traceDispatcher));
@@ -611,14 +604,18 @@ private void setTraceDispatcher() {
     }
 
     public String getCustomizedTraceTopic() {
-        return customizedTraceTopic;
+        return traceTopic;
+    }
+
+    public void setCustomizedTraceTopic(String customizedTraceTopic) {
+        this.traceTopic = customizedTraceTopic;
     }
 
     public boolean isEnableMsgTrace() {
-        return enableMsgTrace;
+        return enableTrace;
     }
 
     public void setEnableMsgTrace(boolean enableMsgTrace) {
-        this.enableMsgTrace = enableMsgTrace;
+        this.enableTrace = enableMsgTrace;
     }
 }
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java
index 499f7731915..38841e41287 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java
@@ -16,9 +16,11 @@
  */
 package org.apache.rocketmq.client.consumer;
 
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.QueryResult;
 import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;
@@ -33,11 +35,13 @@
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
+import org.apache.rocketmq.remoting.protocol.filter.FilterAPI;
 import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
 
 /**
- * @deprecated Default pulling consumer. This class will be removed in 2022, and a better implementation {@link
- * DefaultLitePullConsumer} is recommend to use in the scenario of actively pulling messages.
+ * @deprecated Default pulling consumer. This class will be removed in 2022, and a better implementation
+ * {@link DefaultLitePullConsumer} is recommend to use in the scenario of actively pulling messages.
  */
 @Deprecated
 public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsumer {
@@ -77,6 +81,8 @@ public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsume
      * Topic set you want to register
      */
     private Set<String> registerTopics = new HashSet<>();
+
+    private final Set<SubscriptionData> registerSubscriptions = Collections.newSetFromMap(new ConcurrentHashMap<>());
     /**
      * Queue allocation algorithm
      */
@@ -88,6 +94,8 @@ public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsume
 
     private int maxReconsumeTimes = 16;
 
+    private boolean enableRebalance = true;
+
     public DefaultMQPullConsumer() {
         this(MixAll.DEFAULT_CONSUMER_GROUP, null);
     }
@@ -175,16 +183,6 @@ public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {
         return this.defaultMQPullConsumerImpl.earliestMsgStoreTime(queueWithNamespace(mq));
     }
 
-    /**
-     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.
-     */
-    @Deprecated
-    @Override
-    public MessageExt viewMessage(String offsetMsgId) throws RemotingException, MQBrokerException,
-        InterruptedException, MQClientException {
-        return this.defaultMQPullConsumerImpl.viewMessage(offsetMsgId);
-    }
-
     /**
      * This method will be removed in a certain version after April 5, 2020, so please do not use this method.
      */
@@ -263,6 +261,29 @@ public void setRegisterTopics(Set<String> registerTopics) {
         this.registerTopics = withNamespace(registerTopics);
     }
 
+    public Set<SubscriptionData> getRegisterSubscriptions() {
+        return registerSubscriptions;
+    }
+
+    public void addRegisterSubscriptions(String topic, MessageSelector messageSelector) throws MQClientException {
+        try {
+            if (messageSelector == null) {
+                messageSelector = MessageSelector.byTag(SubscriptionData.SUB_ALL);
+            }
+
+            SubscriptionData subscriptionData = FilterAPI.build(withNamespace(topic),
+                messageSelector.getExpression(), messageSelector.getExpressionType());
+
+            this.registerSubscriptions.add(subscriptionData);
+        } catch (Exception e) {
+            throw new MQClientException("add subscription exception", e);
+        }
+    }
+
+    public void clearRegisterSubscriptions() {
+        this.registerSubscriptions.clear();
+    }
+
     /**
      * This method will be removed or it's visibility will be changed in a certain version after April 5, 2020, so
      * please do not use this method.
@@ -272,7 +293,7 @@ public void setRegisterTopics(Set<String> registerTopics) {
     public void sendMessageBack(MessageExt msg, int delayLevel)
         throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
         msg.setTopic(withNamespace(msg.getTopic()));
-        this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, null);
+        this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, msg.getBrokerName());
     }
 
     /**
@@ -385,6 +406,20 @@ public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offs
         this.defaultMQPullConsumerImpl.pullBlockIfNotFound(queueWithNamespace(mq), subExpression, offset, maxNums, pullCallback);
     }
 
+    @Override
+    public void pullBlockIfNotFoundWithMessageSelector(MessageQueue mq, MessageSelector selector,
+        long offset, int maxNums,
+        PullCallback pullCallback) throws MQClientException, RemotingException, InterruptedException {
+        this.defaultMQPullConsumerImpl.pullBlockIfNotFoundWithMessageSelector(mq, selector, offset, maxNums, pullCallback);
+    }
+
+    @Override
+    public PullResult pullBlockIfNotFoundWithMessageSelector(MessageQueue mq, MessageSelector selector,
+        long offset,
+        int maxNums) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
+        return this.defaultMQPullConsumerImpl.pullBlockIfNotFoundWithMessageSelector(mq, selector, offset, maxNums);
+    }
+
     @Override
     public void updateConsumeOffset(MessageQueue mq, long offset) throws MQClientException {
         this.defaultMQPullConsumerImpl.updateConsumeOffset(queueWithNamespace(mq), offset);
@@ -405,7 +440,7 @@ public MessageExt viewMessage(String topic,
         String uniqKey) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
         try {
             MessageDecoder.decodeMessageId(uniqKey);
-            return this.viewMessage(uniqKey);
+            return this.defaultMQPullConsumerImpl.viewMessage(topic, uniqKey);
         } catch (Exception e) {
             // Ignore
         }
@@ -464,4 +499,12 @@ public void setMaxReconsumeTimes(final int maxReconsumeTimes) {
     public void persist(MessageQueue mq) {
         this.getOffsetStore().persist(queueWithNamespace(mq));
     }
+
+    public boolean isEnableRebalance() {
+        return enableRebalance;
+    }
+
+    public void setEnableRebalance(boolean enableRebalance) {
+        this.enableRebalance = enableRebalance;
+    }
 }
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java
index 224ea67d5fb..5df5cc8fa1a 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java
@@ -16,10 +16,6 @@
  */
 package org.apache.rocketmq.client.consumer;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.QueryResult;
 import org.apache.rocketmq.client.consumer.listener.MessageListener;
@@ -40,21 +36,24 @@
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
 import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
 
 /**
  * In most scenarios, this is the mostly recommended class to consume messages.
  * </p>
- *
  * Technically speaking, this push client is virtually a wrapper of the underlying pull service. Specifically, on
  * arrival of messages pulled from brokers, it roughly invokes the registered callback handler to feed the messages.
  * </p>
- *
  * See quickstart/Consumer in the example module for a typical usage.
  * </p>
  *
@@ -75,7 +74,6 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
      * Consumers of the same role is required to have exactly same subscriptions and consumerGroup to correctly achieve
      * load balance. It's required and needs to be globally unique.
      * </p>
-     *
      * See <a href="https://rocketmq.apache.org/docs/introduction/02concepts">here</a> for further discussion.
      */
     private String consumerGroup;
@@ -83,13 +81,11 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
     /**
      * Message model defines the way how messages are delivered to each consumer clients.
      * </p>
-     *
      * RocketMQ supports two message models: clustering and broadcasting. If clustering is set, consumer clients with
      * the same {@link #consumerGroup} would only consume shards of the messages subscribed, which achieves load
      * balances; Conversely, if the broadcasting is set, each consumer client will consume all subscribed messages
      * separately.
      * </p>
-     *
      * This field defaults to clustering.
      */
     private MessageModel messageModel = MessageModel.CLUSTERING;
@@ -97,7 +93,6 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
     /**
      * Consuming point on consumer booting.
      * </p>
-     *
      * There are three consuming points:
      * <ul>
      * <li>
@@ -238,7 +233,6 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
      */
     private int pullBatchSize = 32;
 
-
     private int pullBatchSizeInBytes = 256 * 1024;
 
     /**
@@ -255,7 +249,6 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
      * Max re-consume times.
      * In concurrently mode, -1 means 16;
      * In orderly mode, -1 means Integer.MAX_VALUE.
-     *
      * If messages are re-consumed more than {@link #maxReconsumeTimes} before success.
      */
     private int maxReconsumeTimes = -1;
@@ -293,6 +286,8 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
     // force to use client rebalance
     private boolean clientRebalance = true;
 
+    private RPCHook rpcHook = null;
+
     /**
      * Default constructor.
      */
@@ -309,7 +304,6 @@ public DefaultMQPushConsumer(final String consumerGroup) {
         this(consumerGroup, null, new AllocateMessageQueueAveragely());
     }
 
-
     /**
      * Constructor specifying RPC hook.
      *
@@ -323,70 +317,60 @@ public DefaultMQPushConsumer(RPCHook rpcHook) {
      * Constructor specifying consumer group, RPC hook.
      *
      * @param consumerGroup Consumer group.
-     * @param rpcHook RPC hook to execute before each remoting command.
+     * @param rpcHook       RPC hook to execute before each remoting command.
      */
     public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook) {
-        this.consumerGroup = consumerGroup;
-        this.allocateMessageQueueStrategy = new AllocateMessageQueueAveragely();
-        defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook);
+        this(consumerGroup, rpcHook, new AllocateMessageQueueAveragely());
     }
 
-
     /**
      * Constructor specifying consumer group, enabled msg trace flag and customized trace topic name.
      *
-     * @param consumerGroup Consumer group.
-     * @param enableMsgTrace Switch flag instance for message trace.
+     * @param consumerGroup        Consumer group.
+     * @param enableMsgTrace       Switch flag instance for message trace.
      * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name.
      */
-    public DefaultMQPushConsumer(final String consumerGroup, boolean enableMsgTrace, final String customizedTraceTopic) {
+    public DefaultMQPushConsumer(final String consumerGroup, boolean enableMsgTrace,
+        final String customizedTraceTopic) {
         this(consumerGroup, null, new AllocateMessageQueueAveragely(), enableMsgTrace, customizedTraceTopic);
     }
 
     /**
      * Constructor specifying consumer group, RPC hook and message queue allocating algorithm.
      *
-     * @param consumerGroup Consume queue.
-     * @param rpcHook RPC hook to execute before each remoting command.
+     * @param consumerGroup                Consumer group.
+     * @param rpcHook                      RPC hook to execute before each remoting command.
      * @param allocateMessageQueueStrategy Message queue allocating algorithm.
      */
     public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook,
         AllocateMessageQueueStrategy allocateMessageQueueStrategy) {
-        this.consumerGroup = consumerGroup;
-        this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;
-        defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook);
+        this(consumerGroup, rpcHook, allocateMessageQueueStrategy, false, null);
     }
 
     /**
      * Constructor specifying consumer group, RPC hook, message queue allocating algorithm, enabled msg trace flag and customized trace topic name.
      *
-     * @param consumerGroup Consume queue.
-     * @param rpcHook RPC hook to execute before each remoting command.
+     * @param consumerGroup                Consumer group.
+     * @param rpcHook                      RPC hook to execute before each remoting command.
      * @param allocateMessageQueueStrategy message queue allocating algorithm.
-     * @param enableMsgTrace Switch flag instance for message trace.
-     * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name.
+     * @param enableMsgTrace               Switch flag instance for message trace.
+     * @param customizedTraceTopic         The name value of message trace topic.If you don't config,you can use the default trace topic name.
      */
     public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook,
-        AllocateMessageQueueStrategy allocateMessageQueueStrategy, boolean enableMsgTrace, final String customizedTraceTopic) {
+        AllocateMessageQueueStrategy allocateMessageQueueStrategy, boolean enableMsgTrace,
+        final String customizedTraceTopic) {
         this.consumerGroup = consumerGroup;
+        this.rpcHook = rpcHook;
         this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;
         defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook);
-        if (enableMsgTrace) {
-            try {
-                AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, customizedTraceTopic, rpcHook);
-                dispatcher.setHostConsumer(this.defaultMQPushConsumerImpl);
-                traceDispatcher = dispatcher;
-                this.defaultMQPushConsumerImpl.registerConsumeMessageHook(new ConsumeMessageTraceHookImpl(traceDispatcher));
-            } catch (Throwable e) {
-                log.error("system mqtrace hook init failed ,maybe can't send msg trace data");
-            }
-        }
+        this.enableTrace = enableMsgTrace;
+        this.traceTopic = customizedTraceTopic;
     }
 
     /**
      * Constructor specifying namespace and consumer group.
      *
-     * @param namespace Namespace for this MQ Producer instance.
+     * @param namespace     Namespace for this MQ Producer instance.
      * @param consumerGroup Consumer group.
      */
     @Deprecated
@@ -397,9 +381,9 @@ public DefaultMQPushConsumer(final String namespace, final String consumerGroup)
     /**
      * Constructor specifying namespace, consumer group and RPC hook .
      *
-     * @param namespace Namespace for this MQ Producer instance.
+     * @param namespace     Namespace for this MQ Producer instance.
      * @param consumerGroup Consumer group.
-     * @param rpcHook RPC hook to execute before each remoting command.
+     * @param rpcHook       RPC hook to execute before each remoting command.
      */
     @Deprecated
     public DefaultMQPushConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook) {
@@ -409,9 +393,9 @@ public DefaultMQPushConsumer(final String namespace, final String consumerGroup,
     /**
      * Constructor specifying namespace, consumer group, RPC hook and message queue allocating algorithm.
      *
-     * @param namespace Namespace for this MQ Producer instance.
-     * @param consumerGroup Consume queue.
-     * @param rpcHook RPC hook to execute before each remoting command.
+     * @param namespace                    Namespace for this MQ Producer instance.
+     * @param consumerGroup                Consumer group.
+     * @param rpcHook                      RPC hook to execute before each remoting command.
      * @param allocateMessageQueueStrategy Message queue allocating algorithm.
      */
     @Deprecated
@@ -419,6 +403,7 @@ public DefaultMQPushConsumer(final String namespace, final String consumerGroup,
         AllocateMessageQueueStrategy allocateMessageQueueStrategy) {
         this.consumerGroup = consumerGroup;
         this.namespace = namespace;
+        this.rpcHook = rpcHook;
         this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;
         defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook);
     }
@@ -426,30 +411,24 @@ public DefaultMQPushConsumer(final String namespace, final String consumerGroup,
     /**
      * Constructor specifying namespace, consumer group, RPC hook, message queue allocating algorithm, enabled msg trace flag and customized trace topic name.
      *
-     * @param namespace Namespace for this MQ Producer instance.
-     * @param consumerGroup Consume queue.
-     * @param rpcHook RPC hook to execute before each remoting command.
+     * @param namespace                    Namespace for this MQ Producer instance.
+     * @param consumerGroup                Consumer group.
+     * @param rpcHook                      RPC hook to execute before each remoting command.
      * @param allocateMessageQueueStrategy message queue allocating algorithm.
-     * @param enableMsgTrace Switch flag instance for message trace.
-     * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name.
+     * @param enableMsgTrace               Switch flag instance for message trace.
+     * @param customizedTraceTopic         The name value of message trace topic.If you don't config,you can use the default trace topic name.
      */
     @Deprecated
     public DefaultMQPushConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook,
-        AllocateMessageQueueStrategy allocateMessageQueueStrategy, boolean enableMsgTrace, final String customizedTraceTopic) {
+        AllocateMessageQueueStrategy allocateMessageQueueStrategy, boolean enableMsgTrace,
+        final String customizedTraceTopic) {
         this.consumerGroup = consumerGroup;
         this.namespace = namespace;
+        this.rpcHook = rpcHook;
         this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;
         defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook);
-        if (enableMsgTrace) {
-            try {
-                AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, customizedTraceTopic, rpcHook);
-                dispatcher.setHostConsumer(this.defaultMQPushConsumerImpl);
-                traceDispatcher = dispatcher;
-                this.defaultMQPushConsumerImpl.registerConsumeMessageHook(new ConsumeMessageTraceHookImpl(traceDispatcher));
-            } catch (Throwable e) {
-                log.error("system mqtrace hook init failed ,maybe can't send msg trace data");
-            }
-        }
+        this.enableTrace = enableMsgTrace;
+        this.traceTopic = customizedTraceTopic;
     }
 
     /**
@@ -457,16 +436,14 @@ public DefaultMQPushConsumer(final String namespace, final String consumerGroup,
      */
     @Deprecated
     @Override
-    public void createTopic(String key, String newTopic, int queueNum, Map<String, String> attributes) throws MQClientException {
+    public void createTopic(String key, String newTopic, int queueNum,
+        Map<String, String> attributes) throws MQClientException {
         createTopic(key, withNamespace(newTopic), queueNum, 0, null);
     }
 
     @Override
     public void setUseTLS(boolean useTLS) {
         super.setUseTLS(useTLS);
-        if (traceDispatcher instanceof AsyncTraceDispatcher) {
-            ((AsyncTraceDispatcher) traceDispatcher).getTraceProducer().setUseTLS(useTLS);
-        }
     }
 
     /**
@@ -474,7 +451,8 @@ public void setUseTLS(boolean useTLS) {
      */
     @Deprecated
     @Override
-    public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag, Map<String, String> attributes) throws MQClientException {
+    public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag,
+        Map<String, String> attributes) throws MQClientException {
         this.defaultMQPushConsumerImpl.createTopic(key, withNamespace(newTopic), queueNum, topicSysFlag);
     }
 
@@ -514,16 +492,6 @@ public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {
         return this.defaultMQPushConsumerImpl.earliestMsgStoreTime(queueWithNamespace(mq));
     }
 
-    /**
-     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.
-     */
-    @Deprecated
-    @Override
-    public MessageExt viewMessage(
-        String offsetMsgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        return this.defaultMQPushConsumerImpl.viewMessage(offsetMsgId);
-    }
-
     /**
      * This method will be removed in a certain version after April 5, 2020, so please do not use this method.
      */
@@ -543,7 +511,7 @@ public MessageExt viewMessage(String topic,
         String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
         try {
             MessageDecoder.decodeMessageId(msgId);
-            return this.viewMessage(msgId);
+            return this.defaultMQPushConsumerImpl.viewMessage(withNamespace(topic), msgId);
         } catch (Exception e) {
             // Ignore
         }
@@ -704,39 +672,39 @@ public void setSubscription(Map<String, String> subscription) {
 
     /**
      * Send message back to broker which will be re-delivered in future.
-     *
+     * <p>
      * This method will be removed or it's visibility will be changed in a certain version after April 5, 2020, so
      * please do not use this method.
      *
-     * @param msg Message to send back.
+     * @param msg        Message to send back.
      * @param delayLevel delay level.
-     * @throws RemotingException if there is any network-tier error.
-     * @throws MQBrokerException if there is any broker error.
+     * @throws RemotingException    if there is any network-tier error.
+     * @throws MQBrokerException    if there is any broker error.
      * @throws InterruptedException if the thread is interrupted.
-     * @throws MQClientException if there is any client error.
+     * @throws MQClientException    if there is any client error.
      */
     @Deprecated
     @Override
     public void sendMessageBack(MessageExt msg, int delayLevel)
         throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
         msg.setTopic(withNamespace(msg.getTopic()));
-        this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, (String) null);
+        this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, msg.getBrokerName());
     }
 
     /**
      * Send message back to the broker whose name is <code>brokerName</code> and the message will be re-delivered in
      * future.
-     *
+     * <p>
      * This method will be removed or it's visibility will be changed in a certain version after April 5, 2020, so
      * please do not use this method.
      *
-     * @param msg Message to send back.
+     * @param msg        Message to send back.
      * @param delayLevel delay level.
      * @param brokerName broker name.
-     * @throws RemotingException if there is any network-tier error.
-     * @throws MQBrokerException if there is any broker error.
+     * @throws RemotingException    if there is any network-tier error.
+     * @throws MQBrokerException    if there is any broker error.
      * @throws InterruptedException if the thread is interrupted.
-     * @throws MQClientException if there is any client error.
+     * @throws MQClientException    if there is any client error.
      */
     @Deprecated
     @Override
@@ -760,7 +728,21 @@ public Set<MessageQueue> fetchSubscribeMessageQueues(String topic) throws MQClie
     public void start() throws MQClientException {
         setConsumerGroup(NamespaceUtil.wrapNamespace(this.getNamespace(), this.consumerGroup));
         this.defaultMQPushConsumerImpl.start();
+        if (enableTrace) {
+            try {
+                AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, getTraceMsgBatchNum(), traceTopic, rpcHook);
+                dispatcher.setHostConsumer(this.defaultMQPushConsumerImpl);
+                dispatcher.setNamespaceV2(namespaceV2);
+                traceDispatcher = dispatcher;
+                this.defaultMQPushConsumerImpl.registerConsumeMessageHook(new ConsumeMessageTraceHookImpl(traceDispatcher));
+            } catch (Throwable e) {
+                log.error("system mqtrace hook init failed ,maybe can't send msg trace data");
+            }
+        }
         if (null != traceDispatcher) {
+            if (traceDispatcher instanceof AsyncTraceDispatcher) {
+                ((AsyncTraceDispatcher) traceDispatcher).getTraceProducer().setUseTLS(isUseTLS());
+            }
             try {
                 traceDispatcher.start(this.getNamesrvAddr(), this.getAccessChannel());
             } catch (MQClientException e) {
@@ -812,9 +794,9 @@ public void registerMessageListener(MessageListenerOrderly messageListener) {
     /**
      * Subscribe a topic to consuming subscription.
      *
-     * @param topic topic to subscribe.
+     * @param topic         topic to subscribe.
      * @param subExpression subscription expression.it only support or operation such as "tag1 || tag2 || tag3" <br>
-     * if null or * expression,meaning subscribe all
+     *                      if null or * expression,meaning subscribe all
      * @throws MQClientException if there is any client error.
      */
     @Override
@@ -825,8 +807,8 @@ public void subscribe(String topic, String subExpression) throws MQClientExcepti
     /**
      * Subscribe a topic to consuming subscription.
      *
-     * @param topic topic to consume.
-     * @param fullClassName full class name,must extend org.apache.rocketmq.common.filter. MessageFilter
+     * @param topic             topic to consume.
+     * @param fullClassName     full class name,must extend org.apache.rocketmq.common.filter. MessageFilter
      * @param filterClassSource class source code,used UTF-8 file encoding,must be responsible for your code safety
      */
     @Override
@@ -837,7 +819,7 @@ public void subscribe(String topic, String fullClassName, String filterClassSour
     /**
      * Subscribe a topic by message selector.
      *
-     * @param topic topic to consume.
+     * @param topic           topic to consume.
      * @param messageSelector {@link org.apache.rocketmq.client.consumer.MessageSelector}
      * @see org.apache.rocketmq.client.consumer.MessageSelector#bySql
      * @see org.apache.rocketmq.client.consumer.MessageSelector#byTag
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java
index 868ee93ff8a..ee77b12bbc8 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java
@@ -47,8 +47,7 @@ public interface MQPullConsumer extends MQConsumer {
      *
      * @param mq from which message queue
      * @param subExpression subscription expression.it only support or operation such as "tag1 || tag2 || tag3" <br> if
-     * null or * expression,meaning subscribe
-     * all
+     * null or * expression,meaning subscribe all
      * @param offset from where to pull
      * @param maxNums max pulling numbers
      * @return The resulting {@code PullRequest}
@@ -121,7 +120,7 @@ void pull(final MessageQueue mq, final String subExpression, final long offset,
         InterruptedException;
 
     /**
-     * Pulling the messages in a async. way. Support message selection
+     * Pulling the messages in a async way. Support message selection
      */
     void pull(final MessageQueue mq, final MessageSelector selector, final long offset, final int maxNums,
         final PullCallback pullCallback) throws MQClientException, RemotingException,
@@ -150,6 +149,23 @@ void pullBlockIfNotFound(final MessageQueue mq, final String subExpression, fina
         final int maxNums, final PullCallback pullCallback) throws MQClientException, RemotingException,
         InterruptedException;
 
+    /**
+     * Pulling the messages through callback function,if no message arrival,blocking. Support message selection
+     */
+    void pullBlockIfNotFoundWithMessageSelector(final MessageQueue mq, final MessageSelector selector,
+        final long offset, final int maxNums,
+        final PullCallback pullCallback) throws MQClientException, RemotingException,
+        InterruptedException;
+
+    /**
+     * Pulling the messages,if no message arrival,blocking some time. Support message selection
+     *
+     * @return The resulting {@code PullRequest}
+     */
+    PullResult pullBlockIfNotFoundWithMessageSelector(final MessageQueue mq, final MessageSelector selector,
+        final long offset, final int maxNums) throws MQClientException, RemotingException,
+        MQBrokerException, InterruptedException;
+
     /**
      * Update the offset
      */
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/NotifyResult.java b/client/src/main/java/org/apache/rocketmq/client/consumer/NotifyResult.java
new file mode 100644
index 00000000000..4bd8b281752
--- /dev/null
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/NotifyResult.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.consumer;
+
+public class NotifyResult {
+    private boolean hasMsg;
+    private boolean pollingFull;
+
+    public boolean isHasMsg() {
+        return hasMsg;
+    }
+
+    public boolean isPollingFull() {
+        return pollingFull;
+    }
+
+    public void setHasMsg(boolean hasMsg) {
+        this.hasMsg = hasMsg;
+    }
+
+    public void setPollingFull(boolean pollingFull) {
+        this.pollingFull = pollingFull;
+    }
+
+    @Override public String toString() {
+        return "NotifyResult{" +
+            "hasMsg=" + hasMsg +
+            ", pollingFull=" + pollingFull +
+            '}';
+    }
+}
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/PopStatus.java b/client/src/main/java/org/apache/rocketmq/client/consumer/PopStatus.java
index 17dda9a2001..57fbe67bcca 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/PopStatus.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/PopStatus.java
@@ -23,7 +23,7 @@ public enum PopStatus {
     FOUND,
     /**
      * No new message can be pull after polling time out
-     * delete after next realease
+     * delete after next release
      */
     NO_NEW_MSG,
     /**
diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java
index 75e5d1c218b..6f63a6fc607 100644
--- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java
+++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java
@@ -42,7 +42,7 @@ public List<MessageQueue> allocate(String consumerGroup, String currentCID, List
         int startIndex = (mod > 0 && index < mod) ? index * averageSize : index * averageSize + mod;
         int range = Math.min(averageSize, mqAll.size() - startIndex);
         for (int i = 0; i < range; i++) {
-            result.add(mqAll.get((startIndex + i) % mqAll.size()));
+            result.add(mqAll.get(startIndex + i));
         }
         return result;
     }
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java
index 2f18c610c14..e46c651f928 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java
@@ -288,8 +288,8 @@ private void processReplyMessage(MessageExt replyMsg) {
             }
         } else {
             String bornHost = replyMsg.getBornHostString();
-            logger.warn(String.format("receive reply message, but not matched any request, CorrelationId: %s , reply from host: %s",
-                correlationId, bornHost));
+            logger.warn("receive reply message, but not matched any request, CorrelationId: {} , reply from host: {}",
+                correlationId, bornHost);
         }
     }
 }
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java
index 83835bd3d3e..c1e3ee33dc1 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java
@@ -28,6 +28,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.client.QueryResult;
 import org.apache.rocketmq.client.Validators;
 import org.apache.rocketmq.client.exception.MQBrokerException;
@@ -43,6 +44,7 @@
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageId;
 import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.common.utils.NetworkUtil;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
@@ -181,7 +183,7 @@ public Set<MessageQueue> fetchSubscribeMessageQueues(String topic) throws MQClie
                 e);
         }
 
-        throw new MQClientException("Unknow why, Can not find Message Queue for this topic, " + topic, null);
+        throw new MQClientException("Unknown why, Can not find Message Queue for this topic, " + topic, null);
     }
 
     public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException {
@@ -199,7 +201,7 @@ public long searchOffset(MessageQueue mq, long timestamp, BoundaryType boundaryT
         if (brokerAddr != null) {
             try {
                 return this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq, timestamp,
-                        boundaryType, timeoutMillis);
+                    boundaryType, timeoutMillis);
             } catch (Exception e) {
                 throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e);
             }
@@ -262,28 +264,35 @@ public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {
         throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
     }
 
-    public MessageExt viewMessage(String msgId)
+    public MessageExt viewMessage(String topic, String msgId)
         throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        MessageId messageId = null;
+        MessageId messageId;
         try {
             messageId = MessageDecoder.decodeMessageId(msgId);
         } catch (Exception e) {
             throw new MQClientException(ResponseCode.NO_MESSAGE, "query message by id finished, but no message.");
         }
         return this.mQClientFactory.getMQClientAPIImpl().viewMessage(NetworkUtil.socketAddress2String(messageId.getAddress()),
-            messageId.getOffset(), timeoutMillis);
+            topic, messageId.getOffset(), timeoutMillis);
     }
 
     public QueryResult queryMessage(String topic, String key, int maxNum, long begin,
         long end) throws MQClientException,
         InterruptedException {
-        return queryMessage(topic, key, maxNum, begin, end, false);
+        return queryMessage(null, topic, key, maxNum, begin, end, false);
     }
 
     public QueryResult queryMessageByUniqKey(String topic, String uniqKey, int maxNum, long begin, long end)
         throws MQClientException, InterruptedException {
 
-        return queryMessage(topic, uniqKey, maxNum, begin, end, true);
+        return queryMessage(null, topic, uniqKey, maxNum, begin, end, true);
+    }
+
+    public QueryResult queryMessageByUniqKey(String clusterName, String topic, String uniqKey, int maxNum, long begin,
+        long end)
+        throws MQClientException, InterruptedException {
+
+        return queryMessage(clusterName, topic, uniqKey, maxNum, begin, end, true);
     }
 
     public MessageExt queryMessageByUniqKey(String topic,
@@ -311,25 +320,29 @@ public MessageExt queryMessageByUniqKey(String clusterName, String topic,
         }
     }
 
-    protected QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end,
+    public QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, long end,
         boolean isUniqKey) throws MQClientException,
         InterruptedException {
-        return queryMessage(null, topic, key, maxNum, begin, end, isUniqKey);
-    }
+        boolean isLmq = MixAll.isLmq(topic);
+
+        String routeTopic = topic;
+        // if topic is lmq ,then use clusterName as lmq parent topic
+        // Use clusterName or lmq parent topic to get topic route for lmq or rmq_sys_wheel_timer
+        if (!StringUtils.isEmpty(topic) && (isLmq || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer"))
+            && !StringUtils.isEmpty(clusterName)) {
+            routeTopic = clusterName;
+        }
 
-    protected QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, long end,
-        boolean isUniqKey) throws MQClientException,
-        InterruptedException {
-        TopicRouteData topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(topic);
+        TopicRouteData topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(routeTopic);
         if (null == topicRouteData) {
-            this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);
-            topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(topic);
+            this.mQClientFactory.updateTopicRouteInfoFromNameServer(routeTopic);
+            topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(routeTopic);
         }
 
         if (topicRouteData != null) {
             List<String> brokerAddrs = new LinkedList<>();
             for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {
-                if (clusterName != null && !clusterName.isEmpty()
+                if (!isLmq && clusterName != null && !clusterName.isEmpty()
                     && !clusterName.equals(brokerData.getCluster())) {
                     continue;
                 }
@@ -347,7 +360,11 @@ protected QueryResult queryMessage(String clusterName, String topic, String key,
                 for (String addr : brokerAddrs) {
                     try {
                         QueryMessageRequestHeader requestHeader = new QueryMessageRequestHeader();
-                        requestHeader.setTopic(topic);
+                        if (isLmq) {
+                            requestHeader.setTopic(clusterName);
+                        } else {
+                            requestHeader.setTopic(topic);
+                        }
                         requestHeader.setKey(key);
                         requestHeader.setMaxNum(maxNum);
                         requestHeader.setBeginTimestamp(begin);
@@ -436,7 +453,7 @@ public void operationFail(Throwable throwable) {
                                 String[] keyArray = keys.split(MessageConst.KEY_SEPARATOR);
                                 for (String k : keyArray) {
                                     // both topic and key must be equal at the same time
-                                    if (Objects.equals(key, k) && Objects.equals(topic, msgTopic)) {
+                                    if (Objects.equals(key, k) && (isLmq || Objects.equals(topic, msgTopic))) {
                                         matched = true;
                                         break;
                                     }
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
index 1b4b3878c67..c001b33fa98 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
@@ -17,19 +17,7 @@
 package org.apache.rocketmq.client.impl;
 
 import com.alibaba.fastjson.JSON;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.Validators;
@@ -55,10 +43,11 @@
 import org.apache.rocketmq.client.producer.SendStatus;
 import org.apache.rocketmq.client.rpchook.NamespaceRpcHook;
 import org.apache.rocketmq.common.BoundaryType;
+import org.apache.rocketmq.common.CheckRocksdbCqWriteResult;
 import org.apache.rocketmq.common.MQVersion;
 import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.ObjectCreator;
 import org.apache.rocketmq.common.Pair;
-import org.apache.rocketmq.common.PlainAccessConfig;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.attribute.AttributeParser;
@@ -77,6 +66,7 @@
 import org.apache.rocketmq.common.namesrv.TopAddressing;
 import org.apache.rocketmq.common.sysflag.PullSysFlag;
 import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.common.utils.StartAndShutdown;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.ChannelEventListener;
@@ -95,7 +85,6 @@
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
 import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
 import org.apache.rocketmq.remoting.netty.ResponseFuture;
-import org.apache.rocketmq.remoting.protocol.DataVersion;
 import org.apache.rocketmq.remoting.protocol.LanguageCode;
 import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
@@ -104,18 +93,19 @@
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
 import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;
 import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;
+import org.apache.rocketmq.remoting.protocol.body.AclInfo;
 import org.apache.rocketmq.remoting.protocol.body.BatchAck;
 import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody;
 import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;
 import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;
 import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData;
 import org.apache.rocketmq.remoting.protocol.body.CheckClientRequestBody;
-import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo;
 import org.apache.rocketmq.remoting.protocol.body.ClusterInfo;
 import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;
 import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList;
 import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection;
 import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;
+import org.apache.rocketmq.remoting.protocol.body.CreateTopicListRequestBody;
 import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache;
 import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody;
 import org.apache.rocketmq.remoting.protocol.body.GroupList;
@@ -134,26 +124,33 @@
 import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;
 import org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;
 import org.apache.rocketmq.remoting.protocol.body.SetMessageRequestModeRequestBody;
+import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupList;
 import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;
 import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
 import org.apache.rocketmq.remoting.protocol.body.TopicList;
 import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;
+import org.apache.rocketmq.remoting.protocol.body.UserInfo;
 import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.AddBrokerRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.CheckRocksdbCqWriteProgressRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.CloneGroupOffsetRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.CreateAccessConfigRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.CreateAclRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.CreateTopicListRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.DeleteAccessConfigRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.CreateUserRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.DeleteAclRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.DeleteSubscriptionGroupRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.DeleteUserRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ExportRocksDBConfigToJsonRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
+import org.apache.rocketmq.remoting.protocol.header.GetAclRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetAllProducerInfoRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.GetBrokerAclConfigResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsInBrokerHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetConsumerConnectionListRequestHeader;
@@ -172,7 +169,10 @@
 import org.apache.rocketmq.remoting.protocol.header.GetTopicConfigRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.GetTopicsByClusterRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetUserRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.HeartbeatRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ListAclsRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ListUsersRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.LockBatchMqRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader;
@@ -187,6 +187,8 @@
 import org.apache.rocketmq.remoting.protocol.header.QuerySubscriptionByConsumerRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.RemoveBrokerRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ResetMasterFlushOffsetHeader;
 import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;
@@ -198,9 +200,10 @@
 import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.UnlockBatchMqRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateAclRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.UpdateGlobalWhiteAddrsConfigRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateUserRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ViewBrokerStatsDataRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;
@@ -229,9 +232,23 @@
 import org.apache.rocketmq.remoting.rpchook.DynamicalExtFieldRPCHook;
 import org.apache.rocketmq.remoting.rpchook.StreamTypeRPCHook;
 
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
 import static org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode.SUCCESS;
 
-public class MQClientAPIImpl implements NameServerUpdateCallback {
+public class MQClientAPIImpl implements NameServerUpdateCallback, StartAndShutdown {
     private final static Logger log = LoggerFactory.getLogger(MQClientAPIImpl.class);
     private static boolean sendSmartMsg =
         Boolean.parseBoolean(System.getProperty("org.apache.rocketmq.client.sendSmartMsg", "true"));
@@ -246,19 +263,43 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
     private String nameSrvAddr = null;
     private ClientConfig clientConfig;
 
-    public MQClientAPIImpl(final NettyClientConfig nettyClientConfig,
+    public MQClientAPIImpl(
+        final NettyClientConfig nettyClientConfig,
         final ClientRemotingProcessor clientRemotingProcessor,
-        RPCHook rpcHook, final ClientConfig clientConfig) {
+        final RPCHook rpcHook,
+        final ClientConfig clientConfig
+    ) {
         this(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, null);
     }
 
-    public MQClientAPIImpl(final NettyClientConfig nettyClientConfig,
+    public MQClientAPIImpl(
+        final NettyClientConfig nettyClientConfig,
         final ClientRemotingProcessor clientRemotingProcessor,
-        RPCHook rpcHook, final ClientConfig clientConfig, final ChannelEventListener channelEventListener) {
+        final RPCHook rpcHook,
+        final ClientConfig clientConfig,
+        final ChannelEventListener channelEventListener
+    ) {
+        this(
+            nettyClientConfig,
+            clientRemotingProcessor,
+            rpcHook,
+            clientConfig,
+            channelEventListener,
+            null
+        );
+    }
+
+    public MQClientAPIImpl(final NettyClientConfig nettyClientConfig,
+                           final ClientRemotingProcessor clientRemotingProcessor,
+                           RPCHook rpcHook, final ClientConfig clientConfig,
+                           final ChannelEventListener channelEventListener,
+                           final ObjectCreator<RemotingClient> remotingClientCreator) {
         this.clientConfig = clientConfig;
         topAddressing = new DefaultTopAddressing(MixAll.getWSAddr(), clientConfig.getUnitName());
         topAddressing.registerChangeCallBack(this);
-        this.remotingClient = new NettyRemotingClient(nettyClientConfig, channelEventListener);
+        this.remotingClient = remotingClientCreator != null
+            ? remotingClientCreator.create(nettyClientConfig, channelEventListener)
+            : new NettyRemotingClient(nettyClientConfig, channelEventListener);
         this.clientRemotingProcessor = clientRemotingProcessor;
 
         this.remotingClient.registerRPCHook(new NamespaceRpcHook(clientConfig));
@@ -385,6 +426,22 @@ public void createSubscriptionGroup(final String addr, final SubscriptionGroupCo
 
     }
 
+    public void createSubscriptionGroupList(final String address, final List<SubscriptionGroupConfig> configs,
+        final long timeoutMillis) throws RemotingException, InterruptedException, MQClientException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP_LIST, null);
+        SubscriptionGroupList requestBody = new SubscriptionGroupList(configs);
+        request.setBody(requestBody.encode());
+
+        RemotingCommand response = this.remotingClient.invokeSync(
+            MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), address), request, timeoutMillis);
+        assert response != null;
+        if (response.getCode() == ResponseCode.SUCCESS) {
+            return;
+        }
+
+        throw new MQClientException(response.getCode(), response.getRemark());
+    }
+
     public void createTopic(final String addr, final String defaultTopic, final TopicConfig topicConfig,
         final long timeoutMillis)
         throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
@@ -417,112 +474,24 @@ public void createTopic(final String addr, final String defaultTopic, final Topi
         throw new MQClientException(response.getCode(), response.getRemark());
     }
 
-    public void createPlainAccessConfig(final String addr, final PlainAccessConfig plainAccessConfig,
-        final long timeoutMillis)
-        throws RemotingException, InterruptedException, MQClientException {
-        CreateAccessConfigRequestHeader requestHeader = new CreateAccessConfigRequestHeader();
-        requestHeader.setAccessKey(plainAccessConfig.getAccessKey());
-        requestHeader.setSecretKey(plainAccessConfig.getSecretKey());
-        requestHeader.setAdmin(plainAccessConfig.isAdmin());
-        requestHeader.setDefaultGroupPerm(plainAccessConfig.getDefaultGroupPerm());
-        requestHeader.setDefaultTopicPerm(plainAccessConfig.getDefaultTopicPerm());
-        requestHeader.setWhiteRemoteAddress(plainAccessConfig.getWhiteRemoteAddress());
-        requestHeader.setTopicPerms(UtilAll.join(plainAccessConfig.getTopicPerms(), ","));
-        requestHeader.setGroupPerms(UtilAll.join(plainAccessConfig.getGroupPerms(), ","));
-
-        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_ACL_CONFIG, requestHeader);
+    public void createTopicList(final String address, final List<TopicConfig> topicConfigList, final long timeoutMillis)
+        throws InterruptedException, RemotingException, MQClientException {
+        CreateTopicListRequestHeader requestHeader = new CreateTopicListRequestHeader();
+        CreateTopicListRequestBody requestBody = new CreateTopicListRequestBody(topicConfigList);
 
-        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),
-            request, timeoutMillis);
-        assert response != null;
-        switch (response.getCode()) {
-            case ResponseCode.SUCCESS: {
-                return;
-            }
-            default:
-                break;
-        }
-
-        throw new MQClientException(response.getCode(), response.getRemark());
-    }
-
-    public void deleteAccessConfig(final String addr, final String accessKey, final long timeoutMillis)
-        throws RemotingException, InterruptedException, MQClientException {
-        DeleteAccessConfigRequestHeader requestHeader = new DeleteAccessConfigRequestHeader();
-        requestHeader.setAccessKey(accessKey);
-
-        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_ACL_CONFIG, requestHeader);
-
-        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),
-            request, timeoutMillis);
-        assert response != null;
-        switch (response.getCode()) {
-            case ResponseCode.SUCCESS: {
-                return;
-            }
-            default:
-                break;
-        }
-
-        throw new MQClientException(response.getCode(), response.getRemark());
-    }
-
-    public void updateGlobalWhiteAddrsConfig(final String addr, final String globalWhiteAddrs, String aclFileFullPath,
-        final long timeoutMillis)
-        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        UpdateGlobalWhiteAddrsConfigRequestHeader requestHeader = new UpdateGlobalWhiteAddrsConfigRequestHeader();
-        requestHeader.setGlobalWhiteAddrs(globalWhiteAddrs);
-        requestHeader.setAclFileFullPath(aclFileFullPath);
-
-        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_GLOBAL_WHITE_ADDRS_CONFIG, requestHeader);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC_LIST, requestHeader);
+        request.setBody(requestBody.encode());
 
-        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),
-            request, timeoutMillis);
+        RemotingCommand response = this.remotingClient.invokeSync(
+            MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), address), request, timeoutMillis);
         assert response != null;
-        switch (response.getCode()) {
-            case ResponseCode.SUCCESS: {
-                return;
-            }
-            default:
-                break;
+        if (response.getCode() == ResponseCode.SUCCESS) {
+            return;
         }
 
         throw new MQClientException(response.getCode(), response.getRemark());
     }
 
-    public ClusterAclVersionInfo getBrokerClusterAclInfo(final String addr,
-        final long timeoutMillis) throws RemotingCommandException, InterruptedException, RemotingTimeoutException,
-        RemotingSendRequestException, RemotingConnectException, MQBrokerException {
-        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CLUSTER_ACL_INFO, null);
-
-        RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis);
-        assert response != null;
-        switch (response.getCode()) {
-            case ResponseCode.SUCCESS: {
-                GetBrokerAclConfigResponseHeader responseHeader =
-                    (GetBrokerAclConfigResponseHeader) response.decodeCommandCustomHeader(GetBrokerAclConfigResponseHeader.class);
-
-                ClusterAclVersionInfo clusterAclVersionInfo = new ClusterAclVersionInfo();
-                clusterAclVersionInfo.setClusterName(responseHeader.getClusterName());
-                clusterAclVersionInfo.setBrokerName(responseHeader.getBrokerName());
-                clusterAclVersionInfo.setBrokerAddr(responseHeader.getBrokerAddr());
-                clusterAclVersionInfo.setAclConfigDataVersion(DataVersion.fromJson(responseHeader.getVersion(), DataVersion.class));
-                HashMap<String, Object> dataVersionMap = JSON.parseObject(responseHeader.getAllAclFileVersion(), HashMap.class);
-                Map<String, DataVersion> allAclConfigDataVersion = new HashMap<>(dataVersionMap.size(), 1);
-                for (Map.Entry<String, Object> entry : dataVersionMap.entrySet()) {
-                    allAclConfigDataVersion.put(entry.getKey(), DataVersion.fromJson(JSON.toJSONString(entry.getValue()), DataVersion.class));
-                }
-                clusterAclVersionInfo.setAllAclConfigDataVersion(allAclConfigDataVersion);
-                return clusterAclVersionInfo;
-            }
-            default:
-                break;
-        }
-
-        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);
-
-    }
-
     public SendResult sendMessage(
         final String addr,
         final String brokerName,
@@ -691,9 +660,10 @@ public void operationFail(Throwable throwable) {
                         onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
                             retryTimesWhenSendFailed, times, ex, context, true, producer);
                     } else {
-                        MQClientException ex = new MQClientException("unknow reseaon", throwable);
+                        MQClientException ex = new MQClientException("unknown reason", throwable);
+                        boolean needRetry = !(throwable instanceof RemotingTooMuchRequestException);
                         onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
-                            retryTimesWhenSendFailed, times, ex, context, true, producer);
+                            retryTimesWhenSendFailed, times, ex, context, needRetry, producer);
                     }
                 }
             });
@@ -720,7 +690,7 @@ private void onExceptionImpl(final String brokerName,
         final DefaultMQProducerImpl producer
     ) {
         int tmp = curTimes.incrementAndGet();
-        if (needRetry && tmp <= timesTotal) {
+        if (needRetry && tmp <= timesTotal && timeoutMillis > 0) {
             String retryBrokerName = brokerName;//by default, it will send to the same broker
             if (topicPublishInfo != null) { //select one message queue accordingly, in order to determine which broker to send
                 MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName, false);
@@ -799,6 +769,7 @@ protected SendResult processSendResponse(
             uniqMsgId,
             responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset());
         sendResult.setTransactionId(responseHeader.getTransactionId());
+        sendResult.setRecallHandle(responseHeader.getRecallHandle());
         String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION);
         if (regionId == null || regionId.isEmpty()) {
             regionId = MixAll.DEFAULT_TRACE_REGION_ID;
@@ -1143,16 +1114,19 @@ private PopResult processPopResponse(final String brokerName, final RemotingComm
                         final Long msgQueueOffset;
                         if (MixAll.isLmq(topic) && messageExt.getReconsumeTimes() == 0 && StringUtils.isNotEmpty(
                             messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))) {
-                            // process LMQ, LMQ topic has only 1 queue, which queue id is 0
+                            // process LMQ
+                            String[] queues = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)
+                                .split(MixAll.LMQ_DISPATCH_SEPARATOR);
+                            String[] queueOffsets = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET)
+                                .split(MixAll.LMQ_DISPATCH_SEPARATOR);
+                            long offset = Long.parseLong(queueOffsets[ArrayUtils.indexOf(queues, topic)]);
+                            // LMQ topic has only 1 queue, which queue id is 0
                             queueIdKey = ExtraInfoUtil.getStartOffsetInfoMapKey(topic, MixAll.LMQ_QUEUE_ID);
-                            queueOffsetKey = ExtraInfoUtil.getQueueOffsetMapKey(topic, MixAll.LMQ_QUEUE_ID, Long.parseLong(
-                                messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET)));
-                            index = sortMap.get(queueIdKey).indexOf(
-                                Long.parseLong(messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET)));
+                            queueOffsetKey = ExtraInfoUtil.getQueueOffsetMapKey(topic, MixAll.LMQ_QUEUE_ID, offset);
+                            index = sortMap.get(queueIdKey).indexOf(offset);
                             msgQueueOffset = msgOffsetInfo.get(queueIdKey).get(index);
-                            if (msgQueueOffset != Long.parseLong(
-                                messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET))) {
-                                log.warn("Queue offset[%d] of msg is strange, not equal to the stored in msg, %s",
+                            if (msgQueueOffset != offset) {
+                                log.warn("Queue offset[{}] of msg is strange, not equal to the stored in msg, {}",
                                     msgQueueOffset, messageExt);
                             }
                             messageExt.getProperties().put(MessageConst.PROPERTY_POP_CK,
@@ -1165,7 +1139,7 @@ private PopResult processPopResponse(final String brokerName, final RemotingComm
                             index = sortMap.get(queueIdKey).indexOf(messageExt.getQueueOffset());
                             msgQueueOffset = msgOffsetInfo.get(queueIdKey).get(index);
                             if (msgQueueOffset != messageExt.getQueueOffset()) {
-                                log.warn("Queue offset[%d] of msg is strange, not equal to the stored in msg, %s", msgQueueOffset, messageExt);
+                                log.warn("Queue offset[{}] of msg is strange, not equal to the stored in msg, {}", msgQueueOffset, messageExt);
                             }
                             messageExt.getProperties().put(MessageConst.PROPERTY_POP_CK,
                                 ExtraInfoUtil.buildExtraInfo(startOffsetInfo.get(queueIdKey), responseHeader.getPopTime(), responseHeader.getInvisibleTime(),
@@ -1205,14 +1179,15 @@ private static Map<String, List<Long>> buildQueueOffsetSortedMap(String topic, L
             final String key;
             if (MixAll.isLmq(topic) && messageExt.getReconsumeTimes() == 0
                 && StringUtils.isNotEmpty(messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))) {
-                // process LMQ, LMQ topic has only 1 queue, which queue id is 0
-                key = ExtraInfoUtil.getStartOffsetInfoMapKey(
-                    messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH), 0);
-                if (!sortMap.containsKey(key)) {
-                    sortMap.put(key, new ArrayList<>(4));
-                }
-                sortMap.get(key).add(
-                    Long.parseLong(messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET)));
+                // process LMQ
+                String[] queues = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)
+                    .split(MixAll.LMQ_DISPATCH_SEPARATOR);
+                String[] queueOffsets = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET)
+                    .split(MixAll.LMQ_DISPATCH_SEPARATOR);
+                // LMQ topic has only 1 queue, which queue id is 0
+                key = ExtraInfoUtil.getStartOffsetInfoMapKey(topic, MixAll.LMQ_QUEUE_ID);
+                sortMap.putIfAbsent(key, new ArrayList<>(4));
+                sortMap.get(key).add(Long.parseLong(queueOffsets[ArrayUtils.indexOf(queues, topic)]));
                 continue;
             }
             // Value of POP_CK is used to determine whether it is a pop retry,
@@ -1227,9 +1202,10 @@ private static Map<String, List<Long>> buildQueueOffsetSortedMap(String topic, L
         return sortMap;
     }
 
-    public MessageExt viewMessage(final String addr, final long phyoffset, final long timeoutMillis)
+    public MessageExt viewMessage(final String addr, final String topic, final long phyoffset, final long timeoutMillis)
         throws RemotingException, MQBrokerException, InterruptedException {
         ViewMessageRequestHeader requestHeader = new ViewMessageRequestHeader();
+        requestHeader.setTopic(topic);
         requestHeader.setOffset(phyoffset);
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.VIEW_MESSAGE_BY_ID, requestHeader);
 
@@ -1560,10 +1536,10 @@ public void queryMessage(
         final QueryMessageRequestHeader requestHeader,
         final long timeoutMillis,
         final InvokeCallback invokeCallback,
-        final Boolean isUnqiueKey
+        final Boolean isUniqueKey
     ) throws RemotingException, MQBrokerException, InterruptedException {
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, requestHeader);
-        request.addExtField(MixAll.UNIQUE_MSG_QUERY_FLAG, isUnqiueKey.toString());
+        request.addExtField(MixAll.UNIQUE_MSG_QUERY_FLAG, isUniqueKey.toString());
         this.remotingClient.invokeAsync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis,
             invokeCallback);
     }
@@ -1685,16 +1661,27 @@ public TopicStatsTable getTopicStatsInfo(final String addr, final String topic,
     public ConsumeStats getConsumeStats(final String addr, final String consumerGroup, final long timeoutMillis)
         throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException,
         MQBrokerException {
-        return getConsumeStats(addr, consumerGroup, null, timeoutMillis);
+        return getConsumeStats(addr, consumerGroup, null, null, timeoutMillis);
+    }
+
+    public ConsumeStats getConsumeStats(final String addr, final String consumerGroup, final List<String> topicList,
+        final long timeoutMillis) throws RemotingSendRequestException, RemotingConnectException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        return getConsumeStats(addr, consumerGroup, null, topicList, timeoutMillis);
     }
 
     public ConsumeStats getConsumeStats(final String addr, final String consumerGroup, final String topic,
-        final long timeoutMillis)
+        final long timeoutMillis) throws RemotingSendRequestException, RemotingConnectException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        return getConsumeStats(addr, consumerGroup, topic, null, timeoutMillis);
+    }
+
+    public ConsumeStats getConsumeStats(final String addr, final String consumerGroup, final String topic,
+        final List<String> topicList, final long timeoutMillis)
         throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException,
         MQBrokerException {
         GetConsumeStatsRequestHeader requestHeader = new GetConsumeStatsRequestHeader();
         requestHeader.setConsumerGroup(consumerGroup);
         requestHeader.setTopic(topic);
+        requestHeader.updateTopicList(topicList);
 
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUME_STATS, requestHeader);
 
@@ -2960,6 +2947,35 @@ public QueryConsumeQueueResponseBody queryConsumeQueue(final String brokerAddr,
         throw new MQClientException(response.getCode(), response.getRemark());
     }
 
+    public CheckRocksdbCqWriteResult checkRocksdbCqWriteProgress(final String brokerAddr, final String topic, final long checkStoreTime, final long timeoutMillis) throws InterruptedException,
+        RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {
+        CheckRocksdbCqWriteProgressRequestHeader header = new CheckRocksdbCqWriteProgressRequestHeader();
+        header.setTopic(topic);
+        header.setCheckStoreTime(checkStoreTime);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS, header);
+        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
+        assert response != null;
+        if (ResponseCode.SUCCESS == response.getCode()) {
+            return JSON.parseObject(response.getBody(), CheckRocksdbCqWriteResult.class);
+        }
+        throw new MQClientException(response.getCode(), response.getRemark());
+    }
+
+    public void exportRocksDBConfigToJson(final String brokerAddr,
+        final List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configType,
+        final long timeoutMillis) throws InterruptedException,
+        RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {
+        ExportRocksDBConfigToJsonRequestHeader header = new ExportRocksDBConfigToJsonRequestHeader();
+        header.updateConfigType(configType);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXPORT_ROCKSDB_CONFIG_TO_JSON, header);
+        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
+        assert response != null;
+
+        if (ResponseCode.SUCCESS != response.getCode()) {
+            throw new MQClientException(response.getCode(), response.getRemark());
+        }
+    }
+
     public void checkClientInBroker(final String brokerAddr, final String consumerGroup,
         final String clientId, final SubscriptionData subscriptionData,
         final long timeoutMillis)
@@ -2983,9 +2999,10 @@ public void checkClientInBroker(final String brokerAddr, final String consumerGr
         }
     }
 
-    public boolean resumeCheckHalfMessage(final String addr, String msgId,
+    public boolean resumeCheckHalfMessage(final String addr, String topic, String msgId,
         final long timeoutMillis) throws RemotingException, InterruptedException {
         ResumeCheckHalfMessageRequestHeader requestHeader = new ResumeCheckHalfMessageRequestHeader();
+        requestHeader.setTopic(topic);
         requestHeader.setMsgId(msgId);
 
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RESUME_CHECK_HALF_MESSAGE, requestHeader);
@@ -3297,4 +3314,215 @@ public void cleanControllerBrokerData(String controllerAddr, String clusterName,
         }
         throw new MQBrokerException(response.getCode(), response.getRemark());
     }
+
+    public void createUser(String addr, UserInfo userInfo, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
+        CreateUserRequestHeader requestHeader = new CreateUserRequestHeader(userInfo.getUsername());
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_CREATE_USER, requestHeader);
+        request.setBody(RemotingSerializable.encode(userInfo));
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return;
+            }
+            default:
+                break;
+        }
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public void updateUser(String addr, UserInfo userInfo, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
+        UpdateUserRequestHeader requestHeader = new UpdateUserRequestHeader(userInfo.getUsername());
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_UPDATE_USER, requestHeader);
+        request.setBody(RemotingSerializable.encode(userInfo));
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return;
+            }
+            default:
+                break;
+        }
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public void deleteUser(String addr, String username, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
+        DeleteUserRequestHeader requestHeader = new DeleteUserRequestHeader(username);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_DELETE_USER, requestHeader);
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return;
+            }
+            default:
+                break;
+        }
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public UserInfo getUser(String addr, String username, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
+        GetUserRequestHeader requestHeader = new GetUserRequestHeader(username);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_GET_USER, requestHeader);
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return RemotingSerializable.decode(response.getBody(), UserInfo.class);
+            }
+            default:
+                break;
+        }
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public List<UserInfo> listUser(String addr, String filter, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
+        ListUsersRequestHeader requestHeader = new ListUsersRequestHeader(filter);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_LIST_USER, requestHeader);
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return RemotingSerializable.decodeList(response.getBody(), UserInfo.class);
+            }
+            default:
+                break;
+        }
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public void createAcl(String addr, AclInfo aclInfo, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
+        CreateAclRequestHeader requestHeader = new CreateAclRequestHeader(aclInfo.getSubject());
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_CREATE_ACL, requestHeader);
+        request.setBody(RemotingSerializable.encode(aclInfo));
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return;
+            }
+            default:
+                break;
+        }
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public void updateAcl(String addr, AclInfo aclInfo, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
+        UpdateAclRequestHeader requestHeader = new UpdateAclRequestHeader(aclInfo.getSubject());
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_UPDATE_ACL, requestHeader);
+        request.setBody(RemotingSerializable.encode(aclInfo));
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return;
+            }
+            default:
+                break;
+        }
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public void deleteAcl(String addr, String subject, String resource, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
+        DeleteAclRequestHeader requestHeader = new DeleteAclRequestHeader(subject, resource);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_DELETE_ACL, requestHeader);
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return;
+            }
+            default:
+                break;
+        }
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public AclInfo getAcl(String addr, String subject, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
+        GetAclRequestHeader requestHeader = new GetAclRequestHeader(subject);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_GET_ACL, requestHeader);
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return RemotingSerializable.decode(response.getBody(), AclInfo.class);
+            }
+            default:
+                break;
+        }
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public List<AclInfo> listAcl(String addr, String subjectFilter, String resourceFilter, long millis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
+        ListAclsRequestHeader requestHeader = new ListAclsRequestHeader(subjectFilter, resourceFilter);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.AUTH_LIST_ACL, requestHeader);
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, millis);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return RemotingSerializable.decodeList(response.getBody(), AclInfo.class);
+            }
+            default:
+                break;
+        }
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public String recallMessage(
+        final String addr,
+        RecallMessageRequestHeader requestHeader,
+        final long timeoutMillis
+    ) throws RemotingException, MQBrokerException, InterruptedException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader);
+
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis);
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                RecallMessageResponseHeader responseHeader =
+                    response.decodeCommandCustomHeader(RecallMessageResponseHeader.class);
+                return responseHeader.getMsgId();
+            }
+            default:
+                break;
+        }
+        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);
+    }
+
+    public void recallMessageAsync(
+        final String addr,
+        final RecallMessageRequestHeader requestHeader,
+        final long timeoutMillis,
+        final InvokeCallback invokeCallback
+    ) throws RemotingException, InterruptedException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader);
+
+        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
+            @Override
+            public void operationComplete(ResponseFuture responseFuture) {
+            }
+
+            @Override
+            public void operationSucceed(RemotingCommand response) {
+                invokeCallback.operationSucceed(response);
+            }
+
+            @Override
+            public void operationFail(Throwable throwable) {
+                invokeCallback.operationFail(throwable);
+            }
+        });
+    }
+
+    public void exportPopRecord(String brokerAddr, long timeout) throws RemotingConnectException,
+        RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(
+            RequestCode.POP_ROLLBACK, null);
+        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeout);
+        assert response != null;
+        if (response.getCode() == SUCCESS) {
+            return;
+        }
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
 }
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java
index 02eaa66e99a..ca6f4617456 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java
@@ -85,4 +85,8 @@ public ProduceAccumulator getOrCreateProduceAccumulator(final ClientConfig clien
     public void removeClientFactory(final String clientId) {
         this.factoryTable.remove(clientId);
     }
+
+    public ConcurrentMap<String, MQClientInstance> getFactoryTable() {
+        return factoryTable;
+    }
 }
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
index ea6c8072b57..b151fefbbb3 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
@@ -169,11 +169,11 @@ public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, Strin
             result.setConsumeResult(CMResult.CR_THROW_EXCEPTION);
             result.setRemark(UtilAll.exceptionSimpleDesc(e));
 
-            log.warn(String.format("consumeMessageDirectly exception: %s Group: %s Msgs: %s MQ: %s",
+            log.warn("consumeMessageDirectly exception: {} Group: {} Msgs: {} MQ: {}",
                 UtilAll.exceptionSimpleDesc(e),
                 ConsumeMessageConcurrentlyService.this.consumerGroup,
                 msgs,
-                mq), e);
+                mq, e);
         }
 
         result.setSpentTimeMills(System.currentTimeMillis() - beginTime);
@@ -410,11 +410,11 @@ public void run() {
                 }
                 status = listener.consumeMessage(Collections.unmodifiableList(msgs), context);
             } catch (Throwable e) {
-                log.warn(String.format("consumeMessage exception: %s Group: %s Msgs: %s MQ: %s",
+                log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",
                     UtilAll.exceptionSimpleDesc(e),
                     ConsumeMessageConcurrentlyService.this.consumerGroup,
                     msgs,
-                    messageQueue), e);
+                    messageQueue, e);
                 hasException = true;
             }
             long consumeRT = System.currentTimeMillis() - beginTimestamp;
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java
index cab4fe5d69f..3ca465da70d 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java
@@ -85,6 +85,7 @@ public ConsumeMessageOrderlyService(DefaultMQPushConsumerImpl defaultMQPushConsu
         this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("ConsumeMessageScheduledThread_" + consumerGroupTag));
     }
 
+    @Override
     public void start() {
         if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())) {
             this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
@@ -96,10 +97,11 @@ public void run() {
                         log.error("scheduleAtFixedRate lockMQPeriodically exception", e);
                     }
                 }
-            }, 1000 * 1, ProcessQueue.REBALANCE_LOCK_INTERVAL, TimeUnit.MILLISECONDS);
+            }, 1000, ProcessQueue.REBALANCE_LOCK_INTERVAL, TimeUnit.MILLISECONDS);
         }
     }
 
+    @Override
     public void shutdown(long awaitTerminateMillis) {
         this.stopped = true;
         this.scheduledExecutorService.shutdown();
@@ -181,11 +183,11 @@ public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, Strin
             result.setConsumeResult(CMResult.CR_THROW_EXCEPTION);
             result.setRemark(UtilAll.exceptionSimpleDesc(e));
 
-            log.warn(String.format("consumeMessageDirectly exception: %s Group: %s Msgs: %s MQ: %s",
+            log.warn("consumeMessageDirectly exception: {} Group: {} Msgs: {} MQ: {}",
                 UtilAll.exceptionSimpleDesc(e),
                 ConsumeMessageOrderlyService.this.consumerGroup,
                 msgs,
-                mq), e);
+                mq, e);
         }
 
         result.setAutoCommit(context.isAutoCommit());
@@ -201,8 +203,8 @@ public void submitConsumeRequest(
         final List<MessageExt> msgs,
         final ProcessQueue processQueue,
         final MessageQueue messageQueue,
-        final boolean dispathToConsume) {
-        if (dispathToConsume) {
+        final boolean dispatchToConsume) {
+        if (dispatchToConsume) {
             ConsumeRequest consumeRequest = new ConsumeRequest(processQueue, messageQueue);
             this.consumeExecutor.submit(consumeRequest);
         }
@@ -497,11 +499,11 @@ public void run() {
 
                                 status = messageListener.consumeMessage(Collections.unmodifiableList(msgs), context);
                             } catch (Throwable e) {
-                                log.warn(String.format("consumeMessage exception: %s Group: %s Msgs: %s MQ: %s",
+                                log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",
                                     UtilAll.exceptionSimpleDesc(e),
                                     ConsumeMessageOrderlyService.this.consumerGroup,
                                     msgs,
-                                    messageQueue), e);
+                                    messageQueue, e);
                                 hasException = true;
                             } finally {
                                 this.processQueue.getConsumeLock().readLock().unlock();
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyService.java
index a61454f5955..d5191871106 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyService.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyService.java
@@ -153,11 +153,11 @@ public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, Strin
             result.setConsumeResult(CMResult.CR_THROW_EXCEPTION);
             result.setRemark(UtilAll.exceptionSimpleDesc(e));
 
-            log.warn(String.format("consumeMessageDirectly exception: %s Group: %s Msgs: %s MQ: %s",
+            log.warn("consumeMessageDirectly exception: {} Group: {} Msgs: {} MQ: {}",
                 UtilAll.exceptionSimpleDesc(e),
                 ConsumeMessagePopConcurrentlyService.this.consumerGroup,
                 msgs,
-                mq), e);
+                mq, e);
         }
 
         result.setSpentTimeMills(System.currentTimeMillis() - beginTime);
@@ -471,7 +471,7 @@ public void run() {
                     processQueue.decFoundMsg(-msgs.size());
                 }
 
-                log.warn("processQueue invalid. isDropped={}, isPopTimeout={}, messageQueue={}, msgs={}",
+                log.warn("processQueue invalid or popTimeout. isDropped={}, isPopTimeout={}, messageQueue={}, msgs={}",
                         processQueue.isDropped(), isPopTimeout(), messageQueue, msgs);
             }
         }
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyService.java
index ae6adfea5df..4eab1ccf664 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyService.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyService.java
@@ -175,11 +175,11 @@ public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, Strin
             result.setConsumeResult(CMResult.CR_THROW_EXCEPTION);
             result.setRemark(UtilAll.exceptionSimpleDesc(e));
 
-            log.warn(String.format("consumeMessageDirectly exception: %s Group: %s Msgs: %s MQ: %s",
+            log.warn("consumeMessageDirectly exception: {} Group: {} Msgs: {} MQ: {}",
                 UtilAll.exceptionSimpleDesc(e),
                 ConsumeMessagePopOrderlyService.this.consumerGroup,
                 msgs,
-                mq), e);
+                mq, e);
         }
 
         result.setAutoCommit(context.isAutoCommit());
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java
index 9350970a07e..f85dcc7b459 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java
@@ -164,10 +164,6 @@ private enum SubscriptionType {
     public DefaultLitePullConsumerImpl(final DefaultLitePullConsumer defaultLitePullConsumer, final RPCHook rpcHook) {
         this.defaultLitePullConsumer = defaultLitePullConsumer;
         this.rpcHook = rpcHook;
-        this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(
-            this.defaultLitePullConsumer.getPullThreadNums(),
-            new ThreadFactoryImpl("PullMsgThread-" + this.defaultLitePullConsumer.getConsumerGroup())
-        );
         this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("MonitorMessageQueueChangeThread"));
         this.pullTimeDelayMillsWhenException = defaultLitePullConsumer.getPullTimeDelayMillsWhenException();
     }
@@ -293,6 +289,8 @@ public synchronized void start() throws MQClientException {
                     this.defaultLitePullConsumer.changeInstanceNameToPID();
                 }
 
+                initScheduledThreadPoolExecutor();
+
                 initMQClientFactory();
 
                 initRebalanceImpl();
@@ -309,7 +307,12 @@ public synchronized void start() throws MQClientException {
 
                 log.info("the consumer [{}] start OK", this.defaultLitePullConsumer.getConsumerGroup());
 
-                operateAfterRunning();
+                try {
+                    operateAfterRunning();
+                } catch (Exception e) {
+                    shutdown();
+                    throw e;
+                }
 
                 break;
             case RUNNING:
@@ -324,6 +327,13 @@ public synchronized void start() throws MQClientException {
         }
     }
 
+    private void initScheduledThreadPoolExecutor() {
+        this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(
+                this.defaultLitePullConsumer.getPullThreadNums(),
+                new ThreadFactoryImpl("PullMsgThread-" + this.defaultLitePullConsumer.getConsumerGroup())
+        );
+    }
+
     private void initMQClientFactory() throws MQClientException {
         this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultLitePullConsumer, this.rpcHook);
         boolean registerOK = mQClientFactory.registerConsumer(this.defaultLitePullConsumer.getConsumerGroup(), this);
@@ -901,7 +911,9 @@ public void run() {
                 if ((long) consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchSize() > defaultLitePullConsumer.getPullThresholdForAll()) {
                     scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_CACHE_FLOW_CONTROL, TimeUnit.MILLISECONDS);
                     if ((consumeRequestFlowControlTimes++ % 1000) == 0) {
-                        log.warn("The consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}", consumeRequestCache.size(), consumeRequestFlowControlTimes);
+                        log.warn("The consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}",
+                                (int)Math.ceil((double)defaultLitePullConsumer.getPullThresholdForAll() / defaultLitePullConsumer.getPullBatchSize()),
+                                consumeRequestCache.size(), consumeRequestFlowControlTimes);
                     }
                     return;
                 }
@@ -1072,12 +1084,12 @@ private void resetTopic(List<MessageExt> msgList) {
         }
 
         //If namespace not null , reset Topic without namespace.
-        for (MessageExt messageExt : msgList) {
-            if (null != this.defaultLitePullConsumer.getNamespace()) {
-                messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), this.defaultLitePullConsumer.getNamespace()));
+        String namespace = this.defaultLitePullConsumer.getNamespace();
+        if (namespace != null) {
+            for (MessageExt messageExt : msgList) {
+                messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), namespace));
             }
         }
-
     }
 
     public void updateConsumeOffset(MessageQueue mq, long offset) {
@@ -1235,18 +1247,16 @@ private boolean isSetEqual(Set<MessageQueue> set1, Set<MessageQueue> set2) {
             return true;
         }
 
-        if (set1 == null || set2 == null || set1.size() != set2.size() || set1.size() == 0) {
+        if (set1 == null || set2 == null || set1.size() != set2.size()) {
             return false;
         }
 
-        Iterator<MessageQueue> iter = set2.iterator();
-        boolean isEqual = true;
-        while (iter.hasNext()) {
-            if (!set1.contains(iter.next())) {
-                isEqual = false;
+        for (MessageQueue messageQueue : set2) {
+            if (!set1.contains(messageQueue)) {
+                return false;
             }
         }
-        return isEqual;
+        return true;
     }
 
     public AssignedMessageQueue getAssignedMessageQueue() {
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java
index f5d326071d2..9d46e28f5d4 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java
@@ -54,6 +54,8 @@
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.sysflag.PullSysFlag;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.exception.RemotingException;
@@ -63,8 +65,6 @@
 import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
 import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
 import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 
 /**
  * This class will be removed in 2022, and a better implementation {@link DefaultLitePullConsumerImpl} is recommend to use
@@ -290,12 +290,12 @@ public void resetTopic(List<MessageExt> msgList) {
         }
 
         //If namespace not null , reset Topic without namespace.
-        for (MessageExt messageExt : msgList) {
-            if (null != this.getDefaultMQPullConsumer().getNamespace()) {
-                messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), this.defaultMQPullConsumer.getNamespace()));
+        String namespace = this.getDefaultMQPullConsumer().getNamespace();
+        if (namespace != null) {
+            for (MessageExt messageExt : msgList) {
+                messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), namespace));
             }
         }
-
     }
 
     public void subscriptionAutomatically(final String topic) {
@@ -356,6 +356,11 @@ public ConsumeFromWhere consumeFromWhere() {
 
     @Override
     public Set<SubscriptionData> subscriptions() {
+        Set<SubscriptionData> registerSubscriptions = defaultMQPullConsumer.getRegisterSubscriptions();
+        if (registerSubscriptions != null && !registerSubscriptions.isEmpty()) {
+            return registerSubscriptions;
+        }
+
         Set<SubscriptionData> result = new HashSet<>();
 
         Set<String> topics = this.defaultMQPullConsumer.getRegisterTopics();
@@ -381,6 +386,9 @@ public Set<SubscriptionData> subscriptions() {
 
     @Override
     public void doRebalance() {
+        if (!defaultMQPullConsumer.isEnableRebalance()) {
+            return;
+        }
         if (this.rebalanceImpl != null) {
             this.rebalanceImpl.doRebalance(false);
         }
@@ -388,6 +396,10 @@ public void doRebalance() {
 
     @Override
     public boolean tryRebalance() {
+        if (!defaultMQPullConsumer.isEnableRebalance()) {
+            return true;
+        }
+
         if (this.rebalanceImpl != null) {
             return this.rebalanceImpl.doRebalance(false);
         }
@@ -589,6 +601,21 @@ public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offs
             this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis());
     }
 
+    public void pullBlockIfNotFoundWithMessageSelector(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums,
+        PullCallback pullCallback)
+        throws MQClientException, RemotingException, InterruptedException {
+        SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector);
+        this.pullAsyncImpl(mq, subscriptionData, offset, maxNums, pullCallback, true,
+            this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis());
+    }
+
+    public PullResult pullBlockIfNotFoundWithMessageSelector(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums)
+        throws MQClientException, RemotingException, InterruptedException, MQBrokerException {
+        SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector);
+        return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, true, this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis());
+    }
+
+
     public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end)
         throws MQClientException, InterruptedException {
         this.isRunning();
@@ -627,6 +654,10 @@ public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerN
             String brokerAddr = (null != destBrokerName) ? this.mQClientFactory.findBrokerAddressInPublish(destBrokerName)
                 : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost());
 
+            if (UtilAll.isBlank(brokerAddr)) {
+                throw new MQClientException("Broker[" + destBrokerName + "] master node does not exist", null);
+            }
+
             if (UtilAll.isBlank(consumerGroup)) {
                 consumerGroup = this.defaultMQPullConsumer.getConsumerGroup();
             }
@@ -804,10 +835,10 @@ public void updateConsumeOffset(MessageQueue mq, long offset) throws MQClientExc
         this.offsetStore.updateOffset(mq, offset, false);
     }
 
-    public MessageExt viewMessage(String msgId)
+    public MessageExt viewMessage(String topic, String msgId)
         throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
         this.isRunning();
-        return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId);
+        return this.mQClientFactory.getMQAdminImpl().viewMessage(topic, msgId);
     }
 
     public void registerFilterMessageHook(final FilterMessageHook hook) {
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
index 6666c4335eb..c93cff42452 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
@@ -510,7 +510,7 @@ void popMessage(final PopRequest popRequest) {
         try {
             this.makeSureStateOK();
         } catch (MQClientException e) {
-            log.warn("pullMessage exception, consumer state not ok", e);
+            log.warn("popMessage exception, consumer state not ok", e);
             this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException);
             return;
         }
@@ -581,8 +581,6 @@ public void onSuccess(PopResult popResult) {
                         DefaultMQPushConsumerImpl.this.executePopPullRequestImmediately(popRequest);
                         break;
                     case POLLING_FULL:
-                        DefaultMQPushConsumerImpl.this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException);
-                        break;
                     default:
                         DefaultMQPushConsumerImpl.this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException);
                         break;
@@ -623,10 +621,9 @@ public void onException(Throwable e) {
     private PopResult processPopResult(final PopResult popResult, final SubscriptionData subscriptionData) {
         if (PopStatus.FOUND == popResult.getPopStatus()) {
             List<MessageExt> msgFoundList = popResult.getMsgFoundList();
-            List<MessageExt> msgListFilterAgain = msgFoundList;
+            List<MessageExt> msgListFilterAgain = new ArrayList<>(popResult.getMsgFoundList().size());
             if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode()
                 && popResult.getMsgFoundList().size() > 0) {
-                msgListFilterAgain = new ArrayList<>(popResult.getMsgFoundList().size());
                 for (MessageExt msg : popResult.getMsgFoundList()) {
                     if (msg.getTags() != null) {
                         if (subscriptionData.getTagsSet().contains(msg.getTags())) {
@@ -634,6 +631,8 @@ private PopResult processPopResult(final PopResult popResult, final Subscription
                         }
                     }
                 }
+            } else {
+                msgListFilterAgain.addAll(msgFoundList);
             }
 
             if (!this.filterMessageHookList.isEmpty()) {
@@ -651,6 +650,15 @@ private PopResult processPopResult(final PopResult popResult, final Subscription
                 }
             }
 
+            Iterator<MessageExt> iterator = msgListFilterAgain.iterator();
+            while (iterator.hasNext()) {
+                MessageExt msg = iterator.next();
+                if (msg.getReconsumeTimes() > getMaxReconsumeTimes()) {
+                    iterator.remove();
+                    log.info("Reconsume times has reached {}, so ack msg={}", msg.getReconsumeTimes(), msg);
+                }
+            }
+
             if (msgFoundList.size() != msgListFilterAgain.size()) {
                 for (MessageExt msg : msgFoundList) {
                     if (!msgListFilterAgain.contains(msg)) {
@@ -744,7 +752,7 @@ public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerN
 
     public void sendMessageBack(MessageExt msg, int delayLevel, final MessageQueue mq)
             throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        sendMessageBack(msg, delayLevel, null, mq);
+        sendMessageBack(msg, delayLevel, msg.getBrokerName(), mq);
     }
 
 
@@ -759,6 +767,9 @@ private void sendMessageBack(MessageExt msg, int delayLevel, final String broker
             } else {
                 String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName)
                     : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost());
+                if (UtilAll.isBlank(brokerAddr)) {
+                    throw new MQClientException("Broker[" + brokerName + "] master node does not exist", null);
+                }
                 this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, brokerName, msg,
                     this.defaultMQPushConsumer.getConsumerGroup(), delayLevel, 5000, getMaxReconsumeTimes());
             }
@@ -998,10 +1009,16 @@ public synchronized void start() throws MQClientException {
                 break;
         }
 
-        this.updateTopicSubscribeInfoWhenSubscriptionChanged();
-        this.mQClientFactory.checkClientInBroker();
-        if (this.mQClientFactory.sendHeartbeatToAllBrokerWithLock()) {
-            this.mQClientFactory.rebalanceImmediately();
+        try {
+            this.updateTopicSubscribeInfoWhenSubscriptionChanged();
+            this.mQClientFactory.checkClientInBroker();
+            if (this.mQClientFactory.sendHeartbeatToAllBrokerWithLock()) {
+                this.mQClientFactory.rebalanceImmediately();
+            }
+        } catch (Exception e) {
+            log.warn("Start the consumer {} fail.", this.defaultMQPushConsumer.getConsumerGroup(), e);
+            shutdown();
+            throw e;
         }
     }
 
@@ -1309,9 +1326,9 @@ public void updateCorePoolSize(int corePoolSize) {
         this.consumeMessageService.updateCorePoolSize(corePoolSize);
     }
 
-    public MessageExt viewMessage(String msgId)
+    public MessageExt viewMessage(String topic, String msgId)
         throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId);
+        return this.mQClientFactory.getMQAdminImpl().viewMessage(topic, msgId);
     }
 
     public RebalanceImpl getRebalanceImpl() {
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java
index 33e698b00c1..bc1b5eff2f9 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java
@@ -137,7 +137,7 @@ public boolean putMessage(final List<MessageExt> msgs) {
                     if (null == old) {
                         validMsgCnt++;
                         this.queueOffsetMax = msg.getQueueOffset();
-                        msgSize.addAndGet(msg.getBody().length);
+                        msgSize.addAndGet(null == msg.getBody() ? 0 : msg.getBody().length);
                     }
                 }
                 msgCount.addAndGet(validMsgCnt);
@@ -198,7 +198,10 @@ public long removeMessage(final List<MessageExt> msgs) {
                         MessageExt prev = msgTreeMap.remove(msg.getQueueOffset());
                         if (prev != null) {
                             removedCnt--;
-                            msgSize.addAndGet(-msg.getBody().length);
+                            long bodySize = null == msg.getBody() ? 0 : msg.getBody().length;
+                            if (bodySize > 0) {
+                                msgSize.addAndGet(-bodySize);
+                            }
                         }
                     }
                     if (msgCount.addAndGet(removedCnt) == 0) {
@@ -270,7 +273,10 @@ public long commit() {
                     msgSize.set(0);
                 } else {
                     for (MessageExt msg : this.consumingMsgOrderlyTreeMap.values()) {
-                        msgSize.addAndGet(-msg.getBody().length);
+                        int bodySize = null == msg.getBody() ? 0 : msg.getBody().length;
+                        if (bodySize > 0) {
+                            msgSize.addAndGet(-bodySize);
+                        }
                     }
                 }
                 this.consumingMsgOrderlyTreeMap.clear();
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java
index ec6ede6bdea..4300f20a765 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullMessageService.java
@@ -135,7 +135,7 @@ public void run() {
                     this.pullMessage((PullRequest) messageRequest);
                 }
             } catch (InterruptedException ignored) {
-            } catch (Exception e) {
+            } catch (Throwable e) {
                 logger.error("Pull Message Service Run Method exception", e);
             }
         }
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java
index 53addc5f50c..b6f1d99b1c7 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java
@@ -36,7 +36,6 @@
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.message.MessageQueueAssignment;
 import org.apache.rocketmq.common.message.MessageRequestMode;
-import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;
 import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;
 import org.apache.rocketmq.remoting.protocol.filter.FilterAPI;
@@ -60,12 +59,8 @@ public abstract class RebalanceImpl {
     protected MessageModel messageModel;
     protected AllocateMessageQueueStrategy allocateMessageQueueStrategy;
     protected MQClientInstance mQClientFactory;
-    private static final int TIMEOUT_CHECK_TIMES = 3;
     private static final int QUERY_ASSIGNMENT_TIMEOUT = 3000;
 
-    private Map<String, String> topicBrokerRebalance = new ConcurrentHashMap<>();
-    private Map<String, String> topicClientRebalance = new ConcurrentHashMap<>();
-
     public RebalanceImpl(String consumerGroup, MessageModel messageModel,
         AllocateMessageQueueStrategy allocateMessageQueueStrategy,
         MQClientInstance mQClientFactory) {
@@ -241,7 +236,7 @@ public boolean doRebalance(final boolean isOrder) {
             for (final Map.Entry<String, SubscriptionData> entry : subTable.entrySet()) {
                 final String topic = entry.getKey();
                 try {
-                    if (!clientRebalance(topic) && tryQueryAssignment(topic)) {
+                    if (!clientRebalance(topic)) {
                         boolean result = this.getRebalanceResultFromBroker(topic, isOrder);
                         if (!result) {
                             balanced = false;
@@ -266,38 +261,6 @@ public boolean doRebalance(final boolean isOrder) {
         return balanced;
     }
 
-    private boolean tryQueryAssignment(String topic) {
-        if (topicClientRebalance.containsKey(topic)) {
-            return false;
-        }
-
-        if (topicBrokerRebalance.containsKey(topic)) {
-            return true;
-        }
-        String strategyName = allocateMessageQueueStrategy != null ? allocateMessageQueueStrategy.getName() : null;
-        int retryTimes = 0;
-        while (retryTimes++ < TIMEOUT_CHECK_TIMES) {
-            try {
-                Set<MessageQueueAssignment> resultSet = mQClientFactory.queryAssignment(topic, consumerGroup,
-                    strategyName, messageModel, QUERY_ASSIGNMENT_TIMEOUT / TIMEOUT_CHECK_TIMES * retryTimes);
-                topicBrokerRebalance.put(topic, topic);
-                return true;
-            } catch (Throwable t) {
-                if (!(t instanceof RemotingTimeoutException)) {
-                    log.error("tryQueryAssignment error.", t);
-                    topicClientRebalance.put(topic, topic);
-                    return false;
-                }
-            }
-        }
-        if (retryTimes >= TIMEOUT_CHECK_TIMES) {
-            // if never success before and timeout exceed TIMEOUT_CHECK_TIMES, force client rebalance
-            topicClientRebalance.put(topic, topic);
-            return false;
-        }
-        return true;
-    }
-
     public ConcurrentMap<String, SubscriptionData> getSubscriptionInner() {
         return subscriptionInner;
     }
@@ -308,7 +271,7 @@ private boolean rebalanceByTopic(final String topic, final boolean isOrder) {
             case BROADCASTING: {
                 Set<MessageQueue> mqSet = this.topicSubscribeInfoTable.get(topic);
                 if (mqSet != null) {
-                    boolean changed = this.updateProcessQueueTableInRebalance(topic, mqSet, isOrder);
+                    boolean changed = this.updateProcessQueueTableInRebalance(topic, mqSet, false);
                     if (changed) {
                         this.messageQueueChanged(topic, mqSet, mqSet);
                         log.info("messageQueueChanged {} {} {} {}", consumerGroup, topic, mqSet, mqSet);
@@ -460,24 +423,10 @@ private void truncateMessageQueueNotMyTopic() {
                 }
             }
         }
-
-        Iterator<Map.Entry<String, String>> clientIter = topicClientRebalance.entrySet().iterator();
-        while (clientIter.hasNext()) {
-            if (!subTable.containsKey(clientIter.next().getKey())) {
-                clientIter.remove();
-            }
-        }
-
-        Iterator<Map.Entry<String, String>> brokerIter = topicBrokerRebalance.entrySet().iterator();
-        while (brokerIter.hasNext()) {
-            if (!subTable.containsKey(brokerIter.next().getKey())) {
-                brokerIter.remove();
-            }
-        }
     }
 
     private boolean updateProcessQueueTableInRebalance(final String topic, final Set<MessageQueue> mqSet,
-        final boolean isOrder) {
+        final boolean needLockMq) {
         boolean changed = false;
 
         // drop process queues no longer belong me
@@ -518,14 +467,14 @@ private boolean updateProcessQueueTableInRebalance(final String topic, final Set
         List<PullRequest> pullRequestList = new ArrayList<>();
         for (MessageQueue mq : mqSet) {
             if (!this.processQueueTable.containsKey(mq)) {
-                if (isOrder && !this.lock(mq)) {
+                if (needLockMq && !this.lock(mq)) {
                     log.warn("doRebalance, {}, add a new mq failed, {}, because lock failed", consumerGroup, mq);
                     allMQLocked = false;
                     continue;
                 }
 
                 this.removeDirtyOffset(mq);
-                ProcessQueue pq = createProcessQueue(topic);
+                ProcessQueue pq = createProcessQueue();
                 pq.setLocked(true);
                 long nextOffset = this.computePullFromWhere(mq);
                 if (nextOffset >= 0) {
@@ -779,8 +728,6 @@ public boolean removeUnnecessaryPopMessageQueue(final MessageQueue mq, final Pop
 
     public abstract PopProcessQueue createPopProcessQueue();
 
-    public abstract ProcessQueue createProcessQueue(String topicName);
-
     public void removeProcessQueue(final MessageQueue mq) {
         ProcessQueue prev = this.processQueueTable.remove(mq);
         if (prev != null) {
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java
index 335d89b7877..330772f22bb 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java
@@ -177,7 +177,4 @@ public PopProcessQueue createPopProcessQueue() {
         return null;
     }
 
-    public ProcessQueue createProcessQueue(String topicName) {
-        return createProcessQueue();
-    }
 }
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java
index 1b5f9766174..e0b682868ab 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java
@@ -103,8 +103,4 @@ public PopProcessQueue createPopProcessQueue() {
         return null;
     }
 
-    public ProcessQueue createProcessQueue(String topicName) {
-        return createProcessQueue();
-    }
-
 }
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java
index f28890d306f..fe2f19b2f9a 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java
@@ -140,10 +140,6 @@ public boolean clientRebalance(String topic) {
         return defaultMQPushConsumerImpl.getDefaultMQPushConsumer().isClientRebalance() || defaultMQPushConsumerImpl.isConsumeOrderly() || MessageModel.BROADCASTING.equals(messageModel);
     }
 
-    public boolean removeUnnecessaryPopMessageQueue(final MessageQueue mq, final PopProcessQueue pq) {
-        return true;
-    }
-
     @Override
     public ConsumeType consumeType() {
         return ConsumeType.CONSUME_PASSIVELY;
@@ -288,11 +284,6 @@ public ProcessQueue createProcessQueue() {
         return new ProcessQueue();
     }
 
-    @Override
-    public ProcessQueue createProcessQueue(String topicName) {
-        return createProcessQueue();
-    }
-
     @Override
     public PopProcessQueue createPopProcessQueue() {
         return new PopProcessQueue();
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
index 436782efd33..3055f2cdee1 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
@@ -16,6 +16,7 @@
  */
 package org.apache.rocketmq.client.impl.factory;
 
+import com.alibaba.fastjson.JSON;
 import io.netty.channel.Channel;
 import java.util.Collections;
 import java.util.HashMap;
@@ -35,7 +36,6 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
-import com.alibaba.fastjson.JSON;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.admin.MQAdminExtInner;
@@ -66,6 +66,8 @@
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.message.MessageQueueAssignment;
 import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.ChannelEventListener;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.common.HeartbeatV2Result;
@@ -83,8 +85,6 @@
 import org.apache.rocketmq.remoting.protocol.route.BrokerData;
 import org.apache.rocketmq.remoting.protocol.route.QueueData;
 import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 
 import static org.apache.rocketmq.remoting.rpc.ClientMetadata.topicRouteData2EndpointsForStaticTopic;
 
@@ -118,15 +118,15 @@ public class MQClientInstance {
     private final Lock lockHeartbeat = new ReentrantLock();
 
     /**
-     * The container which stores the brokerClusterInfo. The key of the map is the brokerCluster name.
+     * The container which stores the brokerClusterInfo. The key of the map is the broker name.
      * And the value is the broker instance list that belongs to the broker cluster.
      * For the sub map, the key is the id of single broker instance, and the value is the address.
      */
     private final ConcurrentMap<String, HashMap<Long, String>> brokerAddrTable = new ConcurrentHashMap<>();
 
     private final ConcurrentMap<String/* Broker Name */, HashMap<String/* address */, Integer>> brokerVersionTable = new ConcurrentHashMap<>();
-    private final Set<String/* Broker address */> brokerSupportV2HeartbeatSet = new HashSet();
-    private final ConcurrentMap<String, Integer> brokerAddrHeartbeatFingerprintTable = new ConcurrentHashMap();
+    private final Set<String/* Broker address */> brokerSupportV2HeartbeatSet = new HashSet<>();
+    private final ConcurrentMap<String, Integer> brokerAddrHeartbeatFingerprintTable = new ConcurrentHashMap<>();
     private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "MQClientFactoryScheduledThread"));
     private final ScheduledExecutorService fetchRemoteConfigExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
         @Override
@@ -152,11 +152,14 @@ public MQClientInstance(ClientConfig clientConfig, int instanceIndex, String cli
         this.nettyClientConfig.setClientCallbackExecutorThreads(clientConfig.getClientCallbackExecutorThreads());
         this.nettyClientConfig.setUseTLS(clientConfig.isUseTLS());
         this.nettyClientConfig.setSocksProxyConfig(clientConfig.getSocksProxyConfig());
+        this.nettyClientConfig.setScanAvailableNameSrv(false);
         ClientRemotingProcessor clientRemotingProcessor = new ClientRemotingProcessor(this);
         ChannelEventListener channelEventListener;
         if (clientConfig.isEnableHeartbeatChannelEventListener()) {
             channelEventListener = new ChannelEventListener() {
+                
                 private final ConcurrentMap<String, HashMap<Long, String>> brokerAddrTable = MQClientInstance.this.brokerAddrTable;
+                
                 @Override
                 public void onChannelConnect(String remoteAddr, Channel channel) {
                 }
@@ -181,7 +184,7 @@ public void onChannelActive(String remoteAddr, Channel channel) {
                             if (addr.equals(remoteAddr)) {
                                 long id = entry.getKey();
                                 String brokerName = addressEntry.getKey();
-                                if (sendHeartbeatToBroker(id, brokerName, addr)) {
+                                if (sendHeartbeatToBroker(id, brokerName, addr, false)) {
                                     rebalanceImmediately();
                                 }
                                 break;
@@ -333,8 +336,8 @@ private void startScheduledTask() {
             this.scheduledExecutorService.scheduleAtFixedRate(() -> {
                 try {
                     MQClientInstance.this.mQClientAPIImpl.fetchNameServerAddr();
-                } catch (Exception e) {
-                    log.error("ScheduledTask fetchNameServerAddr exception", e);
+                } catch (Throwable t) {
+                    log.error("ScheduledTask fetchNameServerAddr exception", t);
                 }
             }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS);
         }
@@ -342,8 +345,8 @@ private void startScheduledTask() {
         this.scheduledExecutorService.scheduleAtFixedRate(() -> {
             try {
                 MQClientInstance.this.updateTopicRouteInfoFromNameServer();
-            } catch (Exception e) {
-                log.error("ScheduledTask updateTopicRouteInfoFromNameServer exception", e);
+            } catch (Throwable t) {
+                log.error("ScheduledTask updateTopicRouteInfoFromNameServer exception", t);
             }
         }, 10, this.clientConfig.getPollNameServerInterval(), TimeUnit.MILLISECONDS);
 
@@ -351,24 +354,24 @@ private void startScheduledTask() {
             try {
                 MQClientInstance.this.cleanOfflineBroker();
                 MQClientInstance.this.sendHeartbeatToAllBrokerWithLock();
-            } catch (Exception e) {
-                log.error("ScheduledTask sendHeartbeatToAllBroker exception", e);
+            } catch (Throwable t) {
+                log.error("ScheduledTask sendHeartbeatToAllBroker exception", t);
             }
         }, 1000, this.clientConfig.getHeartbeatBrokerInterval(), TimeUnit.MILLISECONDS);
 
         this.scheduledExecutorService.scheduleAtFixedRate(() -> {
             try {
                 MQClientInstance.this.persistAllConsumerOffset();
-            } catch (Exception e) {
-                log.error("ScheduledTask persistAllConsumerOffset exception", e);
+            } catch (Throwable t) {
+                log.error("ScheduledTask persistAllConsumerOffset exception", t);
             }
         }, 1000 * 10, this.clientConfig.getPersistConsumerOffsetInterval(), TimeUnit.MILLISECONDS);
 
         this.scheduledExecutorService.scheduleAtFixedRate(() -> {
             try {
                 MQClientInstance.this.adjustThreadPool();
-            } catch (Exception e) {
-                log.error("ScheduledTask adjustThreadPool exception", e);
+            } catch (Throwable t) {
+                log.error("ScheduledTask adjustThreadPool exception", t);
             }
         }, 1, 1, TimeUnit.MINUTES);
     }
@@ -590,6 +593,18 @@ private boolean isBrokerAddrExistInTopicRouteTable(final String addr) {
     }
 
     public boolean sendHeartbeatToBroker(long id, String brokerName, String addr) {
+        return sendHeartbeatToBroker(id, brokerName, addr, true);
+    }
+
+    /**
+     * @param id
+     * @param brokerName
+     * @param addr
+     * @param strictLockMode When the connection is initially established, sending a heartbeat will simultaneously trigger the onChannelActive event to acquire the lock again, causing an exception. Therefore,
+     *                       the exception that occurs when sending the heartbeat during the initial onChannelActive event can be ignored.
+     * @return
+     */
+    public boolean sendHeartbeatToBroker(long id, String brokerName, String addr, boolean strictLockMode) {
         if (this.lockHeartbeat.tryLock()) {
             final HeartbeatData heartbeatDataWithSub = this.prepareHeartbeatData(false);
             final boolean producerEmpty = heartbeatDataWithSub.getProducerDataSet().isEmpty();
@@ -614,7 +629,9 @@ public boolean sendHeartbeatToBroker(long id, String brokerName, String addr) {
                 this.lockHeartbeat.unlock();
             }
         } else {
-            log.warn("lock heartBeat, but failed. [{}]", this.clientId);
+            if (strictLockMode) {
+                log.warn("lock heartBeat, but failed. [{}]", this.clientId);
+            }
         }
         return false;
     }
@@ -868,7 +885,6 @@ private HeartbeatData prepareHeartbeatData(boolean isWithoutSub) {
                 consumerData.setConsumeType(impl.consumeType());
                 consumerData.setMessageModel(impl.messageModel());
                 consumerData.setConsumeFromWhere(impl.consumeFromWhere());
-                consumerData.getSubscriptionDataSet().addAll(impl.subscriptions());
                 consumerData.setUnitMode(impl.isUnitMode());
                 if (!isWithoutSub) {
                     consumerData.getSubscriptionDataSet().addAll(impl.subscriptions());
@@ -1070,7 +1086,7 @@ public boolean doRebalance() {
                         balanced = false;
                     }
                 } catch (Throwable e) {
-                    log.error("doRebalance exception", e);
+                    log.error("doRebalance for consumer group [{}] exception", entry.getKey(), e);
                 }
             }
         }
@@ -1161,7 +1177,7 @@ public FindBrokerResult findBrokerAddressInSubscribe(
                 Entry<Long, String> entry = map.entrySet().iterator().next();
                 brokerAddr = entry.getValue();
                 slave = entry.getKey() != MixAll.MASTER_ID;
-                found = true;
+                found = brokerAddr != null;
             }
         }
 
@@ -1222,8 +1238,7 @@ public String findBrokerAddrByTopic(final String topic) {
         if (topicRouteData != null) {
             List<BrokerData> brokers = topicRouteData.getBrokerDatas();
             if (!brokers.isEmpty()) {
-                int index = random.nextInt(brokers.size());
-                BrokerData bd = brokers.get(index % brokers.size());
+                BrokerData bd = brokers.get(random.nextInt(brokers.size()));
                 return bd.selectBrokerAddr();
             }
         }
@@ -1238,7 +1253,7 @@ public synchronized void resetOffset(String topic, String group, Map<MessageQueu
             if (impl instanceof DefaultMQPushConsumerImpl) {
                 consumer = (DefaultMQPushConsumerImpl) impl;
             } else {
-                log.info("[reset-offset] consumer dose not exist. group={}", group);
+                log.info("[reset-offset] consumer does not exist. group={}", group);
                 return;
             }
             consumer.suspend();
@@ -1380,6 +1395,14 @@ public ClientConfig getClientConfig() {
         return clientConfig;
     }
 
+    public ConcurrentMap<String, MQProducerInner> getProducerTable() {
+        return producerTable;
+    }
+
+    public ConcurrentMap<String, MQConsumerInner> getConsumerTable() {
+        return consumerTable;
+    }
+
     public TopicRouteData queryTopicRouteData(String topic) {
         TopicRouteData data = this.getAnExistTopicRouteData(topic);
         if (data == null) {
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
index b97e00c577f..90895034070 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
@@ -24,6 +24,7 @@
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.consumer.AckCallback;
 import org.apache.rocketmq.client.consumer.AckResult;
+import org.apache.rocketmq.client.consumer.NotifyResult;
 import org.apache.rocketmq.client.consumer.PopCallback;
 import org.apache.rocketmq.client.consumer.PopResult;
 import org.apache.rocketmq.client.consumer.PullCallback;
@@ -37,6 +38,7 @@
 import org.apache.rocketmq.client.impl.admin.MqClientAdminImpl;
 import org.apache.rocketmq.client.impl.consumer.PullResultExt;
 import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.ObjectCreator;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.common.message.MessageBatch;
@@ -48,6 +50,7 @@
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.InvokeCallback;
 import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.RemotingClient;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
 import org.apache.rocketmq.remoting.netty.ResponseFuture;
@@ -74,6 +77,8 @@
 import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.SearchOffsetRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
@@ -95,7 +100,17 @@ public MQClientAPIExt(
         ClientRemotingProcessor clientRemotingProcessor,
         RPCHook rpcHook
     ) {
-        super(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig);
+        this(clientConfig, nettyClientConfig, clientRemotingProcessor, rpcHook, null);
+    }
+
+    public MQClientAPIExt(
+        ClientConfig clientConfig,
+        NettyClientConfig nettyClientConfig,
+        ClientRemotingProcessor clientRemotingProcessor,
+        RPCHook rpcHook,
+        ObjectCreator<RemotingClient> remotingClientCreator
+    ) {
+        super(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, null, remotingClientCreator);
         this.clientConfig = clientConfig;
         this.mqClientAdmin = new MqClientAdminImpl(getRemotingClient());
     }
@@ -400,6 +415,33 @@ public CompletableFuture<Void> updateConsumerOffsetOneWay(
         return future;
     }
 
+    public CompletableFuture<Void> updateConsumerOffsetAsync(
+        String brokerAddr,
+        UpdateConsumerOffsetRequestHeader header,
+        long timeoutMillis
+    ) {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, header);
+        CompletableFuture<Void> future = new CompletableFuture<>();
+        invoke(brokerAddr, request, timeoutMillis).whenComplete((response, t) -> {
+            if (t != null) {
+                log.error("updateConsumerOffsetAsync failed, brokerAddr={}, requestHeader={}", brokerAddr, header, t);
+                future.completeExceptionally(t);
+                return;
+            }
+            switch (response.getCode()) {
+                case ResponseCode.SUCCESS: {
+                    future.complete(null);
+                }
+                case ResponseCode.SYSTEM_ERROR:
+                case ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST:
+                case ResponseCode.TOPIC_NOT_EXIST: {
+                    future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
+                }
+            }
+        });
+        return future;
+    }
+
     public CompletableFuture<List<String>> getConsumerListByGroupAsync(
         String brokerAddr,
         GetConsumerListByGroupRequestHeader requestHeader,
@@ -579,14 +621,23 @@ public CompletableFuture<Void> unlockBatchMQOneway(String brokerAddr,
     }
 
     public CompletableFuture<Boolean> notification(String brokerAddr, NotificationRequestHeader requestHeader,
+        long timeoutMillis) {
+        return notificationWithPollingStats(brokerAddr, requestHeader, timeoutMillis).thenApply(NotifyResult::isHasMsg);
+    }
+
+    public CompletableFuture<NotifyResult> notificationWithPollingStats(String brokerAddr,
+        NotificationRequestHeader requestHeader,
         long timeoutMillis) {
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFICATION, requestHeader);
         return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
-            CompletableFuture<Boolean> future0 = new CompletableFuture<>();
+            CompletableFuture<NotifyResult> future0 = new CompletableFuture<>();
             if (response.getCode() == ResponseCode.SUCCESS) {
                 try {
                     NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class);
-                    future0.complete(responseHeader.isHasMsg());
+                    NotifyResult notifyResult = new NotifyResult();
+                    notifyResult.setHasMsg(responseHeader.isHasMsg());
+                    notifyResult.setPollingFull(responseHeader.isPollingFull());
+                    future0.complete(notifyResult);
                 } catch (Throwable t) {
                     future0.completeExceptionally(t);
                 }
@@ -597,6 +648,26 @@ public CompletableFuture<Boolean> notification(String brokerAddr, NotificationRe
         });
     }
 
+    public CompletableFuture<String> recallMessageAsync(String brokerAddr,
+        RecallMessageRequestHeader requestHeader, long timeoutMillis) {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader);
+        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
+            CompletableFuture<String> future = new CompletableFuture<>();
+            if (ResponseCode.SUCCESS == response.getCode()) {
+                try {
+                    RecallMessageResponseHeader responseHeader =
+                        response.decodeCommandCustomHeader(RecallMessageResponseHeader.class);
+                    future.complete(responseHeader.getMsgId());
+                } catch (Throwable t) {
+                    future.completeExceptionally(t);
+                }
+            } else {
+                future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr));
+            }
+            return future;
+        });
+    }
+
     public CompletableFuture<RemotingCommand> invoke(String brokerAddr, RemotingCommand request, long timeoutMillis) {
         return getRemotingClient().invoke(brokerAddr, request, timeoutMillis);
     }
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java
index c68859b2889..d85dcc70a55 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java
@@ -17,17 +17,22 @@
 package org.apache.rocketmq.client.impl.mqclient;
 
 import com.google.common.base.Strings;
+
 import java.time.Duration;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.common.NameserverAccessConfig;
 import org.apache.rocketmq.client.impl.ClientRemotingProcessor;
 import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.utils.AsyncShutdownHelper;
+import org.apache.rocketmq.common.ObjectCreator;
 import org.apache.rocketmq.common.utils.StartAndShutdown;
 import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.RemotingClient;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
 
 public class MQClientAPIFactory implements StartAndShutdown {
@@ -39,16 +44,35 @@ public class MQClientAPIFactory implements StartAndShutdown {
     private final RPCHook rpcHook;
     private final ScheduledExecutorService scheduledExecutorService;
     private final NameserverAccessConfig nameserverAccessConfig;
+    private final ObjectCreator<RemotingClient> remotingClientCreator;
+
+    public MQClientAPIFactory(
+        NameserverAccessConfig nameserverAccessConfig,
+        String namePrefix,
+        int clientNum,
+        ClientRemotingProcessor clientRemotingProcessor,
+        RPCHook rpcHook,
+        ScheduledExecutorService scheduledExecutorService
+    ) {
+        this(nameserverAccessConfig, namePrefix, clientNum, clientRemotingProcessor, rpcHook, scheduledExecutorService, null);
+    }
 
-    public MQClientAPIFactory(NameserverAccessConfig nameserverAccessConfig, String namePrefix, int clientNum,
+    public MQClientAPIFactory(
+        NameserverAccessConfig nameserverAccessConfig,
+        String namePrefix,
+        int clientNum,
         ClientRemotingProcessor clientRemotingProcessor,
-        RPCHook rpcHook, ScheduledExecutorService scheduledExecutorService) {
+        RPCHook rpcHook,
+        ScheduledExecutorService scheduledExecutorService,
+        ObjectCreator<RemotingClient> remotingClientCreator
+    ) {
         this.nameserverAccessConfig = nameserverAccessConfig;
         this.namePrefix = namePrefix;
         this.clientNum = clientNum;
         this.clientRemotingProcessor = clientRemotingProcessor;
         this.rpcHook = rpcHook;
         this.scheduledExecutorService = scheduledExecutorService;
+        this.remotingClientCreator = remotingClientCreator;
 
         this.init();
     }
@@ -85,9 +109,11 @@ public void start() throws Exception {
 
     @Override
     public void shutdown() throws Exception {
+        AsyncShutdownHelper helper = new AsyncShutdownHelper();
         for (int i = 0; i < this.clientNum; i++) {
-            clients[i].shutdown();
+            helper.addTarget(clients[i]);
         }
+        helper.shutdown().await(Integer.MAX_VALUE, TimeUnit.SECONDS);
     }
 
     protected MQClientAPIExt createAndStart(String instanceName) {
@@ -99,9 +125,13 @@ protected MQClientAPIExt createAndStart(String instanceName) {
         NettyClientConfig nettyClientConfig = new NettyClientConfig();
         nettyClientConfig.setDisableCallbackExecutor(true);
 
-        MQClientAPIExt mqClientAPIExt = new MQClientAPIExt(clientConfig, nettyClientConfig,
+        MQClientAPIExt mqClientAPIExt = new MQClientAPIExt(
+            clientConfig,
+            nettyClientConfig,
             clientRemotingProcessor,
-            rpcHook);
+            rpcHook,
+            remotingClientCreator
+        );
 
         if (!mqClientAPIExt.updateNameServerAddressList()) {
             mqClientAPIExt.fetchNameServerAddr();
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
index 26e6297a8c7..4aa605821f0 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
@@ -24,6 +24,7 @@
 import java.util.List;
 import java.util.Random;
 import java.util.Set;
+import java.util.Optional;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -34,7 +35,7 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.base.Optional;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.client.QueryResult;
 import org.apache.rocketmq.client.Validators;
 import org.apache.rocketmq.client.common.ClientErrorCode;
@@ -70,9 +71,6 @@
 import org.apache.rocketmq.common.ServiceState;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.UtilAll;
-import org.apache.rocketmq.common.compression.CompressionType;
-import org.apache.rocketmq.common.compression.Compressor;
-import org.apache.rocketmq.common.compression.CompressorFactory;
 import org.apache.rocketmq.common.help.FAQUrl;
 import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.common.message.MessageAccessor;
@@ -84,6 +82,7 @@
 import org.apache.rocketmq.common.message.MessageId;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.message.MessageType;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.common.utils.CorrelationIdUtil;
 import org.apache.rocketmq.remoting.RPCHook;
@@ -94,6 +93,7 @@
 import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
 import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
@@ -118,11 +118,6 @@ public class DefaultMQProducerImpl implements MQProducerInner {
     private MQFaultStrategy mqFaultStrategy;
     private ExecutorService asyncSenderExecutor;
 
-    // compression related
-    private int compressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, "5"));
-    private CompressionType compressType = CompressionType.of(System.getProperty(MixAll.MESSAGE_COMPRESS_TYPE, "ZLIB"));
-    private final Compressor compressor = CompressorFactory.getCompressor(compressType);
-
     // backpressure related
     private Semaphore semaphoreAsyncSendNum;
     private Semaphore semaphoreAsyncSendSize;
@@ -184,7 +179,7 @@ public String resolve(String name) {
     }
     private Optional<String> pickTopic() {
         if (topicPublishInfoTable.isEmpty()) {
-            return Optional.absent();
+            return Optional.empty();
         }
         return Optional.of(topicPublishInfoTable.keySet().iterator().next());
     }
@@ -202,6 +197,14 @@ public void setSemaphoreAsyncSendSize(int size) {
         semaphoreAsyncSendSize = new Semaphore(size, true);
     }
 
+    public int getSemaphoreAsyncSendNumAvailablePermits() {
+        return semaphoreAsyncSendNum == null ? 0 : semaphoreAsyncSendNum.availablePermits();
+    }
+
+    public int getSemaphoreAsyncSendSizeAvailablePermits() {
+        return semaphoreAsyncSendSize == null ? 0 : semaphoreAsyncSendSize.availablePermits();
+    }
+
     public void initTransactionEnv() {
         TransactionMQProducer producer = (TransactionMQProducer) this.defaultMQProducer;
         if (producer.getExecutorService() != null) {
@@ -250,6 +253,8 @@ public void start(final boolean startFactory) throws MQClientException {
 
                 this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQProducer, rpcHook);
 
+                defaultMQProducer.initProduceAccumulator();
+
                 boolean registerOK = mQClientFactory.registerProducer(this.defaultMQProducer.getProducerGroup(), this);
                 if (!registerOK) {
                     this.serviceState = ServiceState.CREATE_JUST;
@@ -386,6 +391,7 @@ public void run() {
                     }
 
                     this.processTransactionState(
+                        checkRequestHeader.getTopic(),
                         localTransactionState,
                         group,
                         exception);
@@ -395,10 +401,12 @@ public void run() {
             }
 
             private void processTransactionState(
+                final String topic,
                 final LocalTransactionState localTransactionState,
                 final String producerGroup,
                 final Throwable exception) {
                 final EndTransactionRequestHeader thisHeader = new EndTransactionRequestHeader();
+                thisHeader.setTopic(topic);
                 thisHeader.setCommitLogOffset(checkRequestHeader.getCommitLogOffset());
                 thisHeader.setProducerGroup(producerGroup);
                 thisHeader.setTranStateTableOffset(checkRequestHeader.getTranStateTableOffset());
@@ -506,11 +514,11 @@ public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {
         return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq);
     }
 
-    public MessageExt viewMessage(
+    public MessageExt viewMessage(String topic,
         String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
         this.makeSureStateOK();
 
-        return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId);
+        return this.mQClientFactory.getMQAdminImpl().viewMessage(topic, msgId);
     }
 
     public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end)
@@ -567,8 +575,8 @@ public void run() {
     }
 
     class BackpressureSendCallBack implements SendCallback {
-        public boolean isSemaphoreAsyncSizeAquired = false;
-        public boolean isSemaphoreAsyncNumAquired = false;
+        public boolean isSemaphoreAsyncSizeAcquired = false;
+        public boolean isSemaphoreAsyncNumAcquired = false;
         public int msgLen;
         private final SendCallback sendCallback;
 
@@ -578,24 +586,49 @@ public BackpressureSendCallBack(final SendCallback sendCallback) {
 
         @Override
         public void onSuccess(SendResult sendResult) {
-            if (isSemaphoreAsyncSizeAquired) {
-                semaphoreAsyncSendSize.release(msgLen);
-            }
-            if (isSemaphoreAsyncNumAquired) {
-                semaphoreAsyncSendNum.release();
-            }
+            semaphoreProcessor();
             sendCallback.onSuccess(sendResult);
         }
 
         @Override
         public void onException(Throwable e) {
-            if (isSemaphoreAsyncSizeAquired) {
+            semaphoreProcessor();
+            sendCallback.onException(e);
+        }
+
+        public void semaphoreProcessor() {
+            if (isSemaphoreAsyncSizeAcquired) {
+                defaultMQProducer.acquireBackPressureForAsyncSendSizeLock();
                 semaphoreAsyncSendSize.release(msgLen);
+                defaultMQProducer.releaseBackPressureForAsyncSendSizeLock();
             }
-            if (isSemaphoreAsyncNumAquired) {
+            if (isSemaphoreAsyncNumAcquired) {
+                defaultMQProducer.acquireBackPressureForAsyncSendNumLock();
                 semaphoreAsyncSendNum.release();
+                defaultMQProducer.releaseBackPressureForAsyncSendNumLock();
             }
-            sendCallback.onException(e);
+        }
+
+        public void semaphoreAsyncAdjust(int semaphoreAsyncNum, int semaphoreAsyncSize) throws InterruptedException {
+            defaultMQProducer.acquireBackPressureForAsyncSendNumLock();
+            if (semaphoreAsyncNum > 0) {
+                semaphoreAsyncSendNum.release(semaphoreAsyncNum);
+            } else {
+                semaphoreAsyncSendNum.acquire(- semaphoreAsyncNum);
+            }
+            defaultMQProducer.setBackPressureForAsyncSendNumInsideAdjust(defaultMQProducer.getBackPressureForAsyncSendNum()
+                    + semaphoreAsyncNum);
+            defaultMQProducer.releaseBackPressureForAsyncSendNumLock();
+
+            defaultMQProducer.acquireBackPressureForAsyncSendSizeLock();
+            if (semaphoreAsyncSize > 0) {
+                semaphoreAsyncSendSize.release(semaphoreAsyncSize);
+            } else {
+                semaphoreAsyncSendSize.acquire(- semaphoreAsyncSize);
+            }
+            defaultMQProducer.setBackPressureForAsyncSendSizeInsideAdjust(defaultMQProducer.getBackPressureForAsyncSendSize()
+                    + semaphoreAsyncSize);
+            defaultMQProducer.releaseBackPressureForAsyncSendSizeLock();
         }
     }
 
@@ -604,32 +637,40 @@ public void executeAsyncMessageSend(Runnable runnable, final Message msg, final
         throws MQClientException, InterruptedException {
         ExecutorService executor = this.getAsyncSenderExecutor();
         boolean isEnableBackpressureForAsyncMode = this.getDefaultMQProducer().isEnableBackpressureForAsyncMode();
-        boolean isSemaphoreAsyncNumAquired = false;
-        boolean isSemaphoreAsyncSizeAquired = false;
+        boolean isSemaphoreAsyncNumAcquired = false;
+        boolean isSemaphoreAsyncSizeAcquired = false;
         int msgLen = msg.getBody() == null ? 1 : msg.getBody().length;
+        sendCallback.msgLen = msgLen;
 
         try {
             if (isEnableBackpressureForAsyncMode) {
+                defaultMQProducer.acquireBackPressureForAsyncSendNumLock();
                 long costTime = System.currentTimeMillis() - beginStartTime;
-                isSemaphoreAsyncNumAquired = timeout - costTime > 0
+
+                isSemaphoreAsyncNumAcquired = timeout - costTime > 0
                     && semaphoreAsyncSendNum.tryAcquire(timeout - costTime, TimeUnit.MILLISECONDS);
-                if (!isSemaphoreAsyncNumAquired) {
+                sendCallback.isSemaphoreAsyncNumAcquired = isSemaphoreAsyncNumAcquired;
+                defaultMQProducer.releaseBackPressureForAsyncSendNumLock();
+                if (!isSemaphoreAsyncNumAcquired) {
                     sendCallback.onException(
                         new RemotingTooMuchRequestException("send message tryAcquire semaphoreAsyncNum timeout"));
                     return;
                 }
+
+                defaultMQProducer.acquireBackPressureForAsyncSendSizeLock();
                 costTime = System.currentTimeMillis() - beginStartTime;
-                isSemaphoreAsyncSizeAquired = timeout - costTime > 0
+
+                isSemaphoreAsyncSizeAcquired = timeout - costTime > 0
                     && semaphoreAsyncSendSize.tryAcquire(msgLen, timeout - costTime, TimeUnit.MILLISECONDS);
-                if (!isSemaphoreAsyncSizeAquired) {
+                sendCallback.isSemaphoreAsyncSizeAcquired = isSemaphoreAsyncSizeAcquired;
+                defaultMQProducer.releaseBackPressureForAsyncSendSizeLock();
+                if (!isSemaphoreAsyncSizeAcquired) {
                     sendCallback.onException(
                         new RemotingTooMuchRequestException("send message tryAcquire semaphoreAsyncSize timeout"));
                     return;
                 }
             }
-            sendCallback.isSemaphoreAsyncSizeAquired = isSemaphoreAsyncSizeAquired;
-            sendCallback.isSemaphoreAsyncNumAquired = isSemaphoreAsyncNumAquired;
-            sendCallback.msgLen = msgLen;
+
             executor.submit(runnable);
         } catch (RejectedExecutionException e) {
             if (isEnableBackpressureForAsyncMode) {
@@ -736,8 +777,16 @@ private SendResult sendDefaultImpl(
                             callTimeout = true;
                             break;
                         }
-
-                        sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime);
+                        long curTimeout = timeout - costTime;
+                        // Get the maximum timeout allowed per request
+                        long maxSendTimeoutPerRequest = defaultMQProducer.getSendMsgMaxTimeoutPerRequest();
+                        // Determine if retries are still possible
+                        boolean canRetryAgain = times + 1 < timesTotal;
+                        // If retries are possible, and the current timeout exceeds the max allowed timeout, set the current timeout to the max allowed
+                        if (maxSendTimeoutPerRequest > -1 && canRetryAgain && curTimeout > maxSendTimeoutPerRequest) {
+                            curTimeout = maxSendTimeoutPerRequest;
+                        }
+                        sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, curTimeout);
                         endTimestamp = System.currentTimeMillis();
                         this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true);
                         switch (communicationMode) {
@@ -759,7 +808,7 @@ private SendResult sendDefaultImpl(
                     } catch (MQClientException e) {
                         endTimestamp = System.currentTimeMillis();
                         this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true);
-                        log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e);
+                        log.warn("sendKernelImpl exception, resend at once, InvokeID: {}, RT: {}ms, Broker: {}", invokeID, endTimestamp - beginTimestampPrev, mq, e);
                         log.warn(msg.toString());
                         exception = e;
                         continue;
@@ -772,7 +821,7 @@ private SendResult sendDefaultImpl(
                             // Otherwise, isolate this broker.
                             this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, true);
                         }
-                        log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e);
+                        log.warn("sendKernelImpl exception, resend at once, InvokeID: {}, RT: {}ms, Broker: {}", invokeID, endTimestamp - beginTimestampPrev, mq, e);
                         if (log.isDebugEnabled()) {
                             log.debug(msg.toString());
                         }
@@ -781,7 +830,7 @@ private SendResult sendDefaultImpl(
                     } catch (MQBrokerException e) {
                         endTimestamp = System.currentTimeMillis();
                         this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, false);
-                        log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e);
+                        log.warn("sendKernelImpl exception, resend at once, InvokeID: {}, RT: {}ms, Broker: {}", invokeID, endTimestamp - beginTimestampPrev, mq, e);
                         if (log.isDebugEnabled()) {
                             log.debug(msg.toString());
                         }
@@ -798,7 +847,7 @@ private SendResult sendDefaultImpl(
                     } catch (InterruptedException e) {
                         endTimestamp = System.currentTimeMillis();
                         this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true);
-                        log.warn("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e);
+                        log.warn("sendKernelImpl exception, throw exception, InvokeID: {}, RT: {}ms, Broker: {}", invokeID, endTimestamp - beginTimestampPrev, mq, e);
                         if (log.isDebugEnabled()) {
                             log.debug(msg.toString());
                         }
@@ -897,7 +946,7 @@ private SendResult sendKernelImpl(final Message msg,
                 boolean msgBodyCompressed = false;
                 if (this.tryToCompressMessage(msg)) {
                     sysFlag |= MessageSysFlag.COMPRESSED_FLAG;
-                    sysFlag |= compressType.getCompressionFlag();
+                    sysFlag |= this.defaultMQProducer.getCompressType().getCompressionFlag();
                     msgBodyCompressed = true;
                 }
 
@@ -974,7 +1023,7 @@ private SendResult sendKernelImpl(final Message msg,
                         boolean messageCloned = false;
                         if (msgBodyCompressed) {
                             //If msg body was compressed, msgbody should be reset using prevBody.
-                            //Clone new message using commpressed message body and recover origin massage.
+                            //Clone new message using compressed message body and recover origin massage.
                             //Fix bug:https://github.com/apache/rocketmq-externals/issues/66
                             tmpMessage = MessageAccessor.cloneMessage(msg);
                             messageCloned = true;
@@ -1067,7 +1116,7 @@ private boolean tryToCompressMessage(final Message msg) {
         if (body != null) {
             if (body.length >= this.defaultMQProducer.getCompressMsgBodyOverHowmuch()) {
                 try {
-                    byte[] data = compressor.compress(body, compressLevel);
+                    byte[] data = this.defaultMQProducer.getCompressor().compress(body, this.defaultMQProducer.getCompressLevel());
                     if (data != null) {
                         msg.setBody(data);
                         return true;
@@ -1484,6 +1533,7 @@ public void endTransaction(
         final String destBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(defaultMQProducer.queueWithNamespace(sendResult.getMessageQueue()));
         final String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(destBrokerName);
         EndTransactionRequestHeader requestHeader = new EndTransactionRequestHeader();
+        requestHeader.setTopic(msg.getTopic());
         requestHeader.setTransactionId(transactionId);
         requestHeader.setCommitLogOffset(id.getOffset());
         requestHeader.setBrokerName(destBrokerName);
@@ -1510,6 +1560,40 @@ public void endTransaction(
             this.defaultMQProducer.getSendMsgTimeout());
     }
 
+    public String recallMessage(
+        String topic,
+        String recallHandle) throws RemotingException, MQClientException, MQBrokerException, InterruptedException {
+        makeSureStateOK();
+        Validators.checkTopic(topic);
+        if (NamespaceUtil.isRetryTopic(topic) || NamespaceUtil.isDLQTopic(topic)) {
+            throw new MQClientException("topic is not supported", null);
+        }
+        RecallMessageHandle.HandleV1 handleEntity;
+        try {
+            handleEntity = (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(recallHandle);
+        } catch (Exception e) {
+            throw new MQClientException(e.getMessage(), null);
+        }
+
+        tryToFindTopicPublishInfo(topic);
+        String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(handleEntity.getBrokerName());
+        brokerAddr = StringUtils.isNotEmpty(brokerAddr) ?
+            // find another address to support multi proxy endpoints,
+            // may cause failure request in proxy-less mode when the broker is temporarily unavailable
+            brokerAddr : this.mQClientFactory.findBrokerAddrByTopic(topic);
+        if (StringUtils.isEmpty(brokerAddr)) {
+            log.warn("can't find broker service address. {}", handleEntity.getBrokerName());
+            throw new MQClientException("The broker service address not found", null);
+        }
+        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();
+        requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup());
+        requestHeader.setTopic(topic);
+        requestHeader.setRecallHandle(recallHandle);
+        requestHeader.setBrokerName(handleEntity.getBrokerName());
+        return this.mQClientFactory.getMQClientAPIImpl().recallMessage(brokerAddr,
+            requestHeader, this.defaultMQProducer.getSendMsgTimeout());
+    }
+
     public void setCallbackExecutor(final ExecutorService callbackExecutor) {
         this.mQClientFactory.getMQClientAPIImpl().getRemotingClient().setCallbackExecutor(callbackExecutor);
     }
@@ -1542,6 +1626,7 @@ public Message request(final Message msg,
                 @Override
                 public void onSuccess(SendResult sendResult) {
                     requestResponseFuture.setSendRequestOk(true);
+                    requestResponseFuture.acquireCountDownLatch();
                 }
 
                 @Override
@@ -1599,6 +1684,7 @@ public Message request(final Message msg, final MessageQueueSelector selector, f
                 @Override
                 public void onSuccess(SendResult sendResult) {
                     requestResponseFuture.setSendRequestOk(true);
+                    requestResponseFuture.acquireCountDownLatch();
                 }
 
                 @Override
@@ -1656,6 +1742,7 @@ public Message request(final Message msg, final MessageQueue mq, final long time
                 @Override
                 public void onSuccess(SendResult sendResult) {
                     requestResponseFuture.setSendRequestOk(true);
+                    requestResponseFuture.acquireCountDownLatch();
                 }
 
                 @Override
@@ -1759,22 +1846,6 @@ public ConcurrentMap<String, TopicPublishInfo> getTopicPublishInfoTable() {
         return topicPublishInfoTable;
     }
 
-    public int getCompressLevel() {
-        return compressLevel;
-    }
-
-    public void setCompressLevel(int compressLevel) {
-        this.compressLevel = compressLevel;
-    }
-
-    public CompressionType getCompressType() {
-        return compressType;
-    }
-
-    public void setCompressType(CompressionType compressType) {
-        this.compressType = compressType;
-    }
-
     public ServiceState getServiceState() {
         return serviceState;
     }
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java
index 37b1f3252f7..917fe57aa87 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java
@@ -111,9 +111,7 @@ public MessageQueue selectOneMessageQueue(final String lastBrokerName) {
             return selectOneMessageQueue();
         } else {
             for (int i = 0; i < this.messageQueueList.size(); i++) {
-                int index = this.sendWhichQueue.incrementAndGet();
-                int pos = index % this.messageQueueList.size();
-                MessageQueue mq = this.messageQueueList.get(pos);
+                MessageQueue mq = selectOneMessageQueue();
                 if (!mq.getBrokerName().equals(lastBrokerName)) {
                     return mq;
                 }
diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java
index f629fe44a87..db8bbd66ef2 100644
--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java
@@ -55,6 +55,7 @@ public LatencyFaultToleranceImpl(Resolver resolver, ServiceDetector serviceDetec
         this.serviceDetector = serviceDetector;
     }
 
+    @Override
     public void detectByOneRound() {
         for (Map.Entry<String, FaultItem> item : this.faultItemTable.entrySet()) {
             FaultItem brokerItem = item.getValue();
@@ -77,6 +78,7 @@ public void detectByOneRound() {
         }
     }
 
+    @Override
     public void startDetector() {
         this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
             @Override
@@ -92,6 +94,7 @@ public void run() {
         }, 3, 3, TimeUnit.SECONDS);
     }
 
+    @Override
     public void shutdown() {
         this.scheduledExecutorService.shutdown();
     }
@@ -128,6 +131,7 @@ public boolean isAvailable(final String name) {
         return true;
     }
 
+    @Override
     public boolean isReachable(final String name) {
         final FaultItem faultItem = this.faultItemTable.get(name);
         if (faultItem != null) {
@@ -141,10 +145,12 @@ public void remove(final String name) {
         this.faultItemTable.remove(name);
     }
 
+    @Override
     public boolean isStartDetectorEnable() {
         return startDetectorEnable;
     }
 
+    @Override
     public void setStartDetectorEnable(boolean startDetectorEnable) {
         this.startDetectorEnable = startDetectorEnable;
     }
@@ -177,10 +183,12 @@ public String toString() {
                 '}';
     }
 
+    @Override
     public void setDetectTimeout(final int detectTimeout) {
         this.detectTimeout = detectTimeout;
     }
 
+    @Override
     public void setDetectInterval(final int detectInterval) {
         this.detectInterval = detectInterval;
     }
diff --git a/client/src/main/java/org/apache/rocketmq/client/lock/ReadWriteCASLock.java b/client/src/main/java/org/apache/rocketmq/client/lock/ReadWriteCASLock.java
new file mode 100644
index 00000000000..3d157313715
--- /dev/null
+++ b/client/src/main/java/org/apache/rocketmq/client/lock/ReadWriteCASLock.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.lock;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class ReadWriteCASLock {
+    //true : can lock ; false : not lock
+    private final AtomicBoolean writeLock = new AtomicBoolean(true);
+
+    private final AtomicInteger readLock = new AtomicInteger(0);
+
+    public void acquireWriteLock() {
+        boolean isLock = false;
+        do {
+            isLock = writeLock.compareAndSet(true, false);
+        } while (!isLock);
+
+        do {
+            isLock = readLock.get() == 0;
+        } while (!isLock);
+    }
+
+    public void releaseWriteLock() {
+        this.writeLock.compareAndSet(false, true);
+    }
+
+    public void acquireReadLock() {
+        boolean isLock = false;
+        do {
+            isLock = writeLock.get();
+        } while (!isLock);
+        readLock.getAndIncrement();
+    }
+
+    public void releaseReadLock() {
+        this.readLock.getAndDecrement();
+    }
+
+    public boolean getWriteLock() {
+        return this.writeLock.get() && this.readLock.get() == 0;
+    }
+
+    public boolean getReadLock() {
+        return this.writeLock.get();
+    }
+
+}
diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java
index 13be47c79da..11edcaa4410 100644
--- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java
+++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java
@@ -16,13 +16,6 @@
  */
 package org.apache.rocketmq.client.producer;
 
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArraySet;
-import java.util.concurrent.ExecutorService;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.QueryResult;
 import org.apache.rocketmq.client.Validators;
@@ -31,11 +24,16 @@
 import org.apache.rocketmq.client.exception.RequestTimeoutException;
 import org.apache.rocketmq.client.impl.MQClientManager;
 import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
+import org.apache.rocketmq.client.lock.ReadWriteCASLock;
+import org.apache.rocketmq.client.trace.hook.DefaultRecallMessageTraceHook;
 import org.apache.rocketmq.client.trace.AsyncTraceDispatcher;
 import org.apache.rocketmq.client.trace.TraceDispatcher;
 import org.apache.rocketmq.client.trace.hook.EndTransactionTraceHookImpl;
 import org.apache.rocketmq.client.trace.hook.SendMessageTraceHookImpl;
 import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.compression.CompressionType;
+import org.apache.rocketmq.common.compression.Compressor;
+import org.apache.rocketmq.common.compression.CompressorFactory;
 import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.common.message.MessageBatch;
 import org.apache.rocketmq.common.message.MessageClientIDSetter;
@@ -43,11 +41,19 @@
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.concurrent.ExecutorService;
 
 /**
  * This class is the entry point for applications intending to send messages. </p>
@@ -72,9 +78,11 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
         ResponseCode.TOPIC_NOT_EXIST,
         ResponseCode.SERVICE_NOT_AVAILABLE,
         ResponseCode.SYSTEM_ERROR,
+        ResponseCode.SYSTEM_BUSY,
         ResponseCode.NO_PERMISSION,
         ResponseCode.NO_BUYER_ID,
-        ResponseCode.NOT_IN_CURRENT_UNIT
+        ResponseCode.NOT_IN_CURRENT_UNIT,
+        ResponseCode.GO_AWAY
     ));
 
     /**
@@ -107,6 +115,11 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
      */
     private int sendMsgTimeout = 3000;
 
+    /**
+     * Max timeout for sending messages per request.
+     */
+    private int sendMsgMaxTimeoutPerRequest = -1;
+
     /**
      * Compress message body threshold, namely, message body larger than 4k will be compressed on default.
      */
@@ -167,6 +180,48 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
      */
     private int backPressureForAsyncSendSize = 100 * 1024 * 1024;
 
+    /**
+     * Maximum hold time of accumulator.
+     */
+    private int batchMaxDelayMs = -1;
+
+    /**
+     * Maximum accumulation message body size for a single messageAccumulation.
+     */
+    private long batchMaxBytes = -1;
+
+    /**
+     * Maximum message body size for produceAccumulator.
+     */
+    private long totalBatchMaxBytes = -1;
+
+    private RPCHook rpcHook = null;
+
+    /**
+     *  backPressureForAsyncSendNum is guaranteed to be modified at runtime and no new requests are allowed
+     */
+    private final ReadWriteCASLock backPressureForAsyncSendNumLock = new ReadWriteCASLock();
+
+    /**
+     * backPressureForAsyncSendSize is guaranteed to be modified at runtime and no new requests are allowed
+     */
+    private final ReadWriteCASLock backPressureForAsyncSendSizeLock = new ReadWriteCASLock();
+
+    /**
+     * Compress level of compress algorithm.
+     */
+    private int compressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, "5"));
+
+    /**
+     * Compress type of compress algorithm, default using ZLIB.
+     */
+    private CompressionType compressType = CompressionType.of(System.getProperty(MixAll.MESSAGE_COMPRESS_TYPE, "ZLIB"));
+
+    /**
+     * Compressor of compress algorithm.
+     */
+    private Compressor compressor = CompressorFactory.getCompressor(compressType);
+
     /**
      * Default constructor.
      */
@@ -189,9 +244,7 @@ public DefaultMQProducer(RPCHook rpcHook) {
      * @param producerGroup Producer group, see the name-sake field.
      */
     public DefaultMQProducer(final String producerGroup) {
-        this.producerGroup = producerGroup;
-        defaultMQProducerImpl = new DefaultMQProducerImpl(this, null);
-        produceAccumulator = MQClientManager.getInstance().getOrCreateProduceAccumulator(this);
+        this(producerGroup, (RPCHook) null);
     }
 
     /**
@@ -201,9 +254,7 @@ public DefaultMQProducer(final String producerGroup) {
      * @param rpcHook       RPC hook to execute per each remoting command execution.
      */
     public DefaultMQProducer(final String producerGroup, RPCHook rpcHook) {
-        this.producerGroup = producerGroup;
-        defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook);
-        produceAccumulator = MQClientManager.getInstance().getOrCreateProduceAccumulator(this);
+        this(producerGroup, rpcHook, null);
     }
 
     /**
@@ -215,8 +266,7 @@ public DefaultMQProducer(final String producerGroup, RPCHook rpcHook) {
      */
     public DefaultMQProducer(final String producerGroup, RPCHook rpcHook,
         final List<String> topics) {
-        this(producerGroup, rpcHook);
-        this.topics = topics;
+        this(producerGroup, rpcHook, topics, false, null);
     }
 
     /**
@@ -242,21 +292,7 @@ public DefaultMQProducer(final String producerGroup, boolean enableMsgTrace, fin
      */
     public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace,
         final String customizedTraceTopic) {
-        this(producerGroup, rpcHook);
-        //if client open the message trace feature
-        if (enableMsgTrace) {
-            try {
-                AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(producerGroup, TraceDispatcher.Type.PRODUCE, customizedTraceTopic, rpcHook);
-                dispatcher.setHostProducer(this.defaultMQProducerImpl);
-                traceDispatcher = dispatcher;
-                this.defaultMQProducerImpl.registerSendMessageHook(
-                    new SendMessageTraceHookImpl(traceDispatcher));
-                this.defaultMQProducerImpl.registerEndTransactionHook(
-                    new EndTransactionTraceHookImpl(traceDispatcher));
-            } catch (Throwable e) {
-                logger.error("system mqtrace hook init failed ,maybe can't send msg trace data");
-            }
-        }
+        this(producerGroup, rpcHook, null, enableMsgTrace, customizedTraceTopic);
     }
 
     /**
@@ -272,8 +308,12 @@ public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean en
      */
     public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, final List<String> topics,
         boolean enableMsgTrace, final String customizedTraceTopic) {
-        this(producerGroup, rpcHook, enableMsgTrace, customizedTraceTopic);
+        this.producerGroup = producerGroup;
+        this.rpcHook = rpcHook;
         this.topics = topics;
+        this.enableTrace = enableMsgTrace;
+        this.traceTopic = customizedTraceTopic;
+        defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook);
     }
 
     /**
@@ -298,8 +338,8 @@ public DefaultMQProducer(final String namespace, final String producerGroup) {
     public DefaultMQProducer(final String namespace, final String producerGroup, RPCHook rpcHook) {
         this.namespace = namespace;
         this.producerGroup = producerGroup;
+        this.rpcHook = rpcHook;
         defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook);
-        produceAccumulator = MQClientManager.getInstance().getOrCreateProduceAccumulator(this);
     }
 
     /**
@@ -318,27 +358,8 @@ public DefaultMQProducer(final String namespace, final String producerGroup, RPC
         boolean enableMsgTrace, final String customizedTraceTopic) {
         this(namespace, producerGroup, rpcHook);
         //if client open the message trace feature
-        if (enableMsgTrace) {
-            try {
-                AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(producerGroup, TraceDispatcher.Type.PRODUCE, customizedTraceTopic, rpcHook);
-                dispatcher.setHostProducer(this.defaultMQProducerImpl);
-                traceDispatcher = dispatcher;
-                this.defaultMQProducerImpl.registerSendMessageHook(
-                    new SendMessageTraceHookImpl(traceDispatcher));
-                this.defaultMQProducerImpl.registerEndTransactionHook(
-                    new EndTransactionTraceHookImpl(traceDispatcher));
-            } catch (Throwable e) {
-                logger.error("system mqtrace hook init failed ,maybe can't send msg trace data");
-            }
-        }
-    }
-
-    @Override
-    public void setUseTLS(boolean useTLS) {
-        super.setUseTLS(useTLS);
-        if (traceDispatcher instanceof AsyncTraceDispatcher) {
-            ((AsyncTraceDispatcher) traceDispatcher).getTraceProducer().setUseTLS(useTLS);
-        }
+        this.enableTrace = enableMsgTrace;
+        this.traceTopic = customizedTraceTopic;
     }
 
     /**
@@ -356,7 +377,26 @@ public void start() throws MQClientException {
         if (this.produceAccumulator != null) {
             this.produceAccumulator.start();
         }
+        if (enableTrace) {
+            try {
+                AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(producerGroup, TraceDispatcher.Type.PRODUCE, getTraceMsgBatchNum(), traceTopic, rpcHook);
+                dispatcher.setHostProducer(this.defaultMQProducerImpl);
+                dispatcher.setNamespaceV2(this.namespaceV2);
+                traceDispatcher = dispatcher;
+                this.defaultMQProducerImpl.registerSendMessageHook(
+                    new SendMessageTraceHookImpl(traceDispatcher));
+                this.defaultMQProducerImpl.registerEndTransactionHook(
+                    new EndTransactionTraceHookImpl(traceDispatcher));
+                this.defaultMQProducerImpl.getMqClientFactory().getMQClientAPIImpl().getRemotingClient()
+                    .registerRPCHook(new DefaultRecallMessageTraceHook(traceDispatcher));
+            } catch (Throwable e) {
+                logger.error("system mqtrace hook init failed ,maybe can't send msg trace data");
+            }
+        }
         if (null != traceDispatcher) {
+            if (traceDispatcher instanceof AsyncTraceDispatcher) {
+                ((AsyncTraceDispatcher) traceDispatcher).getTraceProducer().setUseTLS(isUseTLS());
+            }
             try {
                 traceDispatcher.start(this.getNamesrvAddr(), this.getAccessChannel());
             } catch (MQClientException e) {
@@ -1002,25 +1042,6 @@ public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {
         return this.defaultMQProducerImpl.earliestMsgStoreTime(queueWithNamespace(mq));
     }
 
-    /**
-     * Query message of the given offset message ID.
-     * <p>
-     * This method will be removed in a certain version after April 5, 2020, so please do not use this method.
-     *
-     * @param offsetMsgId message id
-     * @return Message specified.
-     * @throws MQBrokerException    if there is any broker error.
-     * @throws MQClientException    if there is any client error.
-     * @throws RemotingException    if there is any network-tier error.
-     * @throws InterruptedException if the sending thread is interrupted.
-     */
-    @Deprecated
-    @Override
-    public MessageExt viewMessage(
-        String offsetMsgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        return this.defaultMQProducerImpl.viewMessage(offsetMsgId);
-    }
-
     /**
      * Query message by key.
      * <p>
@@ -1060,7 +1081,7 @@ public QueryResult queryMessage(String topic, String key, int maxNum, long begin
     public MessageExt viewMessage(String topic,
         String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
         try {
-            return this.viewMessage(msgId);
+            return this.defaultMQProducerImpl.viewMessage(topic, msgId);
         } catch (Exception ignored) {
         }
         return this.defaultMQProducerImpl.queryMessageByUniqKey(withNamespace(topic), msgId);
@@ -1115,6 +1136,12 @@ public void send(Collection<Message> msgs, MessageQueue mq,
         this.defaultMQProducerImpl.send(batch(msgs), queueWithNamespace(mq), sendCallback, timeout);
     }
 
+    @Override
+    public String recallMessage(String topic, String recallHandle)
+        throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
+        return this.defaultMQProducerImpl.recallMessage(withNamespace(topic), recallHandle);
+    }
+
     /**
      * Sets an Executor to be used for executing callback methods.
      *
@@ -1168,10 +1195,10 @@ public int getBatchMaxDelayMs() {
     }
 
     public void batchMaxDelayMs(int holdMs) {
-        if (this.produceAccumulator == null) {
-            throw new UnsupportedOperationException("The currently constructed producer does not support autoBatch");
+        this.batchMaxDelayMs = holdMs;
+        if (this.produceAccumulator != null) {
+            this.produceAccumulator.batchMaxDelayMs(holdMs);
         }
-        this.produceAccumulator.batchMaxDelayMs(holdMs);
     }
 
     public long getBatchMaxBytes() {
@@ -1182,10 +1209,10 @@ public long getBatchMaxBytes() {
     }
 
     public void batchMaxBytes(long holdSize) {
-        if (this.produceAccumulator == null) {
-            throw new UnsupportedOperationException("The currently constructed producer does not support autoBatch");
+        this.batchMaxBytes = holdSize;
+        if (this.produceAccumulator != null) {
+            this.produceAccumulator.batchMaxBytes(holdSize);
         }
-        this.produceAccumulator.batchMaxBytes(holdSize);
     }
 
     public long getTotalBatchMaxBytes() {
@@ -1196,10 +1223,10 @@ public long getTotalBatchMaxBytes() {
     }
 
     public void totalBatchMaxBytes(long totalHoldSize) {
-        if (this.produceAccumulator == null) {
-            throw new UnsupportedOperationException("The currently constructed producer does not support autoBatch");
+        this.totalBatchMaxBytes = totalHoldSize;
+        if (this.produceAccumulator != null) {
+            this.produceAccumulator.totalBatchMaxBytes(totalHoldSize);
         }
-        this.produceAccumulator.totalBatchMaxBytes(totalHoldSize);
     }
 
     public boolean getAutoBatch() {
@@ -1210,9 +1237,6 @@ public boolean getAutoBatch() {
     }
 
     public void setAutoBatch(boolean autoBatch) {
-        if (this.produceAccumulator == null) {
-            throw new UnsupportedOperationException("The currently constructed producer does not support autoBatch");
-        }
         this.autoBatch = autoBatch;
     }
 
@@ -1240,6 +1264,14 @@ public void setSendMsgTimeout(int sendMsgTimeout) {
         this.sendMsgTimeout = sendMsgTimeout;
     }
 
+    public int getSendMsgMaxTimeoutPerRequest() {
+        return sendMsgMaxTimeoutPerRequest;
+    }
+
+    public void setSendMsgMaxTimeoutPerRequest(int sendMsgMaxTimeoutPerRequest) {
+        this.sendMsgMaxTimeoutPerRequest = sendMsgMaxTimeoutPerRequest;
+    }
+
     public int getCompressMsgBodyOverHowmuch() {
         return compressMsgBodyOverHowmuch;
     }
@@ -1345,18 +1377,64 @@ public int getBackPressureForAsyncSendNum() {
         return backPressureForAsyncSendNum;
     }
 
+    /**
+     * For user modify backPressureForAsyncSendNum at runtime
+     */
     public void setBackPressureForAsyncSendNum(int backPressureForAsyncSendNum) {
+        this.backPressureForAsyncSendNumLock.acquireWriteLock();
+        backPressureForAsyncSendNum = Math.max(backPressureForAsyncSendNum, 10);
+        int acquiredBackPressureForAsyncSendNum = this.backPressureForAsyncSendNum
+                - defaultMQProducerImpl.getSemaphoreAsyncSendNumAvailablePermits();
         this.backPressureForAsyncSendNum = backPressureForAsyncSendNum;
-        defaultMQProducerImpl.setSemaphoreAsyncSendNum(backPressureForAsyncSendNum);
+        defaultMQProducerImpl.setSemaphoreAsyncSendNum(backPressureForAsyncSendNum - acquiredBackPressureForAsyncSendNum);
+        this.backPressureForAsyncSendNumLock.releaseWriteLock();
     }
 
     public int getBackPressureForAsyncSendSize() {
         return backPressureForAsyncSendSize;
     }
 
+    /**
+    * For user modify backPressureForAsyncSendSize at runtime
+     */
     public void setBackPressureForAsyncSendSize(int backPressureForAsyncSendSize) {
+        this.backPressureForAsyncSendSizeLock.acquireWriteLock();
+        backPressureForAsyncSendSize = Math.max(backPressureForAsyncSendSize, 1024 * 1024);
+        int acquiredBackPressureForAsyncSendSize = this.backPressureForAsyncSendSize
+                - defaultMQProducerImpl.getSemaphoreAsyncSendSizeAvailablePermits();
+        this.backPressureForAsyncSendSize = backPressureForAsyncSendSize;
+        defaultMQProducerImpl.setSemaphoreAsyncSendSize(backPressureForAsyncSendSize - acquiredBackPressureForAsyncSendSize);
+        this.backPressureForAsyncSendSizeLock.releaseWriteLock();
+    }
+
+    /**
+     * Used for system internal adjust backPressureForAsyncSendSize
+     */
+    public void setBackPressureForAsyncSendSizeInsideAdjust(int backPressureForAsyncSendSize) {
         this.backPressureForAsyncSendSize = backPressureForAsyncSendSize;
-        defaultMQProducerImpl.setSemaphoreAsyncSendSize(backPressureForAsyncSendSize);
+    }
+
+    /**
+     * Used for system internal adjust backPressureForAsyncSendNum
+     */
+    public void setBackPressureForAsyncSendNumInsideAdjust(int backPressureForAsyncSendNum) {
+        this.backPressureForAsyncSendNum = backPressureForAsyncSendNum;
+    }
+
+    public void acquireBackPressureForAsyncSendSizeLock() {
+        this.backPressureForAsyncSendSizeLock.acquireReadLock();
+    }
+
+    public void releaseBackPressureForAsyncSendSizeLock() {
+        this.backPressureForAsyncSendSizeLock.releaseReadLock();
+    }
+
+    public void acquireBackPressureForAsyncSendNumLock() {
+        this.backPressureForAsyncSendNumLock.acquireReadLock();
+    }
+
+    public void releaseBackPressureForAsyncSendNumLock() {
+        this.backPressureForAsyncSendNumLock.releaseReadLock();
     }
 
     public List<String> getTopics() {
@@ -1372,4 +1450,42 @@ public void setStartDetectorEnable(boolean startDetectorEnable) {
         super.setStartDetectorEnable(startDetectorEnable);
         this.defaultMQProducerImpl.getMqFaultStrategy().setStartDetectorEnable(startDetectorEnable);
     }
+
+    public int getCompressLevel() {
+        return compressLevel;
+    }
+
+    public void setCompressLevel(int compressLevel) {
+        this.compressLevel = compressLevel;
+    }
+
+    public CompressionType getCompressType() {
+        return compressType;
+    }
+
+    public void setCompressType(CompressionType compressType) {
+        this.compressType = compressType;
+        this.compressor = CompressorFactory.getCompressor(compressType);
+    }
+
+    public Compressor getCompressor() {
+        return compressor;
+    }
+
+    public void initProduceAccumulator() {
+        this.produceAccumulator = MQClientManager.getInstance().getOrCreateProduceAccumulator(this);
+
+        if (this.batchMaxDelayMs > -1) {
+            this.produceAccumulator.batchMaxDelayMs(this.batchMaxDelayMs);
+        }
+
+        if (this.batchMaxBytes > -1) {
+            this.produceAccumulator.batchMaxBytes(this.batchMaxBytes);
+        }
+
+        if (this.totalBatchMaxBytes > -1) {
+            this.produceAccumulator.totalBatchMaxBytes(this.totalBatchMaxBytes);
+        }
+
+    }
 }
diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java
index 8bd30e98d7b..4286fdd7f96 100644
--- a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java
+++ b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java
@@ -113,6 +113,9 @@ void send(final Collection<Message> msgs, final MessageQueue mq, final SendCallb
         final long timeout) throws MQClientException,
         RemotingException, MQBrokerException, InterruptedException;
 
+    String recallMessage(String topic, String recallHandle)
+        throws MQClientException, RemotingException, MQBrokerException, InterruptedException;
+
     //for rpc
     Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException,
         RemotingException, MQBrokerException, InterruptedException;
diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/ProduceAccumulator.java b/client/src/main/java/org/apache/rocketmq/client/producer/ProduceAccumulator.java
index 46dfcf71d29..809830e4641 100644
--- a/client/src/main/java/org/apache/rocketmq/client/producer/ProduceAccumulator.java
+++ b/client/src/main/java/org/apache/rocketmq/client/producer/ProduceAccumulator.java
@@ -52,9 +52,9 @@ public class ProduceAccumulator {
     private final Logger log = LoggerFactory.getLogger(DefaultMQProducer.class);
     private final GuardForSyncSendService guardThreadForSyncSend;
     private final GuardForAsyncSendService guardThreadForAsyncSend;
-    private Map<AggregateKey, MessageAccumulation> syncSendBatchs = new ConcurrentHashMap<AggregateKey, MessageAccumulation>();
-    private Map<AggregateKey, MessageAccumulation> asyncSendBatchs = new ConcurrentHashMap<AggregateKey, MessageAccumulation>();
-    private AtomicLong currentlyHoldSize = new AtomicLong(0);
+    private final Map<AggregateKey, MessageAccumulation> syncSendBatchs = new ConcurrentHashMap<AggregateKey, MessageAccumulation>();
+    private final Map<AggregateKey, MessageAccumulation> asyncSendBatchs = new ConcurrentHashMap<AggregateKey, MessageAccumulation>();
+    private final AtomicLong currentlyHoldSize = new AtomicLong(0);
     private final String instanceName;
 
     public ProduceAccumulator(String instanceName) {
@@ -70,11 +70,13 @@ public GuardForSyncSendService(String clientInstanceName) {
             serviceName = String.format("Client_%s_GuardForSyncSend", clientInstanceName);
         }
 
-        @Override public String getServiceName() {
+        @Override
+        public String getServiceName() {
             return serviceName;
         }
 
-        @Override public void run() {
+        @Override
+        public void run() {
             log.info(this.getServiceName() + " service started");
 
             while (!this.isStopped()) {
@@ -115,11 +117,13 @@ public GuardForAsyncSendService(String clientInstanceName) {
             serviceName = String.format("Client_%s_GuardForAsyncSend", clientInstanceName);
         }
 
-        @Override public String getServiceName() {
+        @Override
+        public String getServiceName() {
             return serviceName;
         }
 
-        @Override public void run() {
+        @Override
+        public void run() {
             log.info(this.getServiceName() + " service started");
 
             while (!this.isStopped()) {
@@ -276,7 +280,10 @@ void send(Message msg, MessageQueue mq,
     boolean tryAddMessage(Message message) {
         synchronized (currentlyHoldSize) {
             if (currentlyHoldSize.get() < totalHoldSize) {
-                currentlyHoldSize.addAndGet(message.getBody().length);
+                int bodySize = null == message.getBody() ? 0 : message.getBody().length;
+                if (bodySize > 0) {
+                    currentlyHoldSize.addAndGet(bodySize);
+                }
                 return true;
             } else {
                 return false;
@@ -305,7 +312,8 @@ public AggregateKey(String topic, MessageQueue mq, boolean waitStoreMsgOK, Strin
             this.tag = tag;
         }
 
-        @Override public boolean equals(Object o) {
+        @Override
+        public boolean equals(Object o) {
             if (this == o)
                 return true;
             if (o == null || getClass() != o.getClass())
@@ -314,7 +322,8 @@ public AggregateKey(String topic, MessageQueue mq, boolean waitStoreMsgOK, Strin
             return waitStoreMsgOK == key.waitStoreMsgOK && topic.equals(key.topic) && Objects.equals(mq, key.mq) && Objects.equals(tag, key.tag);
         }
 
-        @Override public int hashCode() {
+        @Override
+        public int hashCode() {
             return Objects.hash(topic, mq, waitStoreMsgOK, tag);
         }
     }
@@ -324,7 +333,7 @@ private class MessageAccumulation {
         private LinkedList<Message> messages;
         private LinkedList<SendCallback> sendCallbacks;
         private Set<String> keys;
-        private AtomicBoolean closed;
+        private final AtomicBoolean closed;
         private SendResult[] sendResults;
         private AggregateKey aggregateKey;
         private AtomicInteger messagesSize;
@@ -351,8 +360,7 @@ private boolean readyToSend() {
             return false;
         }
 
-        public int add(
-            Message msg) throws InterruptedException, MQBrokerException, RemotingException, MQClientException {
+        public int add(Message msg) throws InterruptedException, MQBrokerException, RemotingException, MQClientException {
             int ret = -1;
             synchronized (this.closed) {
                 if (this.closed.get()) {
@@ -360,7 +368,10 @@ public int add(
                 }
                 ret = this.count++;
                 this.messages.add(msg);
-                messagesSize.addAndGet(msg.getBody().length);
+                int bodySize = null == msg.getBody() ? 0 : msg.getBody().length;
+                if (bodySize > 0) {
+                    messagesSize.addAndGet(bodySize);
+                }
                 String msgKeys = msg.getKeys();
                 if (msgKeys != null) {
                     this.keys.addAll(Arrays.asList(msgKeys.split(MessageConst.KEY_SEPARATOR)));
@@ -388,7 +399,10 @@ public boolean add(Message msg,
                 this.count++;
                 this.messages.add(msg);
                 this.sendCallbacks.add(sendCallback);
-                messagesSize.getAndAdd(msg.getBody().length);
+                int bodySize = null == msg.getBody() ? 0 : msg.getBody().length;
+                if (bodySize > 0) {
+                    messagesSize.addAndGet(bodySize);
+                }
             }
             if (readyToSend()) {
                 this.send(sendCallback);
@@ -472,7 +486,8 @@ private void send(SendCallback sendCallback) {
                 if (defaultMQProducer != null) {
                     final int size = messagesSize.get();
                     defaultMQProducer.sendDirect(messageBatch, aggregateKey.mq, new SendCallback() {
-                        @Override public void onSuccess(SendResult sendResult) {
+                        @Override
+                        public void onSuccess(SendResult sendResult) {
                             try {
                                 splitSendResults(sendResult);
                                 int i = 0;
@@ -490,7 +505,8 @@ private void send(SendCallback sendCallback) {
                             }
                         }
 
-                        @Override public void onException(Throwable e) {
+                        @Override
+                        public void onException(Throwable e) {
                             for (SendCallback v : sendCallbacks) {
                                 v.onException(e);
                             }
diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java
index e66c08fdc53..203f92907a4 100644
--- a/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java
+++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java
@@ -107,6 +107,10 @@ public void setSendRequestOk(boolean sendRequestOk) {
         this.sendRequestOk = sendRequestOk;
     }
 
+    public void acquireCountDownLatch() {
+        this.countDownLatch.countDown();
+    }
+
     public Message getRequestMsg() {
         return requestMsg;
     }
diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java b/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java
index dd7ea1cdc5f..d160eb4eae9 100644
--- a/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java
+++ b/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java
@@ -29,6 +29,7 @@ public class SendResult {
     private String regionId;
     private boolean traceOn = true;
     private byte[] rawRespBody;
+    private String recallHandle;
 
     public SendResult() {
     }
@@ -126,10 +127,18 @@ public void setOffsetMsgId(String offsetMsgId) {
         this.offsetMsgId = offsetMsgId;
     }
 
+    public String getRecallHandle() {
+        return recallHandle;
+    }
+
+    public void setRecallHandle(String recallHandle) {
+        this.recallHandle = recallHandle;
+    }
+
     @Override
     public String toString() {
         return "SendResult [sendStatus=" + sendStatus + ", msgId=" + msgId + ", offsetMsgId=" + offsetMsgId + ", messageQueue=" + messageQueue
-            + ", queueOffset=" + queueOffset + "]";
+            + ", queueOffset=" + queueOffset + ", recallHandle=" + recallHandle + "]";
     }
 
     public void setRawRespBody(byte[] body) {
diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java
index ea423b71766..c76fea74921 100644
--- a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java
+++ b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java
@@ -21,8 +21,8 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
-import java.util.UUID;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -44,67 +44,63 @@
 import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.topic.TopicValidator;
-import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.common.utils.ThreadUtils;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.RPCHook;
 
 import static org.apache.rocketmq.client.trace.TraceConstants.TRACE_INSTANCE_NAME;
 
 public class AsyncTraceDispatcher implements TraceDispatcher {
-    private final static Logger log = LoggerFactory.getLogger(AsyncTraceDispatcher.class);
-    private final static AtomicInteger COUNTER = new AtomicInteger();
-    private final static short MAX_MSG_KEY_SIZE = Short.MAX_VALUE - 10000;
-    private final int queueSize;
-    private final int batchSize;
+    private static final Logger log = LoggerFactory.getLogger(AsyncTraceDispatcher.class);
+    private static final AtomicInteger COUNTER = new AtomicInteger();
+    private static final AtomicInteger INSTANCE_NUM = new AtomicInteger(0);
+    private static final long WAIT_FOR_SHUTDOWN = 5000L;
+    private volatile boolean stopped = false;
+    private final int traceInstanceId = INSTANCE_NUM.getAndIncrement();
+    private final int batchNum;
     private final int maxMsgSize;
-    private final long pollingTimeMil;
-    private final long waitTimeThresholdMil;
     private final DefaultMQProducer traceProducer;
-    private final ThreadPoolExecutor traceExecutor;
-    // The last discard number of log
     private AtomicLong discardCount;
     private Thread worker;
+    private final ThreadPoolExecutor traceExecutor;
     private final ArrayBlockingQueue<TraceContext> traceContextQueue;
-    private final HashMap<String, TraceDataSegment> taskQueueByTopic;
-    private ArrayBlockingQueue<Runnable> appenderQueue;
+    private final ArrayBlockingQueue<Runnable> appenderQueue;
     private volatile Thread shutDownHook;
-    private volatile boolean stopped = false;
+
     private DefaultMQProducerImpl hostProducer;
     private DefaultMQPushConsumerImpl hostConsumer;
     private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex();
-    private String dispatcherId = UUID.randomUUID().toString();
     private volatile String traceTopicName;
     private AtomicBoolean isStarted = new AtomicBoolean(false);
     private volatile AccessChannel accessChannel = AccessChannel.LOCAL;
     private String group;
     private Type type;
+    private String namespaceV2;
+    private final int flushTraceInterval = 5000;
+
+    private long lastFlushTime = System.currentTimeMillis();
 
-    public AsyncTraceDispatcher(String group, Type type, String traceTopicName, RPCHook rpcHook) {
-        // queueSize is greater than or equal to the n power of 2 of value
-        this.queueSize = 2048;
-        this.batchSize = 100;
+    public AsyncTraceDispatcher(String group, Type type, int batchNum, String traceTopicName, RPCHook rpcHook) {
+        this.batchNum = Math.min(batchNum, 20);/* max value 20*/
         this.maxMsgSize = 128000;
-        this.pollingTimeMil = 100;
-        this.waitTimeThresholdMil = 500;
         this.discardCount = new AtomicLong(0L);
-        this.traceContextQueue = new ArrayBlockingQueue<>(1024);
-        this.taskQueueByTopic = new HashMap();
+        this.traceContextQueue = new ArrayBlockingQueue<>(2048);
         this.group = group;
         this.type = type;
-
-        this.appenderQueue = new ArrayBlockingQueue<>(queueSize);
+        this.appenderQueue = new ArrayBlockingQueue<>(2048);
         if (!UtilAll.isBlank(traceTopicName)) {
             this.traceTopicName = traceTopicName;
         } else {
             this.traceTopicName = TopicValidator.RMQ_SYS_TRACE_TOPIC;
         }
         this.traceExecutor = new ThreadPoolExecutor(//
-            10, //
-            20, //
+            2, //
+            4, //
             1000 * 60, //
             TimeUnit.MILLISECONDS, //
             this.appenderQueue, //
-            new ThreadFactoryImpl("MQTraceSendThread_"));
+            new ThreadFactoryImpl("MQTraceSendThread_" + traceInstanceId + "_"));
         traceProducer = getAndCreateTraceProducer(rpcHook);
     }
 
@@ -144,14 +140,25 @@ public void setHostConsumer(DefaultMQPushConsumerImpl hostConsumer) {
         this.hostConsumer = hostConsumer;
     }
 
+    public String getNamespaceV2() {
+        return namespaceV2;
+    }
+
+    public void setNamespaceV2(String namespaceV2) {
+        this.namespaceV2 = namespaceV2;
+    }
+
     public void start(String nameSrvAddr, AccessChannel accessChannel) throws MQClientException {
         if (isStarted.compareAndSet(false, true)) {
             traceProducer.setNamesrvAddr(nameSrvAddr);
             traceProducer.setInstanceName(TRACE_INSTANCE_NAME + "_" + nameSrvAddr);
+            traceProducer.setNamespaceV2(namespaceV2);
+            traceProducer.setEnableTrace(false);
             traceProducer.start();
         }
         this.accessChannel = accessChannel;
-        this.worker = new Thread(new AsyncRunnable(), "MQ-AsyncTraceDispatcher-Thread-" + dispatcherId);
+        this.worker = new ThreadFactoryImpl("MQ-AsyncArrayDispatcher-Thread" + traceInstanceId, true)
+            .newThread(new AsyncRunnable());
         this.worker.setDaemon(true);
         this.worker.start();
         this.registerShutDownHook();
@@ -185,37 +192,24 @@ public boolean append(final Object ctx) {
 
     @Override
     public void flush() {
-        // The maximum waiting time for refresh,avoid being written all the time, resulting in failure to return.
-        long end = System.currentTimeMillis() + 500;
-        while (System.currentTimeMillis() <= end) {
-            synchronized (taskQueueByTopic) {
-                for (TraceDataSegment taskInfo : taskQueueByTopic.values()) {
-                    taskInfo.sendAllData();
-                }
-            }
-            synchronized (traceContextQueue) {
-                if (traceContextQueue.size() == 0 && appenderQueue.size() == 0) {
-                    break;
-                }
-            }
+        while (traceContextQueue.size() > 0) {
             try {
-                Thread.sleep(1);
-            } catch (InterruptedException e) {
-                break;
+                flushTraceContext(true);
+            } catch (Throwable throwable) {
+                log.error("flushTraceContext error", throwable);
             }
         }
-        log.info("------end trace send " + traceContextQueue.size() + "   " + appenderQueue.size());
     }
 
     @Override
     public void shutdown() {
-        this.stopped = true;
         flush();
-        this.traceExecutor.shutdown();
+        ThreadUtils.shutdownGracefully(this.traceExecutor, WAIT_FOR_SHUTDOWN, TimeUnit.MILLISECONDS);
         if (isStarted.get()) {
             traceProducer.shutdown();
         }
         this.removeShutdownHook();
+        stopped = true;
     }
 
     public void registerShutDownHook() {
@@ -247,152 +241,122 @@ public void removeShutdownHook() {
     }
 
     class AsyncRunnable implements Runnable {
-        private boolean stopped;
+        private volatile boolean stopped = false;
 
         @Override
         public void run() {
             while (!stopped) {
-                synchronized (traceContextQueue) {
-                    long endTime = System.currentTimeMillis() + pollingTimeMil;
-                    while (System.currentTimeMillis() < endTime) {
-                        try {
-                            TraceContext traceContext = traceContextQueue.poll(
-                                endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS
-                            );
-
-                            if (traceContext != null && !traceContext.getTraceBeans().isEmpty()) {
-                                // get the topic which the trace message will send to
-                                String traceTopicName = this.getTraceTopicName(traceContext.getRegionId());
-
-                                // get the traceDataSegment which will save this trace message, create if null
-                                TraceDataSegment traceDataSegment = taskQueueByTopic.get(traceTopicName);
-                                if (traceDataSegment == null) {
-                                    traceDataSegment = new TraceDataSegment(traceTopicName, traceContext.getRegionId());
-                                    taskQueueByTopic.put(traceTopicName, traceDataSegment);
-                                }
-
-                                // encode traceContext and save it into traceDataSegment
-                                // NOTE if data size in traceDataSegment more than maxMsgSize,
-                                //  a AsyncDataSendTask will be created and submitted
-                                TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(traceContext);
-                                traceDataSegment.addTraceTransferBean(traceTransferBean);
-                            }
-                        } catch (InterruptedException ignore) {
-                            log.debug("traceContextQueue#poll exception");
-                        }
-                    }
-
-                    // NOTE send the data in traceDataSegment which the first TraceTransferBean
-                    //  is longer than waitTimeThreshold
-                    sendDataByTimeThreshold();
+                try {
+                    flushTraceContext(false);
+                } catch (Throwable e) {
+                    log.error("flushTraceContext error", e);
+                }
 
-                    if (AsyncTraceDispatcher.this.stopped) {
-                        this.stopped = true;
-                    }
+                if (AsyncTraceDispatcher.this.stopped) {
+                    this.stopped = true;
                 }
             }
-
         }
+    }
 
-        private void sendDataByTimeThreshold() {
-            long now = System.currentTimeMillis();
-            for (TraceDataSegment taskInfo : taskQueueByTopic.values()) {
-                if (now - taskInfo.firstBeanAddTime >= waitTimeThresholdMil) {
-                    taskInfo.sendAllData();
+    private void flushTraceContext(boolean forceFlush) throws InterruptedException {
+        List<TraceContext> contextList = new ArrayList<>(batchNum);
+        int size = traceContextQueue.size();
+        if (size != 0) {
+            if (forceFlush || size >= batchNum || System.currentTimeMillis() - lastFlushTime > flushTraceInterval) {
+                for (int i = 0; i < batchNum; i++) {
+                    TraceContext context = traceContextQueue.poll();
+                    if (context != null) {
+                        contextList.add(context);
+                    } else {
+                        break;
+                    }
                 }
+                asyncSendTraceMessage(contextList);
+                return;
             }
         }
+        // To prevent an infinite loop, add a wait time between each two task executions
+        Thread.sleep(5);
+    }
 
-        private String getTraceTopicName(String regionId) {
-            AccessChannel accessChannel = AsyncTraceDispatcher.this.getAccessChannel();
-            if (AccessChannel.CLOUD == accessChannel) {
-                return TraceConstants.TRACE_TOPIC_PREFIX + regionId;
-            }
-
-            return AsyncTraceDispatcher.this.getTraceTopicName();
-        }
+    private void asyncSendTraceMessage(List<TraceContext> contextList) {
+        AsyncDataSendTask request = new AsyncDataSendTask(contextList);
+        traceExecutor.submit(request);
+        lastFlushTime = System.currentTimeMillis();
     }
 
-    class TraceDataSegment {
-        private long firstBeanAddTime;
-        private int currentMsgSize;
-        private int currentMsgKeySize;
-        private final String traceTopicName;
-        private final String regionId;
-        private final List<TraceTransferBean> traceTransferBeanList = new ArrayList();
+    class AsyncDataSendTask implements Runnable {
+        private final List<TraceContext> contextList;
 
-        TraceDataSegment(String traceTopicName, String regionId) {
-            this.traceTopicName = traceTopicName;
-            this.regionId = regionId;
+        public AsyncDataSendTask(List<TraceContext> contextList) {
+            this.contextList = contextList;
         }
 
-        public void addTraceTransferBean(TraceTransferBean traceTransferBean) {
-            initFirstBeanAddTime();
-            this.traceTransferBeanList.add(traceTransferBean);
-            this.currentMsgSize += traceTransferBean.getTransData().length();
-
-            this.currentMsgKeySize = traceTransferBean.getTransKey().stream()
-                .reduce(currentMsgKeySize, (acc, x) -> acc + x.length(), Integer::sum);
-            if (currentMsgSize >= traceProducer.getMaxMessageSize() - 10 * 1000 || currentMsgKeySize >= MAX_MSG_KEY_SIZE) {
-                List<TraceTransferBean> dataToSend = new ArrayList(traceTransferBeanList);
-                AsyncDataSendTask asyncDataSendTask = new AsyncDataSendTask(traceTopicName, regionId, dataToSend);
-                traceExecutor.submit(asyncDataSendTask);
-                this.clear();
-            }
+        @Override
+        public void run() {
+            sendTraceData(contextList);
         }
 
-        public void sendAllData() {
-            if (this.traceTransferBeanList.isEmpty()) {
-                return;
-            }
-            List<TraceTransferBean> dataToSend = new ArrayList(traceTransferBeanList);
-            AsyncDataSendTask asyncDataSendTask = new AsyncDataSendTask(traceTopicName, regionId, dataToSend);
-            traceExecutor.submit(asyncDataSendTask);
-
-            this.clear();
-        }
+        public void sendTraceData(List<TraceContext> contextList) {
+            Map<String, List<TraceTransferBean>> transBeanMap = new HashMap<>(16);
+            String traceTopic;
+            for (TraceContext context : contextList) {
+                AccessChannel accessChannel = context.getAccessChannel();
+                if (accessChannel == null) {
+                    accessChannel = AsyncTraceDispatcher.this.accessChannel;
+                }
+                String currentRegionId = context.getRegionId();
+                if (currentRegionId == null || context.getTraceBeans().isEmpty()) {
+                    continue;
+                }
+                if (AccessChannel.CLOUD == accessChannel) {
+                    traceTopic = TraceConstants.TRACE_TOPIC_PREFIX + currentRegionId;
+                } else {
+                    traceTopic = traceTopicName;
+                }
 
-        private void initFirstBeanAddTime() {
-            if (firstBeanAddTime == 0) {
-                firstBeanAddTime = System.currentTimeMillis();
+                String topic = context.getTraceBeans().get(0).getTopic();
+                String key = topic + TraceConstants.CONTENT_SPLITOR + traceTopic;
+                List<TraceTransferBean> transBeanList = transBeanMap.computeIfAbsent(key, k -> new ArrayList<>());
+                TraceTransferBean traceData = TraceDataEncoder.encoderFromContextBean(context);
+                transBeanList.add(traceData);
+            }
+            for (Map.Entry<String, List<TraceTransferBean>> entry : transBeanMap.entrySet()) {
+                String[] key = entry.getKey().split(String.valueOf(TraceConstants.CONTENT_SPLITOR));
+                flushData(entry.getValue(), key[0], key[1]);
             }
         }
 
-        private void clear() {
-            this.firstBeanAddTime = 0;
-            this.currentMsgSize = 0;
-            this.currentMsgKeySize = 0;
-            this.traceTransferBeanList.clear();
-        }
-    }
-
-    class AsyncDataSendTask implements Runnable {
-        private final String traceTopicName;
-        private final String regionId;
-        private final List<TraceTransferBean> traceTransferBeanList;
-
-        public AsyncDataSendTask(String traceTopicName, String regionId, List<TraceTransferBean> traceTransferBeanList) {
-            this.traceTopicName = traceTopicName;
-            this.regionId = regionId;
-            this.traceTransferBeanList = traceTransferBeanList;
-        }
-
-        @Override
-        public void run() {
+        private void flushData(List<TraceTransferBean> transBeanList, String topic, String traceTopic) {
+            if (transBeanList.size() == 0) {
+                return;
+            }
             StringBuilder buffer = new StringBuilder(1024);
-            Set<String> keySet = new HashSet<>();
-            for (TraceTransferBean bean : traceTransferBeanList) {
+            int count = 0;
+            Set<String> keySet = new HashSet<String>();
+            for (TraceTransferBean bean : transBeanList) {
                 keySet.addAll(bean.getTransKey());
                 buffer.append(bean.getTransData());
+                count++;
+                if (buffer.length() >= traceProducer.getMaxMessageSize()) {
+                    sendTraceDataByMQ(keySet, buffer.toString(), traceTopic);
+                    buffer.delete(0, buffer.length());
+                    keySet.clear();
+                    count = 0;
+                }
+            }
+            if (count > 0) {
+                sendTraceDataByMQ(keySet, buffer.toString(), traceTopic);
             }
-            sendTraceDataByMQ(keySet, buffer.toString(), traceTopicName);
+            transBeanList.clear();
         }
 
         /**
          * Send message trace data
          *
-         * @param keySet the keyset in this batch(including msgId in original message not offsetMsgId)
-         * @param data   the message trace data in this batch
+         * @param keySet     the keyset in this batch(including msgId in original message not offsetMsgId)
+         * @param data       the message trace data in this batch
          * @param traceTopic the topic which message trace data will send to
          */
         private void sendTraceDataByMQ(Set<String> keySet, final String data, String traceTopic) {
@@ -455,4 +419,4 @@ private Set<String> tryGetMessageQueueBrokerSet(DefaultMQProducerImpl producer,
         }
     }
 
-}
+}
\ No newline at end of file
diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java
index 70c147e1eb3..17db1fbfa15 100644
--- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java
+++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java
@@ -21,7 +21,7 @@
 import org.apache.rocketmq.common.message.MessageType;
 
 public class TraceBean {
-    private static final String LOCAL_ADDRESS = UtilAll.ipToIPv4Str(UtilAll.getIP());
+    private static final String LOCAL_ADDRESS;
     private String topic = "";
     private String msgId = "";
     private String offsetMsgId = "";
@@ -37,6 +37,15 @@ public class TraceBean {
     private String transactionId;
     private boolean fromTransactionCheck;
 
+    static {
+        byte[] ip = UtilAll.getIP();
+        if (ip.length == 4) {
+            LOCAL_ADDRESS = UtilAll.ipToIPv4Str(ip);
+        } else {
+            LOCAL_ADDRESS = UtilAll.ipToIPv6Str(ip);
+        }
+    }
+
     public MessageType getMsgType() {
         return msgType;
     }
diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java
index 1ad4b610515..67f7ab3f8fb 100644
--- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java
+++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java
@@ -32,7 +32,7 @@ public class TraceConstants {
     public static final String ROCKETMQ_SUCCESS = "rocketmq.success";
     public static final String ROCKETMQ_TAGS = "rocketmq.tags";
     public static final String ROCKETMQ_KEYS = "rocketmq.keys";
-    public static final String ROCKETMQ_SOTRE_HOST = "rocketmq.store_host";
+    public static final String ROCKETMQ_STORE_HOST = "rocketmq.store_host";
     public static final String ROCKETMQ_BODY_LENGTH = "rocketmq.body_length";
     public static final String ROCKETMQ_MSG_ID = "rocketmq.mgs_id";
     public static final String ROCKETMQ_MSG_TYPE = "rocketmq.mgs_type";
diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java
index 0fdd95243a5..1e66aa0498d 100644
--- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java
+++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java
@@ -132,6 +132,19 @@ public static List<TraceContext> decoderFromTraceDataString(String traceData) {
                 endTransactionContext.setTraceBeans(new ArrayList<>(1));
                 endTransactionContext.getTraceBeans().add(bean);
                 resList.add(endTransactionContext);
+            } else if (line[0].equals(TraceType.Recall.name())) {
+                TraceContext recallContext = new TraceContext();
+                recallContext.setTraceType(TraceType.Recall);
+                recallContext.setTimeStamp(Long.parseLong(line[1]));
+                recallContext.setRegionId(line[2]);
+                recallContext.setGroupName(line[3]);
+                TraceBean bean = new TraceBean();
+                bean.setTopic(line[4]);
+                bean.setMsgId(line[5]);
+                recallContext.setSuccess(Boolean.parseBoolean(line[6]));
+                recallContext.setTraceBeans(new ArrayList<>(1));
+                recallContext.getTraceBeans().add(bean);
+                resList.add(recallContext);
             }
         }
         return resList;
@@ -193,9 +206,10 @@ public static TraceTransferBean encoderFromContextBean(TraceContext ctx) {
                         .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)//
                         .append(ctx.getContextCode()).append(TraceConstants.CONTENT_SPLITOR);
                     if (!ctx.getAccessChannel().equals(AccessChannel.CLOUD)) {
-                        sb.append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)
-                            .append(ctx.getGroupName()).append(TraceConstants.FIELD_SPLITOR);
+                        sb.append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR);
+                        sb.append(ctx.getGroupName());
                     }
+                    sb.append(TraceConstants.FIELD_SPLITOR);
                 }
             }
             break;
@@ -216,6 +230,17 @@ public static TraceTransferBean encoderFromContextBean(TraceContext ctx) {
                     .append(bean.isFromTransactionCheck()).append(TraceConstants.FIELD_SPLITOR);
             }
             break;
+            case Recall: {
+                TraceBean bean = ctx.getTraceBeans().get(0);
+                sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)
+                    .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)
+                    .append(ctx.getRegionId()).append(TraceConstants.CONTENT_SPLITOR)
+                    .append(ctx.getGroupName()).append(TraceConstants.CONTENT_SPLITOR)
+                    .append(bean.getTopic()).append(TraceConstants.CONTENT_SPLITOR)
+                    .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)
+                    .append(ctx.isSuccess()).append(TraceConstants.FIELD_SPLITOR);//
+            }
+            break;
             default:
         }
         transferBean.setTransData(sb.toString());
diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java
index 8870ddcbdb3..4c0e7d8ab26 100644
--- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java
+++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java
@@ -18,6 +18,7 @@
 
 public enum TraceType {
     Pub,
+    Recall,
     SubBefore,
     SubAfter,
     EndTransaction,
diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/DefaultRecallMessageTraceHook.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/DefaultRecallMessageTraceHook.java
new file mode 100644
index 00000000000..c490a7b3599
--- /dev/null
+++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/DefaultRecallMessageTraceHook.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.client.trace.hook;
+
+import org.apache.rocketmq.client.trace.TraceBean;
+import org.apache.rocketmq.client.trace.TraceContext;
+import org.apache.rocketmq.client.trace.TraceDispatcher;
+import org.apache.rocketmq.client.trace.TraceType;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+
+import java.util.ArrayList;
+
+public class DefaultRecallMessageTraceHook implements RPCHook {
+
+    private static final String RECALL_TRACE_ENABLE_KEY = "com.rocketmq.recall.default.trace.enable";
+    private boolean enableDefaultTrace = Boolean.parseBoolean(System.getProperty(RECALL_TRACE_ENABLE_KEY, "false"));
+    private TraceDispatcher traceDispatcher;
+
+    public DefaultRecallMessageTraceHook(TraceDispatcher traceDispatcher) {
+        this.traceDispatcher = traceDispatcher;
+    }
+
+    @Override
+    public void doBeforeRequest(String remoteAddr, RemotingCommand request) {
+    }
+
+    @Override
+    public void doAfterResponse(String remoteAddr, RemotingCommand request, RemotingCommand response) {
+        if (request.getCode() != RequestCode.RECALL_MESSAGE
+            || !enableDefaultTrace
+            || null == response.getExtFields()
+            || null == response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION)
+            || null == traceDispatcher) {
+            return;
+        }
+
+        try {
+            String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION);
+            RecallMessageRequestHeader requestHeader =
+                request.decodeCommandCustomHeader(RecallMessageRequestHeader.class);
+            String topic = NamespaceUtil.withoutNamespace(requestHeader.getTopic());
+            String group = NamespaceUtil.withoutNamespace(requestHeader.getProducerGroup());
+            String recallHandle = requestHeader.getRecallHandle();
+            RecallMessageHandle.HandleV1 handleV1 =
+                (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(recallHandle);
+
+            TraceBean traceBean = new TraceBean();
+            traceBean.setTopic(topic);
+            traceBean.setMsgId(handleV1.getMessageId());
+
+            TraceContext traceContext = new TraceContext();
+            traceContext.setRegionId(regionId);
+            traceContext.setTraceBeans(new ArrayList<>(1));
+            traceContext.setTraceType(TraceType.Recall);
+            traceContext.setGroupName(group);
+            traceContext.getTraceBeans().add(traceBean);
+            traceContext.setSuccess(ResponseCode.SUCCESS == response.getCode());
+
+            traceDispatcher.append(traceContext);
+        } catch (Exception e) {
+        }
+    }
+}
diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionOpenTracingHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionOpenTracingHookImpl.java
index 62d310f1961..44e4b69776e 100644
--- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionOpenTracingHookImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionOpenTracingHookImpl.java
@@ -60,7 +60,7 @@ public void endTransaction(EndTransactionContext context) {
         span.setTag(Tags.MESSAGE_BUS_DESTINATION, msg.getTopic());
         span.setTag(TraceConstants.ROCKETMQ_TAGS, msg.getTags());
         span.setTag(TraceConstants.ROCKETMQ_KEYS, msg.getKeys());
-        span.setTag(TraceConstants.ROCKETMQ_SOTRE_HOST, context.getBrokerAddr());
+        span.setTag(TraceConstants.ROCKETMQ_STORE_HOST, context.getBrokerAddr());
         span.setTag(TraceConstants.ROCKETMQ_MSG_ID, context.getMsgId());
         span.setTag(TraceConstants.ROCKETMQ_MSG_TYPE, MessageType.Trans_msg_Commit.name());
         span.setTag(TraceConstants.ROCKETMQ_TRANSACTION_ID, context.getTransactionId());
diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java
index 60c18a22a77..0f828f2b4e1 100644
--- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java
@@ -48,8 +48,8 @@ public void sendMessageBefore(SendMessageContext context) {
         }
         Message msg = context.getMessage();
         Tracer.SpanBuilder spanBuilder = tracer
-                .buildSpan(TraceConstants.TO_PREFIX + msg.getTopic())
-                .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_PRODUCER);
+            .buildSpan(TraceConstants.TO_PREFIX + msg.getTopic())
+            .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_PRODUCER);
         SpanContext spanContext = tracer.extract(Format.Builtin.TEXT_MAP, new TextMapAdapter(msg.getProperties()));
         if (spanContext != null) {
             spanBuilder.asChildOf(spanContext);
@@ -60,9 +60,9 @@ public void sendMessageBefore(SendMessageContext context) {
         span.setTag(Tags.MESSAGE_BUS_DESTINATION, msg.getTopic());
         span.setTag(TraceConstants.ROCKETMQ_TAGS, msg.getTags());
         span.setTag(TraceConstants.ROCKETMQ_KEYS, msg.getKeys());
-        span.setTag(TraceConstants.ROCKETMQ_SOTRE_HOST, context.getBrokerAddr());
+        span.setTag(TraceConstants.ROCKETMQ_STORE_HOST, context.getBrokerAddr());
         span.setTag(TraceConstants.ROCKETMQ_MSG_TYPE, context.getMsgType().name());
-        span.setTag(TraceConstants.ROCKETMQ_BODY_LENGTH, msg.getBody().length);
+        span.setTag(TraceConstants.ROCKETMQ_BODY_LENGTH, null == msg.getBody() ? 0 : msg.getBody().length);
         context.setMqTraceContext(span);
     }
 
diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java
index dba04b593f2..61738928bb3 100644
--- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java
@@ -58,7 +58,8 @@ public void sendMessageBefore(SendMessageContext context) {
         traceBean.setTags(context.getMessage().getTags());
         traceBean.setKeys(context.getMessage().getKeys());
         traceBean.setStoreHost(context.getBrokerAddr());
-        traceBean.setBodyLength(context.getMessage().getBody().length);
+        int bodyLength = null == context.getMessage().getBody() ? 0 : context.getMessage().getBody().length;
+        traceBean.setBodyLength(bodyLength);
         traceBean.setMsgType(context.getMsgType());
         traceContext.getTraceBeans().add(traceBean);
     }
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/AclClientRPCHookTest.java b/client/src/test/java/org/apache/rocketmq/acl/common/AclClientRPCHookTest.java
similarity index 99%
rename from acl/src/test/java/org/apache/rocketmq/acl/common/AclClientRPCHookTest.java
rename to client/src/test/java/org/apache/rocketmq/acl/common/AclClientRPCHookTest.java
index 9789ed191c4..8d99407cfb9 100644
--- a/acl/src/test/java/org/apache/rocketmq/acl/common/AclClientRPCHookTest.java
+++ b/client/src/test/java/org/apache/rocketmq/acl/common/AclClientRPCHookTest.java
@@ -17,10 +17,6 @@
 
 package org.apache.rocketmq.acl.common;
 
-import java.lang.reflect.Field;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
@@ -29,6 +25,11 @@
 import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
 import org.junit.Test;
 
+import java.lang.reflect.Field;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+
 import static org.apache.rocketmq.acl.common.SessionCredentials.ACCESS_KEY;
 import static org.apache.rocketmq.acl.common.SessionCredentials.SECURITY_TOKEN;
 import static org.assertj.core.api.Assertions.assertThat;
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/AclSignerTest.java b/client/src/test/java/org/apache/rocketmq/acl/common/AclSignerTest.java
similarity index 100%
rename from acl/src/test/java/org/apache/rocketmq/acl/common/AclSignerTest.java
rename to client/src/test/java/org/apache/rocketmq/acl/common/AclSignerTest.java
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java b/client/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java
similarity index 72%
rename from acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java
rename to client/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java
index 03bceade770..2169144c88d 100644
--- a/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java
+++ b/client/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java
@@ -16,10 +16,8 @@
  */
 package org.apache.rocketmq.acl.common;
 
-import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson2.JSONObject;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.acl.plain.PlainAccessData;
-import org.apache.rocketmq.common.PlainAccessConfig;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.junit.Assert;
 import org.junit.Test;
@@ -31,9 +29,13 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
+import java.util.Objects;
 import java.util.UUID;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
 public class AclUtilsTest {
 
     @Test
@@ -201,16 +203,6 @@ public void testExpandIP() {
         Assert.assertEquals(AclUtils.expandIP("5::7:6", 6), "0005:0000:0000:0000:0007:0006");
     }
 
-    @SuppressWarnings("unchecked")
-    @Test
-    public void testGetYamlDataObject() throws IOException {
-        try (InputStream is = AclUtilsTest.class.getClassLoader().getResourceAsStream("conf/plain_acl_correct.yml")) {
-            Map<String, Object> map = AclUtils.getYamlDataObject(is, Map.class);
-            Assert.assertNotNull(map);
-            Assert.assertFalse(map.isEmpty());
-        }
-    }
-
     private static String randomTmpFile() {
         String tmpFileName = System.getProperty("java.io.tmpdir");
         // https://rationalpi.wordpress.com/2007/01/26/javaiotmpdir-inconsitency/
@@ -221,67 +213,6 @@ private static String randomTmpFile() {
         return tmpFileName;
     }
 
-    @Test
-    public void writeDataObject2YamlFileTest() throws IOException {
-        String targetFileName = randomTmpFile();
-        File transport = new File(targetFileName);
-        Assert.assertTrue(transport.createNewFile());
-        transport.deleteOnExit();
-
-        PlainAccessData aclYamlMap = new PlainAccessData();
-
-        // For globalWhiteRemoteAddrs element in acl yaml config file
-        List<String> globalWhiteRemoteAddrs = new ArrayList<>();
-        globalWhiteRemoteAddrs.add("10.10.103.*");
-        globalWhiteRemoteAddrs.add("192.168.0.*");
-        aclYamlMap.setGlobalWhiteRemoteAddresses(globalWhiteRemoteAddrs);
-
-        // For accounts element in acl yaml config file
-        List<PlainAccessConfig> accounts = new ArrayList<>();
-        PlainAccessConfig accountsMap = new PlainAccessConfig() {
-            {
-                setAccessKey("RocketMQ");
-                setSecretKey("12345678");
-                setWhiteRemoteAddress("whiteRemoteAddress");
-                setAdmin(true);
-            }
-        };
-        accounts.add(accountsMap);
-        aclYamlMap.setAccounts(accounts);
-        Assert.assertTrue(AclUtils.writeDataObject(targetFileName, aclYamlMap));
-    }
-
-    @Test
-    public void updateExistedYamlFileTest() throws IOException {
-        String targetFileName = randomTmpFile();
-        File transport = new File(targetFileName);
-        Assert.assertTrue(transport.createNewFile());
-        transport.deleteOnExit();
-
-        PlainAccessData aclYamlMap = new PlainAccessData();
-
-        // For globalWhiteRemoteAddrs element in acl yaml config file
-        List<String> globalWhiteRemoteAddrs = new ArrayList<>();
-        globalWhiteRemoteAddrs.add("10.10.103.*");
-        globalWhiteRemoteAddrs.add("192.168.0.*");
-        aclYamlMap.setGlobalWhiteRemoteAddresses(globalWhiteRemoteAddrs);
-
-        // Write file to yaml file
-        AclUtils.writeDataObject(targetFileName, aclYamlMap);
-
-        PlainAccessData updatedMap = AclUtils.getYamlDataObject(targetFileName, PlainAccessData.class);
-        List<String> globalWhiteRemoteAddrList = updatedMap.getGlobalWhiteRemoteAddresses();
-        globalWhiteRemoteAddrList.clear();
-        globalWhiteRemoteAddrList.add("192.168.1.2");
-
-        // Update file and flush to yaml file
-        AclUtils.writeDataObject(targetFileName, updatedMap);
-
-        PlainAccessData readableMap = AclUtils.getYamlDataObject(targetFileName, PlainAccessData.class);
-        List<String> updatedGlobalWhiteRemoteAddrs = readableMap.getGlobalWhiteRemoteAddresses();
-        Assert.assertEquals("192.168.1.2", updatedGlobalWhiteRemoteAddrs.get(0));
-    }
-
     @Test
     public void getYamlDataIgnoreFileNotFoundExceptionTest() {
 
@@ -296,4 +227,25 @@ public void getAclRPCHookTest() throws IOException {
             Assert.assertNull(incompleteContRPCHook);
         }
     }
+
+    @Test
+    public void testGetAclRPCHookByFileName() {
+        RPCHook actual = AclUtils.getAclRPCHook(Objects.requireNonNull(AclUtilsTest.class.getResource("/acl_hook/plain_acl.yml")).getPath());
+        assertNotNull(actual);
+        assertTrue(actual instanceof AclClientRPCHook);
+        assertAclClientRPCHook((AclClientRPCHook) actual);
+    }
+
+    @Test
+    public void testGetAclRPCHookByInputStream() {
+        RPCHook actual = AclUtils.getAclRPCHook(Objects.requireNonNull(AclUtilsTest.class.getResourceAsStream("/acl_hook/plain_acl.yml")));
+        assertNotNull(actual);
+        assertTrue(actual instanceof AclClientRPCHook);
+        assertAclClientRPCHook((AclClientRPCHook) actual);
+    }
+
+    private void assertAclClientRPCHook(final AclClientRPCHook actual) {
+        assertEquals("rocketmq2", actual.getSessionCredentials().getAccessKey());
+        assertEquals("12345678", actual.getSessionCredentials().getSecretKey());
+    }
 }
diff --git a/client/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java b/client/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java
new file mode 100644
index 00000000000..d23f11b6807
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.acl.common;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class PermissionTest {
+
+    @Test
+    public void fromStringGetPermissionTest() {
+        byte perm = Permission.parsePermFromString("PUB");
+        Assert.assertEquals(perm, Permission.PUB);
+
+        perm = Permission.parsePermFromString("SUB");
+        Assert.assertEquals(perm, Permission.SUB);
+
+        perm = Permission.parsePermFromString("PUB|SUB");
+        Assert.assertEquals(perm, Permission.PUB | Permission.SUB);
+
+        perm = Permission.parsePermFromString("SUB|PUB");
+        Assert.assertEquals(perm, Permission.PUB | Permission.SUB);
+
+        perm = Permission.parsePermFromString("DENY");
+        Assert.assertEquals(perm, Permission.DENY);
+
+        perm = Permission.parsePermFromString("1");
+        Assert.assertEquals(perm, Permission.DENY);
+
+        perm = Permission.parsePermFromString(null);
+        Assert.assertEquals(perm, Permission.DENY);
+
+    }
+
+    @Test
+    public void AclExceptionTest() {
+        AclException aclException = new AclException("CAL_SIGNATURE_FAILED",10015);
+        AclException aclExceptionWithMessage = new AclException("CAL_SIGNATURE_FAILED",10015,"CAL_SIGNATURE_FAILED Exception");
+        Assert.assertEquals(aclException.getCode(),10015);
+        Assert.assertEquals(aclExceptionWithMessage.getStatus(),"CAL_SIGNATURE_FAILED");
+        aclException.setCode(10016);
+        Assert.assertEquals(aclException.getCode(),10016);
+        aclException.setStatus("netAddress examine scope Exception netAddress");
+        Assert.assertEquals(aclException.getStatus(),"netAddress examine scope Exception netAddress");
+    }
+}
diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/SessionCredentialsTest.java b/client/src/test/java/org/apache/rocketmq/acl/common/SessionCredentialsTest.java
similarity index 100%
rename from acl/src/test/java/org/apache/rocketmq/acl/common/SessionCredentialsTest.java
rename to client/src/test/java/org/apache/rocketmq/acl/common/SessionCredentialsTest.java
diff --git a/client/src/test/java/org/apache/rocketmq/client/ClientConfigTest.java b/client/src/test/java/org/apache/rocketmq/client/ClientConfigTest.java
new file mode 100644
index 00000000000..5afe9cc011d
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/ClientConfigTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client;
+
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.protocol.LanguageCode;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ClientConfigTest {
+
+    private ClientConfig clientConfig;
+
+    private final String resource = "resource";
+
+    @Before
+    public void init() {
+        clientConfig = createClientConfig();
+    }
+
+    @Test
+    public void testWithNamespace() {
+        Set<String> resources = clientConfig.withNamespace(Collections.singleton(resource));
+        assertTrue(resources.contains("lmq%resource"));
+    }
+
+    @Test
+    public void testWithoutNamespace() {
+        String actual = clientConfig.withoutNamespace(resource);
+        assertEquals(resource, actual);
+        Set<String> resources = clientConfig.withoutNamespace(Collections.singleton(resource));
+        assertTrue(resources.contains(resource));
+    }
+
+    @Test
+    public void testQueuesWithNamespace() {
+        MessageQueue messageQueue = new MessageQueue();
+        messageQueue.setTopic("defaultTopic");
+        Collection<MessageQueue> messageQueues = clientConfig.queuesWithNamespace(Collections.singleton(messageQueue));
+        assertTrue(messageQueues.contains(messageQueue));
+        assertEquals("lmq%defaultTopic", messageQueues.iterator().next().getTopic());
+    }
+
+    private ClientConfig createClientConfig() {
+        ClientConfig result = new ClientConfig();
+        result.setUnitName("unitName");
+        result.setClientIP("127.0.0.1");
+        result.setClientCallbackExecutorThreads(1);
+        result.setPollNameServerInterval(1000 * 30);
+        result.setHeartbeatBrokerInterval(1000 * 30);
+        result.setPersistConsumerOffsetInterval(1000 * 5);
+        result.setPullTimeDelayMillsWhenException(1000);
+        result.setUnitMode(true);
+        result.setSocksProxyConfig("{}");
+        result.setLanguage(LanguageCode.JAVA);
+        result.setDecodeReadBody(true);
+        result.setDecodeDecompressBody(true);
+        result.setAccessChannel(AccessChannel.LOCAL);
+        result.setMqClientApiTimeout(1000 * 3);
+        result.setEnableStreamRequestType(true);
+        result.setSendLatencyEnable(true);
+        result.setEnableHeartbeatChannelEventListener(true);
+        result.setDetectTimeout(200);
+        result.setDetectInterval(1000 * 2);
+        result.setUseHeartbeatV2(false);
+        result.buildMQClientId();
+        result.setNamespace("lmq");
+        return result;
+    }
+}
diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java
index 24e39f56689..592c247057b 100644
--- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java
@@ -74,11 +74,12 @@
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 
-@RunWith(MockitoJUnitRunner.class)
+@RunWith(MockitoJUnitRunner.Silent.class)
 public class DefaultLitePullConsumerTest {
     @Spy
     private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig());
@@ -743,7 +744,7 @@ public PullResult answer(InvocationOnMock mock) throws Throwable {
                 }
             });
 
-        when(mQClientFactory.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(new FindBrokerResult("127.0.0.1:10911", false));
+        doAnswer(x -> new FindBrokerResult("127.0.0.1:10911", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean());
 
         doReturn(Collections.singletonList(mQClientFactory.getClientId())).when(mQClientFactory).findConsumerIdList(anyString(), anyString());
 
diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java
index 3943b922899..834be5cf16f 100644
--- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java
@@ -16,20 +16,6 @@
  */
 package org.apache.rocketmq.client.consumer;
 
-import java.io.ByteArrayOutputStream;
-import java.lang.reflect.Field;
-import java.net.InetSocketAddress;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
 import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
@@ -37,6 +23,8 @@
 import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
 import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
 import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
+import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;
+import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragelyByCircle;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.impl.CommunicationMode;
@@ -53,6 +41,7 @@
 import org.apache.rocketmq.client.impl.consumer.PullResultExt;
 import org.apache.rocketmq.client.impl.consumer.RebalanceImpl;
 import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.message.MessageClientExt;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
@@ -62,7 +51,6 @@
 import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
 import org.junit.AfterClass;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -71,8 +59,27 @@
 import org.mockito.junit.MockitoJUnitRunner;
 import org.mockito.stubbing.Answer;
 
+import java.io.ByteArrayOutputStream;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anyInt;
@@ -80,6 +87,7 @@
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.nullable;
 import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 
@@ -201,12 +209,14 @@ public PullResult answer(InvocationOnMock mock) throws Throwable {
 
     @AfterClass
     public static void terminate() {
-        pushConsumer.shutdown();
+        if (pushConsumer != null) {
+            pushConsumer.shutdown();
+        }
     }
 
     @Test
     public void testStart_OffsetShouldNotNUllAfterStart() {
-        Assert.assertNotNull(pushConsumer.getOffsetStore());
+        assertNotNull(pushConsumer.getOffsetStore());
     }
 
     @Test
@@ -388,4 +398,22 @@ public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
         pullMessageService.executePullRequestImmediately(createPullRequest());
         assertThat(messageExts[0]).isNull();
     }
+
+    @Test
+    public void assertCreatePushConsumer() {
+        DefaultMQPushConsumer pushConsumer1 = new DefaultMQPushConsumer(consumerGroup, mock(RPCHook.class));
+        assertNotNull(pushConsumer1);
+        assertEquals(consumerGroup, pushConsumer1.getConsumerGroup());
+        assertTrue(pushConsumer1.getAllocateMessageQueueStrategy() instanceof AllocateMessageQueueAveragely);
+        assertNotNull(pushConsumer1.defaultMQPushConsumerImpl);
+        assertFalse(pushConsumer1.isEnableTrace());
+        assertTrue(UtilAll.isBlank(pushConsumer1.getTraceTopic()));
+        DefaultMQPushConsumer pushConsumer2 = new DefaultMQPushConsumer(consumerGroup, mock(RPCHook.class), new AllocateMessageQueueAveragelyByCircle());
+        assertNotNull(pushConsumer2);
+        assertEquals(consumerGroup, pushConsumer2.getConsumerGroup());
+        assertTrue(pushConsumer2.getAllocateMessageQueueStrategy() instanceof AllocateMessageQueueAveragelyByCircle);
+        assertNotNull(pushConsumer2.defaultMQPushConsumerImpl);
+        assertFalse(pushConsumer2.isEnableTrace());
+        assertTrue(UtilAll.isBlank(pushConsumer2.getTraceTopic()));
+    }
 }
diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/store/ControllableOffsetTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/store/ControllableOffsetTest.java
new file mode 100644
index 00000000000..23a56ca51ca
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/consumer/store/ControllableOffsetTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.client.consumer.store;
+
+
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
+public class ControllableOffsetTest {
+
+    private ControllableOffset controllableOffset;
+
+    @Before
+    public void setUp() {
+        controllableOffset = new ControllableOffset(0);
+    }
+
+    @Test
+    public void testUpdateAndFreeze_ShouldFreezeOffsetAtTargetValue() {
+        controllableOffset.updateAndFreeze(100);
+        assertEquals(100, controllableOffset.getOffset());
+        controllableOffset.update(200);
+        assertEquals(100, controllableOffset.getOffset());
+    }
+
+    @Test
+    public void testUpdate_ShouldUpdateOffsetWhenNotFrozen() {
+        controllableOffset.update(200);
+        assertEquals(200, controllableOffset.getOffset());
+    }
+
+    @Test
+    public void testUpdate_ShouldNotUpdateOffsetWhenFrozen() {
+        controllableOffset.updateAndFreeze(100);
+        controllableOffset.update(200);
+        assertEquals(100, controllableOffset.getOffset());
+    }
+
+    @Test
+    public void testUpdate_ShouldNotDecreaseOffsetWhenIncreaseOnly() {
+        controllableOffset.update(200);
+        controllableOffset.update(100, true);
+        assertEquals(200, controllableOffset.getOffset());
+    }
+
+    @Test
+    public void testUpdate_ShouldUpdateOffsetToGreaterValueWhenIncreaseOnly() {
+        controllableOffset.update(100);
+        controllableOffset.update(200, true);
+        assertEquals(200, controllableOffset.getOffset());
+    }
+
+}
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/ClientRemotingProcessorTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/ClientRemotingProcessorTest.java
new file mode 100644
index 00000000000..ed31aa10430
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/ClientRemotingProcessorTest.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.impl;
+
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.client.ClientConfig;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+import org.apache.rocketmq.client.impl.producer.MQProducerInner;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;
+import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;
+import org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;
+import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.NotifyConsumerIdsChangedRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ReplyMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+
+import static org.apache.rocketmq.common.message.MessageDecoder.NAME_VALUE_SEPARATOR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ClientRemotingProcessorTest {
+
+    @Mock
+    private MQClientInstance mQClientFactory;
+
+    private ClientRemotingProcessor processor;
+
+    private final String defaultTopic = "defaultTopic";
+
+    private final String defaultBroker = "defaultBroker";
+
+    private final String defaultGroup = "defaultGroup";
+
+    @Before
+    public void init() throws RemotingException, InterruptedException, MQClientException {
+        processor = new ClientRemotingProcessor(mQClientFactory);
+        ClientConfig clientConfig = mock(ClientConfig.class);
+        when(clientConfig.getNamespace()).thenReturn("namespace");
+        when(mQClientFactory.getClientConfig()).thenReturn(clientConfig);
+        MQProducerInner producerInner = mock(MQProducerInner.class);
+        when(mQClientFactory.selectProducer(defaultGroup)).thenReturn(producerInner);
+    }
+
+    @Test
+    public void testCheckTransactionState() throws Exception {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        RemotingCommand request = mock(RemotingCommand.class);
+        when(request.getCode()).thenReturn(RequestCode.CHECK_TRANSACTION_STATE);
+        when(request.getBody()).thenReturn(getMessageResult());
+        CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader();
+        when(request.decodeCommandCustomHeader(CheckTransactionStateRequestHeader.class)).thenReturn(requestHeader);
+        assertNull(processor.processRequest(ctx, request));
+    }
+
+    @Test
+    public void testNotifyConsumerIdsChanged() throws Exception {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        RemotingCommand request = mock(RemotingCommand.class);
+        when(request.getCode()).thenReturn(RequestCode.NOTIFY_CONSUMER_IDS_CHANGED);
+        NotifyConsumerIdsChangedRequestHeader requestHeader = new NotifyConsumerIdsChangedRequestHeader();
+        when(request.decodeCommandCustomHeader(NotifyConsumerIdsChangedRequestHeader.class)).thenReturn(requestHeader);
+        assertNull(processor.processRequest(ctx, request));
+    }
+
+    @Test
+    public void testResetOffset() throws Exception {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        RemotingCommand request = mock(RemotingCommand.class);
+        when(request.getCode()).thenReturn(RequestCode.RESET_CONSUMER_CLIENT_OFFSET);
+        ResetOffsetBody offsetBody = new ResetOffsetBody();
+        when(request.getBody()).thenReturn(RemotingSerializable.encode(offsetBody));
+        ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader();
+        when(request.decodeCommandCustomHeader(ResetOffsetRequestHeader.class)).thenReturn(requestHeader);
+        assertNull(processor.processRequest(ctx, request));
+    }
+
+    @Test
+    public void testGetConsumeStatus() throws Exception {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        RemotingCommand request = mock(RemotingCommand.class);
+        when(request.getCode()).thenReturn(RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT);
+        GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader();
+        when(request.decodeCommandCustomHeader(GetConsumerStatusRequestHeader.class)).thenReturn(requestHeader);
+        assertNotNull(processor.processRequest(ctx, request));
+    }
+
+    @Test
+    public void testGetConsumerRunningInfo() throws Exception {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        RemotingCommand request = mock(RemotingCommand.class);
+        when(request.getCode()).thenReturn(RequestCode.GET_CONSUMER_RUNNING_INFO);
+        ConsumerRunningInfo consumerRunningInfo = new ConsumerRunningInfo();
+        consumerRunningInfo.setJstack("jstack");
+        when(mQClientFactory.consumerRunningInfo(anyString())).thenReturn(consumerRunningInfo);
+        GetConsumerRunningInfoRequestHeader requestHeader = new GetConsumerRunningInfoRequestHeader();
+        requestHeader.setJstackEnable(true);
+        requestHeader.setConsumerGroup(defaultGroup);
+        when(request.decodeCommandCustomHeader(GetConsumerRunningInfoRequestHeader.class)).thenReturn(requestHeader);
+        RemotingCommand command = processor.processRequest(ctx, request);
+        assertNotNull(command);
+        assertEquals(ResponseCode.SUCCESS, command.getCode());
+    }
+
+    @Test
+    public void testConsumeMessageDirectly() throws Exception {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        RemotingCommand request = mock(RemotingCommand.class);
+        when(request.getCode()).thenReturn(RequestCode.CONSUME_MESSAGE_DIRECTLY);
+        when(request.getBody()).thenReturn(getMessageResult());
+        ConsumeMessageDirectlyResult directlyResult = mock(ConsumeMessageDirectlyResult.class);
+        when(mQClientFactory.consumeMessageDirectly(any(MessageExt.class), anyString(), anyString())).thenReturn(directlyResult);
+        ConsumeMessageDirectlyResultRequestHeader requestHeader = new ConsumeMessageDirectlyResultRequestHeader();
+        requestHeader.setConsumerGroup(defaultGroup);
+        requestHeader.setBrokerName(defaultBroker);
+        when(request.decodeCommandCustomHeader(ConsumeMessageDirectlyResultRequestHeader.class)).thenReturn(requestHeader);
+        RemotingCommand command = processor.processRequest(ctx, request);
+        assertNotNull(command);
+        assertEquals(ResponseCode.SUCCESS, command.getCode());
+    }
+
+    @Test
+    public void testReceiveReplyMessage() throws Exception {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        RemotingCommand request = mock(RemotingCommand.class);
+        when(request.getCode()).thenReturn(RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT);
+        when(request.getBody()).thenReturn(getMessageResult());
+        when(request.decodeCommandCustomHeader(ReplyMessageRequestHeader.class)).thenReturn(createReplyMessageRequestHeader());
+        when(request.getBody()).thenReturn(new byte[1]);
+        RemotingCommand command = processor.processRequest(ctx, request);
+        assertNotNull(command);
+        assertEquals(ResponseCode.SUCCESS, command.getCode());
+    }
+
+    private ReplyMessageRequestHeader createReplyMessageRequestHeader() {
+        ReplyMessageRequestHeader result = new ReplyMessageRequestHeader();
+        result.setTopic(defaultTopic);
+        result.setQueueId(0);
+        result.setStoreTimestamp(System.currentTimeMillis());
+        result.setBornTimestamp(System.currentTimeMillis());
+        result.setReconsumeTimes(1);
+        result.setBornHost("127.0.0.1:12911");
+        result.setStoreHost("127.0.0.1:10911");
+        result.setSysFlag(1);
+        result.setFlag(1);
+        result.setProperties("CORRELATION_ID" + NAME_VALUE_SEPARATOR + "1");
+        return result;
+    }
+
+    private byte[] getMessageResult() throws Exception {
+        byte[] bytes = MessageDecoder.encode(createMessageExt(), false);
+        ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length);
+        byteBuffer.put(bytes);
+        return byteBuffer.array();
+    }
+
+    private MessageExt createMessageExt() {
+        MessageExt result = new MessageExt();
+        result.setBody("body".getBytes(StandardCharsets.UTF_8));
+        result.setTopic(defaultTopic);
+        result.setBrokerName(defaultBroker);
+        result.putUserProperty("key", "value");
+        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup);
+        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1");
+        result.setKeys("keys");
+        SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911);
+        SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911);
+        result.setBornHost(bornHost);
+        result.setStoreHost(storeHost);
+        return result;
+    }
+}
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java
new file mode 100644
index 00000000000..f52aba2dc00
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.impl;
+
+import org.apache.rocketmq.client.ClientConfig;
+import org.apache.rocketmq.client.QueryResult;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.sysflag.MessageSysFlag;
+import org.apache.rocketmq.remoting.InvokeCallback;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.QueryMessageResponseHeader;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.QueueData;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class MQAdminImplTest {
+
+    @Mock
+    private MQClientInstance mQClientFactory;
+
+    @Mock
+    private MQClientAPIImpl mQClientAPIImpl;
+
+    private MQAdminImpl mqAdminImpl;
+
+    private final String defaultTopic = "defaultTopic";
+
+    private final String defaultBroker = "defaultBroker";
+
+    private final String defaultCluster = "defaultCluster";
+
+    private final String defaultBrokerAddr = "127.0.0.1:10911";
+
+    private final long defaultTimeout = 3000L;
+
+    @Before
+    public void init() throws RemotingException, InterruptedException, MQClientException {
+        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mQClientAPIImpl);
+        when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createRouteData());
+        ClientConfig clientConfig = mock(ClientConfig.class);
+        when(clientConfig.getNamespace()).thenReturn("namespace");
+        when(mQClientFactory.getClientConfig()).thenReturn(clientConfig);
+        when(mQClientFactory.findBrokerAddressInPublish(any())).thenReturn(defaultBrokerAddr);
+        when(mQClientFactory.getAnExistTopicRouteData(any())).thenReturn(createRouteData());
+        mqAdminImpl = new MQAdminImpl(mQClientFactory);
+    }
+
+    @Test
+    public void assertTimeoutMillis() {
+        assertEquals(6000L, mqAdminImpl.getTimeoutMillis());
+        mqAdminImpl.setTimeoutMillis(defaultTimeout);
+        assertEquals(defaultTimeout, mqAdminImpl.getTimeoutMillis());
+    }
+
+    @Test
+    public void testCreateTopic() throws MQClientException {
+        mqAdminImpl.createTopic("", defaultTopic, 6);
+    }
+
+    @Test
+    public void assertFetchPublishMessageQueues() throws MQClientException {
+        List<MessageQueue> queueList = mqAdminImpl.fetchPublishMessageQueues(defaultTopic);
+        assertNotNull(queueList);
+        assertEquals(6, queueList.size());
+        for (MessageQueue each : queueList) {
+            assertEquals(defaultTopic, each.getTopic());
+            assertEquals(defaultBroker, each.getBrokerName());
+        }
+    }
+
+    @Test
+    public void assertFetchSubscribeMessageQueues() throws MQClientException {
+        Set<MessageQueue> queueList = mqAdminImpl.fetchSubscribeMessageQueues(defaultTopic);
+        assertNotNull(queueList);
+        assertEquals(6, queueList.size());
+        for (MessageQueue each : queueList) {
+            assertEquals(defaultTopic, each.getTopic());
+            assertEquals(defaultBroker, each.getBrokerName());
+        }
+    }
+
+    @Test
+    public void assertSearchOffset() throws MQClientException {
+        assertEquals(0, mqAdminImpl.searchOffset(new MessageQueue(), defaultTimeout));
+    }
+
+    @Test
+    public void assertMaxOffset() throws MQClientException {
+        assertEquals(0, mqAdminImpl.maxOffset(new MessageQueue()));
+    }
+
+    @Test
+    public void assertMinOffset() throws MQClientException {
+        assertEquals(0, mqAdminImpl.minOffset(new MessageQueue()));
+    }
+
+    @Test
+    public void assertEarliestMsgStoreTime() throws MQClientException {
+        assertEquals(0, mqAdminImpl.earliestMsgStoreTime(new MessageQueue()));
+    }
+
+    @Test(expected = MQClientException.class)
+    public void assertViewMessage() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {
+        MessageExt actual = mqAdminImpl.viewMessage(defaultTopic, "1");
+        assertNotNull(actual);
+    }
+
+    @Test
+    public void assertQueryMessage() throws InterruptedException, MQClientException, MQBrokerException, RemotingException {
+        doAnswer(invocation -> {
+            InvokeCallback callback = invocation.getArgument(3);
+            QueryMessageResponseHeader responseHeader = new QueryMessageResponseHeader();
+            responseHeader.setIndexLastUpdatePhyoffset(1L);
+            responseHeader.setIndexLastUpdateTimestamp(System.currentTimeMillis());
+            RemotingCommand response = mock(RemotingCommand.class);
+            when(response.decodeCommandCustomHeader(QueryMessageResponseHeader.class)).thenReturn(responseHeader);
+            when(response.getBody()).thenReturn(getMessageResult());
+            when(response.getCode()).thenReturn(ResponseCode.SUCCESS);
+            callback.operationSucceed(response);
+            return null;
+        }).when(mQClientAPIImpl).queryMessage(anyString(), any(), anyLong(), any(InvokeCallback.class), any());
+        QueryResult actual = mqAdminImpl.queryMessage(defaultTopic, "keys", 100, 1L, 50L);
+        assertNotNull(actual);
+        assertEquals(1, actual.getMessageList().size());
+        assertEquals(defaultTopic, actual.getMessageList().get(0).getTopic());
+    }
+
+    @Test
+    public void assertQueryMessageByUniqKey() throws InterruptedException, MQClientException, MQBrokerException, RemotingException {
+        doAnswer(invocation -> {
+            InvokeCallback callback = invocation.getArgument(3);
+            QueryMessageResponseHeader responseHeader = new QueryMessageResponseHeader();
+            responseHeader.setIndexLastUpdatePhyoffset(1L);
+            responseHeader.setIndexLastUpdateTimestamp(System.currentTimeMillis());
+            RemotingCommand response = mock(RemotingCommand.class);
+            when(response.decodeCommandCustomHeader(QueryMessageResponseHeader.class)).thenReturn(responseHeader);
+            when(response.getBody()).thenReturn(getMessageResult());
+            when(response.getCode()).thenReturn(ResponseCode.SUCCESS);
+            callback.operationSucceed(response);
+            return null;
+        }).when(mQClientAPIImpl).queryMessage(anyString(), any(), anyLong(), any(InvokeCallback.class), any());
+        String msgId = buildMsgId();
+        MessageExt actual = mqAdminImpl.queryMessageByUniqKey(defaultTopic, msgId);
+        assertNotNull(actual);
+        assertEquals(msgId, actual.getMsgId());
+        assertEquals(defaultTopic, actual.getTopic());
+        actual = mqAdminImpl.queryMessageByUniqKey(defaultCluster, defaultTopic, msgId);
+        assertNotNull(actual);
+        assertEquals(msgId, actual.getMsgId());
+        assertEquals(defaultTopic, actual.getTopic());
+        QueryResult queryResult = mqAdminImpl.queryMessageByUniqKey(defaultTopic, msgId, 1, 0L, 1L);
+        assertNotNull(queryResult);
+        assertEquals(1, queryResult.getMessageList().size());
+        assertEquals(defaultTopic, queryResult.getMessageList().get(0).getTopic());
+    }
+
+    private String buildMsgId() {
+        MessageExt msgExt = createMessageExt();
+        int storeHostIPLength = (msgExt.getFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 : 16;
+        int msgIDLength = storeHostIPLength + 4 + 8;
+        ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength);
+        return MessageDecoder.createMessageId(byteBufferMsgId, msgExt.getStoreHostBytes(), msgExt.getCommitLogOffset());
+    }
+
+    private TopicRouteData createRouteData() {
+        TopicRouteData result = new TopicRouteData();
+        result.setBrokerDatas(createBrokerData());
+        result.setQueueDatas(createQueueData());
+        return result;
+    }
+
+    private List<BrokerData> createBrokerData() {
+        HashMap<Long, String> brokerAddrs = new HashMap<>();
+        brokerAddrs.put(MixAll.MASTER_ID, defaultBrokerAddr);
+        return Collections.singletonList(new BrokerData(defaultCluster, defaultBroker, brokerAddrs));
+    }
+
+    private List<QueueData> createQueueData() {
+        QueueData queueData = new QueueData();
+        queueData.setPerm(6);
+        queueData.setBrokerName(defaultBroker);
+        queueData.setReadQueueNums(6);
+        queueData.setWriteQueueNums(6);
+        return Collections.singletonList(queueData);
+    }
+
+    private byte[] getMessageResult() throws Exception {
+        byte[] bytes = MessageDecoder.encode(createMessageExt(), false);
+        ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length);
+        byteBuffer.put(bytes);
+        return byteBuffer.array();
+    }
+
+    private MessageExt createMessageExt() {
+        MessageExt result = new MessageExt();
+        result.setBody("body".getBytes(StandardCharsets.UTF_8));
+        result.setTopic(defaultTopic);
+        result.setBrokerName(defaultBroker);
+        result.putUserProperty("key", "value");
+        result.setKeys("keys");
+        SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911);
+        SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911);
+        result.setBornHost(bornHost);
+        result.setStoreHost(storeHost);
+        return result;
+    }
+}
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
index 9f1a71c92f6..c12b23cb0db 100644
--- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
@@ -16,12 +16,6 @@
  */
 package org.apache.rocketmq.client.impl;
 
-import java.lang.reflect.Field;
-import java.net.InetSocketAddress;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.consumer.AckCallback;
 import org.apache.rocketmq.client.consumer.AckResult;
@@ -29,6 +23,9 @@
 import org.apache.rocketmq.client.consumer.PopCallback;
 import org.apache.rocketmq.client.consumer.PopResult;
 import org.apache.rocketmq.client.consumer.PopStatus;
+import org.apache.rocketmq.client.consumer.PullCallback;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.PullStatus;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.hook.SendMessageContext;
@@ -38,28 +35,76 @@
 import org.apache.rocketmq.client.producer.SendResult;
 import org.apache.rocketmq.client.producer.SendStatus;
 import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.common.PlainAccessConfig;
+import org.apache.rocketmq.common.ObjectCreator;
+import org.apache.rocketmq.common.Pair;
 import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
 import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageClientIDSetter;
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.message.MessageQueueAssignment;
 import org.apache.rocketmq.common.message.MessageRequestMode;
+import org.apache.rocketmq.common.namesrv.TopAddressing;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.InvokeCallback;
 import org.apache.rocketmq.remoting.RemotingClient;
+import org.apache.rocketmq.remoting.common.HeartbeatV2Result;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.exception.RemotingConnectException;
 import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
 import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
 import org.apache.rocketmq.remoting.netty.ResponseFuture;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
 import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;
+import org.apache.rocketmq.remoting.protocol.admin.TopicOffset;
+import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;
+import org.apache.rocketmq.remoting.protocol.body.AclInfo;
+import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;
+import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;
+import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData;
+import org.apache.rocketmq.remoting.protocol.body.BrokerStatsItem;
+import org.apache.rocketmq.remoting.protocol.body.ClusterInfo;
+import org.apache.rocketmq.remoting.protocol.body.Connection;
+import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;
+import org.apache.rocketmq.remoting.protocol.body.ConsumeQueueData;
+import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList;
+import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection;
+import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;
+import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache;
+import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody;
+import org.apache.rocketmq.remoting.protocol.body.GroupList;
+import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
+import org.apache.rocketmq.remoting.protocol.body.KVTable;
+import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody;
+import org.apache.rocketmq.remoting.protocol.body.ProducerConnection;
+import org.apache.rocketmq.remoting.protocol.body.ProducerInfo;
+import org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo;
 import org.apache.rocketmq.remoting.protocol.body.QueryAssignmentResponseBody;
+import org.apache.rocketmq.remoting.protocol.body.QueryConsumeQueueResponseBody;
+import org.apache.rocketmq.remoting.protocol.body.QueryConsumeTimeSpanBody;
+import org.apache.rocketmq.remoting.protocol.body.QueryCorrectionOffsetBody;
+import org.apache.rocketmq.remoting.protocol.body.QuerySubscriptionResponseBody;
+import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;
+import org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;
+import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;
+import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
+import org.apache.rocketmq.remoting.protocol.body.TopicList;
+import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;
+import org.apache.rocketmq.remoting.protocol.body.UserInfo;
 import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
 import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseBody;
 import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseHeader;
@@ -68,50 +113,112 @@
 import org.apache.rocketmq.remoting.protocol.header.GetMinOffsetResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.namesrv.GetKVConfigResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader;
+import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;
 import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.QueueData;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;
+import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail;
+import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;
+import org.apache.rocketmq.remoting.protocol.subscription.GroupForbidden;
 import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
 import org.assertj.core.api.Assertions;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
-import org.mockito.Matchers;
+import org.mockito.ArgumentMatchers;
 import org.mockito.Mock;
-import org.mockito.invocation.InvocationOnMock;
 import org.mockito.junit.MockitoJUnitRunner;
 import org.mockito.stubbing.Answer;
 
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
 public class MQClientAPIImplTest {
+
     private MQClientAPIImpl mqClientAPI = new MQClientAPIImpl(new NettyClientConfig(), null, null, new ClientConfig());
+
     @Mock
     private RemotingClient remotingClient;
+
     @Mock
     private DefaultMQProducerImpl defaultMQProducerImpl;
 
-    private String brokerAddr = "127.0.0.1";
-    private String brokerName = "DefaultBroker";
-    private String clusterName = "DefaultCluster";
-    private static String group = "FooBarGroup";
-    private static String topic = "FooBar";
-    private Message msg = new Message("FooBar", new byte[] {});
-    private static String clientId = "127.0.0.2@UnitTest";
+    @Mock
+    private RemotingCommand response;
+
+    private final String brokerAddr = "127.0.0.1";
+
+    private final String brokerName = "DefaultBroker";
+
+    private final String clusterName = "DefaultCluster";
+
+    private final String group = "FooBarGroup";
+
+    private final String topic = "FooBar";
+
+    private final Message msg = new Message("FooBar", new byte[]{});
+
+    private final String clientId = "127.0.0.2@UnitTest";
+
+    private final String defaultTopic = "defaultTopic";
+
+    private final String defaultBrokerAddr = "127.0.0.1:10911";
+
+    private final String defaultNsAddr = "127.0.0.1:9876";
+
+    private final long defaultTimeout = 3000L;
 
     @Before
     public void init() throws Exception {
@@ -151,12 +258,9 @@ public void testSendMessageOneWay_WithException() throws RemotingException, Inte
 
     @Test
     public void testSendMessageSync_Success() throws InterruptedException, RemotingException, MQBrokerException {
-        doAnswer(new Answer() {
-            @Override
-            public Object answer(InvocationOnMock mock) throws Throwable {
-                RemotingCommand request = mock.getArgument(1);
-                return createSendMessageSuccessResponse(request);
-            }
+        doAnswer(mock -> {
+            RemotingCommand request = mock.getArgument(1);
+            return createSendMessageSuccessResponse(request);
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
         SendMessageRequestHeader requestHeader = createSendMessageRequestHeader();
@@ -171,17 +275,14 @@ public Object answer(InvocationOnMock mock) throws Throwable {
     }
 
     @Test
-    public void testSendMessageSync_WithException() throws InterruptedException, RemotingException, MQBrokerException {
-        doAnswer(new Answer() {
-            @Override
-            public Object answer(InvocationOnMock mock) throws Throwable {
-                RemotingCommand request = mock.getArgument(1);
-                RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
-                response.setCode(ResponseCode.SYSTEM_ERROR);
-                response.setOpaque(request.getOpaque());
-                response.setRemark("Broker is broken.");
-                return response;
-            }
+    public void testSendMessageSync_WithException() throws InterruptedException, RemotingException {
+        doAnswer(mock -> {
+            RemotingCommand request = mock.getArgument(1);
+            RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setOpaque(request.getOpaque());
+            response.setRemark("Broker is broken.");
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
         SendMessageRequestHeader requestHeader = createSendMessageRequestHeader();
@@ -202,16 +303,13 @@ public void testSendMessageAsync_Success() throws RemotingException, Interrupted
             3 * 1000, CommunicationMode.ASYNC, new SendMessageContext(), defaultMQProducerImpl);
         assertThat(sendResult).isNull();
 
-        doAnswer(new Answer() {
-            @Override
-            public Object answer(InvocationOnMock mock) throws Throwable {
-                InvokeCallback callback = mock.getArgument(3);
-                RemotingCommand request = mock.getArgument(1);
-                ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
-                responseFuture.setResponseCommand(createSendMessageSuccessResponse(request));
-                callback.operationSucceed(responseFuture.getResponseCommand());
-                return null;
-            }
+        doAnswer(mock -> {
+            InvokeCallback callback = mock.getArgument(3);
+            RemotingCommand request = mock.getArgument(1);
+            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
+            responseFuture.setResponseCommand(createSendMessageSuccessResponse(request));
+            callback.operationSucceed(responseFuture.getResponseCommand());
+            return null;
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
         SendMessageContext sendMessageContext = new SendMessageContext();
         sendMessageContext.setProducer(new DefaultMQProducerImpl(new DefaultMQProducer()));
@@ -264,129 +362,42 @@ public void onException(Throwable e) {
     }
 
     @Test
-    public void testCreatePlainAccessConfig_Success() throws InterruptedException, RemotingException, MQBrokerException {
-
-        doAnswer(new Answer() {
-            @Override
-            public Object answer(InvocationOnMock mock) throws Throwable {
-                RemotingCommand request = mock.getArgument(1);
-                return createSuccessResponse4UpdateAclConfig(request);
-            }
-        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
-
-        PlainAccessConfig config = createUpdateAclConfig();
-
-        try {
-            mqClientAPI.createPlainAccessConfig(brokerAddr, config, 3 * 1000);
-        } catch (MQClientException ex) {
-
-        }
-    }
-
-    @Test
-    public void testCreatePlainAccessConfig_Exception() throws InterruptedException, RemotingException, MQBrokerException {
-
-        doAnswer(new Answer() {
-            @Override
-            public Object answer(InvocationOnMock mock) throws Throwable {
-                RemotingCommand request = mock.getArgument(1);
-                return createErrorResponse4UpdateAclConfig(request);
-            }
-        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
-
-        PlainAccessConfig config = createUpdateAclConfig();
-        try {
-            mqClientAPI.createPlainAccessConfig(brokerAddr, config, 3 * 1000);
-        } catch (MQClientException ex) {
-            assertThat(ex.getResponseCode()).isEqualTo(209);
-            assertThat(ex.getErrorMessage()).isEqualTo("corresponding to accessConfig has been updated failed");
-        }
-    }
-
-    @Test
-    public void testDeleteAccessConfig_Success() throws InterruptedException, RemotingException, MQBrokerException {
-
-        doAnswer(new Answer() {
-            @Override
-            public Object answer(InvocationOnMock mock) throws Throwable {
-                RemotingCommand request = mock.getArgument(1);
-                return createSuccessResponse4DeleteAclConfig(request);
-            }
+    public void testResumeCheckHalfMessage_WithException() throws RemotingException, InterruptedException {
+        doAnswer(mock -> {
+            RemotingCommand request = mock.getArgument(1);
+            RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setOpaque(request.getOpaque());
+            response.setRemark("Put message back to RMQ_SYS_TRANS_HALF_TOPIC failed.");
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
-        String accessKey = "1234567";
-        try {
-            mqClientAPI.deleteAccessConfig(brokerAddr, accessKey, 3 * 1000);
-        } catch (MQClientException ex) {
-
-        }
-    }
-
-    @Test
-    public void testDeleteAccessConfig_Exception() throws InterruptedException, RemotingException, MQBrokerException {
-
-        doAnswer(new Answer() {
-            @Override
-            public Object answer(InvocationOnMock mock) throws Throwable {
-                RemotingCommand request = mock.getArgument(1);
-                return createErrorResponse4DeleteAclConfig(request);
-            }
-        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
-
-        try {
-            mqClientAPI.deleteAccessConfig(brokerAddr, "11111", 3 * 1000);
-        } catch (MQClientException ex) {
-            assertThat(ex.getResponseCode()).isEqualTo(210);
-            assertThat(ex.getErrorMessage()).isEqualTo("corresponding to accessConfig has been deleted failed");
-        }
-    }
-
-    @Test
-    public void testResumeCheckHalfMessage_WithException() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
-        doAnswer(new Answer() {
-            @Override
-            public Object answer(InvocationOnMock mock) throws Throwable {
-                RemotingCommand request = mock.getArgument(1);
-                RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
-                response.setCode(ResponseCode.SYSTEM_ERROR);
-                response.setOpaque(request.getOpaque());
-                response.setRemark("Put message back to RMQ_SYS_TRANS_HALF_TOPIC failed.");
-                return response;
-            }
-        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
-
-        boolean result = mqClientAPI.resumeCheckHalfMessage(brokerAddr, "test", 3000);
+        boolean result = mqClientAPI.resumeCheckHalfMessage(brokerAddr, "topic,", "test", 3000);
         assertThat(result).isEqualTo(false);
     }
 
     @Test
-    public void testResumeCheckHalfMessage_Success() throws InterruptedException, RemotingException, MQBrokerException, MQClientException {
-        doAnswer(new Answer() {
-            @Override
-            public Object answer(InvocationOnMock mock) throws Throwable {
-                RemotingCommand request = mock.getArgument(1);
-                return createResumeSuccessResponse(request);
-            }
+    public void testResumeCheckHalfMessage_Success() throws InterruptedException, RemotingException {
+        doAnswer(mock -> {
+            RemotingCommand request = mock.getArgument(1);
+            return createResumeSuccessResponse(request);
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
-        boolean result = mqClientAPI.resumeCheckHalfMessage(brokerAddr, "test", 3000);
+        boolean result = mqClientAPI.resumeCheckHalfMessage(brokerAddr, "topic", "test", 3000);
 
         assertThat(result).isEqualTo(true);
     }
 
     @Test
     public void testSendMessageTypeofReply() throws Exception {
-        doAnswer(new Answer() {
-            @Override
-            public Object answer(InvocationOnMock mock) throws Throwable {
-                InvokeCallback callback = mock.getArgument(3);
-                RemotingCommand request = mock.getArgument(1);
-                ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
-                responseFuture.setResponseCommand(createSendMessageSuccessResponse(request));
-                callback.operationSucceed(responseFuture.getResponseCommand());
-                return null;
-            }
-        }).when(remotingClient).invokeAsync(Matchers.anyString(), Matchers.any(RemotingCommand.class), Matchers.anyLong(), Matchers.any(InvokeCallback.class));
+        doAnswer(mock -> {
+            InvokeCallback callback = mock.getArgument(3);
+            RemotingCommand request = mock.getArgument(1);
+            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
+            responseFuture.setResponseCommand(createSendMessageSuccessResponse(request));
+            callback.operationSucceed(responseFuture.getResponseCommand());
+            return null;
+        }).when(remotingClient).invokeAsync(ArgumentMatchers.anyString(), ArgumentMatchers.any(RemotingCommand.class), ArgumentMatchers.anyLong(), ArgumentMatchers.any(InvokeCallback.class));
         SendMessageContext sendMessageContext = new SendMessageContext();
         sendMessageContext.setProducer(new DefaultMQProducerImpl(new DefaultMQProducer()));
         msg.getProperties().put("MSG_TYPE", "reply");
@@ -408,19 +419,16 @@ public void onException(Throwable e) {
 
     @Test
     public void testQueryAssignment_Success() throws Exception {
-        doAnswer(new Answer<RemotingCommand>() {
-            @Override
-            public RemotingCommand answer(InvocationOnMock mock) {
-                RemotingCommand request = mock.getArgument(1);
-
-                RemotingCommand response = RemotingCommand.createResponseCommand(null);
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                QueryAssignmentResponseBody b = new QueryAssignmentResponseBody();
-                b.setMessageQueueAssignments(Collections.singleton(new MessageQueueAssignment()));
-                response.setBody(b.encode());
-                return response;
-            }
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
+
+            RemotingCommand response = RemotingCommand.createResponseCommand(null);
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            QueryAssignmentResponseBody b = new QueryAssignmentResponseBody();
+            b.setMessageQueueAssignments(Collections.singleton(new MessageQueueAssignment()));
+            response.setBody(b.encode());
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
         Set<MessageQueueAssignment> assignments = mqClientAPI.queryAssignment(brokerAddr, topic, group, clientId, null, MessageModel.CLUSTERING, 10 * 1000);
         assertThat(assignments).size().isEqualTo(1);
@@ -430,48 +438,45 @@ public RemotingCommand answer(InvocationOnMock mock) {
     public void testPopMessageAsync_Success() throws Exception {
         final long popTime = System.currentTimeMillis();
         final int invisibleTime = 10 * 1000;
-        doAnswer(new Answer<Void>() {
-            @Override
-            public Void answer(InvocationOnMock mock) throws Throwable {
-                InvokeCallback callback = mock.getArgument(3);
-                RemotingCommand request = mock.getArgument(1);
-                ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
-                RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-
-                PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();
-                responseHeader.setInvisibleTime(invisibleTime);
-                responseHeader.setPopTime(popTime);
-                responseHeader.setReviveQid(0);
-                responseHeader.setRestNum(1);
-                StringBuilder startOffsetInfo = new StringBuilder(64);
-                ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L);
-                responseHeader.setStartOffsetInfo(startOffsetInfo.toString());
-                StringBuilder msgOffsetInfo = new StringBuilder(64);
-                ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L));
-                responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString());
-                response.setRemark("FOUND");
-                response.makeCustomHeaderToNet();
-
-                MessageExt message = new MessageExt();
-                message.setQueueId(0);
-                message.setFlag(12);
-                message.setQueueOffset(0L);
-                message.setCommitLogOffset(100L);
-                message.setSysFlag(0);
-                message.setBornTimestamp(System.currentTimeMillis());
-                message.setBornHost(new InetSocketAddress("127.0.0.1", 10));
-                message.setStoreTimestamp(System.currentTimeMillis());
-                message.setStoreHost(new InetSocketAddress("127.0.0.1", 11));
-                message.setBody("body".getBytes());
-                message.setTopic(topic);
-                message.putUserProperty("key", "value");
-                response.setBody(MessageDecoder.encode(message, false));
-                responseFuture.setResponseCommand(response);
-                callback.operationSucceed(responseFuture.getResponseCommand());
-                return null;
-            }
+        doAnswer((Answer<Void>) mock -> {
+            InvokeCallback callback = mock.getArgument(3);
+            RemotingCommand request = mock.getArgument(1);
+            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
+            RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+
+            PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();
+            responseHeader.setInvisibleTime(invisibleTime);
+            responseHeader.setPopTime(popTime);
+            responseHeader.setReviveQid(0);
+            responseHeader.setRestNum(1);
+            StringBuilder startOffsetInfo = new StringBuilder(64);
+            ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L);
+            responseHeader.setStartOffsetInfo(startOffsetInfo.toString());
+            StringBuilder msgOffsetInfo = new StringBuilder(64);
+            ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L));
+            responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString());
+            response.setRemark("FOUND");
+            response.makeCustomHeaderToNet();
+
+            MessageExt message = new MessageExt();
+            message.setQueueId(0);
+            message.setFlag(12);
+            message.setQueueOffset(0L);
+            message.setCommitLogOffset(100L);
+            message.setSysFlag(0);
+            message.setBornTimestamp(System.currentTimeMillis());
+            message.setBornHost(new InetSocketAddress("127.0.0.1", 10));
+            message.setStoreTimestamp(System.currentTimeMillis());
+            message.setStoreHost(new InetSocketAddress("127.0.0.1", 11));
+            message.setBody("body".getBytes());
+            message.setTopic(topic);
+            message.putUserProperty("key", "value");
+            response.setBody(MessageDecoder.encode(message, false));
+            responseFuture.setResponseCommand(response);
+            callback.operationSucceed(responseFuture.getResponseCommand());
+            return null;
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
         final CountDownLatch done = new CountDownLatch(1);
         mqClientAPI.popMessageAsync(brokerName, brokerAddr, new PopMessageRequestHeader(), 10 * 1000, new PopCallback() {
@@ -499,50 +504,47 @@ public void testPopLmqMessage_async() throws Exception {
         final long popTime = System.currentTimeMillis();
         final int invisibleTime = 10 * 1000;
         final String lmqTopic = MixAll.LMQ_PREFIX + "lmq1";
-        doAnswer(new Answer<Void>() {
-            @Override
-            public Void answer(InvocationOnMock mock) throws Throwable {
-                InvokeCallback callback = mock.getArgument(3);
-                RemotingCommand request = mock.getArgument(1);
-                ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
-                RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-
-                PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();
-                responseHeader.setInvisibleTime(invisibleTime);
-                responseHeader.setPopTime(popTime);
-                responseHeader.setReviveQid(0);
-                responseHeader.setRestNum(1);
-                StringBuilder startOffsetInfo = new StringBuilder(64);
-                ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L);
-                responseHeader.setStartOffsetInfo(startOffsetInfo.toString());
-                StringBuilder msgOffsetInfo = new StringBuilder(64);
-                ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L));
-                responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString());
-                response.setRemark("FOUND");
-                response.makeCustomHeaderToNet();
-
-                MessageExt message = new MessageExt();
-                message.setQueueId(3);
-                message.setFlag(0);
-                message.setQueueOffset(5L);
-                message.setCommitLogOffset(11111L);
-                message.setSysFlag(0);
-                message.setBornTimestamp(System.currentTimeMillis());
-                message.setBornHost(new InetSocketAddress("127.0.0.1", 10));
-                message.setStoreTimestamp(System.currentTimeMillis());
-                message.setStoreHost(new InetSocketAddress("127.0.0.1", 11));
-                message.setBody("body".getBytes());
-                message.setTopic(topic);
-                message.putUserProperty("key", "value");
-                message.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqTopic);
-                message.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, String.valueOf(0));
-                response.setBody(MessageDecoder.encode(message, false));
-                responseFuture.setResponseCommand(response);
-                callback.operationSucceed(responseFuture.getResponseCommand());
-                return null;
-            }
+        doAnswer((Answer<Void>) mock -> {
+            InvokeCallback callback = mock.getArgument(3);
+            RemotingCommand request = mock.getArgument(1);
+            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
+            RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+
+            PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();
+            responseHeader.setInvisibleTime(invisibleTime);
+            responseHeader.setPopTime(popTime);
+            responseHeader.setReviveQid(0);
+            responseHeader.setRestNum(1);
+            StringBuilder startOffsetInfo = new StringBuilder(64);
+            ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L);
+            responseHeader.setStartOffsetInfo(startOffsetInfo.toString());
+            StringBuilder msgOffsetInfo = new StringBuilder(64);
+            ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L));
+            responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString());
+            response.setRemark("FOUND");
+            response.makeCustomHeaderToNet();
+
+            MessageExt message = new MessageExt();
+            message.setQueueId(3);
+            message.setFlag(0);
+            message.setQueueOffset(5L);
+            message.setCommitLogOffset(11111L);
+            message.setSysFlag(0);
+            message.setBornTimestamp(System.currentTimeMillis());
+            message.setBornHost(new InetSocketAddress("127.0.0.1", 10));
+            message.setStoreTimestamp(System.currentTimeMillis());
+            message.setStoreHost(new InetSocketAddress("127.0.0.1", 11));
+            message.setBody("body".getBytes());
+            message.setTopic(topic);
+            message.putUserProperty("key", "value");
+            message.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqTopic);
+            message.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, String.valueOf(0));
+            response.setBody(MessageDecoder.encode(message, false));
+            responseFuture.setResponseCommand(response);
+            callback.operationSucceed(responseFuture.getResponseCommand());
+            return null;
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
         final CountDownLatch done = new CountDownLatch(1);
         final PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();
@@ -571,20 +573,94 @@ public void onException(Throwable e) {
     }
 
     @Test
-    public void testAckMessageAsync_Success() throws Exception {
-        doAnswer(new Answer<Void>() {
+    public void testPopMultiLmqMessage_async() throws Exception {
+        final long popTime = System.currentTimeMillis();
+        final int invisibleTime = 10 * 1000;
+        final String lmqTopic = MixAll.LMQ_PREFIX + "lmq1";
+        final String lmqTopic2 = MixAll.LMQ_PREFIX + "lmq2";
+        final String multiDispatch = String.join(MixAll.LMQ_DISPATCH_SEPARATOR, lmqTopic, lmqTopic2);
+        final String multiOffset = String.join(MixAll.LMQ_DISPATCH_SEPARATOR, "0", "0");
+        doAnswer((Answer<Void>) mock -> {
+            InvokeCallback callback = mock.getArgument(3);
+            RemotingCommand request = mock.getArgument(1);
+            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
+            RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+
+            PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();
+            responseHeader.setInvisibleTime(invisibleTime);
+            responseHeader.setPopTime(popTime);
+            responseHeader.setReviveQid(0);
+            responseHeader.setRestNum(1);
+            StringBuilder startOffsetInfo = new StringBuilder(64);
+            ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L);
+            responseHeader.setStartOffsetInfo(startOffsetInfo.toString());
+            StringBuilder msgOffsetInfo = new StringBuilder(64);
+            ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L));
+            responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString());
+            response.setRemark("FOUND");
+            response.makeCustomHeaderToNet();
+
+            MessageExt message = new MessageExt();
+            message.setQueueId(0);
+            message.setFlag(0);
+            message.setQueueOffset(10L);
+            message.setCommitLogOffset(10000L);
+            message.setSysFlag(0);
+            message.setBornTimestamp(System.currentTimeMillis());
+            message.setBornHost(new InetSocketAddress("127.0.0.1", 10));
+            message.setStoreTimestamp(System.currentTimeMillis());
+            message.setStoreHost(new InetSocketAddress("127.0.0.1", 11));
+            message.setBody("body".getBytes());
+            message.setTopic(topic);
+            MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_DISPATCH, multiDispatch);
+            MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, multiOffset);
+            response.setBody(MessageDecoder.encode(message, false));
+            responseFuture.setResponseCommand(response);
+            callback.operationSucceed(responseFuture.getResponseCommand());
+            return null;
+        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
+        final CountDownLatch done = new CountDownLatch(1);
+        final PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();
+        requestHeader.setTopic(lmqTopic);
+        mqClientAPI.popMessageAsync(brokerName, brokerAddr, requestHeader, 10 * 1000, new PopCallback() {
             @Override
-            public Void answer(InvocationOnMock mock) throws Throwable {
-                InvokeCallback callback = mock.getArgument(3);
-                RemotingCommand request = mock.getArgument(1);
-                ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
-                RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);
-                response.setOpaque(request.getOpaque());
-                response.setCode(ResponseCode.SUCCESS);
-                responseFuture.setResponseCommand(response);
-                callback.operationSucceed(responseFuture.getResponseCommand());
-                return null;
+            public void onSuccess(PopResult popResult) {
+                assertThat(popResult.getPopStatus()).isEqualTo(PopStatus.FOUND);
+                assertThat(popResult.getRestNum()).isEqualTo(1);
+                assertThat(popResult.getInvisibleTime()).isEqualTo(invisibleTime);
+                assertThat(popResult.getPopTime()).isEqualTo(popTime);
+                assertThat(popResult.getMsgFoundList()).size().isEqualTo(1);
+                assertThat(popResult.getMsgFoundList().get(0).getTopic()).isEqualTo(lmqTopic);
+                assertThat(popResult.getMsgFoundList().get(0).getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))
+                    .isEqualTo(multiDispatch);
+                assertThat(popResult.getMsgFoundList().get(0).getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET))
+                    .isEqualTo(multiOffset);
+                done.countDown();
             }
+
+            @Override
+            public void onException(Throwable e) {
+                Assertions.fail("want no exception but got one", e);
+                done.countDown();
+            }
+        });
+        done.await();
+    }
+
+    @Test
+    public void testAckMessageAsync_Success() throws Exception {
+        doAnswer((Answer<Void>) mock -> {
+            InvokeCallback callback = mock.getArgument(3);
+            RemotingCommand request = mock.getArgument(1);
+            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
+            RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);
+            response.setOpaque(request.getOpaque());
+            response.setCode(ResponseCode.SUCCESS);
+            responseFuture.setResponseCommand(response);
+            callback.operationSucceed(responseFuture.getResponseCommand());
+            return null;
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
 
         final CountDownLatch done = new CountDownLatch(1);
@@ -606,22 +682,19 @@ public void onException(Throwable e) {
 
     @Test
     public void testChangeInvisibleTimeAsync_Success() throws Exception {
-        doAnswer(new Answer<Void>() {
-            @Override
-            public Void answer(InvocationOnMock mock) throws Throwable {
-                InvokeCallback callback = mock.getArgument(3);
-                RemotingCommand request = mock.getArgument(1);
-                ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
-                RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);
-                response.setOpaque(request.getOpaque());
-                response.setCode(ResponseCode.SUCCESS);
-                ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.readCustomHeader();
-                responseHeader.setPopTime(System.currentTimeMillis());
-                responseHeader.setInvisibleTime(10 * 1000L);
-                responseFuture.setResponseCommand(response);
-                callback.operationSucceed(responseFuture.getResponseCommand());
-                return null;
-            }
+        doAnswer((Answer<Void>) mock -> {
+            InvokeCallback callback = mock.getArgument(3);
+            RemotingCommand request = mock.getArgument(1);
+            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
+            RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);
+            response.setOpaque(request.getOpaque());
+            response.setCode(ResponseCode.SUCCESS);
+            ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.readCustomHeader();
+            responseHeader.setPopTime(System.currentTimeMillis());
+            responseHeader.setInvisibleTime(10 * 1000L);
+            responseFuture.setResponseCommand(response);
+            callback.operationSucceed(responseFuture.getResponseCommand());
+            return null;
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
 
         final CountDownLatch done = new CountDownLatch(1);
@@ -648,16 +721,13 @@ public void onException(Throwable e) {
 
     @Test
     public void testSetMessageRequestMode_Success() throws Exception {
-        doAnswer(new Answer<RemotingCommand>() {
-            @Override
-            public RemotingCommand answer(InvocationOnMock mock) {
-                RemotingCommand request = mock.getArgument(1);
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
 
-                RemotingCommand response = RemotingCommand.createResponseCommand(null);
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                return response;
-            }
+            RemotingCommand response = RemotingCommand.createResponseCommand(null);
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
         mqClientAPI.setMessageRequestMode(brokerAddr, topic, group, MessageRequestMode.POP, 8, 10 * 1000L);
@@ -665,16 +735,13 @@ public RemotingCommand answer(InvocationOnMock mock) {
 
     @Test
     public void testCreateSubscriptionGroup_Success() throws Exception {
-        doAnswer(new Answer<RemotingCommand>() {
-            @Override
-            public RemotingCommand answer(InvocationOnMock mock) {
-                RemotingCommand request = mock.getArgument(1);
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
 
-                RemotingCommand response = RemotingCommand.createResponseCommand(null);
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                return response;
-            }
+            RemotingCommand response = RemotingCommand.createResponseCommand(null);
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
         mqClientAPI.createSubscriptionGroup(brokerAddr, new SubscriptionGroupConfig(), 10000);
@@ -682,16 +749,13 @@ public RemotingCommand answer(InvocationOnMock mock) {
 
     @Test
     public void testCreateTopic_Success() throws Exception {
-        doAnswer(new Answer<RemotingCommand>() {
-            @Override
-            public RemotingCommand answer(InvocationOnMock mock) {
-                RemotingCommand request = mock.getArgument(1);
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
 
-                RemotingCommand response = RemotingCommand.createResponseCommand(null);
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                return response;
-            }
+            RemotingCommand response = RemotingCommand.createResponseCommand(null);
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
         mqClientAPI.createTopic(brokerAddr, topic, new TopicConfig(), 10000);
@@ -699,52 +763,46 @@ public RemotingCommand answer(InvocationOnMock mock) {
 
     @Test
     public void testViewMessage() throws Exception {
-        doAnswer(new Answer<RemotingCommand>() {
-            @Override
-            public RemotingCommand answer(InvocationOnMock mock) throws Exception {
-                RemotingCommand request = mock.getArgument(1);
-
-                RemotingCommand response = RemotingCommand.createResponseCommand(null);
-                MessageExt message = new MessageExt();
-                message.setQueueId(0);
-                message.setFlag(12);
-                message.setQueueOffset(0L);
-                message.setCommitLogOffset(100L);
-                message.setSysFlag(0);
-                message.setBornTimestamp(System.currentTimeMillis());
-                message.setBornHost(new InetSocketAddress("127.0.0.1", 10));
-                message.setStoreTimestamp(System.currentTimeMillis());
-                message.setStoreHost(new InetSocketAddress("127.0.0.1", 11));
-                message.setBody("body".getBytes());
-                message.setTopic(topic);
-                message.putUserProperty("key", "value");
-                response.setBody(MessageDecoder.encode(message, false));
-                response.makeCustomHeaderToNet();
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                return response;
-            }
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
+
+            RemotingCommand response = RemotingCommand.createResponseCommand(null);
+            MessageExt message = new MessageExt();
+            message.setQueueId(0);
+            message.setFlag(12);
+            message.setQueueOffset(0L);
+            message.setCommitLogOffset(100L);
+            message.setSysFlag(0);
+            message.setBornTimestamp(System.currentTimeMillis());
+            message.setBornHost(new InetSocketAddress("127.0.0.1", 10));
+            message.setStoreTimestamp(System.currentTimeMillis());
+            message.setStoreHost(new InetSocketAddress("127.0.0.1", 11));
+            message.setBody("body".getBytes());
+            message.setTopic(topic);
+            message.putUserProperty("key", "value");
+            response.setBody(MessageDecoder.encode(message, false));
+            response.makeCustomHeaderToNet();
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
-        MessageExt messageExt = mqClientAPI.viewMessage(brokerAddr, 100L, 10000);
+        MessageExt messageExt = mqClientAPI.viewMessage(brokerAddr, "topic", 100L, 10000);
         assertThat(messageExt.getTopic()).isEqualTo(topic);
     }
 
     @Test
     public void testSearchOffset() throws Exception {
-        doAnswer(new Answer<RemotingCommand>() {
-            @Override
-            public RemotingCommand answer(InvocationOnMock mock) {
-                RemotingCommand request = mock.getArgument(1);
-
-                final RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class);
-                final SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader();
-                responseHeader.setOffset(100L);
-                response.makeCustomHeaderToNet();
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                return response;
-            }
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
+
+            final RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class);
+            final SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader();
+            responseHeader.setOffset(100L);
+            response.makeCustomHeaderToNet();
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
         long offset = mqClientAPI.searchOffset(brokerAddr, topic, 0, System.currentTimeMillis() - 1000, 10000);
@@ -753,19 +811,16 @@ public RemotingCommand answer(InvocationOnMock mock) {
 
     @Test
     public void testGetMaxOffset() throws Exception {
-        doAnswer(new Answer<RemotingCommand>() {
-            @Override
-            public RemotingCommand answer(InvocationOnMock mock) {
-                RemotingCommand request = mock.getArgument(1);
-
-                final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class);
-                final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader();
-                responseHeader.setOffset(100L);
-                response.makeCustomHeaderToNet();
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                return response;
-            }
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
+
+            final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class);
+            final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader();
+            responseHeader.setOffset(100L);
+            response.makeCustomHeaderToNet();
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
         long offset = mqClientAPI.getMaxOffset(brokerAddr, new MessageQueue(topic, brokerName, 0), 10000);
@@ -774,19 +829,16 @@ public RemotingCommand answer(InvocationOnMock mock) {
 
     @Test
     public void testGetMinOffset() throws Exception {
-        doAnswer(new Answer<RemotingCommand>() {
-            @Override
-            public RemotingCommand answer(InvocationOnMock mock) {
-                RemotingCommand request = mock.getArgument(1);
-
-                final RemotingCommand response = RemotingCommand.createResponseCommand(GetMinOffsetResponseHeader.class);
-                final GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.readCustomHeader();
-                responseHeader.setOffset(100L);
-                response.makeCustomHeaderToNet();
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                return response;
-            }
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
+
+            final RemotingCommand response = RemotingCommand.createResponseCommand(GetMinOffsetResponseHeader.class);
+            final GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.readCustomHeader();
+            responseHeader.setOffset(100L);
+            response.makeCustomHeaderToNet();
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
         long offset = mqClientAPI.getMinOffset(brokerAddr, new MessageQueue(topic, brokerName, 0), 10000);
@@ -795,19 +847,16 @@ public RemotingCommand answer(InvocationOnMock mock) {
 
     @Test
     public void testGetEarliestMsgStoretime() throws Exception {
-        doAnswer(new Answer<RemotingCommand>() {
-            @Override
-            public RemotingCommand answer(InvocationOnMock mock) {
-                RemotingCommand request = mock.getArgument(1);
-
-                final RemotingCommand response = RemotingCommand.createResponseCommand(GetEarliestMsgStoretimeResponseHeader.class);
-                final GetEarliestMsgStoretimeResponseHeader responseHeader = (GetEarliestMsgStoretimeResponseHeader) response.readCustomHeader();
-                responseHeader.setTimestamp(100L);
-                response.makeCustomHeaderToNet();
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                return response;
-            }
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
+
+            final RemotingCommand response = RemotingCommand.createResponseCommand(GetEarliestMsgStoretimeResponseHeader.class);
+            final GetEarliestMsgStoretimeResponseHeader responseHeader = (GetEarliestMsgStoretimeResponseHeader) response.readCustomHeader();
+            responseHeader.setTimestamp(100L);
+            response.makeCustomHeaderToNet();
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
         long t = mqClientAPI.getEarliestMsgStoretime(brokerAddr, new MessageQueue(topic, brokerName, 0), 10000);
@@ -816,21 +865,18 @@ public RemotingCommand answer(InvocationOnMock mock) {
 
     @Test
     public void testQueryConsumerOffset() throws Exception {
-        doAnswer(new Answer<RemotingCommand>() {
-            @Override
-            public RemotingCommand answer(InvocationOnMock mock) {
-                RemotingCommand request = mock.getArgument(1);
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
 
-                final RemotingCommand response =
+            final RemotingCommand response =
                     RemotingCommand.createResponseCommand(QueryConsumerOffsetResponseHeader.class);
-                final QueryConsumerOffsetResponseHeader responseHeader =
+            final QueryConsumerOffsetResponseHeader responseHeader =
                     (QueryConsumerOffsetResponseHeader) response.readCustomHeader();
-                responseHeader.setOffset(100L);
-                response.makeCustomHeaderToNet();
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                return response;
-            }
+            responseHeader.setOffset(100L);
+            response.makeCustomHeaderToNet();
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
         long t = mqClientAPI.queryConsumerOffset(brokerAddr, new QueryConsumerOffsetRequestHeader(), 1000);
@@ -839,18 +885,15 @@ public RemotingCommand answer(InvocationOnMock mock) {
 
     @Test
     public void testUpdateConsumerOffset() throws Exception {
-        doAnswer(new Answer<RemotingCommand>() {
-            @Override
-            public RemotingCommand answer(InvocationOnMock mock) {
-                RemotingCommand request = mock.getArgument(1);
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
 
-                final RemotingCommand response =
+            final RemotingCommand response =
                     RemotingCommand.createResponseCommand(UpdateConsumerOffsetResponseHeader.class);
-                response.makeCustomHeaderToNet();
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                return response;
-            }
+            response.makeCustomHeaderToNet();
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
         mqClientAPI.updateConsumerOffset(brokerAddr, new UpdateConsumerOffsetRequestHeader(), 1000);
@@ -858,21 +901,18 @@ public RemotingCommand answer(InvocationOnMock mock) {
 
     @Test
     public void testGetConsumerIdListByGroup() throws Exception {
-        doAnswer(new Answer<RemotingCommand>() {
-            @Override
-            public RemotingCommand answer(InvocationOnMock mock) {
-                RemotingCommand request = mock.getArgument(1);
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
 
-                final RemotingCommand response =
+            final RemotingCommand response =
                     RemotingCommand.createResponseCommand(GetConsumerListByGroupResponseHeader.class);
-                GetConsumerListByGroupResponseBody body = new GetConsumerListByGroupResponseBody();
-                body.setConsumerIdList(Collections.singletonList("consumer1"));
-                response.setBody(body.encode());
-                response.makeCustomHeaderToNet();
-                response.setCode(ResponseCode.SUCCESS);
-                response.setOpaque(request.getOpaque());
-                return response;
-            }
+            GetConsumerListByGroupResponseBody body = new GetConsumerListByGroupResponseBody();
+            body.setConsumerIdList(Collections.singletonList("consumer1"));
+            response.setBody(body.encode());
+            response.makeCustomHeaderToNet();
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
         List<String> consumerIdList = mqClientAPI.getConsumerIdListByGroup(brokerAddr, group, 10000);
         assertThat(consumerIdList).size().isGreaterThan(0);
@@ -909,7 +949,6 @@ private RemotingCommand createSuccessResponse4UpdateAclConfig(RemotingCommand re
         response.setOpaque(request.getOpaque());
         response.markResponseType();
         response.setRemark(null);
-
         return response;
     }
 
@@ -919,42 +958,9 @@ private RemotingCommand createSuccessResponse4DeleteAclConfig(RemotingCommand re
         response.setOpaque(request.getOpaque());
         response.markResponseType();
         response.setRemark(null);
-
-        return response;
-    }
-
-    private RemotingCommand createErrorResponse4UpdateAclConfig(RemotingCommand request) {
-        RemotingCommand response = RemotingCommand.createResponseCommand(null);
-        response.setCode(ResponseCode.UPDATE_AND_CREATE_ACL_CONFIG_FAILED);
-        response.setOpaque(request.getOpaque());
-        response.markResponseType();
-        response.setRemark("corresponding to accessConfig has been updated failed");
-
-        return response;
-    }
-
-    private RemotingCommand createErrorResponse4DeleteAclConfig(RemotingCommand request) {
-        RemotingCommand response = RemotingCommand.createResponseCommand(null);
-        response.setCode(ResponseCode.DELETE_ACL_CONFIG_FAILED);
-        response.setOpaque(request.getOpaque());
-        response.markResponseType();
-        response.setRemark("corresponding to accessConfig has been deleted failed");
-
         return response;
     }
 
-    private PlainAccessConfig createUpdateAclConfig() {
-
-        PlainAccessConfig config = new PlainAccessConfig();
-        config.setAccessKey("Rocketmq111");
-        config.setSecretKey("123456789");
-        config.setAdmin(true);
-        config.setWhiteRemoteAddress("127.0.0.1");
-        config.setDefaultTopicPerm("DENY");
-        config.setDefaultGroupPerm("SUB");
-        return config;
-    }
-
     private SendMessageRequestHeader createSendMessageRequestHeader() {
         SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
         requestHeader.setBornTimestamp(System.currentTimeMillis());
@@ -967,24 +973,1108 @@ private SendMessageRequestHeader createSendMessageRequestHeader() {
 
     @Test
     public void testAddWritePermOfBroker() throws Exception {
-        doAnswer(new Answer() {
-            @Override
-            public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
-                RemotingCommand request = invocationOnMock.getArgument(1);
-                if (request.getCode() != RequestCode.ADD_WRITE_PERM_OF_BROKER) {
-                    return null;
-                }
-
-                RemotingCommand response = RemotingCommand.createResponseCommand(AddWritePermOfBrokerResponseHeader.class);
-                AddWritePermOfBrokerResponseHeader responseHeader = (AddWritePermOfBrokerResponseHeader) response.readCustomHeader();
-                response.setCode(ResponseCode.SUCCESS);
-                responseHeader.setAddTopicCount(7);
-                response.addExtField("addTopicCount", String.valueOf(responseHeader.getAddTopicCount()));
-                return response;
+        doAnswer(invocationOnMock -> {
+            RemotingCommand request = invocationOnMock.getArgument(1);
+            if (request.getCode() != RequestCode.ADD_WRITE_PERM_OF_BROKER) {
+                return null;
             }
+
+            RemotingCommand response = RemotingCommand.createResponseCommand(AddWritePermOfBrokerResponseHeader.class);
+            AddWritePermOfBrokerResponseHeader responseHeader = (AddWritePermOfBrokerResponseHeader) response.readCustomHeader();
+            response.setCode(ResponseCode.SUCCESS);
+            responseHeader.setAddTopicCount(7);
+            response.addExtField("addTopicCount", String.valueOf(responseHeader.getAddTopicCount()));
+            return response;
         }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
 
         int topicCnt = mqClientAPI.addWritePermOfBroker("127.0.0.1", "default-broker", 1000);
         assertThat(topicCnt).isEqualTo(7);
     }
+
+    @Test
+    public void testCreateTopicList_Success() throws RemotingException, InterruptedException, MQClientException {
+        doAnswer((Answer<RemotingCommand>) mock -> {
+            RemotingCommand request = mock.getArgument(1);
+
+            RemotingCommand response = RemotingCommand.createResponseCommand(null);
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            return response;
+        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
+
+        final List<TopicConfig> topicConfigList = new LinkedList<>();
+        for (int i = 0; i < 16; i++) {
+            TopicConfig topicConfig = new TopicConfig();
+            topicConfig.setTopicName("Topic" + i);
+            topicConfigList.add(topicConfig);
+        }
+
+        mqClientAPI.createTopicList(brokerAddr, topicConfigList, 10000);
+    }
+
+    @Test
+    public void assertFetchNameServerAddr() throws NoSuchFieldException, IllegalAccessException {
+        setTopAddressing();
+        assertEquals(defaultNsAddr, mqClientAPI.fetchNameServerAddr());
+    }
+
+    @Test
+    public void assertOnNameServerAddressChange() {
+        assertEquals(defaultNsAddr, mqClientAPI.onNameServerAddressChange(defaultNsAddr));
+    }
+
+    @Test
+    public void assertPullMessage() throws MQBrokerException, RemotingException, InterruptedException {
+        PullMessageRequestHeader requestHeader = mock(PullMessageRequestHeader.class);
+        mockInvokeSync();
+        PullCallback callback = mock(PullCallback.class);
+        PullMessageResponseHeader responseHeader = mock(PullMessageResponseHeader.class);
+        setResponseHeader(responseHeader);
+        when(responseHeader.getNextBeginOffset()).thenReturn(1L);
+        when(responseHeader.getMinOffset()).thenReturn(1L);
+        when(responseHeader.getMaxOffset()).thenReturn(10L);
+        when(responseHeader.getSuggestWhichBrokerId()).thenReturn(MixAll.MASTER_ID);
+        PullResult actual = mqClientAPI.pullMessage(defaultBrokerAddr, requestHeader, defaultTimeout, CommunicationMode.SYNC, callback);
+        assertNotNull(actual);
+        assertEquals(1L, actual.getNextBeginOffset());
+        assertEquals(1L, actual.getMinOffset());
+        assertEquals(10L, actual.getMaxOffset());
+        assertEquals(PullStatus.FOUND, actual.getPullStatus());
+        assertNull(actual.getMsgFoundList());
+    }
+
+    @Test
+    public void testBatchAckMessageAsync() throws MQBrokerException, RemotingException, InterruptedException {
+        AckCallback callback = mock(AckCallback.class);
+        List<String> extraInfoList = new ArrayList<>();
+        extraInfoList.add(String.format("%s %s %s %s %s %s %d %d", "1", "2", "3", "4", "5", brokerName, 7, 8));
+        mqClientAPI.batchAckMessageAsync(defaultBrokerAddr, defaultTimeout, callback, defaultTopic, "", extraInfoList);
+    }
+
+    @Test
+    public void assertSearchOffset() throws MQBrokerException, RemotingException, InterruptedException {
+        mockInvokeSync();
+        SearchOffsetResponseHeader responseHeader = mock(SearchOffsetResponseHeader.class);
+        when(responseHeader.getOffset()).thenReturn(1L);
+        setResponseHeader(responseHeader);
+        assertEquals(1L, mqClientAPI.searchOffset(defaultBrokerAddr, new MessageQueue(), System.currentTimeMillis(), defaultTimeout));
+    }
+
+    @Test
+    public void testUpdateConsumerOffsetOneway() throws RemotingException, InterruptedException {
+        UpdateConsumerOffsetRequestHeader requestHeader = mock(UpdateConsumerOffsetRequestHeader.class);
+        mqClientAPI.updateConsumerOffsetOneway(defaultBrokerAddr, requestHeader, defaultTimeout);
+    }
+
+    @Test
+    public void assertSendHeartbeat() throws MQBrokerException, RemotingException, InterruptedException {
+        mockInvokeSync();
+        HeartbeatData heartbeatData = new HeartbeatData();
+        assertEquals(1, mqClientAPI.sendHeartbeat(defaultBrokerAddr, heartbeatData, defaultTimeout));
+    }
+
+    @Test
+    public void assertSendHeartbeatV2() throws MQBrokerException, RemotingException, InterruptedException {
+        mockInvokeSync();
+        HeartbeatData heartbeatData = new HeartbeatData();
+        HeartbeatV2Result actual = mqClientAPI.sendHeartbeatV2(defaultBrokerAddr, heartbeatData, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getVersion());
+        assertFalse(actual.isSubChange());
+        assertFalse(actual.isSupportV2());
+    }
+
+    @Test
+    public void testUnregisterClient() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        mqClientAPI.unregisterClient(defaultBrokerAddr, "", "", "", defaultTimeout);
+    }
+
+    @Test
+    public void testEndTransactionOneway() throws RemotingException, InterruptedException {
+        mockInvokeSync();
+        EndTransactionRequestHeader requestHeader = mock(EndTransactionRequestHeader.class);
+        mqClientAPI.endTransactionOneway(defaultBrokerAddr, requestHeader, "", defaultTimeout);
+    }
+
+    @Test
+    public void testQueryMessage() throws MQBrokerException, RemotingException, InterruptedException {
+        QueryMessageRequestHeader requestHeader = mock(QueryMessageRequestHeader.class);
+        InvokeCallback callback = mock(InvokeCallback.class);
+        mqClientAPI.queryMessage(defaultBrokerAddr, requestHeader, defaultTimeout, callback, false);
+    }
+
+    @Test
+    public void testRegisterClient() throws RemotingException, InterruptedException {
+        mockInvokeSync();
+        HeartbeatData heartbeatData = new HeartbeatData();
+        assertTrue(mqClientAPI.registerClient(defaultBrokerAddr, heartbeatData, defaultTimeout));
+    }
+
+    @Test
+    public void testConsumerSendMessageBack() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        MessageExt messageExt = mock(MessageExt.class);
+        mqClientAPI.consumerSendMessageBack(defaultBrokerAddr, brokerName, messageExt, "", 1, defaultTimeout, 1000);
+    }
+
+    @Test
+    public void assertLockBatchMQ() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        LockBatchRequestBody responseBody = new LockBatchRequestBody();
+        setResponseBody(responseBody);
+        Set<MessageQueue> actual = mqClientAPI.lockBatchMQ(defaultBrokerAddr, responseBody, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(0, actual.size());
+    }
+
+    @Test
+    public void testUnlockBatchMQ() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        UnlockBatchRequestBody unlockBatchRequestBody = new UnlockBatchRequestBody();
+        mqClientAPI.unlockBatchMQ(defaultBrokerAddr, unlockBatchRequestBody, defaultTimeout, false);
+    }
+
+    @Test
+    public void assertGetTopicStatsInfo() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        TopicStatsTable responseBody = new TopicStatsTable();
+        MessageQueue messageQueue = new MessageQueue();
+        TopicOffset topicOffset = new TopicOffset();
+        responseBody.getOffsetTable().put(messageQueue, topicOffset);
+        setResponseBody(responseBody);
+        TopicStatsTable actual = mqClientAPI.getTopicStatsInfo(defaultBrokerAddr, defaultTopic, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getOffsetTable().size());
+    }
+
+    @Test
+    public void assertGetConsumeStats() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        ConsumeStats responseBody = new ConsumeStats();
+        responseBody.setConsumeTps(1000);
+        setResponseBody(responseBody);
+        ConsumeStats actual = mqClientAPI.getConsumeStats(defaultBrokerAddr, "", defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1000, actual.getConsumeTps(), 0.0);
+    }
+
+    @Test
+    public void assertGetProducerConnectionList() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        ProducerConnection responseBody = new ProducerConnection();
+        responseBody.getConnectionSet().add(new Connection());
+        setResponseBody(responseBody);
+        ProducerConnection actual = mqClientAPI.getProducerConnectionList(defaultBrokerAddr, "", defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getConnectionSet().size());
+    }
+
+    @Test
+    public void assertGetAllProducerInfo() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        Map<String, List<ProducerInfo>> data = new HashMap<>();
+        data.put("key", Collections.emptyList());
+        ProducerTableInfo responseBody = new ProducerTableInfo(data);
+        setResponseBody(responseBody);
+        ProducerTableInfo actual = mqClientAPI.getAllProducerInfo(defaultBrokerAddr, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getData().size());
+    }
+
+    @Test
+    public void assertGetConsumerConnectionList() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        ConsumerConnection responseBody = new ConsumerConnection();
+        responseBody.setConsumeType(ConsumeType.CONSUME_ACTIVELY);
+        responseBody.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
+        responseBody.setMessageModel(MessageModel.CLUSTERING);
+        setResponseBody(responseBody);
+        ConsumerConnection actual = mqClientAPI.getConsumerConnectionList(defaultBrokerAddr, "", defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(ConsumeType.CONSUME_ACTIVELY, actual.getConsumeType());
+        assertEquals(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, actual.getConsumeFromWhere());
+        assertEquals(MessageModel.CLUSTERING, actual.getMessageModel());
+    }
+
+    @Test
+    public void assertGetBrokerRuntimeInfo() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        KVTable responseBody = new KVTable();
+        responseBody.getTable().put("key", "value");
+        setResponseBody(responseBody);
+        KVTable actual = mqClientAPI.getBrokerRuntimeInfo(defaultBrokerAddr, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getTable().size());
+    }
+
+    @Test
+    public void testAddBroker() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        mqClientAPI.addBroker(defaultBrokerAddr, "", defaultTimeout);
+    }
+
+    @Test
+    public void testRemoveBroker() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        mqClientAPI.removeBroker(defaultBrokerAddr, clusterName, brokerName, MixAll.MASTER_ID, defaultTimeout);
+    }
+
+    @Test
+    public void testUpdateBrokerConfig() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException, MQClientException {
+        mockInvokeSync();
+        mqClientAPI.updateBrokerConfig(defaultBrokerAddr, createProperties(), defaultTimeout);
+    }
+
+    @Test
+    public void assertGetBrokerConfig() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException {
+        mockInvokeSync();
+        setResponseBody("{\"key\":\"value\"}");
+        Properties actual = mqClientAPI.getBrokerConfig(defaultBrokerAddr, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.size());
+    }
+
+    @Test
+    public void testUpdateColdDataFlowCtrGroupConfig() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException {
+        mockInvokeSync();
+        Properties props = new Properties();
+        mqClientAPI.updateColdDataFlowCtrGroupConfig(defaultBrokerAddr, props, defaultTimeout);
+    }
+
+    @Test
+    public void testRemoveColdDataFlowCtrGroupConfig() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException {
+        mockInvokeSync();
+        mqClientAPI.removeColdDataFlowCtrGroupConfig(defaultBrokerAddr, "", defaultTimeout);
+    }
+
+    @Test
+    public void assertGetColdDataFlowCtrInfo() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException {
+        mockInvokeSync();
+        setResponseBody("{\"key\":\"value\"}");
+        String actual = mqClientAPI.getColdDataFlowCtrInfo(defaultBrokerAddr, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals("\"{\\\"key\\\":\\\"value\\\"}\"", actual);
+    }
+
+    @Test
+    public void assertSetCommitLogReadAheadMode() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        when(response.getRemark()).thenReturn("remark");
+        String actual = mqClientAPI.setCommitLogReadAheadMode(defaultBrokerAddr, "", defaultTimeout);
+        assertNotNull(actual);
+        assertEquals("remark", actual);
+    }
+
+    @Test
+    public void assertGetBrokerClusterInfo() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        ClusterInfo responseBody = new ClusterInfo();
+        Map<String, Set<String>> clusterAddrTable = new HashMap<>();
+        clusterAddrTable.put(clusterName, new HashSet<>());
+        Map<String, BrokerData> brokerAddrTable = new HashMap<>();
+        brokerAddrTable.put(brokerName, new BrokerData());
+        responseBody.setClusterAddrTable(clusterAddrTable);
+        responseBody.setBrokerAddrTable(brokerAddrTable);
+        setResponseBody(responseBody);
+        ClusterInfo actual = mqClientAPI.getBrokerClusterInfo(defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getClusterAddrTable().size());
+        assertEquals(1, actual.getBrokerAddrTable().size());
+    }
+
+    @Test
+    public void assertGetDefaultTopicRouteInfoFromNameServer() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        TopicRouteData responseBody = new TopicRouteData();
+        responseBody.getQueueDatas().add(new QueueData());
+        responseBody.getBrokerDatas().add(new BrokerData());
+        responseBody.getFilterServerTable().put("key", Collections.emptyList());
+        Map<String, TopicQueueMappingInfo> topicQueueMappingByBroker = new HashMap<>();
+        topicQueueMappingByBroker.put("key", new TopicQueueMappingInfo());
+        responseBody.setTopicQueueMappingByBroker(topicQueueMappingByBroker);
+        setResponseBody(responseBody);
+        TopicRouteData actual = mqClientAPI.getDefaultTopicRouteInfoFromNameServer(defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getQueueDatas().size());
+        assertEquals(1, actual.getBrokerDatas().size());
+        assertEquals(1, actual.getFilterServerTable().size());
+        assertEquals(1, actual.getTopicQueueMappingByBroker().size());
+    }
+
+    @Test
+    public void assertGetTopicRouteInfoFromNameServer() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        TopicRouteData responseBody = new TopicRouteData();
+        responseBody.getQueueDatas().add(new QueueData());
+        responseBody.getBrokerDatas().add(new BrokerData());
+        responseBody.getFilterServerTable().put("key", Collections.emptyList());
+        Map<String, TopicQueueMappingInfo> topicQueueMappingByBroker = new HashMap<>();
+        topicQueueMappingByBroker.put("key", new TopicQueueMappingInfo());
+        responseBody.setTopicQueueMappingByBroker(topicQueueMappingByBroker);
+        setResponseBody(responseBody);
+        TopicRouteData actual = mqClientAPI.getTopicRouteInfoFromNameServer(defaultTopic, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getQueueDatas().size());
+        assertEquals(1, actual.getBrokerDatas().size());
+        assertEquals(1, actual.getFilterServerTable().size());
+        assertEquals(1, actual.getTopicQueueMappingByBroker().size());
+    }
+
+    @Test
+    public void assertGetTopicListFromNameServer() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        TopicList responseBody = new TopicList();
+        responseBody.setBrokerAddr(defaultBrokerAddr);
+        responseBody.getTopicList().add(defaultTopic);
+        setResponseBody(responseBody);
+        TopicList actual = mqClientAPI.getTopicListFromNameServer(defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getTopicList().size());
+        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());
+    }
+
+    @Test
+    public void assertWipeWritePermOfBroker() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        WipeWritePermOfBrokerResponseHeader responseHeader = mock(WipeWritePermOfBrokerResponseHeader.class);
+        when(responseHeader.getWipeTopicCount()).thenReturn(1);
+        setResponseHeader(responseHeader);
+        assertEquals(1, mqClientAPI.wipeWritePermOfBroker(defaultNsAddr, brokerName, defaultTimeout));
+    }
+
+    @Test
+    public void testDeleteTopicInBroker() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        mqClientAPI.deleteTopicInBroker(defaultBrokerAddr, defaultTopic, defaultTimeout);
+    }
+
+    @Test
+    public void testDeleteTopicInNameServer() throws RemotingException, InterruptedException, MQClientException, MQBrokerException {
+        mockInvokeSync();
+        mqClientAPI.deleteTopicInNameServer(defaultNsAddr, defaultTopic, defaultTimeout);
+        mqClientAPI.deleteTopicInNameServer(defaultNsAddr, clusterName, defaultTopic, defaultTimeout);
+    }
+
+    @Test
+    public void testDeleteSubscriptionGroup() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        mqClientAPI.deleteSubscriptionGroup(defaultBrokerAddr, "", true, defaultTimeout);
+    }
+
+    @Test
+    public void assertGetKVConfigValue() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        GetKVConfigResponseHeader responseHeader = mock(GetKVConfigResponseHeader.class);
+        when(responseHeader.getValue()).thenReturn("value");
+        setResponseHeader(responseHeader);
+        assertEquals("value", mqClientAPI.getKVConfigValue("", "", defaultTimeout));
+    }
+
+    @Test
+    public void testPutKVConfigValue() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        mqClientAPI.putKVConfigValue("", "", "", defaultTimeout);
+    }
+
+    @Test
+    public void testDeleteKVConfigValue() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        mqClientAPI.deleteKVConfigValue("", "", defaultTimeout);
+    }
+
+    @Test
+    public void assertGetKVListByNamespace() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        KVTable responseBody = new KVTable();
+        responseBody.getTable().put("key", "value");
+        setResponseBody(responseBody);
+        KVTable actual = mqClientAPI.getKVListByNamespace("", defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getTable().size());
+    }
+
+    @Test
+    public void assertInvokeBrokerToResetOffset() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        ResetOffsetBody responseBody = new ResetOffsetBody();
+        responseBody.getOffsetTable().put(new MessageQueue(), 1L);
+        setResponseBody(responseBody);
+        Map<MessageQueue, Long> actual = mqClientAPI.invokeBrokerToResetOffset(defaultBrokerAddr, defaultTopic, "", System.currentTimeMillis(), false, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.size());
+        actual = mqClientAPI.invokeBrokerToResetOffset(defaultBrokerAddr, defaultTopic, "", System.currentTimeMillis(), 1, 1L, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.size());
+    }
+
+    @Test
+    public void assertInvokeBrokerToGetConsumerStatus() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        GetConsumerStatusBody responseBody = new GetConsumerStatusBody();
+        responseBody.getConsumerTable().put("key", new HashMap<>());
+        responseBody.getMessageQueueTable().put(new MessageQueue(), 1L);
+        setResponseBody(responseBody);
+        Map<String, Map<MessageQueue, Long>> actual = mqClientAPI.invokeBrokerToGetConsumerStatus(defaultBrokerAddr, defaultTopic, "", "", defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.size());
+    }
+
+    @Test
+    public void assertQueryTopicConsumeByWho() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        GroupList responseBody = new GroupList();
+        responseBody.getGroupList().add("");
+        setResponseBody(responseBody);
+        GroupList actual = mqClientAPI.queryTopicConsumeByWho(defaultBrokerAddr, defaultTopic, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getGroupList().size());
+    }
+
+    @Test
+    public void assertQueryTopicsByConsumer() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        TopicList responseBody = new TopicList();
+        responseBody.getTopicList().add(defaultTopic);
+        responseBody.setBrokerAddr(defaultBrokerAddr);
+        setResponseBody(responseBody);
+        TopicList actual = mqClientAPI.queryTopicsByConsumer(defaultBrokerAddr, "", defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getTopicList().size());
+        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());
+    }
+
+    @Test
+    public void assertQuerySubscriptionByConsumer() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        QuerySubscriptionResponseBody responseBody = new QuerySubscriptionResponseBody();
+        SubscriptionData subscriptionData = new SubscriptionData();
+        subscriptionData.setTopic(defaultTopic);
+        responseBody.setSubscriptionData(subscriptionData);
+        setResponseBody(responseBody);
+        SubscriptionData actual = mqClientAPI.querySubscriptionByConsumer(defaultBrokerAddr, group, defaultTopic, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(defaultTopic, actual.getTopic());
+    }
+
+    @Test
+    public void assertQueryConsumeTimeSpan() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        QueryConsumeTimeSpanBody responseBody = new QueryConsumeTimeSpanBody();
+        responseBody.getConsumeTimeSpanSet().add(new QueueTimeSpan());
+        setResponseBody(responseBody);
+        List<QueueTimeSpan> actual = mqClientAPI.queryConsumeTimeSpan(defaultBrokerAddr, defaultTopic, group, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.size());
+    }
+
+    @Test
+    public void assertGetTopicsByCluster() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        TopicList responseBody = new TopicList();
+        responseBody.setBrokerAddr(defaultBrokerAddr);
+        responseBody.setTopicList(Collections.singleton(defaultTopic));
+        setResponseBody(responseBody);
+        TopicList actual = mqClientAPI.getTopicsByCluster(clusterName, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());
+        assertEquals(1, actual.getTopicList().size());
+        assertTrue(actual.getTopicList().contains(defaultTopic));
+    }
+
+    @Test
+    public void assertGetSystemTopicList() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        TopicList responseBody = new TopicList();
+        responseBody.setBrokerAddr(defaultBrokerAddr);
+        responseBody.setTopicList(Collections.singleton(defaultTopic));
+        setResponseBody(responseBody);
+        TopicList actual = mqClientAPI.getSystemTopicList(defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());
+        assertEquals(1, actual.getTopicList().size());
+        assertTrue(actual.getTopicList().contains(defaultTopic));
+    }
+
+    @Test
+    public void assertGetSystemTopicListFromBroker() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        TopicList responseBody = new TopicList();
+        responseBody.setBrokerAddr(defaultBrokerAddr);
+        responseBody.setTopicList(Collections.singleton(defaultTopic));
+        setResponseBody(responseBody);
+        TopicList actual = mqClientAPI.getSystemTopicListFromBroker(defaultBrokerAddr, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());
+        assertEquals(1, actual.getTopicList().size());
+        assertTrue(actual.getTopicList().contains(defaultTopic));
+    }
+
+    @Test
+    public void assertCleanExpiredConsumeQueue() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        assertTrue(mqClientAPI.cleanExpiredConsumeQueue(defaultBrokerAddr, defaultTimeout));
+    }
+
+    @Test
+    public void assertDeleteExpiredCommitLog() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        assertTrue(mqClientAPI.deleteExpiredCommitLog(defaultBrokerAddr, defaultTimeout));
+    }
+
+    @Test
+    public void assertCleanUnusedTopicByAddr() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        assertTrue(mqClientAPI.cleanUnusedTopicByAddr(defaultBrokerAddr, defaultTimeout));
+    }
+
+    @Test
+    public void assertGetConsumerRunningInfo() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        ConsumerRunningInfo responseBody = new ConsumerRunningInfo();
+        responseBody.setJstack("jstack");
+        responseBody.getUserConsumerInfo().put("key", "value");
+        setResponseBody(responseBody);
+        ConsumerRunningInfo actual = mqClientAPI.getConsumerRunningInfo(defaultBrokerAddr, group, clientId, false, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals("jstack", actual.getJstack());
+        assertEquals(1, actual.getUserConsumerInfo().size());
+    }
+
+    @Test
+    public void assertConsumeMessageDirectly() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        ConsumeMessageDirectlyResult responseBody = new ConsumeMessageDirectlyResult();
+        responseBody.setAutoCommit(true);
+        responseBody.setRemark("remark");
+        setResponseBody(responseBody);
+        ConsumeMessageDirectlyResult actual = mqClientAPI.consumeMessageDirectly(defaultBrokerAddr, group, clientId, topic, "", defaultTimeout);
+        assertNotNull(actual);
+        assertEquals("remark", actual.getRemark());
+        assertTrue(actual.isAutoCommit());
+    }
+
+    @Test
+    public void assertQueryCorrectionOffset() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        QueryCorrectionOffsetBody responseBody = new QueryCorrectionOffsetBody();
+        responseBody.getCorrectionOffsets().put(1, 1L);
+        setResponseBody(responseBody);
+        Map<Integer, Long> actual = mqClientAPI.queryCorrectionOffset(defaultBrokerAddr, topic, group, new HashSet<>(), defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.size());
+        assertTrue(actual.containsKey(1));
+        assertTrue(actual.containsValue(1L));
+    }
+
+    @Test
+    public void assertGetUnitTopicList() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        TopicList responseBody = new TopicList();
+        responseBody.getTopicList().add(defaultTopic);
+        setResponseBody(responseBody);
+        TopicList actual = mqClientAPI.getUnitTopicList(false, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getTopicList().size());
+    }
+
+    @Test
+    public void assertGetHasUnitSubTopicList() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        TopicList responseBody = new TopicList();
+        responseBody.getTopicList().add(defaultTopic);
+        setResponseBody(responseBody);
+        TopicList actual = mqClientAPI.getHasUnitSubTopicList(false, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getTopicList().size());
+    }
+
+    @Test
+    public void assertGetHasUnitSubUnUnitTopicList() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        TopicList responseBody = new TopicList();
+        responseBody.getTopicList().add(defaultTopic);
+        setResponseBody(responseBody);
+        TopicList actual = mqClientAPI.getHasUnitSubUnUnitTopicList(false, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getTopicList().size());
+    }
+
+    @Test
+    public void testCloneGroupOffset() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        mqClientAPI.cloneGroupOffset(defaultBrokerAddr, "", "", defaultTopic, false, defaultTimeout);
+    }
+
+    @Test
+    public void assertViewBrokerStatsData() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        BrokerStatsData responseBody = new BrokerStatsData();
+        responseBody.setStatsDay(new BrokerStatsItem());
+        setResponseBody(responseBody);
+        BrokerStatsData actual = mqClientAPI.viewBrokerStatsData(defaultBrokerAddr, "", "", defaultTimeout);
+        assertNotNull(actual);
+        assertNotNull(actual.getStatsDay());
+    }
+
+    @Test
+    public void assertGetClusterList() {
+        Set<String> actual = mqClientAPI.getClusterList(topic, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(0, actual.size());
+    }
+
+    @Test
+    public void assertFetchConsumeStatsInBroker() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        ConsumeStatsList responseBody = new ConsumeStatsList();
+        responseBody.setBrokerAddr(defaultBrokerAddr);
+        responseBody.getConsumeStatsList().add(new HashMap<>());
+        setResponseBody(responseBody);
+        ConsumeStatsList actual = mqClientAPI.fetchConsumeStatsInBroker(defaultBrokerAddr, false, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getConsumeStatsList().size());
+        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());
+    }
+
+    @Test
+    public void assertGetAllSubscriptionGroupForSubscriptionGroupWrapper() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        SubscriptionGroupWrapper responseBody = new SubscriptionGroupWrapper();
+        responseBody.getSubscriptionGroupTable().put("key", new SubscriptionGroupConfig());
+        setResponseBody(responseBody);
+        SubscriptionGroupWrapper actual = mqClientAPI.getAllSubscriptionGroup(defaultBrokerAddr, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getSubscriptionGroupTable().size());
+        assertNotNull(actual.getDataVersion());
+        assertEquals(0, actual.getDataVersion().getStateVersion());
+    }
+
+    @Test
+    public void assertGetAllSubscriptionGroupForSubscriptionGroupConfig() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        SubscriptionGroupConfig responseBody = new SubscriptionGroupConfig();
+        responseBody.setGroupName(group);
+        responseBody.setBrokerId(MixAll.MASTER_ID);
+        setResponseBody(responseBody);
+        SubscriptionGroupConfig actual = mqClientAPI.getSubscriptionGroupConfig(defaultBrokerAddr, group, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(group, actual.getGroupName());
+        assertEquals(MixAll.MASTER_ID, actual.getBrokerId());
+    }
+
+    @Test
+    public void assertGetAllTopicConfig() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        TopicConfigSerializeWrapper responseBody = new TopicConfigSerializeWrapper();
+        responseBody.getTopicConfigTable().put("key", new TopicConfig());
+        setResponseBody(responseBody);
+        TopicConfigSerializeWrapper actual = mqClientAPI.getAllTopicConfig(defaultBrokerAddr, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getTopicConfigTable().size());
+        assertNotNull(actual.getDataVersion());
+        assertEquals(0, actual.getDataVersion().getStateVersion());
+    }
+
+    @Test
+    public void testUpdateNameServerConfig() throws RemotingException, InterruptedException, MQClientException, UnsupportedEncodingException {
+        mockInvokeSync();
+        mqClientAPI.updateNameServerConfig(createProperties(), Collections.singletonList(defaultNsAddr), defaultTimeout);
+    }
+
+    @Test
+    public void assertGetNameServerConfig() throws RemotingException, InterruptedException, UnsupportedEncodingException, MQClientException {
+        mockInvokeSync();
+        setResponseBody("{\"key\":\"value\"}");
+        Map<String, Properties> actual = mqClientAPI.getNameServerConfig(Collections.singletonList(defaultNsAddr), defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.size());
+        assertTrue(actual.containsKey(defaultNsAddr));
+    }
+
+    @Test
+    public void assertQueryConsumeQueue() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        QueryConsumeQueueResponseBody responseBody = new QueryConsumeQueueResponseBody();
+        responseBody.setQueueData(Collections.singletonList(new ConsumeQueueData()));
+        setResponseBody(responseBody);
+        QueryConsumeQueueResponseBody actual = mqClientAPI.queryConsumeQueue(defaultBrokerAddr, defaultTopic, 1, 1, 1, group, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1, actual.getQueueData().size());
+    }
+
+    @Test
+    public void testCheckClientInBroker() throws RemotingException, InterruptedException, MQClientException {
+        mockInvokeSync();
+        mqClientAPI.checkClientInBroker(defaultBrokerAddr, group, clientId, new SubscriptionData(), defaultTimeout);
+    }
+
+    @Test
+    public void assertGetTopicConfig() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        TopicConfigAndQueueMapping responseBody = new TopicConfigAndQueueMapping(new TopicConfig(), new TopicQueueMappingDetail());
+        setResponseBody(responseBody);
+        TopicConfigAndQueueMapping actual = mqClientAPI.getTopicConfig(defaultBrokerAddr, defaultTopic, defaultTimeout);
+        assertNotNull(actual);
+        assertNotNull(actual.getMappingDetail());
+    }
+
+    @Test
+    public void testCreateStaticTopic() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        mqClientAPI.createStaticTopic(defaultBrokerAddr, defaultTopic, new TopicConfig(), new TopicQueueMappingDetail(), false, defaultTimeout);
+    }
+
+    @Test
+    public void assertUpdateAndGetGroupForbidden() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        GroupForbidden responseBody = new GroupForbidden();
+        responseBody.setGroup(group);
+        responseBody.setTopic(defaultTopic);
+        setResponseBody(responseBody);
+        GroupForbidden actual = mqClientAPI.updateAndGetGroupForbidden(defaultBrokerAddr, new UpdateGroupForbiddenRequestHeader(), defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(group, actual.getGroup());
+        assertEquals(defaultTopic, actual.getTopic());
+    }
+
+    @Test
+    public void testResetMasterFlushOffset() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        mqClientAPI.resetMasterFlushOffset(defaultBrokerAddr, 1L);
+    }
+
+    @Test
+    public void assertGetBrokerHAStatus() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        HARuntimeInfo responseBody = new HARuntimeInfo();
+        responseBody.setMaster(true);
+        responseBody.setMasterCommitLogMaxOffset(1L);
+        setResponseBody(responseBody);
+        HARuntimeInfo actual = mqClientAPI.getBrokerHAStatus(defaultBrokerAddr, defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1L, actual.getMasterCommitLogMaxOffset());
+        assertTrue(actual.isMaster());
+    }
+
+    @Test
+    public void assertGetControllerMetaData() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        GetMetaDataResponseHeader responseHeader = new GetMetaDataResponseHeader();
+        responseHeader.setGroup(group);
+        responseHeader.setIsLeader(true);
+        setResponseHeader(responseHeader);
+        GetMetaDataResponseHeader actual = mqClientAPI.getControllerMetaData(defaultBrokerAddr);
+        assertNotNull(actual);
+        assertEquals(group, actual.getGroup());
+        assertTrue(actual.isLeader());
+    }
+
+    @Test
+    public void assertGetInSyncStateData() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        BrokerReplicasInfo responseBody = new BrokerReplicasInfo();
+        BrokerReplicasInfo.ReplicasInfo replicasInfo = new BrokerReplicasInfo.ReplicasInfo(MixAll.MASTER_ID, defaultBrokerAddr, 1, 1, Collections.emptyList(), Collections.emptyList());
+        responseBody.getReplicasInfoTable().put("key", replicasInfo);
+        GetMetaDataResponseHeader responseHeader = new GetMetaDataResponseHeader();
+        responseHeader.setControllerLeaderAddress(defaultBrokerAddr);
+        setResponseHeader(responseHeader);
+        setResponseBody(responseBody);
+        BrokerReplicasInfo actual = mqClientAPI.getInSyncStateData(defaultBrokerAddr, Collections.singletonList(defaultBrokerAddr));
+        assertNotNull(actual);
+        assertEquals(1L, actual.getReplicasInfoTable().size());
+    }
+
+    @Test
+    public void assertGetBrokerEpochCache() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        EpochEntryCache responseBody = new EpochEntryCache(clusterName, brokerName, MixAll.MASTER_ID, Collections.emptyList(), 1);
+        setResponseBody(responseBody);
+        EpochEntryCache actual = mqClientAPI.getBrokerEpochCache(defaultBrokerAddr);
+        assertNotNull(actual);
+        assertEquals(1L, actual.getMaxOffset());
+        assertEquals(MixAll.MASTER_ID, actual.getBrokerId());
+        assertEquals(brokerName, actual.getBrokerName());
+        assertEquals(clusterName, actual.getClusterName());
+    }
+
+    @Test
+    public void assertGetControllerConfig() throws RemotingException, InterruptedException, UnsupportedEncodingException, MQClientException {
+        mockInvokeSync();
+        setResponseBody("{\"key\":\"value\"}");
+        Map<String, Properties> actual = mqClientAPI.getControllerConfig(Collections.singletonList(defaultBrokerAddr), defaultTimeout);
+        assertNotNull(actual);
+        assertEquals(1L, actual.size());
+    }
+
+    @Test
+    public void testUpdateControllerConfig() throws RemotingException, InterruptedException, UnsupportedEncodingException, MQClientException {
+        mockInvokeSync();
+        mqClientAPI.updateControllerConfig(createProperties(), Collections.singletonList(defaultBrokerAddr), defaultTimeout);
+    }
+
+    @Test
+    public void assertElectMaster() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        BrokerMemberGroup responseBody = new BrokerMemberGroup();
+        setResponseBody(responseBody);
+        GetMetaDataResponseHeader getMetaDataResponseHeader = new GetMetaDataResponseHeader();
+        getMetaDataResponseHeader.setControllerLeaderAddress(defaultBrokerAddr);
+        when(response.decodeCommandCustomHeader(GetMetaDataResponseHeader.class)).thenReturn(getMetaDataResponseHeader);
+        ElectMasterResponseHeader responseHeader = new ElectMasterResponseHeader();
+        when(response.decodeCommandCustomHeader(ElectMasterResponseHeader.class)).thenReturn(responseHeader);
+        Pair<ElectMasterResponseHeader, BrokerMemberGroup> actual = mqClientAPI.electMaster(defaultBrokerAddr, clusterName, brokerName, MixAll.MASTER_ID);
+        assertNotNull(actual);
+        assertEquals(responseHeader, actual.getObject1());
+        assertEquals(responseBody, actual.getObject2());
+    }
+
+    @Test
+    public void testCleanControllerBrokerData() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        GetMetaDataResponseHeader responseHeader = new GetMetaDataResponseHeader();
+        responseHeader.setControllerLeaderAddress(defaultBrokerAddr);
+        setResponseHeader(responseHeader);
+        mqClientAPI.cleanControllerBrokerData(defaultBrokerAddr, clusterName, brokerName, "", false);
+    }
+
+    @Test
+    public void testCreateUser() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        mqClientAPI.createUser(defaultBrokerAddr, new UserInfo(), defaultTimeout);
+    }
+
+    @Test
+    public void testUpdateUser() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        mqClientAPI.updateUser(defaultBrokerAddr, new UserInfo(), defaultTimeout);
+    }
+
+    @Test
+    public void testDeleteUser() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        mqClientAPI.deleteUser(defaultBrokerAddr, "", defaultTimeout);
+    }
+
+    @Test
+    public void assertGetUser() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        setResponseBody(createUserInfo());
+        UserInfo actual = mqClientAPI.getUser(defaultBrokerAddr, "", defaultTimeout);
+        assertNotNull(actual);
+        assertEquals("username", actual.getUsername());
+        assertEquals("password", actual.getPassword());
+        assertEquals("userStatus", actual.getUserStatus());
+        assertEquals("userType", actual.getUserType());
+    }
+
+    @Test
+    public void assertListUser() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        setResponseBody(Collections.singletonList(createUserInfo()));
+        List<UserInfo> actual = mqClientAPI.listUser(defaultBrokerAddr, "", defaultTimeout);
+        assertNotNull(actual);
+        assertEquals("username", actual.get(0).getUsername());
+        assertEquals("password", actual.get(0).getPassword());
+        assertEquals("userStatus", actual.get(0).getUserStatus());
+        assertEquals("userType", actual.get(0).getUserType());
+    }
+
+    @Test
+    public void testCreateAcl() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        mqClientAPI.createAcl(defaultBrokerAddr, new AclInfo(), defaultTimeout);
+    }
+
+    @Test
+    public void testUpdateAcl() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        mqClientAPI.updateAcl(defaultBrokerAddr, new AclInfo(), defaultTimeout);
+    }
+
+    @Test
+    public void testDeleteAcl() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        mqClientAPI.deleteAcl(defaultBrokerAddr, "", "", defaultTimeout);
+    }
+
+    @Test
+    public void assertGetAcl() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        setResponseBody(createAclInfo());
+        AclInfo actual = mqClientAPI.getAcl(defaultBrokerAddr, "", defaultTimeout);
+        assertNotNull(actual);
+        assertEquals("subject", actual.getSubject());
+        assertEquals(1, actual.getPolicies().size());
+    }
+
+    @Test
+    public void assertListAcl() throws RemotingException, InterruptedException, MQBrokerException {
+        mockInvokeSync();
+        setResponseBody(Collections.singletonList(createAclInfo()));
+        List<AclInfo> actual = mqClientAPI.listAcl(defaultBrokerAddr, "", "", defaultTimeout);
+        assertNotNull(actual);
+        assertEquals("subject", actual.get(0).getSubject());
+        assertEquals(1, actual.get(0).getPolicies().size());
+    }
+
+    @Test
+    public void testRecallMessage() throws RemotingException, InterruptedException, MQBrokerException {
+        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();
+        requestHeader.setProducerGroup(group);
+        requestHeader.setTopic(topic);
+        requestHeader.setRecallHandle("handle");
+        requestHeader.setBrokerName(brokerName);
+
+        // success
+        mockInvokeSync();
+        String msgId = MessageClientIDSetter.createUniqID();
+        RecallMessageResponseHeader responseHeader = new RecallMessageResponseHeader();
+        responseHeader.setMsgId(msgId);
+        setResponseHeader(responseHeader);
+        String result = mqClientAPI.recallMessage(defaultBrokerAddr, requestHeader, defaultTimeout);
+        assertEquals(msgId, result);
+
+        // error
+        when(response.getCode()).thenReturn(ResponseCode.SYSTEM_ERROR);
+        when(response.getRemark()).thenReturn("error");
+        MQBrokerException e = assertThrows(MQBrokerException.class, () -> {
+            mqClientAPI.recallMessage(defaultBrokerAddr, requestHeader, defaultTimeout);
+        });
+        assertEquals(ResponseCode.SYSTEM_ERROR, e.getResponseCode());
+        assertEquals("error", e.getErrorMessage());
+        assertEquals(defaultBrokerAddr, e.getBrokerAddr());
+    }
+
+    @Test
+    public void testRecallMessageAsync() throws RemotingException, InterruptedException {
+        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();
+        requestHeader.setProducerGroup(group);
+        requestHeader.setTopic(topic);
+        requestHeader.setRecallHandle("handle");
+        requestHeader.setBrokerName(brokerName);
+        String msgId = "msgId";
+        doAnswer((Answer<Void>) mock -> {
+            InvokeCallback callback = mock.getArgument(3);
+            RemotingCommand request = mock.getArgument(1);
+            RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class);
+            response.setCode(ResponseCode.SUCCESS);
+            response.setOpaque(request.getOpaque());
+            RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();
+            responseHeader.setMsgId(msgId);
+            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
+            responseFuture.setResponseCommand(response);
+            callback.operationSucceed(responseFuture.getResponseCommand());
+            return null;
+        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
+
+        final CountDownLatch done = new CountDownLatch(1);
+        mqClientAPI.recallMessageAsync(defaultBrokerAddr, requestHeader,
+            defaultTimeout, new InvokeCallback() {
+                @Override
+                public void operationComplete(ResponseFuture responseFuture) {
+                }
+
+                @Override
+                public void operationSucceed(RemotingCommand response) {
+                    RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();
+                    Assert.assertEquals(msgId, responseHeader.getMsgId());
+                    done.countDown();
+                }
+
+                @Override
+                public void operationFail(Throwable throwable) {
+                }
+            });
+        done.await();
+    }
+
+    @Test
+    public void testMQClientAPIImplWithoutObjectCreator() {
+        MQClientAPIImpl clientAPI = new MQClientAPIImpl(
+            new NettyClientConfig(),
+            null,
+            null,
+            new ClientConfig(),
+            null,
+            null
+        );
+        RemotingClient remotingClient1 = clientAPI.getRemotingClient();
+        Assert.assertTrue(remotingClient1 instanceof NettyRemotingClient);
+    }
+
+    @Test
+    public void testMQClientAPIImplWithObjectCreator() {
+        ObjectCreator<RemotingClient> clientObjectCreator = args -> new MockRemotingClientTest((NettyClientConfig) args[0]);
+        final NettyClientConfig nettyClientConfig = new NettyClientConfig();
+        MQClientAPIImpl clientAPI = new MQClientAPIImpl(
+            nettyClientConfig,
+            null,
+            null,
+            new ClientConfig(),
+            null,
+            clientObjectCreator
+        );
+        RemotingClient remotingClient1 = clientAPI.getRemotingClient();
+        Assert.assertTrue(remotingClient1 instanceof MockRemotingClientTest);
+        MockRemotingClientTest remotingClientTest = (MockRemotingClientTest) remotingClient1;
+        Assert.assertSame(remotingClientTest.getNettyClientConfig(), nettyClientConfig);
+    }
+
+    private static class MockRemotingClientTest extends NettyRemotingClient {
+        public MockRemotingClientTest(NettyClientConfig nettyClientConfig) {
+            super(nettyClientConfig);
+        }
+
+        public NettyClientConfig getNettyClientConfig() {
+            return nettyClientConfig;
+        }
+    }
+
+    private Properties createProperties() {
+        Properties result = new Properties();
+        result.put("key", "value");
+        return result;
+    }
+
+    private AclInfo createAclInfo() {
+        return AclInfo.of("subject", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), "");
+    }
+
+    private UserInfo createUserInfo() {
+        UserInfo result = new UserInfo();
+        result.setUsername("username");
+        result.setPassword("password");
+        result.setUserStatus("userStatus");
+        result.setUserType("userType");
+        return result;
+    }
+
+    private void setResponseHeader(CommandCustomHeader responseHeader) throws RemotingCommandException {
+        when(response.decodeCommandCustomHeader(any())).thenReturn(responseHeader);
+    }
+
+    private void setResponseBody(Object responseBody) {
+        when(response.getBody()).thenReturn(RemotingSerializable.encode(responseBody));
+    }
+
+    private void mockInvokeSync() throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException {
+        when(response.getCode()).thenReturn(ResponseCode.SUCCESS);
+        when(response.getVersion()).thenReturn(1);
+        when(remotingClient.invokeSync(any(), any(), anyLong())).thenReturn(response);
+        when(remotingClient.getNameServerAddressList()).thenReturn(Collections.singletonList(defaultNsAddr));
+    }
+
+    private void setTopAddressing() throws NoSuchFieldException, IllegalAccessException {
+        TopAddressing topAddressing = mock(TopAddressing.class);
+        setField(mqClientAPI, "topAddressing", topAddressing);
+        when(topAddressing.fetchNSAddr()).thenReturn(defaultNsAddr);
+    }
+
+    private void setField(final Object target, final String fieldName, final Object newValue) throws NoSuchFieldException, IllegalAccessException {
+        Class<?> clazz = target.getClass();
+        Field field = clazz.getDeclaredField(fieldName);
+        field.setAccessible(true);
+        field.set(target, newValue);
+    }
 }
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/admin/MqClientAdminImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/admin/MqClientAdminImplTest.java
new file mode 100644
index 00000000000..71682fb52c0
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/admin/MqClientAdminImplTest.java
@@ -0,0 +1,559 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.impl.admin;
+
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.RemotingClient;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;
+import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;
+import org.apache.rocketmq.remoting.protocol.body.ClusterInfo;
+import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;
+import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection;
+import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;
+import org.apache.rocketmq.remoting.protocol.body.GroupList;
+import org.apache.rocketmq.remoting.protocol.body.QueryConsumeTimeSpanBody;
+import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;
+import org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;
+import org.apache.rocketmq.remoting.protocol.body.TopicList;
+import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.DeleteSubscriptionGroupRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerConnectionListRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryConsumeTimeSpanRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QuerySubscriptionByConsumerRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteTopicFromNamesrvRequestHeader;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class MqClientAdminImplTest {
+
+    @Mock
+    private RemotingClient remotingClient;
+
+    @Mock
+    private RemotingCommand response;
+
+    private MqClientAdminImpl mqClientAdminImpl;
+
+    private final String defaultTopic = "defaultTopic";
+
+    private final String defaultBrokerAddr = "127.0.0.1:10911";
+
+    private final long defaultTimeout = 3000L;
+
+    @Before
+    public void init() throws RemotingException, InterruptedException, MQClientException {
+        mqClientAdminImpl = new MqClientAdminImpl(remotingClient);
+        when(remotingClient.invoke(any(String.class), any(RemotingCommand.class), any(Long.class))).thenReturn(CompletableFuture.completedFuture(response));
+    }
+
+    @Test
+    public void assertQueryMessageWithSuccess() throws Exception {
+        setResponseSuccess(getMessageResult());
+        QueryMessageRequestHeader requestHeader = mock(QueryMessageRequestHeader.class);
+        when(requestHeader.getTopic()).thenReturn(defaultTopic);
+        when(requestHeader.getKey()).thenReturn("keys");
+        CompletableFuture<List<MessageExt>> actual = mqClientAdminImpl.queryMessage(defaultBrokerAddr, false, false, requestHeader, defaultTimeout);
+        List<MessageExt> messageExtList = actual.get();
+        assertNotNull(messageExtList);
+        assertEquals(1, messageExtList.size());
+    }
+
+    @Test
+    public void assertQueryMessageWithNotFound() throws Exception {
+        when(response.getCode()).thenReturn(ResponseCode.QUERY_NOT_FOUND);
+        QueryMessageRequestHeader requestHeader = mock(QueryMessageRequestHeader.class);
+        CompletableFuture<List<MessageExt>> actual = mqClientAdminImpl.queryMessage(defaultBrokerAddr, false, false, requestHeader, defaultTimeout);
+        List<MessageExt> messageExtList = actual.get();
+        assertNotNull(messageExtList);
+        assertEquals(0, messageExtList.size());
+    }
+
+    @Test
+    public void assertQueryMessageWithError() {
+        setResponseError();
+        QueryMessageRequestHeader requestHeader = mock(QueryMessageRequestHeader.class);
+        CompletableFuture<List<MessageExt>> actual = mqClientAdminImpl.queryMessage(defaultBrokerAddr, false, false, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertGetTopicStatsInfoWithSuccess() throws Exception {
+        TopicStatsTable responseBody = new TopicStatsTable();
+        setResponseSuccess(RemotingSerializable.encode(responseBody));
+        GetTopicStatsInfoRequestHeader requestHeader = mock(GetTopicStatsInfoRequestHeader.class);
+        CompletableFuture<TopicStatsTable> actual = mqClientAdminImpl.getTopicStatsInfo(defaultBrokerAddr, requestHeader, defaultTimeout);
+        TopicStatsTable topicStatsTable = actual.get();
+        assertNotNull(topicStatsTable);
+        assertEquals(0, topicStatsTable.getOffsetTable().size());
+    }
+
+    @Test
+    public void assertGetTopicStatsInfoWithError() {
+        setResponseError();
+        GetTopicStatsInfoRequestHeader requestHeader = mock(GetTopicStatsInfoRequestHeader.class);
+        CompletableFuture<TopicStatsTable> actual = mqClientAdminImpl.getTopicStatsInfo(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertQueryConsumeTimeSpanWithSuccess() throws Exception {
+        QueryConsumeTimeSpanBody responseBody = new QueryConsumeTimeSpanBody();
+        setResponseSuccess(RemotingSerializable.encode(responseBody));
+        QueryConsumeTimeSpanRequestHeader requestHeader = mock(QueryConsumeTimeSpanRequestHeader.class);
+        CompletableFuture<List<QueueTimeSpan>> actual = mqClientAdminImpl.queryConsumeTimeSpan(defaultBrokerAddr, requestHeader, defaultTimeout);
+        List<QueueTimeSpan> queueTimeSpans = actual.get();
+        assertNotNull(queueTimeSpans);
+        assertEquals(0, queueTimeSpans.size());
+    }
+
+    @Test
+    public void assertQueryConsumeTimeSpanWithError() {
+        setResponseError();
+        QueryConsumeTimeSpanRequestHeader requestHeader = mock(QueryConsumeTimeSpanRequestHeader.class);
+        CompletableFuture<List<QueueTimeSpan>> actual = mqClientAdminImpl.queryConsumeTimeSpan(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertUpdateOrCreateTopicWithSuccess() throws Exception {
+        setResponseSuccess(null);
+        CreateTopicRequestHeader requestHeader = mock(CreateTopicRequestHeader.class);
+        CompletableFuture<Void> actual = mqClientAdminImpl.updateOrCreateTopic(defaultBrokerAddr, requestHeader, defaultTimeout);
+        assertNull(actual.get());
+    }
+
+    @Test
+    public void assertUpdateOrCreateTopicWithError() {
+        setResponseError();
+        CreateTopicRequestHeader requestHeader = mock(CreateTopicRequestHeader.class);
+        CompletableFuture<Void> actual = mqClientAdminImpl.updateOrCreateTopic(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertUpdateOrCreateSubscriptionGroupWithSuccess() throws Exception {
+        setResponseSuccess(null);
+        SubscriptionGroupConfig config = mock(SubscriptionGroupConfig.class);
+        CompletableFuture<Void> actual = mqClientAdminImpl.updateOrCreateSubscriptionGroup(defaultBrokerAddr, config, defaultTimeout);
+        assertNull(actual.get());
+    }
+
+    @Test
+    public void assertUpdateOrCreateSubscriptionGroupWithError() {
+        setResponseError();
+        SubscriptionGroupConfig config = mock(SubscriptionGroupConfig.class);
+        CompletableFuture<Void> actual = mqClientAdminImpl.updateOrCreateSubscriptionGroup(defaultBrokerAddr, config, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertDeleteTopicInBrokerWithSuccess() throws Exception {
+        setResponseSuccess(null);
+        DeleteTopicRequestHeader requestHeader = mock(DeleteTopicRequestHeader.class);
+        CompletableFuture<Void> actual = mqClientAdminImpl.deleteTopicInBroker(defaultBrokerAddr, requestHeader, defaultTimeout);
+        assertNull(actual.get());
+    }
+
+    @Test
+    public void assertDeleteTopicInBrokerWithError() {
+        setResponseError();
+        DeleteTopicRequestHeader requestHeader = mock(DeleteTopicRequestHeader.class);
+        CompletableFuture<Void> actual = mqClientAdminImpl.deleteTopicInBroker(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertDeleteTopicInNameserverWithSuccess() throws Exception {
+        setResponseSuccess(null);
+        DeleteTopicFromNamesrvRequestHeader requestHeader = mock(DeleteTopicFromNamesrvRequestHeader.class);
+        CompletableFuture<Void> actual = mqClientAdminImpl.deleteTopicInNameserver(defaultBrokerAddr, requestHeader, defaultTimeout);
+        assertNull(actual.get());
+    }
+
+    @Test
+    public void assertDeleteTopicInNameserverWithError() {
+        setResponseError();
+        DeleteTopicFromNamesrvRequestHeader requestHeader = mock(DeleteTopicFromNamesrvRequestHeader.class);
+        CompletableFuture<Void> actual = mqClientAdminImpl.deleteTopicInNameserver(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertDeleteKvConfigWithSuccess() throws Exception {
+        setResponseSuccess(null);
+        DeleteKVConfigRequestHeader requestHeader = mock(DeleteKVConfigRequestHeader.class);
+        CompletableFuture<Void> actual = mqClientAdminImpl.deleteKvConfig(defaultBrokerAddr, requestHeader, defaultTimeout);
+        assertNull(actual.get());
+    }
+
+    @Test
+    public void assertDeleteKvConfigWithError() {
+        setResponseError();
+        DeleteKVConfigRequestHeader requestHeader = mock(DeleteKVConfigRequestHeader.class);
+        CompletableFuture<Void> actual = mqClientAdminImpl.deleteKvConfig(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertDeleteSubscriptionGroupWithSuccess() throws Exception {
+        setResponseSuccess(null);
+        DeleteSubscriptionGroupRequestHeader requestHeader = mock(DeleteSubscriptionGroupRequestHeader.class);
+        CompletableFuture<Void> actual = mqClientAdminImpl.deleteSubscriptionGroup(defaultBrokerAddr, requestHeader, defaultTimeout);
+        assertNull(actual.get());
+    }
+
+    @Test
+    public void assertDeleteSubscriptionGroupWithError() {
+        setResponseError();
+        DeleteSubscriptionGroupRequestHeader requestHeader = mock(DeleteSubscriptionGroupRequestHeader.class);
+        CompletableFuture<Void> actual = mqClientAdminImpl.deleteSubscriptionGroup(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertInvokeBrokerToResetOffsetWithSuccess() throws Exception {
+        ResetOffsetBody responseBody = new ResetOffsetBody();
+        setResponseSuccess(RemotingSerializable.encode(responseBody));
+        ResetOffsetRequestHeader requestHeader = mock(ResetOffsetRequestHeader.class);
+        CompletableFuture<Map<MessageQueue, Long>> actual = mqClientAdminImpl.invokeBrokerToResetOffset(defaultBrokerAddr, requestHeader, defaultTimeout);
+        assertEquals(0, actual.get().size());
+    }
+
+    @Test
+    public void assertInvokeBrokerToResetOffsetWithError() {
+        setResponseError();
+        ResetOffsetRequestHeader requestHeader = mock(ResetOffsetRequestHeader.class);
+        CompletableFuture<Map<MessageQueue, Long>> actual = mqClientAdminImpl.invokeBrokerToResetOffset(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertViewMessageWithSuccess() throws Exception {
+        setResponseSuccess(getMessageResult());
+        ViewMessageRequestHeader requestHeader = mock(ViewMessageRequestHeader.class);
+        CompletableFuture<MessageExt> actual = mqClientAdminImpl.viewMessage(defaultBrokerAddr, requestHeader, defaultTimeout);
+        MessageExt result = actual.get();
+        assertNotNull(result);
+        assertEquals(defaultTopic, result.getTopic());
+    }
+
+    @Test
+    public void assertViewMessageWithError() {
+        setResponseError();
+        ViewMessageRequestHeader requestHeader = mock(ViewMessageRequestHeader.class);
+        CompletableFuture<MessageExt> actual = mqClientAdminImpl.viewMessage(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertGetBrokerClusterInfoWithSuccess() throws Exception {
+        ClusterInfo responseBody = new ClusterInfo();
+        setResponseSuccess(RemotingSerializable.encode(responseBody));
+        CompletableFuture<ClusterInfo> actual = mqClientAdminImpl.getBrokerClusterInfo(defaultBrokerAddr, defaultTimeout);
+        ClusterInfo result = actual.get();
+        assertNotNull(result);
+    }
+
+    @Test
+    public void assertGetBrokerClusterInfoWithError() {
+        setResponseError();
+        CompletableFuture<ClusterInfo> actual = mqClientAdminImpl.getBrokerClusterInfo(defaultBrokerAddr, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertGetConsumerConnectionListWithSuccess() throws Exception {
+        ConsumerConnection responseBody = new ConsumerConnection();
+        setResponseSuccess(RemotingSerializable.encode(responseBody));
+        GetConsumerConnectionListRequestHeader requestHeader = mock(GetConsumerConnectionListRequestHeader.class);
+        CompletableFuture<ConsumerConnection> actual = mqClientAdminImpl.getConsumerConnectionList(defaultBrokerAddr, requestHeader, defaultTimeout);
+        ConsumerConnection result = actual.get();
+        assertNotNull(result);
+        assertEquals(0, result.getConnectionSet().size());
+    }
+
+    @Test
+    public void assertGetConsumerConnectionListWithError() {
+        setResponseError();
+        GetConsumerConnectionListRequestHeader requestHeader = mock(GetConsumerConnectionListRequestHeader.class);
+        CompletableFuture<ConsumerConnection> actual = mqClientAdminImpl.getConsumerConnectionList(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertQueryTopicsByConsumerWithSuccess() throws Exception {
+        TopicList responseBody = new TopicList();
+        setResponseSuccess(RemotingSerializable.encode(responseBody));
+        QueryTopicsByConsumerRequestHeader requestHeader = mock(QueryTopicsByConsumerRequestHeader.class);
+        CompletableFuture<TopicList> actual = mqClientAdminImpl.queryTopicsByConsumer(defaultBrokerAddr, requestHeader, defaultTimeout);
+        TopicList result = actual.get();
+        assertNotNull(result);
+        assertEquals(0, result.getTopicList().size());
+    }
+
+    @Test
+    public void assertQueryTopicsByConsumerWithError() {
+        setResponseError();
+        QueryTopicsByConsumerRequestHeader requestHeader = mock(QueryTopicsByConsumerRequestHeader.class);
+        CompletableFuture<TopicList> actual = mqClientAdminImpl.queryTopicsByConsumer(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertQuerySubscriptionByConsumerWithSuccess() throws Exception {
+        SubscriptionData responseBody = new SubscriptionData();
+        setResponseSuccess(RemotingSerializable.encode(responseBody));
+        QuerySubscriptionByConsumerRequestHeader requestHeader = mock(QuerySubscriptionByConsumerRequestHeader.class);
+        CompletableFuture<SubscriptionData> actual = mqClientAdminImpl.querySubscriptionByConsumer(defaultBrokerAddr, requestHeader, defaultTimeout);
+        assertNull(actual.get());
+    }
+
+    @Test
+    public void assertQuerySubscriptionByConsumerWithError() {
+        setResponseError();
+        QuerySubscriptionByConsumerRequestHeader requestHeader = mock(QuerySubscriptionByConsumerRequestHeader.class);
+        CompletableFuture<SubscriptionData> actual = mqClientAdminImpl.querySubscriptionByConsumer(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertGetConsumeStatsWithSuccess() throws Exception {
+        ConsumeStats responseBody = new ConsumeStats();
+        setResponseSuccess(RemotingSerializable.encode(responseBody));
+        GetConsumeStatsRequestHeader requestHeader = mock(GetConsumeStatsRequestHeader.class);
+        CompletableFuture<ConsumeStats> actual = mqClientAdminImpl.getConsumeStats(defaultBrokerAddr, requestHeader, defaultTimeout);
+        ConsumeStats result = actual.get();
+        assertNotNull(result);
+        assertEquals(0, result.getOffsetTable().size());
+    }
+
+    @Test
+    public void assertGetConsumeStatsWithError() {
+        setResponseError();
+        GetConsumeStatsRequestHeader requestHeader = mock(GetConsumeStatsRequestHeader.class);
+        CompletableFuture<ConsumeStats> actual = mqClientAdminImpl.getConsumeStats(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertQueryTopicConsumeByWhoWithSuccess() throws Exception {
+        GroupList responseBody = new GroupList();
+        setResponseSuccess(RemotingSerializable.encode(responseBody));
+        QueryTopicConsumeByWhoRequestHeader requestHeader = mock(QueryTopicConsumeByWhoRequestHeader.class);
+        CompletableFuture<GroupList> actual = mqClientAdminImpl.queryTopicConsumeByWho(defaultBrokerAddr, requestHeader, defaultTimeout);
+        GroupList result = actual.get();
+        assertNotNull(result);
+        assertEquals(0, result.getGroupList().size());
+    }
+
+    @Test
+    public void assertQueryTopicConsumeByWhoWithError() {
+        setResponseError();
+        QueryTopicConsumeByWhoRequestHeader requestHeader = mock(QueryTopicConsumeByWhoRequestHeader.class);
+        CompletableFuture<GroupList> actual = mqClientAdminImpl.queryTopicConsumeByWho(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertGetConsumerRunningInfoWithSuccess() throws Exception {
+        ConsumerRunningInfo responseBody = new ConsumerRunningInfo();
+        setResponseSuccess(RemotingSerializable.encode(responseBody));
+        GetConsumerRunningInfoRequestHeader requestHeader = mock(GetConsumerRunningInfoRequestHeader.class);
+        CompletableFuture<ConsumerRunningInfo> actual = mqClientAdminImpl.getConsumerRunningInfo(defaultBrokerAddr, requestHeader, defaultTimeout);
+        ConsumerRunningInfo result = actual.get();
+        assertNotNull(result);
+        assertEquals(0, result.getProperties().size());
+    }
+
+    @Test
+    public void assertGetConsumerRunningInfoWithError() {
+        setResponseError();
+        GetConsumerRunningInfoRequestHeader requestHeader = mock(GetConsumerRunningInfoRequestHeader.class);
+        CompletableFuture<ConsumerRunningInfo> actual = mqClientAdminImpl.getConsumerRunningInfo(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    @Test
+    public void assertConsumeMessageDirectlyWithSuccess() throws Exception {
+        ConsumeMessageDirectlyResult responseBody = new ConsumeMessageDirectlyResult();
+        setResponseSuccess(RemotingSerializable.encode(responseBody));
+        ConsumeMessageDirectlyResultRequestHeader requestHeader = mock(ConsumeMessageDirectlyResultRequestHeader.class);
+        CompletableFuture<ConsumeMessageDirectlyResult> actual = mqClientAdminImpl.consumeMessageDirectly(defaultBrokerAddr, requestHeader, defaultTimeout);
+        ConsumeMessageDirectlyResult result = actual.get();
+        assertNotNull(result);
+        assertTrue(result.isAutoCommit());
+    }
+
+    @Test
+    public void assertConsumeMessageDirectlyWithError() {
+        setResponseError();
+        ConsumeMessageDirectlyResultRequestHeader requestHeader = mock(ConsumeMessageDirectlyResultRequestHeader.class);
+        CompletableFuture<ConsumeMessageDirectlyResult> actual = mqClientAdminImpl.consumeMessageDirectly(defaultBrokerAddr, requestHeader, defaultTimeout);
+        Throwable thrown = assertThrows(ExecutionException.class, actual::get);
+        assertTrue(thrown.getCause() instanceof MQClientException);
+        MQClientException mqException = (MQClientException) thrown.getCause();
+        assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode());
+        assertTrue(mqException.getMessage().contains("CODE: 1  DESC: null"));
+    }
+
+    private byte[] getMessageResult() throws Exception {
+        byte[] bytes = MessageDecoder.encode(createMessageExt(), false);
+        ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length);
+        byteBuffer.put(bytes);
+        return byteBuffer.array();
+    }
+
+    private MessageExt createMessageExt() {
+        MessageExt result = new MessageExt();
+        result.setBody("body".getBytes(StandardCharsets.UTF_8));
+        result.setTopic(defaultTopic);
+        result.setBrokerName("defaultBroker");
+        result.putUserProperty("key", "value");
+        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, "defaultGroup");
+        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1");
+        result.setKeys("keys");
+        SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911);
+        SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911);
+        result.setBornHost(bornHost);
+        result.setStoreHost(storeHost);
+        return result;
+    }
+
+    private void setResponseSuccess(byte[] body) {
+        when(response.getCode()).thenReturn(ResponseCode.SUCCESS);
+        when(response.getBody()).thenReturn(body);
+    }
+
+    private void setResponseError() {
+        when(response.getCode()).thenReturn(ResponseCode.SYSTEM_ERROR);
+    }
+}
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java
index 749201e3c22..395c0ff2335 100644
--- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java
@@ -52,15 +52,14 @@
 import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.remoting.protocol.body.ConsumeStatus;
 import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
-import org.junit.After;
-import org.junit.Before;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.Mock;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.junit.MockitoJUnitRunner;
 import org.mockito.stubbing.Answer;
-
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyBoolean;
@@ -70,49 +69,54 @@
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
+import org.mockito.Mockito;
 
 @RunWith(MockitoJUnitRunner.class)
 public class ConsumeMessageConcurrentlyServiceTest {
-    private String consumerGroup;
-    private String topic = "FooBar";
-    private String brokerName = "BrokerA";
-    private MQClientInstance mQClientFactory;
+
+    private static String consumerGroup;
+
+    private static String topic = "FooBar";
+
+    private static String brokerName = "BrokerA";
+
+    private static MQClientInstance mQClientFactory;
 
     @Mock
-    private MQClientAPIImpl mQClientAPIImpl;
-    private PullAPIWrapper pullAPIWrapper;
-    private RebalancePushImpl rebalancePushImpl;
-    private DefaultMQPushConsumer pushConsumer;
+    private static MQClientAPIImpl mQClientAPIImpl;
+
+    private static PullAPIWrapper pullAPIWrapper;
+
+    private static RebalancePushImpl rebalancePushImpl;
+
+    private static DefaultMQPushConsumer pushConsumer;
 
-    @Before
-    public void init() throws Exception {
+    @BeforeClass
+    public static void init() throws Exception {
+        mQClientAPIImpl = Mockito.mock(MQClientAPIImpl.class);
         ConcurrentMap<String, MQClientInstance> factoryTable = (ConcurrentMap<String, MQClientInstance>) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true);
         Collection<MQClientInstance> instances = factoryTable.values();
         for (MQClientInstance instance : instances) {
             instance.shutdown();
         }
         factoryTable.clear();
-
         consumerGroup = "FooBarGroup" + System.currentTimeMillis();
         pushConsumer = new DefaultMQPushConsumer(consumerGroup);
         pushConsumer.setNamesrvAddr("127.0.0.1:9876");
         pushConsumer.setPullInterval(60 * 1000);
-
         pushConsumer.registerMessageListener(new MessageListenerConcurrently() {
+
             @Override
-            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
-                                                            ConsumeConcurrentlyContext context) {
+            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
                 return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
             }
         });
-
         DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl();
         rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl()));
         Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl");
         field.setAccessible(true);
         field.set(pushConsumerImpl, rebalancePushImpl);
         pushConsumer.subscribe(topic, "*");
-
         // suppress updateTopicRouteInfoFromNameServer
         pushConsumer.changeInstanceNameToPID();
         mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true));
@@ -121,38 +125,32 @@ public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
         field.setAccessible(true);
         field.set(pushConsumerImpl, mQClientFactory);
         factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory);
-
         field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
         field.setAccessible(true);
         field.set(mQClientFactory, mQClientAPIImpl);
-
         pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false));
         field = DefaultMQPushConsumerImpl.class.getDeclaredField("pullAPIWrapper");
         field.setAccessible(true);
         field.set(pushConsumerImpl, pullAPIWrapper);
-
         pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl().setmQClientFactory(mQClientFactory);
+        when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))).thenAnswer(new Answer<PullResult>() {
 
-        when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class),
-                anyLong(), any(CommunicationMode.class), nullable(PullCallback.class)))
-                .thenAnswer(new Answer<PullResult>() {
-                    @Override
-                    public PullResult answer(InvocationOnMock mock) throws Throwable {
-                        PullMessageRequestHeader requestHeader = mock.getArgument(1);
-                        MessageClientExt messageClientExt = new MessageClientExt();
-                        messageClientExt.setTopic(topic);
-                        messageClientExt.setQueueId(0);
-                        messageClientExt.setMsgId("123");
-                        messageClientExt.setBody(new byte[] {'a'});
-                        messageClientExt.setOffsetMsgId("234");
-                        messageClientExt.setBornHost(new InetSocketAddress(8080));
-                        messageClientExt.setStoreHost(new InetSocketAddress(8080));
-                        PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.<MessageExt>singletonList(messageClientExt));
-                        ((PullCallback) mock.getArgument(4)).onSuccess(pullResult);
-                        return pullResult;
-                    }
-                });
-
+            @Override
+            public PullResult answer(InvocationOnMock mock) throws Throwable {
+                PullMessageRequestHeader requestHeader = mock.getArgument(1);
+                MessageClientExt messageClientExt = new MessageClientExt();
+                messageClientExt.setTopic(topic);
+                messageClientExt.setQueueId(0);
+                messageClientExt.setMsgId("123");
+                messageClientExt.setBody(new byte[] { 'a' });
+                messageClientExt.setOffsetMsgId("234");
+                messageClientExt.setBornHost(new InetSocketAddress(8080));
+                messageClientExt.setStoreHost(new InetSocketAddress(8080));
+                PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.<MessageExt>singletonList(messageClientExt));
+                ((PullCallback) mock.getArgument(4)).onSuccess(pullResult);
+                return pullResult;
+            }
+        });
         doReturn(new FindBrokerResult("127.0.0.1:10912", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean());
         doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString());
         Set<MessageQueue> messageQueueSet = new HashSet<>();
@@ -162,54 +160,45 @@ public PullResult answer(InvocationOnMock mock) throws Throwable {
     }
 
     @Test
-    public void testPullMessage_ConsumeSuccess() throws InterruptedException, RemotingException, MQBrokerException, NoSuchFieldException,Exception {
+    public void testPullMessage_ConsumeSuccess() throws InterruptedException, RemotingException, MQBrokerException, NoSuchFieldException, Exception {
         final CountDownLatch countDownLatch = new CountDownLatch(1);
         final AtomicReference<MessageExt> messageAtomic = new AtomicReference<>();
+        ConsumeMessageConcurrentlyService normalServie = new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() {
 
-        ConsumeMessageConcurrentlyService  normalServie = new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() {
             @Override
-            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
-                                                            ConsumeConcurrentlyContext context) {
+            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
                 messageAtomic.set(msgs.get(0));
                 countDownLatch.countDown();
                 return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
             }
         });
         pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(normalServie);
-
         PullMessageService pullMessageService = mQClientFactory.getPullMessageService();
         pullMessageService.executePullRequestImmediately(createPullRequest());
         countDownLatch.await();
-
         Thread.sleep(1000);
-
-        ConsumeStatus stats = normalServie.getConsumerStatsManager().consumeStatus(pushConsumer.getDefaultMQPushConsumerImpl().groupName(),topic);
-
-        ConsumerStatsManager mgr  =   normalServie.getConsumerStatsManager();
-
+        ConsumeStatus stats = normalServie.getConsumerStatsManager().consumeStatus(pushConsumer.getDefaultMQPushConsumerImpl().groupName(), topic);
+        ConsumerStatsManager mgr = normalServie.getConsumerStatsManager();
         Field statItmeSetField = mgr.getClass().getDeclaredField("topicAndGroupConsumeOKTPS");
         statItmeSetField.setAccessible(true);
-
-        StatsItemSet itemSet = (StatsItemSet)statItmeSetField.get(mgr);
+        StatsItemSet itemSet = (StatsItemSet) statItmeSetField.get(mgr);
         StatsItem item = itemSet.getAndCreateStatsItem(topic + "@" + pushConsumer.getDefaultMQPushConsumerImpl().groupName());
-
         assertThat(item.getValue().sum()).isGreaterThan(0L);
         MessageExt msg = messageAtomic.get();
         assertThat(msg).isNotNull();
         assertThat(msg.getTopic()).isEqualTo(topic);
-        assertThat(msg.getBody()).isEqualTo(new byte[] {'a'});
+        assertThat(msg.getBody()).isEqualTo(new byte[] { 'a' });
     }
 
-    @After
-    public void terminate() {
+    @AfterClass
+    public static void terminate() {
         pushConsumer.shutdown();
     }
 
-    private PullRequest createPullRequest() {
+    private static PullRequest createPullRequest() {
         PullRequest pullRequest = new PullRequest();
         pullRequest.setConsumerGroup(consumerGroup);
         pullRequest.setNextOffset(1024);
-
         MessageQueue messageQueue = new MessageQueue();
         messageQueue.setBrokerName(brokerName);
         messageQueue.setQueueId(0);
@@ -219,12 +208,10 @@ private PullRequest createPullRequest() {
         processQueue.setLocked(true);
         processQueue.setLastLockTimestamp(System.currentTimeMillis());
         pullRequest.setProcessQueue(processQueue);
-
         return pullRequest;
     }
 
-    private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus,
-                                           List<MessageExt> messageExtList) throws Exception {
+    private static PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, List<MessageExt> messageExtList) throws Exception {
         ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
         for (MessageExt messageExt : messageExtList) {
             outputStream.write(MessageDecoder.encode(messageExt, false));
@@ -236,23 +223,21 @@ private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, P
     public void testConsumeThreadName() throws Exception {
         final CountDownLatch countDownLatch = new CountDownLatch(1);
         final AtomicReference<String> consumeThreadName = new AtomicReference<>();
-
         StringBuilder consumeGroup2 = new StringBuilder();
         for (int i = 0; i < 101; i++) {
             consumeGroup2.append(i).append("#");
         }
         pushConsumer.setConsumerGroup(consumeGroup2.toString());
-        ConsumeMessageConcurrentlyService  normalServie = new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() {
+        ConsumeMessageConcurrentlyService normalServie = new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() {
+
             @Override
-            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
-                                                            ConsumeConcurrentlyContext context) {
+            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
                 consumeThreadName.set(Thread.currentThread().getName());
                 countDownLatch.countDown();
                 return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
             }
         });
         pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(normalServie);
-
         PullMessageService pullMessageService = mQClientFactory.getPullMessageService();
         pullMessageService.executePullRequestImmediately(createPullRequest());
         countDownLatch.await();
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyServiceTest.java
new file mode 100644
index 00000000000..5097f14ca34
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyServiceTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.impl.consumer;
+
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.client.stat.ConsumerStatsManager;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.protocol.body.CMResult;
+import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ConsumeMessagePopConcurrentlyServiceTest {
+
+    @Mock
+    private DefaultMQPushConsumerImpl defaultMQPushConsumerImpl;
+
+    @Mock
+    private MessageListenerConcurrently messageListener;
+
+    @Mock
+    private DefaultMQPushConsumer defaultMQPushConsumer;
+
+    private ConsumeMessagePopConcurrentlyService popService;
+
+    private final String defaultGroup = "defaultGroup";
+
+    private final String defaultBroker = "defaultBroker";
+
+    private final String defaultTopic = "defaultTopic";
+
+    @Before
+    public void init() throws Exception {
+        when(defaultMQPushConsumer.getConsumerGroup()).thenReturn(defaultGroup);
+        when(defaultMQPushConsumer.getConsumeThreadMin()).thenReturn(1);
+        when(defaultMQPushConsumer.getConsumeThreadMax()).thenReturn(3);
+        when(defaultMQPushConsumer.getConsumeMessageBatchMaxSize()).thenReturn(32);
+        when(defaultMQPushConsumerImpl.getDefaultMQPushConsumer()).thenReturn(defaultMQPushConsumer);
+        ConsumerStatsManager consumerStatsManager = mock(ConsumerStatsManager.class);
+        when(defaultMQPushConsumerImpl.getConsumerStatsManager()).thenReturn(consumerStatsManager);
+        popService = new ConsumeMessagePopConcurrentlyService(defaultMQPushConsumerImpl, messageListener);
+    }
+
+    @Test
+    public void testUpdateCorePoolSize() {
+        popService.updateCorePoolSize(2);
+        popService.incCorePoolSize();
+        popService.decCorePoolSize();
+        assertEquals(2, popService.getCorePoolSize());
+    }
+
+    @Test
+    public void testConsumeMessageDirectly() {
+        when(messageListener.consumeMessage(any(), any(ConsumeConcurrentlyContext.class))).thenReturn(ConsumeConcurrentlyStatus.CONSUME_SUCCESS);
+        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);
+        assertEquals(CMResult.CR_SUCCESS, actual.getConsumeResult());
+    }
+
+    @Test
+    public void testConsumeMessageDirectlyWithCrLater() {
+        when(messageListener.consumeMessage(any(), any(ConsumeConcurrentlyContext.class))).thenReturn(ConsumeConcurrentlyStatus.RECONSUME_LATER);
+        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);
+        assertEquals(CMResult.CR_LATER, actual.getConsumeResult());
+    }
+
+    @Test
+    public void testConsumeMessageDirectlyWithCrReturnNull() {
+        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);
+        assertEquals(CMResult.CR_RETURN_NULL, actual.getConsumeResult());
+    }
+
+    @Test
+    public void testConsumeMessageDirectlyWithCrThrowException() {
+        when(messageListener.consumeMessage(any(), any(ConsumeConcurrentlyContext.class))).thenThrow(new RuntimeException("exception"));
+        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);
+        assertEquals(CMResult.CR_THROW_EXCEPTION, actual.getConsumeResult());
+    }
+
+    @Test
+    public void testShutdown() throws IllegalAccessException {
+        popService.shutdown(3000L);
+        Field scheduledExecutorServiceField = FieldUtils.getDeclaredField(popService.getClass(), "scheduledExecutorService", true);
+        Field consumeExecutorField = FieldUtils.getDeclaredField(popService.getClass(), "consumeExecutor", true);
+        ScheduledExecutorService scheduledExecutorService = (ScheduledExecutorService) scheduledExecutorServiceField.get(popService);
+        ThreadPoolExecutor consumeExecutor = (ThreadPoolExecutor) consumeExecutorField.get(popService);
+        assertTrue(scheduledExecutorService.isShutdown());
+        assertTrue(scheduledExecutorService.isTerminated());
+        assertTrue(consumeExecutor.isShutdown());
+        assertTrue(consumeExecutor.isTerminated());
+    }
+
+    @Test
+    public void testSubmitConsumeRequest() {
+        assertThrows(UnsupportedOperationException.class, () -> {
+            List<MessageExt> msgs = mock(List.class);
+            ProcessQueue processQueue = mock(ProcessQueue.class);
+            MessageQueue messageQueue = mock(MessageQueue.class);
+            popService.submitConsumeRequest(msgs, processQueue, messageQueue, false);
+        });
+    }
+
+    @Test
+    public void testSubmitPopConsumeRequest() throws IllegalAccessException {
+        List<MessageExt> msgs = Collections.singletonList(createMessageExt());
+        PopProcessQueue processQueue = mock(PopProcessQueue.class);
+        MessageQueue messageQueue = mock(MessageQueue.class);
+        ThreadPoolExecutor consumeExecutor = mock(ThreadPoolExecutor.class);
+        FieldUtils.writeDeclaredField(popService, "consumeExecutor", consumeExecutor, true);
+        popService.submitPopConsumeRequest(msgs, processQueue, messageQueue);
+        verify(consumeExecutor, times(1)).submit(any(Runnable.class));
+    }
+
+    @Test
+    public void testSubmitPopConsumeRequestWithMultiMsg() throws IllegalAccessException {
+        List<MessageExt> msgs = Arrays.asList(createMessageExt(), createMessageExt());
+        PopProcessQueue processQueue = mock(PopProcessQueue.class);
+        MessageQueue messageQueue = mock(MessageQueue.class);
+        ThreadPoolExecutor consumeExecutor = mock(ThreadPoolExecutor.class);
+        FieldUtils.writeDeclaredField(popService, "consumeExecutor", consumeExecutor, true);
+        when(defaultMQPushConsumer.getConsumeMessageBatchMaxSize()).thenReturn(1);
+        popService.submitPopConsumeRequest(msgs, processQueue, messageQueue);
+        verify(consumeExecutor, times(2)).submit(any(Runnable.class));
+    }
+
+    @Test
+    public void testProcessConsumeResult() {
+        ConsumeConcurrentlyContext context = mock(ConsumeConcurrentlyContext.class);
+        ConsumeMessagePopConcurrentlyService.ConsumeRequest consumeRequest = mock(ConsumeMessagePopConcurrentlyService.ConsumeRequest.class);
+        when(consumeRequest.getMsgs()).thenReturn(Arrays.asList(createMessageExt(), createMessageExt()));
+        MessageQueue messageQueue = mock(MessageQueue.class);
+        when(messageQueue.getTopic()).thenReturn(defaultTopic);
+        when(consumeRequest.getMessageQueue()).thenReturn(messageQueue);
+        PopProcessQueue processQueue = mock(PopProcessQueue.class);
+        when(processQueue.ack()).thenReturn(0);
+        when(consumeRequest.getPopProcessQueue()).thenReturn(processQueue);
+        when(defaultMQPushConsumerImpl.getPopDelayLevel()).thenReturn(new int[]{1, 10});
+        popService.processConsumeResult(ConsumeConcurrentlyStatus.CONSUME_SUCCESS, context, consumeRequest);
+        verify(defaultMQPushConsumerImpl, times(1)).ackAsync(any(MessageExt.class), any());
+    }
+
+    private MessageExt createMessageExt() {
+        MessageExt result = new MessageExt();
+        result.setBody("body".getBytes(StandardCharsets.UTF_8));
+        result.setTopic(defaultTopic);
+        result.setBrokerName(defaultBroker);
+        result.putUserProperty("key", "value");
+        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup);
+        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1");
+        long curTime = System.currentTimeMillis();
+        result.setBornTimestamp(curTime - 1000);
+        result.getProperties().put(MessageConst.PROPERTY_POP_CK, curTime + " " + curTime + " " + curTime + " " + curTime);
+        result.setKeys("keys");
+        SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911);
+        SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911);
+        result.setBornHost(bornHost);
+        result.setStoreHost(storeHost);
+        return result;
+    }
+}
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyServiceTest.java
new file mode 100644
index 00000000000..257783ecb48
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyServiceTest.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.impl.consumer;
+
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
+import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.stat.ConsumerStatsManager;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.protocol.body.CMResult;
+import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ConsumeMessagePopOrderlyServiceTest {
+
+    @Mock
+    private DefaultMQPushConsumerImpl defaultMQPushConsumerImpl;
+
+    @Mock
+    private MessageListenerOrderly messageListener;
+
+    @Mock
+    private DefaultMQPushConsumer defaultMQPushConsumer;
+
+    @Mock
+    private ConsumerStatsManager consumerStatsManager;
+
+    @Mock
+    private RebalanceImpl rebalanceImpl;
+
+    private ConsumeMessagePopOrderlyService popService;
+
+    private final String defaultGroup = "defaultGroup";
+
+    private final String defaultBroker = "defaultBroker";
+
+    private final String defaultTopic = "defaultTopic";
+
+    @Before
+    public void init() throws Exception {
+        when(defaultMQPushConsumer.getConsumerGroup()).thenReturn(defaultGroup);
+        when(defaultMQPushConsumer.getConsumeThreadMin()).thenReturn(1);
+        when(defaultMQPushConsumer.getConsumeThreadMax()).thenReturn(3);
+        when(defaultMQPushConsumerImpl.getDefaultMQPushConsumer()).thenReturn(defaultMQPushConsumer);
+        when(defaultMQPushConsumerImpl.getRebalanceImpl()).thenReturn(rebalanceImpl);
+        when(defaultMQPushConsumerImpl.getConsumerStatsManager()).thenReturn(consumerStatsManager);
+        MQClientInstance mQClientFactory = mock(MQClientInstance.class);
+        DefaultMQProducer defaultMQProducer = mock(DefaultMQProducer.class);
+        when(mQClientFactory.getDefaultMQProducer()).thenReturn(defaultMQProducer);
+        when(defaultMQPushConsumerImpl.getmQClientFactory()).thenReturn(mQClientFactory);
+        popService = new ConsumeMessagePopOrderlyService(defaultMQPushConsumerImpl, messageListener);
+    }
+
+    @Test
+    public void testShutdown() throws IllegalAccessException {
+        popService.shutdown(3000L);
+        Field scheduledExecutorServiceField = FieldUtils.getDeclaredField(popService.getClass(), "scheduledExecutorService", true);
+        Field consumeExecutorField = FieldUtils.getDeclaredField(popService.getClass(), "consumeExecutor", true);
+        ScheduledExecutorService scheduledExecutorService = (ScheduledExecutorService) scheduledExecutorServiceField.get(popService);
+        ThreadPoolExecutor consumeExecutor = (ThreadPoolExecutor) consumeExecutorField.get(popService);
+        assertTrue(scheduledExecutorService.isShutdown());
+        assertTrue(scheduledExecutorService.isTerminated());
+        assertTrue(consumeExecutor.isShutdown());
+        assertTrue(consumeExecutor.isTerminated());
+    }
+
+    @Test
+    public void testUnlockAllMessageQueues() {
+        popService.unlockAllMessageQueues();
+        verify(rebalanceImpl, times(1)).unlockAll(eq(false));
+    }
+
+    @Test
+    public void testUpdateCorePoolSize() {
+        popService.updateCorePoolSize(2);
+        popService.incCorePoolSize();
+        popService.decCorePoolSize();
+        assertEquals(2, popService.getCorePoolSize());
+    }
+
+    @Test
+    public void testConsumeMessageDirectly() {
+        when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenReturn(ConsumeOrderlyStatus.SUCCESS);
+        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);
+        assertEquals(CMResult.CR_SUCCESS, actual.getConsumeResult());
+        assertTrue(actual.isOrder());
+    }
+
+    @Test
+    public void testConsumeMessageDirectlyWithCommit() {
+        when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenReturn(ConsumeOrderlyStatus.COMMIT);
+        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);
+        assertEquals(CMResult.CR_COMMIT, actual.getConsumeResult());
+        assertTrue(actual.isOrder());
+    }
+
+    @Test
+    public void testConsumeMessageDirectlyWithRollback() {
+        when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenReturn(ConsumeOrderlyStatus.ROLLBACK);
+        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);
+        assertEquals(CMResult.CR_ROLLBACK, actual.getConsumeResult());
+        assertTrue(actual.isOrder());
+    }
+
+    @Test
+    public void testConsumeMessageDirectlyWithCrLater() {
+        when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenReturn(ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT);
+        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);
+        assertEquals(CMResult.CR_LATER, actual.getConsumeResult());
+    }
+
+    @Test
+    public void testConsumeMessageDirectlyWithCrReturnNull() {
+        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);
+        assertEquals(CMResult.CR_RETURN_NULL, actual.getConsumeResult());
+    }
+
+    @Test
+    public void testConsumeMessageDirectlyWithCrThrowException() {
+        when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenThrow(new RuntimeException("exception"));
+        ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker);
+        assertEquals(CMResult.CR_THROW_EXCEPTION, actual.getConsumeResult());
+    }
+
+    @Test
+    public void testSubmitConsumeRequest() {
+        assertThrows(UnsupportedOperationException.class, () -> {
+            List<MessageExt> msgs = mock(List.class);
+            ProcessQueue processQueue = mock(ProcessQueue.class);
+            MessageQueue messageQueue = mock(MessageQueue.class);
+            popService.submitConsumeRequest(msgs, processQueue, messageQueue, false);
+        });
+    }
+
+    @Test
+    public void testSubmitPopConsumeRequest() throws IllegalAccessException {
+        List<MessageExt> msgs = Collections.singletonList(createMessageExt());
+        PopProcessQueue processQueue = mock(PopProcessQueue.class);
+        MessageQueue messageQueue = mock(MessageQueue.class);
+        ThreadPoolExecutor consumeExecutor = mock(ThreadPoolExecutor.class);
+        FieldUtils.writeDeclaredField(popService, "consumeExecutor", consumeExecutor, true);
+        popService.submitPopConsumeRequest(msgs, processQueue, messageQueue);
+        verify(consumeExecutor, times(1)).submit(any(Runnable.class));
+    }
+
+    @Test
+    public void testLockMQPeriodically() {
+        popService.lockMQPeriodically();
+        verify(defaultMQPushConsumerImpl, times(1)).getRebalanceImpl();
+        verify(rebalanceImpl, times(1)).lockAll();
+    }
+
+    @Test
+    public void testGetConsumerStatsManager() {
+        ConsumerStatsManager actual = popService.getConsumerStatsManager();
+        assertNotNull(actual);
+        assertEquals(consumerStatsManager, actual);
+    }
+
+    @Test
+    public void testSendMessageBack() {
+        assertTrue(popService.sendMessageBack(createMessageExt()));
+    }
+
+    @Test
+    public void testProcessConsumeResult() {
+        ConsumeOrderlyContext context = mock(ConsumeOrderlyContext.class);
+        ConsumeMessagePopOrderlyService.ConsumeRequest consumeRequest = mock(ConsumeMessagePopOrderlyService.ConsumeRequest.class);
+        assertTrue(popService.processConsumeResult(Collections.singletonList(createMessageExt()), ConsumeOrderlyStatus.SUCCESS, context, consumeRequest));
+    }
+
+    @Test
+    public void testResetNamespace() {
+        when(defaultMQPushConsumer.getNamespace()).thenReturn("defaultNamespace");
+        List<MessageExt> msgs = Collections.singletonList(createMessageExt());
+        popService.resetNamespace(msgs);
+        assertEquals(defaultTopic, msgs.get(0).getTopic());
+    }
+
+    private MessageExt createMessageExt() {
+        MessageExt result = new MessageExt();
+        result.setBody("body".getBytes(StandardCharsets.UTF_8));
+        result.setTopic(defaultTopic);
+        result.setBrokerName(defaultBroker);
+        result.putUserProperty("key", "value");
+        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup);
+        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1");
+        long curTime = System.currentTimeMillis();
+        result.setBornTimestamp(curTime - 1000);
+        result.getProperties().put(MessageConst.PROPERTY_POP_CK, curTime + " " + curTime + " " + curTime + " " + curTime);
+        result.setKeys("keys");
+        SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911);
+        SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911);
+        result.setBornHost(bornHost);
+        result.setStoreHost(storeHost);
+        return result;
+    }
+}
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImplTest.java
new file mode 100644
index 00000000000..3986c497eb5
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImplTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.impl.consumer;
+
+import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashSet;
+import java.util.Set;
+
+
+public class DefaultLitePullConsumerImplTest {
+    private final DefaultLitePullConsumerImpl consumer = new DefaultLitePullConsumerImpl(new DefaultLitePullConsumer(), null);
+
+    private static Method isSetEqualMethod;
+
+    @BeforeClass
+    public static void initReflectionMethod() throws NoSuchMethodException {
+        Class<DefaultLitePullConsumerImpl> consumerClass = DefaultLitePullConsumerImpl.class;
+        Method testMethod = consumerClass.getDeclaredMethod("isSetEqual", Set.class, Set.class);
+        testMethod.setAccessible(true);
+        isSetEqualMethod = testMethod;
+    }
+
+
+    /**
+     * The two empty sets should be equal
+     */
+    @Test
+    public void testIsSetEqual1() throws InvocationTargetException, IllegalAccessException {
+        Set<MessageQueue> set1 = new HashSet<>();
+        Set<MessageQueue> set2 = new HashSet<>();
+        boolean equalResult = (boolean) isSetEqualMethod.invoke(consumer, set1, set2);
+        Assert.assertTrue(equalResult);
+    }
+
+
+    /**
+     * When a set has elements and one does not, the two sets are not equal
+     */
+    @Test
+    public void testIsSetEqual2() throws InvocationTargetException, IllegalAccessException {
+        Set<MessageQueue> set1 = new HashSet<>();
+        set1.add(new MessageQueue("testTopic","testBroker",111));
+        Set<MessageQueue> set2 = new HashSet<>();
+        boolean equalResult = (boolean) isSetEqualMethod.invoke(consumer, set1, set2);
+        Assert.assertFalse(equalResult);
+    }
+
+    /**
+     * The two null sets should be equal
+     */
+    @Test
+    public void testIsSetEqual3() throws InvocationTargetException, IllegalAccessException {
+        Set<MessageQueue> set1 = null;
+        Set<MessageQueue> set2 = null;
+        boolean equalResult = (boolean) isSetEqualMethod.invoke(consumer, set1, set2);
+        Assert.assertTrue(equalResult);
+    }
+
+    @Test
+    public void testIsSetEqual4() throws InvocationTargetException, IllegalAccessException {
+        Set<MessageQueue> set1 = null;
+        Set<MessageQueue> set2 = new HashSet<>();
+        boolean equalResult = (boolean) isSetEqualMethod.invoke(consumer, set1, set2);
+        Assert.assertFalse(equalResult);
+    }
+
+    @Test
+    public void testIsSetEqual5() throws InvocationTargetException, IllegalAccessException {
+        Set<MessageQueue> set1 = new HashSet<>();
+        set1.add(new MessageQueue("testTopic","testBroker",111));
+        Set<MessageQueue> set2 = new HashSet<>();
+        set2.add(new MessageQueue("testTopic","testBroker",111));
+        boolean equalResult = (boolean) isSetEqualMethod.invoke(consumer, set1, set2);
+        Assert.assertTrue(equalResult);
+    }
+
+}
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java
index 879bbc593c1..2bc9c5a18db 100644
--- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java
@@ -17,17 +17,50 @@
 
 package org.apache.rocketmq.client.impl.consumer;
 
-import java.util.List;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.client.consumer.AckCallback;
+import org.apache.rocketmq.client.consumer.AckResult;
+import org.apache.rocketmq.client.consumer.AckStatus;
 import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
-import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.MessageSelector;
+import org.apache.rocketmq.client.consumer.PopCallback;
+import org.apache.rocketmq.client.consumer.PopResult;
+import org.apache.rocketmq.client.consumer.PopStatus;
+import org.apache.rocketmq.client.consumer.PullCallback;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.PullStatus;
 import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
 import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.client.consumer.store.OffsetStore;
+import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.hook.ConsumeMessageContext;
 import org.apache.rocketmq.client.hook.ConsumeMessageHook;
 import org.apache.rocketmq.client.hook.FilterMessageContext;
 import org.apache.rocketmq.client.hook.FilterMessageHook;
+import org.apache.rocketmq.client.impl.CommunicationMode;
+import org.apache.rocketmq.client.impl.FindBrokerResult;
+import org.apache.rocketmq.client.impl.MQAdminImpl;
+import org.apache.rocketmq.client.impl.MQClientAPIImpl;
+import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+import org.apache.rocketmq.client.stat.ConsumerStatsManager;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.ServiceState;
+import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.remoting.protocol.body.ConsumeStatus;
+import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;
+import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;
+import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
+import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
@@ -36,17 +69,85 @@
 import org.mockito.Mock;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
+
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
 public class DefaultMQPushConsumerImplTest {
+
     @Mock
     private DefaultMQPushConsumer defaultMQPushConsumer;
 
+    @Mock
+    private MQClientInstance mQClientFactory;
+
+    @Mock
+    private RebalanceImpl rebalanceImpl;
+
+    @Mock
+    private PullAPIWrapper pullAPIWrapper;
+
+    @Mock
+    private PullRequest pullRequest;
+
+    @Mock
+    private PopRequest popRequest;
+
+    @Mock
+    private ProcessQueue processQueue;
+
+    @Mock
+    private PopProcessQueue popProcessQueue;
+
+    @Mock
+    private MQClientAPIImpl mqClientAPIImpl;
+
+    @Mock
+    private OffsetStore offsetStore;
+
+    private DefaultMQPushConsumerImpl defaultMQPushConsumerImpl;
+
     @Rule
     public ExpectedException thrown = ExpectedException.none();
 
+    private final String defaultKey = "defaultKey";
+
+    private final String defaultTopic = "defaultTopic";
+
+    private final String defaultBroker = "defaultBroker";
+
+    private final String defaultBrokerAddr = "127.0.0.1:10911";
+
+    private final String defaultGroup = "defaultGroup";
+
+    private final long defaultTimeout = 3000L;
 
     @Test
     public void checkConfigTest() throws MQClientException {
@@ -62,19 +163,14 @@ public void checkConfigTest() throws MQClientException {
         consumer.setConsumeThreadMin(10);
         consumer.setConsumeThreadMax(9);
 
-        consumer.registerMessageListener(new MessageListenerConcurrently() {
-            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
-                ConsumeConcurrentlyContext context) {
-                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
-            }
-        });
+        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> ConsumeConcurrentlyStatus.CONSUME_SUCCESS);
 
         DefaultMQPushConsumerImpl defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(consumer, null);
         defaultMQPushConsumerImpl.start();
     }
 
     @Test
-    public void testHook() throws Exception {
+    public void testHook() {
         DefaultMQPushConsumerImpl defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(defaultMQPushConsumer, null);
         defaultMQPushConsumerImpl.registerConsumeMessageHook(new ConsumeMessageHook() {
             @Override
@@ -110,14 +206,10 @@ public void filterMessage(FilterMessageContext context) {
     @Ignore
     @Test
     public void testPush() throws Exception {
-        when(defaultMQPushConsumer.getMessageListener()).thenReturn(new MessageListenerConcurrently() {
-            @Override
-            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
-                ConsumeConcurrentlyContext context) {
-                assertThat(msgs).size().isGreaterThan(0);
-                assertThat(context).isNotNull();
-                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
-            }
+        when(defaultMQPushConsumer.getMessageListener()).thenReturn((MessageListenerConcurrently) (msgs, context) -> {
+            assertThat(msgs).size().isGreaterThan(0);
+            assertThat(context).isNotNull();
+            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
         });
         DefaultMQPushConsumerImpl defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(defaultMQPushConsumer, null);
         try {
@@ -126,4 +218,615 @@ public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
             defaultMQPushConsumerImpl.shutdown();
         }
     }
+
+    @Before
+    public void init() throws NoSuchFieldException, IllegalAccessException {
+        MQAdminImpl mqAdminImpl = mock(MQAdminImpl.class);
+        when(mQClientFactory.getMQAdminImpl()).thenReturn(mqAdminImpl);
+        ConsumerStatsManager consumerStatsManager = mock(ConsumerStatsManager.class);
+        ConsumeStatus consumeStatus = mock(ConsumeStatus.class);
+        when(consumerStatsManager.consumeStatus(any(), any())).thenReturn(consumeStatus);
+        when(mQClientFactory.getConsumerStatsManager()).thenReturn(consumerStatsManager);
+        when(mQClientFactory.getPullMessageService()).thenReturn(mock(PullMessageService.class));
+        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPIImpl);
+        FindBrokerResult findBrokerResult = mock(FindBrokerResult.class);
+        when(findBrokerResult.getBrokerAddr()).thenReturn(defaultBrokerAddr);
+        when(mQClientFactory.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(findBrokerResult);
+        Set<MessageQueue> messageQueueSet = Collections.singleton(createMessageQueue());
+        ConcurrentMap<String, Set<MessageQueue>> topicMessageQueueMap = new ConcurrentHashMap<>();
+        topicMessageQueueMap.put(defaultTopic, messageQueueSet);
+        when(rebalanceImpl.getTopicSubscribeInfoTable()).thenReturn(topicMessageQueueMap);
+        ConcurrentMap<MessageQueue, ProcessQueue> processQueueTable = new ConcurrentHashMap<>();
+        when(rebalanceImpl.getProcessQueueTable()).thenReturn(processQueueTable);
+        RPCHook rpcHook = mock(RPCHook.class);
+        defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(defaultMQPushConsumer, rpcHook);
+        defaultMQPushConsumerImpl.setOffsetStore(offsetStore);
+        FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, "mQClientFactory", mQClientFactory, true);
+        FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, "rebalanceImpl", rebalanceImpl, true);
+        FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, "pullAPIWrapper", pullAPIWrapper, true);
+        FilterMessageHook filterMessageHook = mock(FilterMessageHook.class);
+        ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<>();
+        filterMessageHookList.add(filterMessageHook);
+        ConsumeMessageService consumeMessagePopService = mock(ConsumeMessageService.class);
+        ConsumeMessageService consumeMessageService = mock(ConsumeMessageService.class);
+        FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, "filterMessageHookList", filterMessageHookList, true);
+        FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, "consumeMessageService", consumeMessageService, true);
+        FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, "consumeMessagePopService", consumeMessagePopService, true);
+        ConcurrentMap<String, SubscriptionData> subscriptionDataMap = new ConcurrentHashMap<>();
+        SubscriptionData subscriptionData = new SubscriptionData();
+        subscriptionData.setTopic(defaultTopic);
+        subscriptionDataMap.put(defaultTopic, subscriptionData);
+        when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap);
+    }
+
+    @Test
+    public void testFetchSubscribeMessageQueues() throws MQClientException {
+        Set<MessageQueue> actual = defaultMQPushConsumerImpl.fetchSubscribeMessageQueues(defaultTopic);
+        assertNotNull(actual);
+        Assert.assertEquals(1, actual.size());
+        MessageQueue next = actual.iterator().next();
+        assertEquals(defaultTopic, next.getTopic());
+        assertEquals(defaultBroker, next.getBrokerName());
+        assertEquals(0, next.getQueueId());
+    }
+
+    @Test
+    public void testEarliestMsgStoreTime() throws MQClientException {
+        assertEquals(0, defaultMQPushConsumerImpl.earliestMsgStoreTime(createMessageQueue()));
+    }
+
+    @Test
+    public void testMaxOffset() throws MQClientException {
+        assertEquals(0, defaultMQPushConsumerImpl.maxOffset(createMessageQueue()));
+    }
+
+    @Test
+    public void testMinOffset() throws MQClientException {
+        assertEquals(0, defaultMQPushConsumerImpl.minOffset(createMessageQueue()));
+    }
+
+    @Test
+    public void testGetOffsetStore() {
+        assertEquals(offsetStore, defaultMQPushConsumerImpl.getOffsetStore());
+    }
+
+    @Test
+    public void testPullMessageWithStateNotOk() {
+        when(pullRequest.getProcessQueue()).thenReturn(processQueue);
+        defaultMQPushConsumerImpl.pullMessage(pullRequest);
+    }
+
+    @Test
+    public void testPullMessageWithIsPause() {
+        when(pullRequest.getProcessQueue()).thenReturn(processQueue);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        defaultMQPushConsumerImpl.setPause(true);
+        defaultMQPushConsumerImpl.pullMessage(pullRequest);
+    }
+
+    @Test
+    public void testPullMessageWithMsgCountFlowControl() {
+        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));
+        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));
+        TreeMap<Long, MessageExt> treeMap = new TreeMap<>();
+        treeMap.put(1L, new MessageExt());
+        when(processQueue.getMsgTreeMap()).thenReturn(treeMap);
+        when(pullRequest.getProcessQueue()).thenReturn(processQueue);
+        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(1);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        defaultMQPushConsumerImpl.pullMessage(pullRequest);
+    }
+
+    @Test
+    public void testPullMessageWithMsgSizeFlowControl() {
+        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));
+        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));
+        TreeMap<Long, MessageExt> treeMap = new TreeMap<>();
+        treeMap.put(1L, new MessageExt());
+        when(processQueue.getMsgTreeMap()).thenReturn(treeMap);
+        when(pullRequest.getProcessQueue()).thenReturn(processQueue);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);
+        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(1);
+        defaultMQPushConsumerImpl.pullMessage(pullRequest);
+    }
+
+    @Test
+    public void testPullMessageWithMaxSpanFlowControl() {
+        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));
+        when(processQueue.getMaxSpan()).thenReturn(2L);
+        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));
+        TreeMap<Long, MessageExt> treeMap = new TreeMap<>();
+        treeMap.put(1L, new MessageExt());
+        when(processQueue.getMsgTreeMap()).thenReturn(treeMap);
+        when(pullRequest.getProcessQueue()).thenReturn(processQueue);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);
+        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10);
+        defaultMQPushConsumerImpl.pullMessage(pullRequest);
+    }
+
+    @Test
+    public void testPullMessageWithNotLocked() {
+        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));
+        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));
+        when(pullRequest.getProcessQueue()).thenReturn(processQueue);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        defaultMQPushConsumerImpl.setConsumeOrderly(true);
+        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);
+        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10);
+        defaultMQPushConsumerImpl.pullMessage(pullRequest);
+    }
+
+    @Test
+    public void testPullMessageWithSubscriptionDataIsNull() {
+        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));
+        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));
+        when(pullRequest.getMessageQueue()).thenReturn(createMessageQueue());
+        when(pullRequest.getProcessQueue()).thenReturn(processQueue);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);
+        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10);
+        defaultMQPushConsumerImpl.pullMessage(pullRequest);
+    }
+
+    @Test
+    public void testPullMessageWithNoMatchedMsg() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {
+        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));
+        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));
+        when(pullRequest.getMessageQueue()).thenReturn(createMessageQueue());
+        when(pullRequest.getProcessQueue()).thenReturn(processQueue);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);
+        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10);
+        PullResult pullResultMock = mock(PullResult.class);
+        when(pullAPIWrapper.processPullResult(any(MessageQueue.class), any(PullResult.class), any(SubscriptionData.class))).thenReturn(pullResultMock);
+        when(pullResultMock.getPullStatus()).thenReturn(PullStatus.NO_MATCHED_MSG);
+        doAnswer(invocation -> {
+            PullCallback callback = invocation.getArgument(12);
+            PullResult pullResult = mock(PullResult.class);
+            callback.onSuccess(pullResult);
+            return null;
+        }).when(pullAPIWrapper).pullKernelImpl(
+                any(MessageQueue.class),
+                any(),
+                any(),
+                anyLong(),
+                anyLong(),
+                anyInt(),
+                anyInt(),
+                anyInt(),
+                anyLong(),
+                anyLong(),
+                anyLong(),
+                any(CommunicationMode.class),
+                any(PullCallback.class));
+        defaultMQPushConsumerImpl.pullMessage(pullRequest);
+    }
+
+    @Test
+    public void testPullMessageWithOffsetIllegal() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {
+        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));
+        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));
+        when(pullRequest.getMessageQueue()).thenReturn(createMessageQueue());
+        when(pullRequest.getProcessQueue()).thenReturn(processQueue);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);
+        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10);
+        PullResult pullResultMock = mock(PullResult.class);
+        when(pullAPIWrapper.processPullResult(any(MessageQueue.class), any(PullResult.class), any(SubscriptionData.class))).thenReturn(pullResultMock);
+        when(pullResultMock.getPullStatus()).thenReturn(PullStatus.OFFSET_ILLEGAL);
+        doAnswer(invocation -> {
+            PullCallback callback = invocation.getArgument(12);
+            PullResult pullResult = mock(PullResult.class);
+            callback.onSuccess(pullResult);
+            return null;
+        }).when(pullAPIWrapper).pullKernelImpl(
+                any(MessageQueue.class),
+                any(),
+                any(),
+                anyLong(),
+                anyLong(),
+                anyInt(),
+                anyInt(),
+                anyInt(),
+                anyLong(),
+                anyLong(),
+                anyLong(),
+                any(CommunicationMode.class),
+                any(PullCallback.class));
+        defaultMQPushConsumerImpl.pullMessage(pullRequest);
+    }
+
+    @Test
+    public void testPullMessageWithException() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {
+        when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2));
+        when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024));
+        when(pullRequest.getMessageQueue()).thenReturn(createMessageQueue());
+        when(pullRequest.getProcessQueue()).thenReturn(processQueue);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3);
+        when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10);
+        doAnswer(invocation -> {
+            PullCallback callback = invocation.getArgument(12);
+            callback.onException(new RuntimeException("exception"));
+            return null;
+        }).when(pullAPIWrapper).pullKernelImpl(
+                any(MessageQueue.class),
+                any(),
+                any(),
+                anyLong(),
+                anyLong(),
+                anyInt(),
+                anyInt(),
+                anyInt(),
+                anyLong(),
+                anyLong(),
+                anyLong(),
+                any(CommunicationMode.class),
+                any(PullCallback.class));
+        defaultMQPushConsumerImpl.pullMessage(pullRequest);
+    }
+
+    @Test
+    public void testPopMessageWithFound() throws RemotingException, InterruptedException, MQClientException {
+        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);
+        when(popRequest.getMessageQueue()).thenReturn(createMessageQueue());
+        when(popRequest.getConsumerGroup()).thenReturn(defaultGroup);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        ConcurrentMap<String, SubscriptionData> subscriptionDataMap = new ConcurrentHashMap<>();
+        SubscriptionData subscriptionData = new SubscriptionData();
+        subscriptionData.setTagsSet(Collections.singleton("*"));
+        subscriptionDataMap.put(defaultTopic, subscriptionData);
+        when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap);
+        doAnswer(invocation -> {
+            PopCallback callback = invocation.getArgument(5);
+            PopResult popResult = mock(PopResult.class);
+            when(popResult.getPopStatus()).thenReturn(PopStatus.FOUND);
+            when(popResult.getMsgFoundList()).thenReturn(Collections.singletonList(createMessageExt()));
+            callback.onSuccess(popResult);
+            return null;
+        }).when(pullAPIWrapper).popAsync(
+                any(MessageQueue.class),
+                anyLong(),
+                anyInt(),
+                any(),
+                anyLong(),
+                any(PopCallback.class),
+                anyBoolean(),
+                anyInt(),
+                anyBoolean(),
+                any(),
+                any());
+        defaultMQPushConsumerImpl.popMessage(popRequest);
+    }
+
+    @Test
+    public void testPopMessageWithException() throws RemotingException, InterruptedException, MQClientException {
+        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);
+        when(popRequest.getMessageQueue()).thenReturn(createMessageQueue());
+        when(popRequest.getConsumerGroup()).thenReturn(defaultGroup);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        ConcurrentMap<String, SubscriptionData> subscriptionDataMap = new ConcurrentHashMap<>();
+        SubscriptionData subscriptionData = new SubscriptionData();
+        subscriptionData.setTagsSet(Collections.singleton("*"));
+        subscriptionDataMap.put(defaultTopic, subscriptionData);
+        when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap);
+        doAnswer(invocation -> {
+            PopCallback callback = invocation.getArgument(5);
+            callback.onException(new RuntimeException("exception"));
+            return null;
+        }).when(pullAPIWrapper).popAsync(
+                any(MessageQueue.class),
+                anyLong(),
+                anyInt(),
+                any(),
+                anyLong(),
+                any(PopCallback.class),
+                anyBoolean(),
+                anyInt(),
+                anyBoolean(),
+                any(),
+                any());
+        defaultMQPushConsumerImpl.popMessage(popRequest);
+    }
+
+    @Test
+    public void testPopMessageWithNoNewMsg() throws RemotingException, InterruptedException, MQClientException {
+        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);
+        when(popRequest.getMessageQueue()).thenReturn(createMessageQueue());
+        when(popRequest.getConsumerGroup()).thenReturn(defaultGroup);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        ConcurrentMap<String, SubscriptionData> subscriptionDataMap = new ConcurrentHashMap<>();
+        SubscriptionData subscriptionData = new SubscriptionData();
+        subscriptionData.setTagsSet(Collections.singleton("*"));
+        subscriptionDataMap.put(defaultTopic, subscriptionData);
+        when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap);
+        doAnswer(invocation -> {
+            PopCallback callback = invocation.getArgument(5);
+            PopResult popResult = mock(PopResult.class);
+            when(popResult.getPopStatus()).thenReturn(PopStatus.NO_NEW_MSG);
+            callback.onSuccess(popResult);
+            return null;
+        }).when(pullAPIWrapper).popAsync(
+                any(MessageQueue.class),
+                anyLong(),
+                anyInt(),
+                any(),
+                anyLong(),
+                any(PopCallback.class),
+                anyBoolean(),
+                anyInt(),
+                anyBoolean(),
+                any(),
+                any());
+        defaultMQPushConsumerImpl.popMessage(popRequest);
+    }
+
+    @Test
+    public void testPopMessageWithPollingFull() throws RemotingException, InterruptedException, MQClientException {
+        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);
+        when(popRequest.getMessageQueue()).thenReturn(createMessageQueue());
+        when(popRequest.getConsumerGroup()).thenReturn(defaultGroup);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        ConcurrentMap<String, SubscriptionData> subscriptionDataMap = new ConcurrentHashMap<>();
+        SubscriptionData subscriptionData = new SubscriptionData();
+        subscriptionData.setTagsSet(Collections.singleton("*"));
+        subscriptionDataMap.put(defaultTopic, subscriptionData);
+        when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap);
+        doAnswer(invocation -> {
+            PopCallback callback = invocation.getArgument(5);
+            PopResult popResult = mock(PopResult.class);
+            when(popResult.getPopStatus()).thenReturn(PopStatus.POLLING_FULL);
+            callback.onSuccess(popResult);
+            return null;
+        }).when(pullAPIWrapper).popAsync(any(
+                        MessageQueue.class),
+                anyLong(),
+                anyInt(),
+                any(),
+                anyLong(),
+                any(PopCallback.class),
+                anyBoolean(),
+                anyInt(),
+                anyBoolean(),
+                any(),
+                any());
+        defaultMQPushConsumerImpl.popMessage(popRequest);
+    }
+
+    @Test
+    public void testPopMessageWithStateNotOk() {
+        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);
+        defaultMQPushConsumerImpl.popMessage(popRequest);
+    }
+
+    @Test
+    public void testPopMessageWithIsPause() {
+        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        defaultMQPushConsumerImpl.setPause(true);
+        defaultMQPushConsumerImpl.popMessage(popRequest);
+    }
+
+    @Test
+    public void testPopMessageWithWaiAckMsgCountFlowControl() {
+        when(popProcessQueue.getWaiAckMsgCount()).thenReturn(2);
+        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);
+        when(defaultMQPushConsumer.getPopThresholdForQueue()).thenReturn(1);
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        defaultMQPushConsumerImpl.popMessage(popRequest);
+    }
+
+    @Test
+    public void testPopMessageWithSubscriptionDataIsNull() throws RemotingException, InterruptedException, MQClientException {
+        when(popProcessQueue.getWaiAckMsgCount()).thenReturn(2);
+        when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue);
+        when(popRequest.getMessageQueue()).thenReturn(createMessageQueue());
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        when(defaultMQPushConsumer.getPopThresholdForQueue()).thenReturn(3);
+        defaultMQPushConsumerImpl.popMessage(popRequest);
+        verify(pullAPIWrapper).popAsync(any(MessageQueue.class),
+                eq(60000L),
+                eq(0),
+                any(),
+                eq(15000L),
+                any(PopCallback.class),
+                eq(true),
+                eq(0),
+                eq(false),
+                any(),
+                any());
+    }
+
+    @Test
+    public void testQueryMessage() throws InterruptedException, MQClientException {
+        assertNull(defaultMQPushConsumerImpl.queryMessage(defaultTopic, defaultKey, 1, 0, 1));
+    }
+
+    @Test
+    public void testQueryMessageByUniqKey() throws InterruptedException, MQClientException {
+        assertNull(defaultMQPushConsumerImpl.queryMessageByUniqKey(defaultTopic, defaultKey));
+    }
+
+    @Test
+    public void testSendMessageBack() throws InterruptedException, MQClientException, MQBrokerException, RemotingException {
+        when(mQClientFactory.findBrokerAddressInPublish(anyString())).thenReturn(defaultBrokerAddr);
+        defaultMQPushConsumerImpl.sendMessageBack(createMessageExt(), 1, createMessageQueue());
+        verify(mqClientAPIImpl).consumerSendMessageBack(
+                eq(defaultBrokerAddr),
+                eq(defaultBroker),
+                any(MessageExt.class),
+                any(),
+                eq(1),
+                eq(5000L),
+                eq(0));
+    }
+
+    @Test
+    public void testAckAsync() throws MQBrokerException, RemotingException, InterruptedException {
+        doAnswer(invocation -> {
+            AckCallback callback = invocation.getArgument(2);
+            AckResult result = mock(AckResult.class);
+            when(result.getStatus()).thenReturn(AckStatus.OK);
+            callback.onSuccess(result);
+            return null;
+        }).when(mqClientAPIImpl).ackMessageAsync(any(),
+                anyLong(),
+                any(AckCallback.class),
+                any(AckMessageRequestHeader.class));
+        defaultMQPushConsumerImpl.ackAsync(createMessageExt(), defaultGroup);
+        verify(mqClientAPIImpl).ackMessageAsync(eq(defaultBrokerAddr),
+                eq(3000L),
+                any(AckCallback.class),
+                any(AckMessageRequestHeader.class));
+    }
+
+    @Test
+    public void testChangePopInvisibleTimeAsync() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {
+        AckCallback callback = mock(AckCallback.class);
+        String extraInfo = createMessageExt().getProperty(MessageConst.PROPERTY_POP_CK);
+        defaultMQPushConsumerImpl.changePopInvisibleTimeAsync(defaultTopic, defaultGroup, extraInfo, defaultTimeout, callback);
+        verify(mqClientAPIImpl).changeInvisibleTimeAsync(eq(defaultBroker),
+                eq(defaultBrokerAddr),
+                any(ChangeInvisibleTimeRequestHeader.class),
+                eq(defaultTimeout),
+                any(AckCallback.class));
+    }
+
+    @Test
+    public void testShutdown() {
+        defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING);
+        defaultMQPushConsumerImpl.shutdown();
+        assertEquals(ServiceState.SHUTDOWN_ALREADY, defaultMQPushConsumerImpl.getServiceState());
+    }
+
+    @Test
+    public void testSubscribe() throws MQClientException {
+        defaultMQPushConsumerImpl.subscribe(defaultTopic, "fullClassname", "filterClassSource");
+        RebalanceImpl actual = defaultMQPushConsumerImpl.getRebalanceImpl();
+        assertEquals(1, actual.getSubscriptionInner().size());
+    }
+
+    @Test
+    public void testSubscribeByMessageSelector() throws MQClientException {
+        MessageSelector messageSelector = mock(MessageSelector.class);
+        defaultMQPushConsumerImpl.subscribe(defaultTopic, messageSelector);
+        RebalanceImpl actual = defaultMQPushConsumerImpl.getRebalanceImpl();
+        assertEquals(1, actual.getSubscriptionInner().size());
+    }
+
+    @Test
+    public void testSuspend() {
+        defaultMQPushConsumerImpl.suspend();
+        assertTrue(defaultMQPushConsumerImpl.isPause());
+    }
+
+    @Test
+    public void testViewMessage() throws InterruptedException, MQClientException, MQBrokerException, RemotingException {
+        assertNull(defaultMQPushConsumerImpl.viewMessage(defaultTopic, createMessageExt().getMsgId()));
+    }
+
+    @Test
+    public void testResetOffsetByTimeStamp() throws MQClientException {
+        ConcurrentMap<String, SubscriptionData> subscriptionDataMap = new ConcurrentHashMap<>();
+        subscriptionDataMap.put(defaultTopic, new SubscriptionData());
+        when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap);
+        defaultMQPushConsumerImpl.resetOffsetByTimeStamp(System.currentTimeMillis());
+        verify(mQClientFactory).resetOffset(eq(defaultTopic), any(), any());
+    }
+
+    @Test
+    public void testSearchOffset() throws MQClientException {
+        assertEquals(0, defaultMQPushConsumerImpl.searchOffset(createMessageQueue(), System.currentTimeMillis()));
+    }
+
+    @Test
+    public void testQueryConsumeTimeSpan() throws InterruptedException, MQClientException, MQBrokerException, RemotingException {
+        TopicRouteData topicRouteData = new TopicRouteData();
+        topicRouteData.getBrokerDatas().add(createBrokerData());
+        when(mqClientAPIImpl.getTopicRouteInfoFromNameServer(any(), anyLong())).thenReturn(topicRouteData);
+        List<QueueTimeSpan> actual = defaultMQPushConsumerImpl.queryConsumeTimeSpan(defaultTopic);
+        assertNotNull(actual);
+        assertEquals(0, actual.size());
+    }
+
+    @Test
+    public void testTryResetPopRetryTopic() {
+        TopicRouteData topicRouteData = new TopicRouteData();
+        topicRouteData.getBrokerDatas().add(createBrokerData());
+        MessageExt messageExt = createMessageExt();
+        List<MessageExt> msgs = new ArrayList<>();
+        messageExt.setTopic(MixAll.RETRY_GROUP_TOPIC_PREFIX + defaultGroup + "_" + defaultTopic);
+        msgs.add(messageExt);
+        defaultMQPushConsumerImpl.tryResetPopRetryTopic(msgs, defaultGroup);
+        assertEquals(defaultTopic, msgs.get(0).getTopic());
+    }
+
+    @Test
+    public void testGetPopDelayLevel() {
+        int[] actual = defaultMQPushConsumerImpl.getPopDelayLevel();
+        int[] expected = new int[]{10, 30, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600, 1200, 1800, 3600, 7200};
+        assertArrayEquals(expected, actual);
+    }
+
+    @Test
+    public void testGetMessageQueueListener() {
+        assertNull(defaultMQPushConsumerImpl.getMessageQueueListener());
+    }
+
+    @Test
+    public void testConsumerRunningInfo() {
+        ConcurrentMap<MessageQueue, ProcessQueue> processQueueMap = new ConcurrentHashMap<>();
+        ConcurrentMap<MessageQueue, PopProcessQueue> popProcessQueueMap = new ConcurrentHashMap<>();
+        processQueueMap.put(createMessageQueue(), new ProcessQueue());
+        popProcessQueueMap.put(createMessageQueue(), new PopProcessQueue());
+        when(rebalanceImpl.getProcessQueueTable()).thenReturn(processQueueMap);
+        when(rebalanceImpl.getPopProcessQueueTable()).thenReturn(popProcessQueueMap);
+        ConsumerRunningInfo actual = defaultMQPushConsumerImpl.consumerRunningInfo();
+        assertNotNull(actual);
+        assertEquals(1, actual.getSubscriptionSet().size());
+        assertEquals(defaultTopic, actual.getSubscriptionSet().iterator().next().getTopic());
+        assertEquals(1, actual.getMqTable().size());
+        assertEquals(1, actual.getMqPopTable().size());
+        assertEquals(1, actual.getStatusTable().size());
+    }
+
+    private BrokerData createBrokerData() {
+        BrokerData result = new BrokerData();
+        HashMap<Long, String> brokerAddrMap = new HashMap<>();
+        brokerAddrMap.put(MixAll.MASTER_ID, defaultBrokerAddr);
+        result.setBrokerAddrs(brokerAddrMap);
+        result.setBrokerName(defaultBroker);
+        return result;
+    }
+
+    private MessageQueue createMessageQueue() {
+        MessageQueue result = new MessageQueue();
+        result.setQueueId(0);
+        result.setBrokerName(defaultBroker);
+        result.setTopic(defaultTopic);
+        return result;
+    }
+
+    private MessageExt createMessageExt() {
+        MessageExt result = new MessageExt();
+        result.setBody("body".getBytes(StandardCharsets.UTF_8));
+        result.setTopic(defaultTopic);
+        result.setBrokerName(defaultBroker);
+        result.putUserProperty("key", "value");
+        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup);
+        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1");
+        long curTime = System.currentTimeMillis();
+        result.setBornTimestamp(curTime - 1000);
+        String popProps = String.format("%d %d %d %d %d %s %d %d %d", curTime, curTime, curTime, curTime, curTime, defaultBroker, 1, 0L, 1L);
+        result.getProperties().put(MessageConst.PROPERTY_POP_CK, popProps);
+        result.setKeys("keys");
+        result.setTags("*");
+        SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911);
+        SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911);
+        result.setBornHost(bornHost);
+        result.setStoreHost(storeHost);
+        return result;
+    }
 }
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ProcessQueueTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ProcessQueueTest.java
index be0bd29f79f..dd7ffa757f8 100644
--- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ProcessQueueTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ProcessQueueTest.java
@@ -16,10 +16,18 @@
  */
 package org.apache.rocketmq.client.impl.consumer;
 
+import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.TreeMap;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.remoting.protocol.body.ProcessQueueInfo;
 import org.assertj.core.util.Lists;
 import org.junit.Test;
@@ -27,6 +35,12 @@
 import org.mockito.junit.MockitoJUnitRunner;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 
 @RunWith(MockitoJUnitRunner.class)
 public class ProcessQueueTest {
@@ -78,7 +92,7 @@ public void testContainsMessage() {
     }
 
     @Test
-    public void testFillProcessQueueInfo() {
+    public void testFillProcessQueueInfo() throws IllegalAccessException {
         ProcessQueue pq = new ProcessQueue();
         pq.putMessage(createMessageList(102400));
 
@@ -101,6 +115,56 @@ public void testFillProcessQueueInfo() {
         pq.commit();
         pq.fillProcessQueueInfo(processQueueInfo);
         assertThat(processQueueInfo.getCachedMsgSizeInMiB()).isEqualTo(0);
+
+        TreeMap<Long, MessageExt> consumingMsgOrderlyTreeMap = new TreeMap<>();
+        consumingMsgOrderlyTreeMap.put(0L, createMessageList(1).get(0));
+        FieldUtils.writeDeclaredField(pq, "consumingMsgOrderlyTreeMap", consumingMsgOrderlyTreeMap, true);
+        pq.fillProcessQueueInfo(processQueueInfo);
+        assertEquals(0, processQueueInfo.getTransactionMsgMinOffset());
+        assertEquals(0, processQueueInfo.getTransactionMsgMaxOffset());
+        assertEquals(1, processQueueInfo.getTransactionMsgCount());
+    }
+
+    @Test
+    public void testPopRequest() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {
+        ProcessQueue processQueue = createProcessQueue();
+        MessageExt messageExt = createMessageList(1).get(0);
+        messageExt.getProperties().put(MessageConst.PROPERTY_CONSUME_START_TIMESTAMP, System.currentTimeMillis() - 20 * 60 * 1000L + "");
+        processQueue.getMsgTreeMap().put(0L, messageExt);
+        DefaultMQPushConsumer pushConsumer = mock(DefaultMQPushConsumer.class);
+        processQueue.cleanExpiredMsg(pushConsumer);
+        verify(pushConsumer).sendMessageBack(any(MessageExt.class), eq(3));
+    }
+
+    @Test
+    public void testRollback() throws IllegalAccessException {
+        ProcessQueue processQueue = createProcessQueue();
+        processQueue.rollback();
+        Field consumingMsgOrderlyTreeMapField = FieldUtils.getDeclaredField(processQueue.getClass(), "consumingMsgOrderlyTreeMap", true);
+        TreeMap<Long, MessageExt> consumingMsgOrderlyTreeMap = (TreeMap<Long, MessageExt>) consumingMsgOrderlyTreeMapField.get(processQueue);
+        assertEquals(0, consumingMsgOrderlyTreeMap.size());
+    }
+
+    @Test
+    public void testHasTempMessage() {
+        ProcessQueue processQueue = createProcessQueue();
+        assertFalse(processQueue.hasTempMessage());
+    }
+
+    @Test
+    public void testProcessQueue() {
+        ProcessQueue processQueue1 = createProcessQueue();
+        ProcessQueue processQueue2 = createProcessQueue();
+        assertEquals(processQueue1.getMsgAccCnt(), processQueue2.getMsgAccCnt());
+        assertEquals(processQueue1.getTryUnlockTimes(), processQueue2.getTryUnlockTimes());
+        assertEquals(processQueue1.getLastPullTimestamp(), processQueue2.getLastPullTimestamp());
+    }
+
+    private ProcessQueue createProcessQueue() {
+        ProcessQueue result = new ProcessQueue();
+        result.setMsgAccCnt(1);
+        result.incTryUnlockTimes();
+        return result;
     }
 
     private List<MessageExt> createMessageList() {
@@ -108,13 +172,15 @@ private List<MessageExt> createMessageList() {
     }
 
     private List<MessageExt> createMessageList(int count) {
-        List<MessageExt> messageExtList = new ArrayList<>();
+        List<MessageExt> result = new ArrayList<>();
         for (int i = 0; i < count; i++) {
             MessageExt messageExt = new MessageExt();
             messageExt.setQueueOffset(i);
             messageExt.setBody(new byte[123]);
-            messageExtList.add(messageExt);
+            messageExt.setKeys("keys" + i);
+            messageExt.getProperties().put(MessageConst.PROPERTY_CONSUME_START_TIMESTAMP, System.currentTimeMillis() + "");
+            result.add(messageExt);
         }
-        return messageExtList;
+        return result;
     }
 }
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapperTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapperTest.java
new file mode 100644
index 00000000000..2ffa8f4f149
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapperTest.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.impl.consumer;
+
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.client.ClientConfig;
+import org.apache.rocketmq.client.consumer.PopCallback;
+import org.apache.rocketmq.client.consumer.PullCallback;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.PullStatus;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.hook.FilterMessageContext;
+import org.apache.rocketmq.client.hook.FilterMessageHook;
+import org.apache.rocketmq.client.impl.CommunicationMode;
+import org.apache.rocketmq.client.impl.FindBrokerResult;
+import org.apache.rocketmq.client.impl.MQClientAPIImpl;
+import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.sysflag.MessageSysFlag;
+import org.apache.rocketmq.common.sysflag.PullSysFlag;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class PullAPIWrapperTest {
+
+    @Mock
+    private MQClientInstance mQClientFactory;
+
+    @Mock
+    private MQClientAPIImpl mqClientAPIImpl;
+
+    private PullAPIWrapper pullAPIWrapper;
+
+    private final String defaultGroup = "defaultGroup";
+
+    private final String defaultBroker = "defaultBroker";
+
+    private final String defaultTopic = "defaultTopic";
+
+    private final String defaultBrokerAddr = "127.0.0.1:10911";
+
+    private final long defaultTimeout = 3000L;
+
+    @Before
+    public void init() throws Exception {
+        ClientConfig clientConfig = mock(ClientConfig.class);
+        when(mQClientFactory.getClientConfig()).thenReturn(clientConfig);
+        MQClientAPIImpl mqClientAPIImpl = mock(MQClientAPIImpl.class);
+        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPIImpl);
+        when(mQClientFactory.getTopicRouteTable()).thenReturn(createTopicRouteTable());
+        FindBrokerResult findBrokerResult = mock(FindBrokerResult.class);
+        when(findBrokerResult.getBrokerAddr()).thenReturn(defaultBrokerAddr);
+        when(mQClientFactory.findBrokerAddressInSubscribe(any(), anyLong(), anyBoolean())).thenReturn(findBrokerResult);
+        pullAPIWrapper = new PullAPIWrapper(mQClientFactory, defaultGroup, false);
+        ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<>();
+        filterMessageHookList.add(mock(FilterMessageHook.class));
+        FieldUtils.writeDeclaredField(pullAPIWrapper, "filterMessageHookList", filterMessageHookList, true);
+    }
+
+    @Test
+    public void testProcessPullResult() throws Exception {
+        PullResultExt pullResult = mock(PullResultExt.class);
+        when(pullResult.getPullStatus()).thenReturn(PullStatus.FOUND);
+        when(pullResult.getMessageBinary()).thenReturn(MessageDecoder.encode(createMessageExt(), false));
+        SubscriptionData subscriptionData = mock(SubscriptionData.class);
+        PullResult actual = pullAPIWrapper.processPullResult(createMessageQueue(), pullResult, subscriptionData);
+        assertNotNull(actual);
+        assertEquals(0, actual.getNextBeginOffset());
+        assertEquals(0, actual.getMsgFoundList().size());
+    }
+
+    @Test
+    public void testExecuteHook() throws IllegalAccessException {
+        FilterMessageContext filterMessageContext = mock(FilterMessageContext.class);
+        ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<>();
+        FilterMessageHook filterMessageHook = mock(FilterMessageHook.class);
+        filterMessageHookList.add(filterMessageHook);
+        FieldUtils.writeDeclaredField(pullAPIWrapper, "filterMessageHookList", filterMessageHookList, true);
+        pullAPIWrapper.executeHook(filterMessageContext);
+        verify(filterMessageHook, times(1)).filterMessage(any(FilterMessageContext.class));
+    }
+
+    @Test
+    public void testPullKernelImpl() throws Exception {
+        PullCallback pullCallback = mock(PullCallback.class);
+        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPIImpl);
+        PullResult actual = pullAPIWrapper.pullKernelImpl(createMessageQueue(),
+                "",
+                "",
+                1L,
+                1L,
+                1,
+                1,
+                PullSysFlag.buildSysFlag(false, false, false, true),
+                1L,
+                System.currentTimeMillis(),
+                defaultTimeout, CommunicationMode.ASYNC, pullCallback);
+        assertNull(actual);
+        verify(mqClientAPIImpl, times(1)).pullMessage(eq(defaultBroker),
+                any(PullMessageRequestHeader.class),
+                eq(defaultTimeout),
+                any(CommunicationMode.class),
+                any(PullCallback.class));
+    }
+
+    @Test
+    public void testSetConnectBrokerByUser() {
+        pullAPIWrapper.setConnectBrokerByUser(true);
+        assertTrue(pullAPIWrapper.isConnectBrokerByUser());
+    }
+
+    @Test
+    public void testRandomNum() {
+        int randomNum = pullAPIWrapper.randomNum();
+        assertTrue(randomNum > 0);
+    }
+
+    @Test
+    public void testSetDefaultBrokerId() {
+        pullAPIWrapper.setDefaultBrokerId(MixAll.MASTER_ID);
+        assertEquals(MixAll.MASTER_ID, pullAPIWrapper.getDefaultBrokerId());
+    }
+
+    @Test
+    public void testPopAsync() throws RemotingException, InterruptedException, MQClientException {
+        PopCallback popCallback = mock(PopCallback.class);
+        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPIImpl);
+        pullAPIWrapper.popAsync(createMessageQueue(),
+                System.currentTimeMillis(),
+                1,
+                defaultGroup,
+                defaultTimeout,
+                popCallback,
+                true,
+                1,
+                false,
+                "",
+                "");
+        verify(mqClientAPIImpl, times(1)).popMessageAsync(eq(defaultBroker),
+                eq(defaultBrokerAddr),
+                any(PopMessageRequestHeader.class),
+                eq(13000L),
+                any(PopCallback.class));
+    }
+
+    private ConcurrentMap<String, TopicRouteData> createTopicRouteTable() {
+        TopicRouteData topicRouteData = new TopicRouteData();
+        List<BrokerData> brokerDatas = new ArrayList<>();
+        BrokerData brokerData = new BrokerData();
+        brokerData.setBrokerName(defaultBroker);
+        HashMap<Long, String> brokerAddrs = new HashMap<>();
+        brokerAddrs.put(MixAll.MASTER_ID, defaultBroker);
+        brokerData.setBrokerAddrs(brokerAddrs);
+        brokerDatas.add(brokerData);
+        topicRouteData.setBrokerDatas(brokerDatas);
+        HashMap<String, List<String>> filterServerTable = new HashMap<>();
+        List<String> filterServers = new ArrayList<>();
+        filterServers.add(defaultBroker);
+        filterServerTable.put(defaultBrokerAddr, filterServers);
+        topicRouteData.setFilterServerTable(filterServerTable);
+        ConcurrentMap<String, TopicRouteData> result = new ConcurrentHashMap<>();
+        result.put(defaultTopic, topicRouteData);
+        return result;
+    }
+
+    private MessageQueue createMessageQueue() {
+        MessageQueue result = new MessageQueue();
+        result.setQueueId(0);
+        result.setBrokerName(defaultBroker);
+        result.setTopic(defaultTopic);
+        return result;
+    }
+
+    private MessageExt createMessageExt() {
+        MessageExt result = new MessageExt();
+        result.setBody("body".getBytes(StandardCharsets.UTF_8));
+        result.setTopic(defaultTopic);
+        result.setBrokerName(defaultBroker);
+        result.putUserProperty("key", "value");
+        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup);
+        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1");
+        long curTime = System.currentTimeMillis();
+        result.setBornTimestamp(curTime - 1000);
+        result.getProperties().put(MessageConst.PROPERTY_POP_CK, curTime + " " + curTime + " " + curTime + " " + curTime);
+        result.setKeys("keys");
+        result.setSysFlag(MessageSysFlag.INNER_BATCH_FLAG);
+        result.setSysFlag(result.getSysFlag() | MessageSysFlag.NEED_UNWRAP_FLAG);
+        SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911);
+        SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911);
+        result.setBornHost(bornHost);
+        result.setStoreHost(storeHost);
+        return result;
+    }
+}
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullMessageServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullMessageServiceTest.java
new file mode 100644
index 00000000000..73fa4e95d69
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullMessageServiceTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.impl.consumer;
+
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+import org.apache.rocketmq.common.message.MessageRequestMode;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class PullMessageServiceTest {
+
+    @Mock
+    private MQClientInstance mQClientFactory;
+
+    @Mock
+    private ScheduledExecutorService executorService;
+
+    private PullMessageService pullMessageService;
+
+    private final long defaultTimeout = 3000L;
+
+    private final String defaultGroup = "defaultGroup";
+
+    @Before
+    public void init() throws Exception {
+        pullMessageService = new PullMessageService(mQClientFactory);
+        FieldUtils.writeDeclaredField(pullMessageService, "scheduledExecutorService", executorService, true);
+        pullMessageService.start();
+    }
+
+    @Test
+    public void testProcessPullResult() {
+        PopRequest popRequest = mock(PopRequest.class);
+        pullMessageService.executePopPullRequestLater(popRequest, defaultTimeout);
+        pullMessageService.makeStop();
+        pullMessageService.executePopPullRequestLater(popRequest, defaultTimeout);
+        verify(executorService, times(1))
+                .schedule(any(Runnable.class),
+                        eq(defaultTimeout),
+                        eq(TimeUnit.MILLISECONDS));
+    }
+
+    @Test
+    public void testExecutePopPullRequestImmediately() throws IllegalAccessException, InterruptedException {
+        PopRequest popRequest = mock(PopRequest.class);
+        LinkedBlockingQueue<MessageRequest> messageRequestQueue = mock(LinkedBlockingQueue.class);
+        FieldUtils.writeDeclaredField(pullMessageService, "messageRequestQueue", messageRequestQueue, true);
+        pullMessageService.executePopPullRequestImmediately(popRequest);
+        verify(messageRequestQueue, times(1)).put(any(PopRequest.class));
+    }
+
+    @Test
+    public void testExecuteTaskLater() {
+        Runnable runnable = mock(Runnable.class);
+        pullMessageService.executeTaskLater(runnable, defaultTimeout);
+        pullMessageService.makeStop();
+        pullMessageService.executeTaskLater(runnable, defaultTimeout);
+        verify(executorService, times(1))
+                .schedule(any(Runnable.class),
+                        eq(defaultTimeout),
+                        eq(TimeUnit.MILLISECONDS));
+    }
+
+    @Test
+    public void testExecuteTask() {
+        Runnable runnable = mock(Runnable.class);
+        pullMessageService.executeTask(runnable);
+        pullMessageService.makeStop();
+        pullMessageService.executeTask(runnable);
+        verify(executorService, times(1)).execute(any(Runnable.class));
+    }
+
+    @Test
+    public void testGetScheduledExecutorService() {
+        assertEquals(executorService, pullMessageService.getScheduledExecutorService());
+    }
+
+    @Test
+    public void testRun() throws InterruptedException, IllegalAccessException {
+        LinkedBlockingQueue<MessageRequest> messageRequestQueue = new LinkedBlockingQueue<>();
+        PopRequest popRequest = mock(PopRequest.class);
+        when(popRequest.getMessageRequestMode()).thenReturn(MessageRequestMode.POP);
+        when(popRequest.getConsumerGroup()).thenReturn(defaultGroup);
+        messageRequestQueue.put(popRequest);
+        DefaultMQPushConsumerImpl defaultMQPushConsumerImpl = mock(DefaultMQPushConsumerImpl.class);
+        when(mQClientFactory.selectConsumer(any())).thenReturn(defaultMQPushConsumerImpl);
+        FieldUtils.writeDeclaredField(pullMessageService, "messageRequestQueue", messageRequestQueue, true);
+        new Thread(() -> pullMessageService.run()).start();
+        TimeUnit.SECONDS.sleep(1);
+        pullMessageService.makeStop();
+        verify(mQClientFactory, times(1)).selectConsumer(eq(defaultGroup));
+        verify(defaultMQPushConsumerImpl).popMessage(any(PopRequest.class));
+    }
+
+    @Test
+    public void testRunWithNullConsumer() throws InterruptedException, IllegalAccessException {
+        LinkedBlockingQueue<MessageRequest> messageRequestQueue = new LinkedBlockingQueue<>();
+        PopRequest popRequest = mock(PopRequest.class);
+        when(popRequest.getMessageRequestMode()).thenReturn(MessageRequestMode.POP);
+        when(popRequest.getConsumerGroup()).thenReturn(defaultGroup);
+        messageRequestQueue.put(popRequest);
+        FieldUtils.writeDeclaredField(pullMessageService, "messageRequestQueue", messageRequestQueue, true);
+        new Thread(() -> pullMessageService.run()).start();
+        TimeUnit.SECONDS.sleep(1);
+        pullMessageService.makeStop();
+        verify(mQClientFactory, times(1)).selectConsumer(eq(defaultGroup));
+    }
+}
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java
index acd792b8625..d71bc25b9b3 100644
--- a/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java
@@ -16,77 +16,116 @@
  */
 package org.apache.rocketmq.client.impl.factory;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.admin.MQAdminExtInner;
+import org.apache.rocketmq.client.consumer.store.OffsetStore;
 import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.impl.FindBrokerResult;
+import org.apache.rocketmq.client.impl.MQClientAPIImpl;
 import org.apache.rocketmq.client.impl.MQClientManager;
+import org.apache.rocketmq.client.impl.consumer.ConsumeMessageService;
+import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl;
 import org.apache.rocketmq.client.impl.consumer.MQConsumerInner;
+import org.apache.rocketmq.client.impl.consumer.ProcessQueue;
+import org.apache.rocketmq.client.impl.consumer.RebalanceImpl;
 import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
 import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.message.MessageQueueAssignment;
+import org.apache.rocketmq.common.sysflag.MessageSysFlag;
+import org.apache.rocketmq.remoting.RemotingClient;
+import org.apache.rocketmq.remoting.common.HeartbeatV2Result;
+import org.apache.rocketmq.remoting.exception.RemotingConnectException;
 import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
+import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;
 import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData;
+import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
 import org.apache.rocketmq.remoting.protocol.route.BrokerData;
 import org.apache.rocketmq.remoting.protocol.route.QueueData;
 import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mockito.Mock;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
 public class MQClientInstanceTest {
-    private MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig());
-    private String topic = "FooBar";
-    private String group = "FooBarGroup";
-    private ConcurrentMap<String, HashMap<Long, String>> brokerAddrTable = new ConcurrentHashMap<>();
 
-    @Before
-    public void init() throws Exception {
-        FieldUtils.writeDeclaredField(mqClientInstance, "brokerAddrTable", brokerAddrTable, true);
-    }
+    @Mock
+    private MQClientAPIImpl mQClientAPIImpl;
 
-    @Test
-    public void testTopicRouteData2TopicPublishInfo() {
-        TopicRouteData topicRouteData = new TopicRouteData();
+    @Mock
+    private RemotingClient remotingClient;
 
-        topicRouteData.setFilterServerTable(new HashMap<>());
-        List<BrokerData> brokerDataList = new ArrayList<>();
-        BrokerData brokerData = new BrokerData();
-        brokerData.setBrokerName("BrokerA");
-        brokerData.setCluster("DefaultCluster");
-        HashMap<Long, String> brokerAddrs = new HashMap<>();
-        brokerAddrs.put(0L, "127.0.0.1:10911");
-        brokerData.setBrokerAddrs(brokerAddrs);
-        brokerDataList.add(brokerData);
-        topicRouteData.setBrokerDatas(brokerDataList);
+    @Mock
+    private ClientConfig clientConfig;
 
-        List<QueueData> queueDataList = new ArrayList<>();
-        QueueData queueData = new QueueData();
-        queueData.setBrokerName("BrokerA");
-        queueData.setPerm(6);
-        queueData.setReadQueueNums(3);
-        queueData.setWriteQueueNums(4);
-        queueData.setTopicSysFlag(0);
-        queueDataList.add(queueData);
-        topicRouteData.setQueueDatas(queueDataList);
+    private final MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig());
+
+    private final String topic = "FooBar";
+
+    private final String group = "FooBarGroup";
+
+    private final String defaultBrokerAddr = "127.0.0.1:10911";
+
+    private final String defaultBroker = "BrokerA";
+
+    private final ConcurrentMap<String, HashMap<Long, String>> brokerAddrTable = new ConcurrentHashMap<>();
 
-        TopicPublishInfo topicPublishInfo = MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData);
+    private final ConcurrentMap<String, MQConsumerInner> consumerTable = new ConcurrentHashMap<>();
 
-        assertThat(topicPublishInfo.isHaveTopicRouterInfo()).isFalse();
-        assertThat(topicPublishInfo.getMessageQueueList().size()).isEqualTo(4);
+    private final ConcurrentMap<String, TopicRouteData> topicRouteTable = new ConcurrentHashMap<>();
+
+    @Before
+    public void init() throws Exception {
+        when(mQClientAPIImpl.getRemotingClient()).thenReturn(remotingClient);
+        FieldUtils.writeDeclaredField(mqClientInstance, "brokerAddrTable", brokerAddrTable, true);
+        FieldUtils.writeDeclaredField(mqClientInstance, "mQClientAPIImpl", mQClientAPIImpl, true);
+        FieldUtils.writeDeclaredField(mqClientInstance, "consumerTable", consumerTable, true);
+        FieldUtils.writeDeclaredField(mqClientInstance, "clientConfig", clientConfig, true);
+        FieldUtils.writeDeclaredField(mqClientInstance, "topicRouteTable", topicRouteTable, true);
     }
 
     @Test
@@ -131,7 +170,7 @@ public void testRegisterProducer() {
     }
 
     @Test
-    public void testRegisterConsumer() throws RemotingException, InterruptedException, MQBrokerException {
+    public void testRegisterConsumer() {
         boolean flag = mqClientInstance.registerConsumer(group, mock(MQConsumerInner.class));
         assertThat(flag).isTrue();
 
@@ -143,7 +182,6 @@ public void testRegisterConsumer() throws RemotingException, InterruptedExceptio
         assertThat(flag).isTrue();
     }
 
-
     @Test
     public void testConsumerRunningInfoWhenConsumersIsEmptyOrNot() throws RemotingException, InterruptedException, MQBrokerException {
         MQConsumerInner mockConsumerInner = mock(MQConsumerInner.class);
@@ -181,4 +219,286 @@ public void testRegisterAdminExt() {
         assertThat(flag).isTrue();
     }
 
+    @Test
+    public void testTopicRouteData2TopicPublishInfo() {
+        TopicPublishInfo actual = MQClientInstance.topicRouteData2TopicPublishInfo(topic, createTopicRouteData());
+        assertThat(actual.isHaveTopicRouterInfo()).isFalse();
+        assertThat(actual.getMessageQueueList().size()).isEqualTo(4);
+    }
+
+    @Test
+    public void testTopicRouteData2TopicPublishInfoWithOrderTopicConf() {
+        TopicRouteData topicRouteData = createTopicRouteData();
+        when(topicRouteData.getOrderTopicConf()).thenReturn("127.0.0.1:4");
+        TopicPublishInfo actual = MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData);
+        assertFalse(actual.isHaveTopicRouterInfo());
+        assertEquals(4, actual.getMessageQueueList().size());
+    }
+
+    @Test
+    public void testTopicRouteData2TopicPublishInfoWithTopicQueueMappingByBroker() {
+        TopicRouteData topicRouteData = createTopicRouteData();
+        when(topicRouteData.getTopicQueueMappingByBroker()).thenReturn(Collections.singletonMap(topic, new TopicQueueMappingInfo()));
+        TopicPublishInfo actual = MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData);
+        assertFalse(actual.isHaveTopicRouterInfo());
+        assertEquals(0, actual.getMessageQueueList().size());
+    }
+
+    @Test
+    public void testTopicRouteData2TopicSubscribeInfo() {
+        TopicRouteData topicRouteData = createTopicRouteData();
+        when(topicRouteData.getTopicQueueMappingByBroker()).thenReturn(Collections.singletonMap(topic, new TopicQueueMappingInfo()));
+        Set<MessageQueue> actual = MQClientInstance.topicRouteData2TopicSubscribeInfo(topic, topicRouteData);
+        assertNotNull(actual);
+        assertEquals(0, actual.size());
+    }
+
+    @Test
+    public void testParseOffsetTableFromBroker() {
+        Map<MessageQueue, Long> offsetTable = new HashMap<>();
+        offsetTable.put(new MessageQueue(), 0L);
+        Map<MessageQueue, Long> actual = mqClientInstance.parseOffsetTableFromBroker(offsetTable, "defaultNamespace");
+        assertNotNull(actual);
+        assertEquals(1, actual.size());
+    }
+
+    @Test
+    public void testCheckClientInBroker() throws MQClientException, RemotingSendRequestException, RemotingConnectException, RemotingTimeoutException, InterruptedException {
+        doThrow(new MQClientException("checkClientInBroker exception", null)).when(mQClientAPIImpl).checkClientInBroker(
+                any(),
+                any(),
+                any(),
+                any(SubscriptionData.class),
+                anyLong());
+        topicRouteTable.put(topic, createTopicRouteData());
+        MQConsumerInner mqConsumerInner = createMQConsumerInner();
+        mqConsumerInner.subscriptions().clear();
+        SubscriptionData subscriptionData = new SubscriptionData();
+        subscriptionData.setTopic(topic);
+        subscriptionData.setExpressionType("type");
+        mqConsumerInner.subscriptions().add(subscriptionData);
+        consumerTable.put(group, mqConsumerInner);
+        Throwable thrown = assertThrows(MQClientException.class, mqClientInstance::checkClientInBroker);
+        assertTrue(thrown.getMessage().contains("checkClientInBroker exception"));
+    }
+
+    @Test
+    public void testSendHeartbeatToBrokerV1() {
+        consumerTable.put(group, createMQConsumerInner());
+        assertTrue(mqClientInstance.sendHeartbeatToBroker(0L, defaultBroker, defaultBrokerAddr));
+    }
+
+    @Test
+    public void testSendHeartbeatToBrokerV2() throws MQBrokerException, RemotingException, InterruptedException {
+        consumerTable.put(group, createMQConsumerInner());
+        when(clientConfig.isUseHeartbeatV2()).thenReturn(true);
+        HeartbeatV2Result heartbeatV2Result = mock(HeartbeatV2Result.class);
+        when(heartbeatV2Result.isSupportV2()).thenReturn(true);
+        when(mQClientAPIImpl.sendHeartbeatV2(any(), any(HeartbeatData.class), anyLong())).thenReturn(heartbeatV2Result);
+        assertTrue(mqClientInstance.sendHeartbeatToBroker(0L, defaultBroker, defaultBrokerAddr));
+    }
+
+    @Test
+    public void testSendHeartbeatToAllBrokerWithLockV1() {
+        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());
+        consumerTable.put(group, createMQConsumerInner());
+        assertTrue(mqClientInstance.sendHeartbeatToAllBrokerWithLock());
+    }
+
+    @Test
+    public void testSendHeartbeatToAllBrokerWithLockV2() {
+        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());
+        consumerTable.put(group, createMQConsumerInner());
+        when(clientConfig.isUseHeartbeatV2()).thenReturn(true);
+        assertTrue(mqClientInstance.sendHeartbeatToAllBrokerWithLock());
+    }
+
+    @Test
+    public void testUpdateTopicRouteInfoFromNameServer() throws RemotingException, InterruptedException, MQClientException {
+        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());
+        consumerTable.put(group, createMQConsumerInner());
+        DefaultMQProducer defaultMQProducer = mock(DefaultMQProducer.class);
+        TopicRouteData topicRouteData = createTopicRouteData();
+        when(mQClientAPIImpl.getDefaultTopicRouteInfoFromNameServer(anyLong())).thenReturn(topicRouteData);
+        assertFalse(mqClientInstance.updateTopicRouteInfoFromNameServer(topic, true, defaultMQProducer));
+    }
+
+    @Test
+    public void testFindBrokerAddressInAdmin() {
+        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());
+        consumerTable.put(group, createMQConsumerInner());
+        FindBrokerResult actual = mqClientInstance.findBrokerAddressInAdmin(defaultBroker);
+        assertNotNull(actual);
+        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());
+    }
+
+    @Test
+    public void testFindBrokerAddressInSubscribeWithOneBroker() throws IllegalAccessException {
+        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());
+        consumerTable.put(group, createMQConsumerInner());
+        ConcurrentMap<String, HashMap<String, Integer>> brokerVersionTable = new ConcurrentHashMap<>();
+        HashMap<String, Integer> addressMap = new HashMap<>();
+        addressMap.put(defaultBrokerAddr, 0);
+        brokerVersionTable.put(defaultBroker, addressMap);
+        FieldUtils.writeDeclaredField(mqClientInstance, "brokerVersionTable", brokerVersionTable, true);
+        FindBrokerResult actual = mqClientInstance.findBrokerAddressInSubscribe(defaultBroker, 1L, false);
+        assertNotNull(actual);
+        assertEquals(defaultBrokerAddr, actual.getBrokerAddr());
+    }
+
+    @Test
+    public void testFindConsumerIdList() {
+        topicRouteTable.put(topic, createTopicRouteData());
+        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());
+        consumerTable.put(group, createMQConsumerInner());
+        List<String> actual = mqClientInstance.findConsumerIdList(topic, group);
+        assertNotNull(actual);
+        assertEquals(0, actual.size());
+    }
+
+    @Test
+    public void testQueryAssignment() throws MQBrokerException, RemotingException, InterruptedException {
+        topicRouteTable.put(topic, createTopicRouteData());
+        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());
+        consumerTable.put(group, createMQConsumerInner());
+        Set<MessageQueueAssignment> actual = mqClientInstance.queryAssignment(topic, group, "", MessageModel.CLUSTERING, 1000);
+        assertNotNull(actual);
+        assertEquals(0, actual.size());
+    }
+
+    @Test
+    public void testResetOffset() throws IllegalAccessException {
+        topicRouteTable.put(topic, createTopicRouteData());
+        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());
+        consumerTable.put(group, createMQConsumerInner());
+        Map<MessageQueue, Long> offsetTable = new HashMap<>();
+        offsetTable.put(createMessageQueue(), 0L);
+        mqClientInstance.resetOffset(topic, group, offsetTable);
+        Field consumerTableField = FieldUtils.getDeclaredField(mqClientInstance.getClass(), "consumerTable", true);
+        ConcurrentMap<String, MQConsumerInner> consumerTable = (ConcurrentMap<String, MQConsumerInner>) consumerTableField.get(mqClientInstance);
+        DefaultMQPushConsumerImpl consumer = (DefaultMQPushConsumerImpl) consumerTable.get(group);
+        verify(consumer).suspend();
+        verify(consumer).resume();
+        verify(consumer, times(1))
+                .updateConsumeOffset(
+                        any(MessageQueue.class),
+                        eq(0L));
+    }
+
+    @Test
+    public void testGetConsumerStatus() {
+        topicRouteTable.put(topic, createTopicRouteData());
+        brokerAddrTable.put(defaultBroker, createBrokerAddrMap());
+        consumerTable.put(group, createMQConsumerInner());
+        Map<MessageQueue, Long> actual = mqClientInstance.getConsumerStatus(topic, group);
+        assertNotNull(actual);
+        assertEquals(0, actual.size());
+    }
+
+    @Test
+    public void testGetAnExistTopicRouteData() {
+        topicRouteTable.put(topic, createTopicRouteData());
+        TopicRouteData actual = mqClientInstance.getAnExistTopicRouteData(topic);
+        assertNotNull(actual);
+        assertNotNull(actual.getQueueDatas());
+        assertNotNull(actual.getBrokerDatas());
+    }
+
+    @Test
+    public void testConsumeMessageDirectly() {
+        consumerTable.put(group, createMQConsumerInner());
+        assertNull(mqClientInstance.consumeMessageDirectly(createMessageExt(), group, defaultBroker));
+    }
+
+    @Test
+    public void testQueryTopicRouteData() {
+        consumerTable.put(group, createMQConsumerInner());
+        topicRouteTable.put(topic, createTopicRouteData());
+        TopicRouteData actual = mqClientInstance.queryTopicRouteData(topic);
+        assertNotNull(actual);
+        assertNotNull(actual.getQueueDatas());
+        assertNotNull(actual.getBrokerDatas());
+    }
+
+    private MessageExt createMessageExt() {
+        MessageExt result = new MessageExt();
+        result.setBody("body".getBytes(StandardCharsets.UTF_8));
+        result.setTopic(topic);
+        result.setBrokerName(defaultBroker);
+        result.putUserProperty("key", "value");
+        result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, group);
+        result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1");
+        long curTime = System.currentTimeMillis();
+        result.setBornTimestamp(curTime - 1000);
+        result.getProperties().put(MessageConst.PROPERTY_POP_CK, curTime + " " + curTime + " " + curTime + " " + curTime);
+        result.setKeys("keys");
+        result.setSysFlag(MessageSysFlag.INNER_BATCH_FLAG);
+        result.setSysFlag(result.getSysFlag() | MessageSysFlag.NEED_UNWRAP_FLAG);
+        SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911);
+        SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911);
+        result.setBornHost(bornHost);
+        result.setStoreHost(storeHost);
+        return result;
+    }
+
+    private MessageQueue createMessageQueue() {
+        MessageQueue result = new MessageQueue();
+        result.setQueueId(0);
+        result.setBrokerName(defaultBroker);
+        result.setTopic(topic);
+        return result;
+    }
+
+    private TopicRouteData createTopicRouteData() {
+        TopicRouteData result = mock(TopicRouteData.class);
+        when(result.getBrokerDatas()).thenReturn(createBrokerDatas());
+        when(result.getQueueDatas()).thenReturn(createQueueDatas());
+        return result;
+    }
+
+    private HashMap<Long, String> createBrokerAddrMap() {
+        HashMap<Long, String> result = new HashMap<>();
+        result.put(0L, defaultBrokerAddr);
+        return result;
+    }
+
+    private MQConsumerInner createMQConsumerInner() {
+        DefaultMQPushConsumerImpl result = mock(DefaultMQPushConsumerImpl.class);
+        Set<SubscriptionData> subscriptionDataSet = new HashSet<>();
+        SubscriptionData subscriptionData = mock(SubscriptionData.class);
+        subscriptionDataSet.add(subscriptionData);
+        when(result.subscriptions()).thenReturn(subscriptionDataSet);
+        RebalanceImpl rebalanceImpl = mock(RebalanceImpl.class);
+        ConcurrentMap<MessageQueue, ProcessQueue> processQueueMap = new ConcurrentHashMap<>();
+        ProcessQueue processQueue = new ProcessQueue();
+        processQueueMap.put(createMessageQueue(), processQueue);
+        when(rebalanceImpl.getProcessQueueTable()).thenReturn(processQueueMap);
+        when(result.getRebalanceImpl()).thenReturn(rebalanceImpl);
+        OffsetStore offsetStore = mock(OffsetStore.class);
+        when(result.getOffsetStore()).thenReturn(offsetStore);
+        ConsumeMessageService consumeMessageService = mock(ConsumeMessageService.class);
+        when(result.getConsumeMessageService()).thenReturn(consumeMessageService);
+        return result;
+    }
+
+    private List<QueueData> createQueueDatas() {
+        QueueData queueData = new QueueData();
+        queueData.setBrokerName(defaultBroker);
+        queueData.setPerm(6);
+        queueData.setReadQueueNums(3);
+        queueData.setWriteQueueNums(4);
+        queueData.setTopicSysFlag(0);
+        return Collections.singletonList(queueData);
+    }
+
+    private List<BrokerData> createBrokerDatas() {
+        BrokerData brokerData = new BrokerData();
+        brokerData.setBrokerName(defaultBroker);
+        String defaultCluster = "defaultCluster";
+        brokerData.setCluster(defaultCluster);
+        HashMap<Long, String> brokerAddrs = new HashMap<>();
+        brokerAddrs.put(MixAll.MASTER_ID, defaultBrokerAddr);
+        brokerData.setBrokerAddrs(brokerAddrs);
+        return Collections.singletonList(brokerData);
+    }
 }
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java
new file mode 100644
index 00000000000..e2a29c9a21f
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.client.impl.mqclient;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
+import org.apache.rocketmq.client.ClientConfig;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.utils.FutureUtils;
+import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doReturn;
+
+@RunWith(MockitoJUnitRunner.class)
+public class MQClientAPIExtTest {
+    MQClientAPIExt mqClientAPIExt;
+    @Mock
+    NettyRemotingClient remotingClientMock;
+
+    @Before
+    public void before() {
+        mqClientAPIExt = Mockito.spy(new MQClientAPIExt(new ClientConfig(), new NettyClientConfig(), null, null));
+        Mockito.when(mqClientAPIExt.getRemotingClient()).thenReturn(remotingClientMock);
+        Mockito.when(remotingClientMock.invoke(anyString(), any(), anyLong())).thenReturn(FutureUtils.completeExceptionally(new RemotingTimeoutException("addr")));
+    }
+
+    @Test
+    public void sendMessageAsync() {
+        String topic = "test";
+        Message msg = new Message(topic, "test".getBytes());
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        requestHeader.setTopic(topic);
+        requestHeader.setProducerGroup("test");
+        requestHeader.setDefaultTopic("test");
+        requestHeader.setDefaultTopicQueueNums(1);
+        requestHeader.setQueueId(0);
+        requestHeader.setSysFlag(0);
+        requestHeader.setBornTimestamp(0L);
+        requestHeader.setFlag(0);
+        requestHeader.setProperties("test");
+        requestHeader.setReconsumeTimes(0);
+        requestHeader.setUnitMode(false);
+        requestHeader.setBatch(false);
+        CompletableFuture<SendResult> future = mqClientAPIExt.sendMessageAsync("127.0.0.1:10911", "test", msg, requestHeader, 10);
+        assertThatThrownBy(future::get).getCause().isInstanceOf(RemotingTimeoutException.class);
+    }
+
+    @Test
+    public void testUpdateConsumerOffsetAsync_Success() throws ExecutionException, InterruptedException {
+        CompletableFuture<RemotingCommand> remotingFuture = new CompletableFuture<>();
+        remotingFuture.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, ""));
+        doReturn(remotingFuture).when(remotingClientMock).invoke(anyString(), any(RemotingCommand.class), anyLong());
+
+        CompletableFuture<Void> future = mqClientAPIExt.updateConsumerOffsetAsync("brokerAddr", new UpdateConsumerOffsetRequestHeader(), 3000L);
+
+        assertNull("Future should be completed without exception", future.get());
+    }
+
+    @Test
+    public void testUpdateConsumerOffsetAsync_Fail() throws InterruptedException {
+
+        CompletableFuture<RemotingCommand> remotingFuture = new CompletableFuture<>();
+        remotingFuture.complete(RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "QueueId is null, topic is testTopic"));
+        doReturn(remotingFuture).when(remotingClientMock).invoke(anyString(), any(RemotingCommand.class), anyLong());
+
+        CompletableFuture<Void> future = mqClientAPIExt.updateConsumerOffsetAsync("brokerAddr", new UpdateConsumerOffsetRequestHeader(), 3000L);
+
+        try {
+            future.get();
+        } catch (ExecutionException e) {
+            MQBrokerException customEx = (MQBrokerException) e.getCause();
+            assertEquals(customEx.getResponseCode(), ResponseCode.SYSTEM_ERROR);
+            assertEquals(customEx.getErrorMessage(), "QueueId is null, topic is testTopic");
+        }
+    }
+
+    @Test
+    public void testRecallMessageAsync_success() {
+        String msgId = "msgId";
+        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();
+        requestHeader.setProducerGroup("group");
+        requestHeader.setTopic("topic");
+        requestHeader.setRecallHandle("handle");
+        requestHeader.setBrokerName("brokerName");
+
+        RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class);
+        response.setCode(ResponseCode.SUCCESS);
+        RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();
+        responseHeader.setMsgId(msgId);
+        response.makeCustomHeaderToNet();
+        CompletableFuture<RemotingCommand> remotingFuture = new CompletableFuture<>();
+        remotingFuture.complete(response);
+        doReturn(remotingFuture).when(remotingClientMock).invoke(anyString(), any(RemotingCommand.class), anyLong());
+
+        String resultId =
+            mqClientAPIExt.recallMessageAsync("brokerAddr", requestHeader, 3000L).join();
+        Assert.assertEquals(msgId, resultId);
+    }
+
+    @Test
+    public void testRecallMessageAsync_fail() {
+        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();
+        requestHeader.setProducerGroup("group");
+        requestHeader.setTopic("topic");
+        requestHeader.setRecallHandle("handle");
+        requestHeader.setBrokerName("brokerName");
+
+        CompletableFuture<RemotingCommand> remotingFuture = new CompletableFuture<>();
+        remotingFuture.complete(RemotingCommand.createResponseCommand(ResponseCode.SERVICE_NOT_AVAILABLE, ""));
+        doReturn(remotingFuture).when(remotingClientMock).invoke(anyString(), any(RemotingCommand.class), anyLong());
+
+        CompletionException exception = Assert.assertThrows(CompletionException.class, () -> {
+            mqClientAPIExt.recallMessageAsync("brokerAddr", requestHeader, 3000L).join();
+        });
+        Assert.assertTrue(exception.getCause() instanceof MQBrokerException);
+        MQBrokerException cause = (MQBrokerException) exception.getCause();
+        Assert.assertEquals(ResponseCode.SERVICE_NOT_AVAILABLE, cause.getResponseCode());
+    }
+}
\ No newline at end of file
diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java
index d4153c7cd97..33cf0df390d 100644
--- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java
@@ -16,19 +16,6 @@
  */
 package org.apache.rocketmq.client.producer;
 
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
@@ -41,9 +28,13 @@
 import org.apache.rocketmq.client.impl.factory.MQClientInstance;
 import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
 import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
+import org.apache.rocketmq.client.latency.MQFaultStrategy;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.compression.CompressionType;
 import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
@@ -58,13 +49,35 @@
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
@@ -73,15 +86,14 @@ public class DefaultMQProducerTest {
     private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig());
     @Mock
     private MQClientAPIImpl mQClientAPIImpl;
-    @Mock
-    private NettyRemotingClient nettyRemotingClient;
 
     private DefaultMQProducer producer;
     private Message message;
     private Message zeroMsg;
     private Message bigMessage;
-    private String topic = "FooBar";
-    private String producerGroupPrefix = "FooBar_PID";
+    private final String topic = "FooBar";
+    private final String producerGroupPrefix = "FooBar_PID";
+    private final long defaultTimeout = 3000L;
 
     @Before
     public void init() throws Exception {
@@ -187,7 +199,7 @@ public void onException(Throwable e) {
                 countDownLatch.countDown();
             }
         });
-        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);
     }
 
     @Test
@@ -231,7 +243,7 @@ public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
         //this message is send success
         producer.send(message, sendCallback, 1000);
 
-        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);
         assertThat(cc.get()).isEqualTo(5);
 
         // off enableBackpressureForAsyncMode
@@ -244,7 +256,7 @@ public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
         //this message is send success
         producer.send(message, sendCallback, 1000);
 
-        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);
         assertThat(cc.get()).isEqualTo(10);
     }
 
@@ -292,7 +304,7 @@ public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
         // this message is send failed
         producer.send(msgs, new MessageQueue(), sendCallback, 1000);
 
-        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);
         assertThat(cc.get()).isEqualTo(1);
 
         // off enableBackpressureForAsyncMode
@@ -303,7 +315,7 @@ public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
         // this message is send failed
         producer.send(msgs, new MessageQueue(), sendCallback, 1000);
 
-        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);
         assertThat(cc.get()).isEqualTo(2);
     }
 
@@ -324,7 +336,7 @@ public void onSuccess(SendResult sendResult) {
             public void onException(Throwable e) {
             }
         });
-        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);
     }
 
     @Test
@@ -463,7 +475,7 @@ public void onException(Throwable e) {
             future.setSendRequestOk(true);
             future.getRequestCallback().onSuccess(responseMsg);
         }
-        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);
     }
 
     @Test
@@ -500,7 +512,7 @@ public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
                 future.getRequestCallback().onException(e);
             }
         }
-        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);
         assertThat(cc.get()).isEqualTo(1);
     }
 
@@ -524,7 +536,7 @@ public void onException(Throwable e) {
             }
         });
 
-        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+        countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS);
         producer.setAutoBatch(false);
     }
 
@@ -540,6 +552,50 @@ public void testBatchSendMessageSync_Success() throws RemotingException, Interru
         producer.setAutoBatch(false);
     }
 
+
+    @Test
+    public void testRunningSetBackCompress() throws RemotingException, InterruptedException, MQClientException {
+        final CountDownLatch countDownLatch = new CountDownLatch(5);
+        SendCallback sendCallback = new SendCallback() {
+            @Override
+            public void onSuccess(SendResult sendResult) {
+                countDownLatch.countDown();
+            }
+
+            @Override
+            public void onException(Throwable e) {
+                e.printStackTrace();
+                countDownLatch.countDown();
+            }
+        };
+
+        // on enableBackpressureForAsyncMode
+        producer.setEnableBackpressureForAsyncMode(true);
+        producer.setBackPressureForAsyncSendNum(10);
+        producer.setBackPressureForAsyncSendSize(50 * 1024 * 1024);
+        Message message = new Message();
+        message.setTopic("test");
+        message.setBody("hello world".getBytes());
+        MessageQueue mq = new MessageQueue("test", "BrokerA", 1);
+        //this message is send success
+        for (int i = 0; i < 5; i++) {
+            new Thread(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        producer.send(message, mq, sendCallback);
+                    } catch (MQClientException | RemotingException | InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+            }).start();
+        }
+        producer.setBackPressureForAsyncSendNum(15);
+        countDownLatch.await(3000L, TimeUnit.MILLISECONDS);
+        assertThat(producer.defaultMQProducerImpl.getSemaphoreAsyncSendNumAvailablePermits() + countDownLatch.getCount()).isEqualTo(15);
+        producer.setEnableBackpressureForAsyncMode(false);
+    }
+
     public static TopicRouteData createTopicRoute() {
         TopicRouteData topicRouteData = new TopicRouteData();
 
@@ -596,4 +652,365 @@ public void run() {
         }
         return assertionErrors[0];
     }
+
+    @Test
+    public void assertCreateDefaultMQProducer() {
+        String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis();
+        DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp);
+        assertNotNull(producer1);
+        assertEquals(producerGroupTemp, producer1.getProducerGroup());
+        assertNotNull(producer1.getDefaultMQProducerImpl());
+        assertEquals(0, producer1.getTotalBatchMaxBytes());
+        assertEquals(0, producer1.getBatchMaxBytes());
+        assertEquals(0, producer1.getBatchMaxDelayMs());
+        assertNull(producer1.getTopics());
+        assertFalse(producer1.isEnableTrace());
+        assertTrue(UtilAll.isBlank(producer1.getTraceTopic()));
+        DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp, mock(RPCHook.class));
+        assertNotNull(producer2);
+        assertEquals(producerGroupTemp, producer2.getProducerGroup());
+        assertNotNull(producer2.getDefaultMQProducerImpl());
+        assertEquals(0, producer2.getTotalBatchMaxBytes());
+        assertEquals(0, producer2.getBatchMaxBytes());
+        assertEquals(0, producer2.getBatchMaxDelayMs());
+        assertNull(producer2.getTopics());
+        assertFalse(producer2.isEnableTrace());
+        assertTrue(UtilAll.isBlank(producer2.getTraceTopic()));
+        DefaultMQProducer producer3 = new DefaultMQProducer(producerGroupTemp, mock(RPCHook.class), Collections.singletonList("custom_topic"));
+        assertNotNull(producer3);
+        assertEquals(producerGroupTemp, producer3.getProducerGroup());
+        assertNotNull(producer3.getDefaultMQProducerImpl());
+        assertEquals(0, producer3.getTotalBatchMaxBytes());
+        assertEquals(0, producer3.getBatchMaxBytes());
+        assertEquals(0, producer3.getBatchMaxDelayMs());
+        assertNotNull(producer3.getTopics());
+        assertEquals(1, producer3.getTopics().size());
+        assertFalse(producer3.isEnableTrace());
+        assertTrue(UtilAll.isBlank(producer3.getTraceTopic()));
+        DefaultMQProducer producer4 = new DefaultMQProducer(producerGroupTemp, mock(RPCHook.class), true, "custom_trace_topic");
+        assertNotNull(producer4);
+        assertEquals(producerGroupTemp, producer4.getProducerGroup());
+        assertNotNull(producer4.getDefaultMQProducerImpl());
+        assertEquals(0, producer4.getTotalBatchMaxBytes());
+        assertEquals(0, producer4.getBatchMaxBytes());
+        assertEquals(0, producer4.getBatchMaxDelayMs());
+        assertNull(producer4.getTopics());
+        assertTrue(producer4.isEnableTrace());
+        assertEquals("custom_trace_topic", producer4.getTraceTopic());
+        DefaultMQProducer producer5 = new DefaultMQProducer(producerGroupTemp, mock(RPCHook.class), Collections.singletonList("custom_topic"), true, "custom_trace_topic");
+        assertNotNull(producer5);
+        assertEquals(producerGroupTemp, producer5.getProducerGroup());
+        assertNotNull(producer5.getDefaultMQProducerImpl());
+        assertEquals(0, producer5.getTotalBatchMaxBytes());
+        assertEquals(0, producer5.getBatchMaxBytes());
+        assertEquals(0, producer5.getBatchMaxDelayMs());
+        assertNotNull(producer5.getTopics());
+        assertEquals(1, producer5.getTopics().size());
+        assertTrue(producer5.isEnableTrace());
+        assertEquals("custom_trace_topic", producer5.getTraceTopic());
+    }
+
+    @Test
+    public void assertSend() throws MQBrokerException, RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException {
+        setDefaultMQProducerImpl();
+        setOtherParam();
+        SendResult send = producer.send(message, defaultTimeout);
+        assertNull(send);
+        Collection<Message> msgs = Collections.singletonList(message);
+        send = producer.send(msgs);
+        assertNull(send);
+        send = producer.send(msgs, defaultTimeout);
+        assertNull(send);
+    }
+
+    @Test
+    public void assertSendOneway() throws RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException {
+        setDefaultMQProducerImpl();
+        producer.sendOneway(message);
+        MessageQueue mq = mock(MessageQueue.class);
+        producer.sendOneway(message, mq);
+        MessageQueueSelector selector = mock(MessageQueueSelector.class);
+        producer.sendOneway(message, selector, 1);
+    }
+
+    @Test
+    public void assertSendByQueue() throws MQBrokerException, RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException {
+        setDefaultMQProducerImpl();
+        MessageQueue mq = mock(MessageQueue.class);
+        SendResult send = producer.send(message, mq);
+        assertNull(send);
+        send = producer.send(message, mq, defaultTimeout);
+        assertNull(send);
+        Collection<Message> msgs = Collections.singletonList(message);
+        send = producer.send(msgs, mq);
+        assertNull(send);
+        send = producer.send(msgs, mq, defaultTimeout);
+        assertNull(send);
+    }
+
+    @Test
+    public void assertSendByQueueSelector() throws MQBrokerException, RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException {
+        setDefaultMQProducerImpl();
+        MessageQueueSelector selector = mock(MessageQueueSelector.class);
+        SendResult send = producer.send(message, selector, 1);
+        assertNull(send);
+        send = producer.send(message, selector, 1, defaultTimeout);
+        assertNull(send);
+    }
+
+    @Test
+    public void assertRequest() throws MQBrokerException, RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException, RequestTimeoutException {
+        setDefaultMQProducerImpl();
+        MessageQueueSelector selector = mock(MessageQueueSelector.class);
+        Message replyNsg = producer.request(message, selector, 1, defaultTimeout);
+        assertNull(replyNsg);
+        RequestCallback requestCallback = mock(RequestCallback.class);
+        producer.request(message, selector, 1, requestCallback, defaultTimeout);
+        MessageQueue mq = mock(MessageQueue.class);
+        producer.request(message, mq, defaultTimeout);
+        producer.request(message, mq, requestCallback, defaultTimeout);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void assertSendMessageInTransaction() throws MQClientException {
+        TransactionSendResult result = producer.sendMessageInTransaction(message, 1);
+        assertNull(result);
+    }
+
+    @Test
+    public void assertSearchOffset() throws MQClientException, NoSuchFieldException, IllegalAccessException {
+        setDefaultMQProducerImpl();
+        MessageQueue mq = mock(MessageQueue.class);
+        long result = producer.searchOffset(mq, System.currentTimeMillis());
+        assertEquals(0L, result);
+    }
+
+    @Test
+    public void assertBatchMaxDelayMs() throws NoSuchFieldException, IllegalAccessException {
+        setProduceAccumulator(true);
+        assertEquals(0, producer.getBatchMaxDelayMs());
+        setProduceAccumulator(false);
+        assertEquals(10, producer.getBatchMaxDelayMs());
+        producer.batchMaxDelayMs(1000);
+        assertEquals(1000, producer.getBatchMaxDelayMs());
+    }
+
+    @Test
+    public void assertBatchMaxBytes() throws NoSuchFieldException, IllegalAccessException {
+        setProduceAccumulator(true);
+        assertEquals(0L, producer.getBatchMaxBytes());
+        setProduceAccumulator(false);
+        assertEquals(32 * 1024L, producer.getBatchMaxBytes());
+        producer.batchMaxBytes(64 * 1024L);
+        assertEquals(64 * 1024L, producer.getBatchMaxBytes());
+    }
+
+    @Test
+    public void assertTotalBatchMaxBytes() throws NoSuchFieldException, IllegalAccessException {
+        setProduceAccumulator(true);
+        assertEquals(0L, producer.getTotalBatchMaxBytes());
+    }
+
+    @Test
+    public void assertProduceAccumulatorStart() throws NoSuchFieldException, IllegalAccessException, MQClientException {
+        String producerGroupTemp = producerGroupPrefix + System.nanoTime();
+        DefaultMQProducer producer = new DefaultMQProducer(producerGroupTemp);
+        assertEquals(0, producer.getTotalBatchMaxBytes());
+        assertEquals(0, producer.getBatchMaxBytes());
+        assertEquals(0, producer.getBatchMaxDelayMs());
+        assertNull(getField(producer, "produceAccumulator", ProduceAccumulator.class));
+        producer.start();
+        assertTrue(producer.getTotalBatchMaxBytes() > 0);
+        assertTrue(producer.getBatchMaxBytes() > 0);
+        assertTrue(producer.getBatchMaxDelayMs() > 0);
+        assertNotNull(getField(producer, "produceAccumulator", ProduceAccumulator.class));
+    }
+
+    @Test
+    public void assertProduceAccumulatorBeforeStartSet() throws NoSuchFieldException, IllegalAccessException, MQClientException {
+        String producerGroupTemp = producerGroupPrefix + System.nanoTime();
+        DefaultMQProducer producer = new DefaultMQProducer(producerGroupTemp);
+        producer.totalBatchMaxBytes(64 * 1024 * 100);
+        producer.batchMaxBytes(64 * 1024);
+        producer.batchMaxDelayMs(10);
+
+        producer.start();
+        assertEquals(64 * 1024, producer.getBatchMaxBytes());
+        assertEquals(10, producer.getBatchMaxDelayMs());
+        assertNotNull(getField(producer, "produceAccumulator", ProduceAccumulator.class));
+    }
+
+    @Test
+    public void assertProduceAccumulatorAfterStartSet() throws NoSuchFieldException, IllegalAccessException, MQClientException {
+        String producerGroupTemp = producerGroupPrefix + System.nanoTime();
+        DefaultMQProducer producer = new DefaultMQProducer(producerGroupTemp);
+        producer.start();
+
+        assertNotNull(getField(producer, "produceAccumulator", ProduceAccumulator.class));
+
+        producer.totalBatchMaxBytes(64 * 1024 * 100);
+        producer.batchMaxBytes(64 * 1024);
+        producer.batchMaxDelayMs(10);
+
+        assertEquals(64 * 1024, producer.getBatchMaxBytes());
+        assertEquals(10, producer.getBatchMaxDelayMs());
+    }
+
+    @Test
+    public void assertProduceAccumulatorUnit() throws NoSuchFieldException, IllegalAccessException, MQClientException {
+        String producerGroupTemp = producerGroupPrefix + System.nanoTime();
+        DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp);
+        producer1.setUnitName("unit1");
+        DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp);
+        producer2.setUnitName("unit2");
+
+        producer1.start();
+        producer2.start();
+
+        ProduceAccumulator producer1Accumulator = getField(producer1, "produceAccumulator", ProduceAccumulator.class);
+        ProduceAccumulator producer2Accumulator = getField(producer2, "produceAccumulator", ProduceAccumulator.class);
+
+        assertNotNull(producer1Accumulator);
+        assertNotNull(producer2Accumulator);
+
+        assertNotEquals(producer1Accumulator, producer2Accumulator);
+    }
+
+    @Test
+    public void assertProduceAccumulator() throws NoSuchFieldException, IllegalAccessException, MQClientException {
+        String producerGroupTemp1 = producerGroupPrefix + System.nanoTime();
+        DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp1);
+        producer1.setInstanceName("instanceName1");
+        String producerGroupTemp2 = producerGroupPrefix + System.nanoTime();
+        DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp2);
+        producer2.setInstanceName("instanceName2");
+
+        producer1.start();
+        producer2.start();
+
+        ProduceAccumulator producer1Accumulator = getField(producer1, "produceAccumulator", ProduceAccumulator.class);
+        ProduceAccumulator producer2Accumulator = getField(producer2, "produceAccumulator", ProduceAccumulator.class);
+
+        assertNotNull(producer1Accumulator);
+        assertNotNull(producer2Accumulator);
+
+        assertNotEquals(producer1Accumulator, producer2Accumulator);
+    }
+
+    @Test
+    public void assertProduceAccumulatorInstanceEqual() throws NoSuchFieldException, IllegalAccessException, MQClientException {
+        String producerGroupTemp1 = producerGroupPrefix + System.nanoTime();
+        DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp1);
+        producer1.setInstanceName("equalInstance");
+        String producerGroupTemp2 = producerGroupPrefix + System.nanoTime();
+        DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp2);
+        producer2.setInstanceName("equalInstance");
+
+        producer1.start();
+        producer2.start();
+
+        ProduceAccumulator producer1Accumulator = getField(producer1, "produceAccumulator", ProduceAccumulator.class);
+        ProduceAccumulator producer2Accumulator = getField(producer2, "produceAccumulator", ProduceAccumulator.class);
+
+        assertNotNull(producer1Accumulator);
+        assertNotNull(producer2Accumulator);
+
+        assertEquals(producer1Accumulator, producer2Accumulator);
+    }
+
+    @Test
+    public void assertProduceAccumulatorInstanceAndUnitNameEqual() throws NoSuchFieldException, IllegalAccessException, MQClientException {
+        String producerGroupTemp1 = producerGroupPrefix + System.nanoTime();
+        DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp1);
+        producer1.setInstanceName("equalInstance");
+        producer1.setUnitName("equalUnitName");
+        String producerGroupTemp2 = producerGroupPrefix + System.nanoTime();
+        DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp2);
+        producer2.setInstanceName("equalInstance");
+        producer2.setUnitName("equalUnitName");
+
+        producer1.start();
+        producer2.start();
+
+        ProduceAccumulator producer1Accumulator = getField(producer1, "produceAccumulator", ProduceAccumulator.class);
+        ProduceAccumulator producer2Accumulator = getField(producer2, "produceAccumulator", ProduceAccumulator.class);
+
+        assertNotNull(producer1Accumulator);
+        assertNotNull(producer2Accumulator);
+
+        assertEquals(producer1Accumulator, producer2Accumulator);
+    }
+
+    @Test
+    public void assertGetRetryResponseCodes() {
+        assertNotNull(producer.getRetryResponseCodes());
+        assertEquals(8, producer.getRetryResponseCodes().size());
+    }
+
+    @Test
+    public void assertIsSendLatencyFaultEnable() {
+        assertFalse(producer.isSendLatencyFaultEnable());
+    }
+
+    @Test
+    public void assertGetLatencyMax() {
+        assertNotNull(producer.getLatencyMax());
+    }
+
+    @Test
+    public void assertGetNotAvailableDuration() {
+        assertNotNull(producer.getNotAvailableDuration());
+    }
+
+    @Test
+    public void assertIsRetryAnotherBrokerWhenNotStoreOK() {
+        assertFalse(producer.isRetryAnotherBrokerWhenNotStoreOK());
+    }
+
+    private void setOtherParam() {
+        producer.setCreateTopicKey("createTopicKey");
+        producer.setRetryAnotherBrokerWhenNotStoreOK(false);
+        producer.setDefaultTopicQueueNums(6);
+        producer.setRetryTimesWhenSendFailed(1);
+        producer.setSendMessageWithVIPChannel(false);
+        producer.setNotAvailableDuration(new long[1]);
+        producer.setLatencyMax(new long[1]);
+        producer.setSendLatencyFaultEnable(false);
+        producer.setRetryTimesWhenSendAsyncFailed(1);
+        producer.setTopics(Collections.singletonList(topic));
+        producer.setStartDetectorEnable(false);
+        producer.setCompressLevel(5);
+        producer.setCompressType(CompressionType.LZ4);
+        producer.addRetryResponseCode(0);
+        ExecutorService executorService = mock(ExecutorService.class);
+        producer.setAsyncSenderExecutor(executorService);
+    }
+
+    private void setProduceAccumulator(final boolean isDefault) throws NoSuchFieldException, IllegalAccessException {
+        ProduceAccumulator accumulator = null;
+        if (!isDefault) {
+            accumulator = new ProduceAccumulator("instanceName");
+        }
+        setField(producer, "produceAccumulator", accumulator);
+    }
+
+    private void setDefaultMQProducerImpl() throws NoSuchFieldException, IllegalAccessException {
+        DefaultMQProducerImpl producerImpl = mock(DefaultMQProducerImpl.class);
+        setField(producer, "defaultMQProducerImpl", producerImpl);
+        when(producerImpl.getMqFaultStrategy()).thenReturn(mock(MQFaultStrategy.class));
+    }
+
+    private void setField(final Object target, final String fieldName, final Object newValue) throws NoSuchFieldException, IllegalAccessException {
+        Class<?> clazz = target.getClass();
+        Field field = clazz.getDeclaredField(fieldName);
+        field.setAccessible(true);
+        field.set(target, newValue);
+    }
+
+    private <T> T getField(final Object target, final String fieldName, final Class<T> fieldClassType) throws NoSuchFieldException, IllegalAccessException {
+        Class<?> targetClazz = target.getClass();
+        Field field = targetClazz.getDeclaredField(fieldName);
+        field.setAccessible(true);
+        return fieldClassType.cast(field.get(target));
+    }
 }
diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/ProduceAccumulatorTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/ProduceAccumulatorTest.java
index 7074fae243d..8e76238d47f 100644
--- a/client/src/test/java/org/apache/rocketmq/client/producer/ProduceAccumulatorTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/producer/ProduceAccumulatorTest.java
@@ -106,6 +106,7 @@ public void testProduceAccumulator_sync() throws MQBrokerException, RemotingExce
         final MockMQProducer mockMQProducer = new MockMQProducer();
 
         final ProduceAccumulator produceAccumulator = new ProduceAccumulator("test");
+        produceAccumulator.batchMaxDelayMs(3000);
         produceAccumulator.start();
 
         List<Message> messages = new ArrayList<Message>();
@@ -134,7 +135,7 @@ public void run() {
                 }
             }).start();
         }
-        assertThat(countDownLatch.await(3000L, TimeUnit.MILLISECONDS)).isTrue();
+        assertThat(countDownLatch.await(5000L, TimeUnit.MILLISECONDS)).isTrue();
         assertThat(mockMQProducer.beSendMessage instanceof MessageBatch).isTrue();
 
         MessageBatch messageBatch1 = (MessageBatch) mockMQProducer.beSendMessage;
diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/selector/DefaultMQProducerImplTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/selector/DefaultMQProducerImplTest.java
new file mode 100644
index 00000000000..77a83af19c0
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/producer/selector/DefaultMQProducerImplTest.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.producer.selector;
+
+import org.apache.rocketmq.client.ClientConfig;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.exception.RequestTimeoutException;
+import org.apache.rocketmq.client.hook.CheckForbiddenContext;
+import org.apache.rocketmq.client.hook.CheckForbiddenHook;
+import org.apache.rocketmq.client.impl.MQAdminImpl;
+import org.apache.rocketmq.client.impl.MQClientAPIImpl;
+import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
+import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
+import org.apache.rocketmq.client.producer.MessageQueueSelector;
+import org.apache.rocketmq.client.producer.RequestCallback;
+import org.apache.rocketmq.client.producer.SendCallback;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.client.producer.TransactionListener;
+import org.apache.rocketmq.client.producer.TransactionMQProducer;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.ServiceState;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertThrows;
+import static org.mockito.AdditionalMatchers.or;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class DefaultMQProducerImplTest {
+
+    @Mock
+    private Message message;
+
+    @Mock
+    private MessageQueue messageQueue;
+
+    @Mock
+    private MessageQueueSelector queueSelector;
+
+    @Mock
+    private RequestCallback requestCallback;
+
+    @Mock
+    private MQClientInstance mQClientFactory;
+
+    @Mock
+    private MQClientAPIImpl mQClientAPIImpl;
+
+    private DefaultMQProducerImpl defaultMQProducerImpl;
+
+    private final long defaultTimeout = 30000L;
+
+    private final String defaultBrokerName = "broker-0";
+
+    private final String defaultBrokerAddr = "127.0.0.1:10911";
+
+    private final String defaultTopic = "testTopic";
+
+    @Before
+    public void init() throws Exception {
+        when(mQClientFactory.getTopicRouteTable()).thenReturn(mock(ConcurrentMap.class));
+        when(mQClientFactory.getClientId()).thenReturn("client-id");
+        when(mQClientFactory.getMQAdminImpl()).thenReturn(mock(MQAdminImpl.class));
+        ClientConfig clientConfig = mock(ClientConfig.class);
+        when(messageQueue.getTopic()).thenReturn(defaultTopic);
+        when(clientConfig.queueWithNamespace(any())).thenReturn(messageQueue);
+        when(mQClientFactory.getClientConfig()).thenReturn(clientConfig);
+        when(mQClientFactory.getTopicRouteTable()).thenReturn(mock(ConcurrentMap.class));
+        when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mQClientAPIImpl);
+        when(mQClientFactory.findBrokerAddressInPublish(or(isNull(), anyString()))).thenReturn(defaultBrokerAddr);
+        when(message.getTopic()).thenReturn(defaultTopic);
+        when(message.getProperty(MessageConst.PROPERTY_CORRELATION_ID)).thenReturn("correlation-id");
+        when(message.getBody()).thenReturn(new byte[1]);
+        TransactionMQProducer producer = new TransactionMQProducer("test-producer-group");
+        producer.setTransactionListener(mock(TransactionListener.class));
+        producer.setTopics(Collections.singletonList(defaultTopic));
+        defaultMQProducerImpl = new DefaultMQProducerImpl(producer);
+        setMQClientFactory();
+        setCheckExecutor();
+        setCheckForbiddenHookList();
+        setTopicPublishInfoTable();
+        defaultMQProducerImpl.setServiceState(ServiceState.RUNNING);
+    }
+
+    @Test
+    public void testRequest() throws Exception {
+        defaultMQProducerImpl.request(message, messageQueue, requestCallback, defaultTimeout);
+        defaultMQProducerImpl.request(message, queueSelector, 1, requestCallback, defaultTimeout);
+    }
+
+    @Test(expected = MQClientException.class)
+    public void testRequestMQClientExceptionByVoid() throws Exception {
+        defaultMQProducerImpl.request(message, requestCallback, defaultTimeout);
+    }
+
+    @Test
+    public void testCheckTransactionState() {
+        defaultMQProducerImpl.checkTransactionState(defaultBrokerAddr, mock(MessageExt.class), mock(CheckTransactionStateRequestHeader.class));
+    }
+
+    @Test
+    public void testCreateTopic() throws MQClientException {
+        defaultMQProducerImpl.createTopic("key", defaultTopic, 0);
+    }
+
+    @Test
+    public void testExecuteCheckForbiddenHook() throws MQClientException {
+        defaultMQProducerImpl.executeCheckForbiddenHook(mock(CheckForbiddenContext.class));
+    }
+
+    @Test(expected = MQClientException.class)
+    public void testSendOneway() throws MQClientException, InterruptedException, RemotingException {
+        defaultMQProducerImpl.sendOneway(message);
+    }
+
+    @Test
+    public void testSendOnewayByQueueSelector() throws MQClientException, InterruptedException, RemotingException {
+        defaultMQProducerImpl.sendOneway(message, mock(MessageQueueSelector.class), 1);
+    }
+
+    @Test
+    public void testSendOnewayByQueue() throws MQClientException, InterruptedException, RemotingException {
+        defaultMQProducerImpl.sendOneway(message, messageQueue);
+    }
+
+    @Test(expected = MQClientException.class)
+    public void testSend() throws RemotingException, InterruptedException, MQClientException, MQBrokerException {
+        assertNull(defaultMQProducerImpl.send(message));
+    }
+
+    @Test
+    public void assertSendByQueue() throws InterruptedException, MQBrokerException, RemotingException, MQClientException {
+        SendResult actual = defaultMQProducerImpl.send(message, messageQueue);
+        assertNull(actual);
+        actual = defaultMQProducerImpl.send(message, messageQueue, defaultTimeout);
+        assertNull(actual);
+    }
+
+    @Test
+    public void assertSendByQueueSelector() throws InterruptedException, MQBrokerException, RemotingException, MQClientException {
+        SendCallback sendCallback = mock(SendCallback.class);
+        defaultMQProducerImpl.send(message, queueSelector, 1, sendCallback);
+        SendResult actual = defaultMQProducerImpl.send(message, queueSelector, 1);
+        assertNull(actual);
+        actual = defaultMQProducerImpl.send(message, queueSelector, 1, defaultTimeout);
+        assertNull(actual);
+    }
+
+    @Test(expected = MQClientException.class)
+    public void assertMQClientException() throws Exception {
+        assertNull(defaultMQProducerImpl.request(message, defaultTimeout));
+    }
+
+    @Test(expected = RequestTimeoutException.class)
+    public void assertRequestRequestTimeoutByQueueSelector() throws Exception {
+        assertNull(defaultMQProducerImpl.request(message, queueSelector, 1, 3000L));
+    }
+
+    @Test(expected = Exception.class)
+    public void assertRequestTimeoutExceptionByQueue() throws Exception {
+        assertNull(defaultMQProducerImpl.request(message, messageQueue, 3000L));
+    }
+
+    @Test
+    public void testRegisterCheckForbiddenHook() {
+        CheckForbiddenHook checkForbiddenHook = mock(CheckForbiddenHook.class);
+        defaultMQProducerImpl.registerCheckForbiddenHook(checkForbiddenHook);
+    }
+
+    @Test
+    public void testInitTopicRoute() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+        Class<?> clazz = defaultMQProducerImpl.getClass();
+        Method method = clazz.getDeclaredMethod("initTopicRoute");
+        method.setAccessible(true);
+        method.invoke(defaultMQProducerImpl);
+    }
+
+    @Test
+    public void assertFetchPublishMessageQueues() throws MQClientException {
+        List<MessageQueue> actual = defaultMQProducerImpl.fetchPublishMessageQueues(defaultTopic);
+        assertNotNull(actual);
+        assertEquals(0, actual.size());
+    }
+
+    @Test
+    public void assertSearchOffset() throws MQClientException {
+        assertEquals(0, defaultMQProducerImpl.searchOffset(messageQueue, System.currentTimeMillis()));
+    }
+
+    @Test
+    public void assertMaxOffset() throws MQClientException {
+        assertEquals(0, defaultMQProducerImpl.maxOffset(messageQueue));
+    }
+
+    @Test
+    public void assertMinOffset() throws MQClientException {
+        assertEquals(0, defaultMQProducerImpl.minOffset(messageQueue));
+    }
+
+    @Test
+    public void assertEarliestMsgStoreTime() throws MQClientException {
+        assertEquals(0, defaultMQProducerImpl.earliestMsgStoreTime(messageQueue));
+    }
+
+    @Test
+    public void assertViewMessage() throws MQClientException, MQBrokerException, RemotingException, InterruptedException {
+        assertNull(defaultMQProducerImpl.viewMessage(defaultTopic, "msgId"));
+    }
+
+    @Test
+    public void assertQueryMessage() throws MQClientException, InterruptedException {
+        assertNull(defaultMQProducerImpl.queryMessage(defaultTopic, "key", 1, 0L, 10L));
+    }
+
+    @Test
+    public void assertQueryMessageByUniqKey() throws MQClientException, InterruptedException {
+        assertNull(defaultMQProducerImpl.queryMessageByUniqKey(defaultTopic, "key"));
+    }
+
+    @Test
+    public void assertSetAsyncSenderExecutor() {
+        ExecutorService asyncSenderExecutor = mock(ExecutorService.class);
+        defaultMQProducerImpl.setAsyncSenderExecutor(asyncSenderExecutor);
+        assertEquals(asyncSenderExecutor, defaultMQProducerImpl.getAsyncSenderExecutor());
+    }
+
+    @Test
+    public void assertServiceState() {
+        ServiceState serviceState = defaultMQProducerImpl.getServiceState();
+        assertNotNull(serviceState);
+        assertEquals(ServiceState.RUNNING, serviceState);
+        defaultMQProducerImpl.setServiceState(ServiceState.SHUTDOWN_ALREADY);
+        serviceState = defaultMQProducerImpl.getServiceState();
+        assertNotNull(serviceState);
+        assertEquals(ServiceState.SHUTDOWN_ALREADY, serviceState);
+    }
+
+    @Test
+    public void assertGetNotAvailableDuration() {
+        long[] notAvailableDuration = defaultMQProducerImpl.getNotAvailableDuration();
+        assertNotNull(notAvailableDuration);
+        defaultMQProducerImpl.setNotAvailableDuration(new long[1]);
+        notAvailableDuration = defaultMQProducerImpl.getNotAvailableDuration();
+        assertNotNull(notAvailableDuration);
+        assertEquals(1, notAvailableDuration.length);
+    }
+
+    @Test
+    public void assertGetLatencyMax() {
+        long[] actual = defaultMQProducerImpl.getLatencyMax();
+        assertNotNull(actual);
+        defaultMQProducerImpl.setLatencyMax(new long[1]);
+        actual = defaultMQProducerImpl.getLatencyMax();
+        assertNotNull(actual);
+        assertEquals(1, actual.length);
+    }
+
+    @Test
+    public void assertIsSendLatencyFaultEnable() {
+        boolean actual = defaultMQProducerImpl.isSendLatencyFaultEnable();
+        assertFalse(actual);
+        defaultMQProducerImpl.setSendLatencyFaultEnable(true);
+        actual = defaultMQProducerImpl.isSendLatencyFaultEnable();
+        assertTrue(actual);
+    }
+
+    @Test
+    public void assertGetMqFaultStrategy() {
+        assertNotNull(defaultMQProducerImpl.getMqFaultStrategy());
+    }
+
+    @Test
+    public void assertCheckListener() {
+        assertNull(defaultMQProducerImpl.checkListener());
+    }
+
+    @Test
+    public void testRecallMessage_invalid() {
+        assertThrows(MQClientException.class, () -> {
+            defaultMQProducerImpl.recallMessage(MixAll.REPLY_TOPIC_POSTFIX + defaultTopic, "handle");
+        });
+        assertThrows(MQClientException.class, () -> {
+            defaultMQProducerImpl.recallMessage(MixAll.DLQ_GROUP_TOPIC_PREFIX + defaultTopic, "handle");
+        });
+        assertThrows(MQClientException.class, () -> {
+            defaultMQProducerImpl.recallMessage(defaultTopic, "handle");
+        });
+    }
+
+    @Test
+    public void testRecallMessage_addressNotFound() {
+        String handle = RecallMessageHandle.HandleV1.buildHandle(defaultTopic, defaultBrokerName, "1", "id");
+        when(mQClientFactory.findBrokerAddressInPublish(defaultBrokerName)).thenReturn(null);
+        MQClientException e = assertThrows(MQClientException.class, () -> {
+            defaultMQProducerImpl.recallMessage(defaultTopic, handle);
+        });
+        assertEquals("The broker service address not found", e.getErrorMessage());
+    }
+
+    @Test
+    public void testRecallMessage_success()
+        throws RemotingException, MQClientException, MQBrokerException, InterruptedException {
+        String handle = RecallMessageHandle.HandleV1.buildHandle(defaultTopic, defaultBrokerName, "1", "id");
+        when(mQClientFactory.findBrokerAddressInPublish(defaultBrokerName)).thenReturn(defaultBrokerAddr);
+        when(mQClientAPIImpl.recallMessage(any(), any(), anyLong())).thenReturn("id");
+        String result = defaultMQProducerImpl.recallMessage(defaultTopic, handle);
+        assertEquals("id", result);
+    }
+
+    private void setMQClientFactory() throws IllegalAccessException, NoSuchFieldException {
+        setField(defaultMQProducerImpl, "mQClientFactory", mQClientFactory);
+    }
+
+    private void setTopicPublishInfoTable() throws IllegalAccessException, NoSuchFieldException {
+        ConcurrentMap<String, TopicPublishInfo> topicPublishInfoTable = new ConcurrentHashMap<>();
+        TopicPublishInfo topicPublishInfo = mock(TopicPublishInfo.class);
+        when(topicPublishInfo.ok()).thenReturn(true);
+        topicPublishInfoTable.put(defaultTopic, topicPublishInfo);
+        setField(defaultMQProducerImpl, "topicPublishInfoTable", topicPublishInfoTable);
+    }
+
+    private void setCheckExecutor() throws NoSuchFieldException, IllegalAccessException {
+        setField(defaultMQProducerImpl, "checkExecutor", mock(ExecutorService.class));
+    }
+
+    private void setCheckForbiddenHookList() throws NoSuchFieldException, IllegalAccessException {
+        ArrayList<CheckForbiddenHook> checkForbiddenHookList = new ArrayList<>();
+        checkForbiddenHookList.add(mock(CheckForbiddenHook.class));
+        setField(defaultMQProducerImpl, "checkForbiddenHookList", checkForbiddenHookList);
+    }
+
+    private void setField(final Object target, final String fieldName, final Object newValue) throws NoSuchFieldException, IllegalAccessException {
+        Class<?> clazz = target.getClass();
+        Field field = clazz.getDeclaredField(fieldName);
+        field.setAccessible(true);
+        field.set(target, newValue);
+    }
+}
diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByRandomTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByRandomTest.java
new file mode 100644
index 00000000000..9443c3f0181
--- /dev/null
+++ b/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByRandomTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.client.producer.selector;
+
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
+
+public class SelectMessageQueueByRandomTest {
+
+    private final SelectMessageQueueByRandom selector = new SelectMessageQueueByRandom();
+
+    private final String defaultBroker = "defaultBroker";
+
+    private final String defaultTopic = "defaultTopic";
+
+    @Test
+    public void testSelectRandomMessageQueue() {
+        List<MessageQueue> messageQueues = createMessageQueues(10);
+        Message message = new Message(defaultTopic, "tag", "key", "body".getBytes());
+        MessageQueue selectedQueue = selector.select(messageQueues, message, null);
+        assertNotNull(selectedQueue);
+        assertEquals(messageQueues.size(), 10);
+        assertEquals(defaultTopic, selectedQueue.getTopic());
+        assertEquals(defaultBroker, selectedQueue.getBrokerName());
+    }
+
+    @Test
+    public void testSelectEmptyMessageQueue() {
+        List<MessageQueue> emptyQueues = new ArrayList<>();
+        Message message = new Message(defaultTopic, "tag", "key", "body".getBytes());
+        assertThrows(IllegalArgumentException.class, () -> selector.select(emptyQueues, message, null));
+    }
+
+    @Test
+    public void testSelectSingleMessageQueue() {
+        List<MessageQueue> singleQueueList = createMessageQueues(1);
+        Message message = new Message(defaultTopic, "tag", "key", "body".getBytes());
+        MessageQueue selectedQueue = selector.select(singleQueueList, message, null);
+        assertNotNull(selectedQueue);
+        assertEquals(defaultTopic, selectedQueue.getTopic());
+        assertEquals(defaultBroker, selectedQueue.getBrokerName());
+        assertEquals(singleQueueList.get(0).getQueueId(), selectedQueue.getQueueId());
+    }
+
+    private List<MessageQueue> createMessageQueues(final int count) {
+        List<MessageQueue> result = new ArrayList<>();
+        for (int i = 0; i < count; i++) {
+            result.add(new MessageQueue(defaultTopic, defaultBroker, i));
+        }
+        return result;
+    }
+}
diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java
index a39ae4a4ded..028445ef2d7 100644
--- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java
@@ -135,6 +135,8 @@ public PullResult answer(InvocationOnMock mock) throws Throwable {
             new ConsumeMessageOpenTracingHookImpl(tracer));
         pushConsumer.setNamesrvAddr("127.0.0.1:9876");
         pushConsumer.setPullInterval(60 * 1000);
+        // disable trace to let mock trace work
+        pushConsumer.setEnableTrace(false);
 
         OffsetStore offsetStore = Mockito.mock(OffsetStore.class);
         Mockito.when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(0L);
diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java
index 60aa446bbe9..fc63cce1ce4 100644
--- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java
@@ -128,11 +128,9 @@ public void init() throws Exception {
         normalPushConsumer = new DefaultMQPushConsumer(consumerGroupNormal, false, "");
         customTraceTopicPushConsumer = new DefaultMQPushConsumer(consumerGroup, true, customerTraceTopic);
         pushConsumer.setNamesrvAddr("127.0.0.1:9876");
+        pushConsumer.setUseTLS(true);
         pushConsumer.setPullInterval(60 * 1000);
 
-        asyncTraceDispatcher = (AsyncTraceDispatcher) pushConsumer.getTraceDispatcher();
-        traceProducer = asyncTraceDispatcher.getTraceProducer();
-
         pushConsumer.registerMessageListener(new MessageListenerConcurrently() {
             @Override
             public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
@@ -157,6 +155,9 @@ public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
 
         pushConsumer.start();
 
+        asyncTraceDispatcher = (AsyncTraceDispatcher) pushConsumer.getTraceDispatcher();
+        traceProducer = asyncTraceDispatcher.getTraceProducer();
+
         mQClientFactory = spy(pushConsumerImpl.getmQClientFactory());
         mQClientTraceFactory = spy(pushConsumerImpl.getmQClientFactory());
 
@@ -242,9 +243,6 @@ public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
 
     @Test
     public void testPushConsumerWithTraceTLS() {
-        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("consumerGroup", true, null);
-        consumer.setUseTLS(true);
-        AsyncTraceDispatcher asyncTraceDispatcher = (AsyncTraceDispatcher) consumer.getTraceDispatcher();
         Assert.assertTrue(asyncTraceDispatcher.getTraceProducer().isUseTLS());
     }
 
diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java
index 8fbc70ea44f..c0eb8568dc6 100644
--- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java
@@ -88,6 +88,8 @@ public void init() throws Exception {
                 new SendMessageOpenTracingHookImpl(tracer));
         producer.setNamesrvAddr("127.0.0.1:9876");
         message = new Message(topic, new byte[] {'a', 'b', 'c'});
+        // disable trace to let mock trace work
+        producer.setEnableTrace(false);
 
         producer.start();
 
@@ -122,7 +124,7 @@ public void testSendMessageSync_WithTrace_Success() throws RemotingException, In
         assertThat(span.tags().get(TraceConstants.ROCKETMQ_BODY_LENGTH)).isEqualTo(3);
         assertThat(span.tags().get(TraceConstants.ROCKETMQ_REGION_ID)).isEqualTo("HZ");
         assertThat(span.tags().get(TraceConstants.ROCKETMQ_MSG_TYPE)).isEqualTo(MessageType.Normal_Msg.name());
-        assertThat(span.tags().get(TraceConstants.ROCKETMQ_SOTRE_HOST)).isEqualTo("127.0.0.1:10911");
+        assertThat(span.tags().get(TraceConstants.ROCKETMQ_STORE_HOST)).isEqualTo("127.0.0.1:10911");
     }
 
     @After
diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java
index ee173351852..ed680d8e6cf 100644
--- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java
@@ -92,14 +92,14 @@ public void init() throws Exception {
         normalProducer.setNamesrvAddr("127.0.0.1:9877");
         customTraceTopicproducer.setNamesrvAddr("127.0.0.1:9878");
         message = new Message(topic, new byte[] {'a', 'b', 'c'});
-        asyncTraceDispatcher = (AsyncTraceDispatcher) producer.getTraceDispatcher();
-        asyncTraceDispatcher.setTraceTopicName(customerTraceTopic);
-        asyncTraceDispatcher.getHostProducer();
-        asyncTraceDispatcher.getHostConsumer();
-        traceProducer = asyncTraceDispatcher.getTraceProducer();
+        producer.setTraceTopic(customerTraceTopic);
+        producer.setUseTLS(true);
 
         producer.start();
 
+        asyncTraceDispatcher = (AsyncTraceDispatcher) producer.getTraceDispatcher();
+        traceProducer = asyncTraceDispatcher.getTraceProducer();
+
         Field field = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory");
         field.setAccessible(true);
         field.set(producer.getDefaultMQProducerImpl(), mQClientFactory);
@@ -150,9 +150,6 @@ public void testSendMessageSync_WithTrace_NoBrokerSet_Exception() throws Remotin
 
     @Test
     public void testProducerWithTraceTLS() {
-        DefaultMQProducer producer = new DefaultMQProducer(producerGroupTemp, true, null);
-        producer.setUseTLS(true);
-        AsyncTraceDispatcher asyncTraceDispatcher = (AsyncTraceDispatcher) producer.getTraceDispatcher();
         Assert.assertTrue(asyncTraceDispatcher.getTraceProducer().isUseTLS());
     }
 
diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java
index 5646a17dbe6..5d4b81d16db 100644
--- a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java
@@ -103,6 +103,8 @@ public LocalTransactionState checkLocalTransaction(MessageExt msg) {
         producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageOpenTracingHookImpl(tracer));
         producer.getDefaultMQProducerImpl().registerEndTransactionHook(new EndTransactionOpenTracingHookImpl(tracer));
         producer.setTransactionListener(transactionListener);
+        // disable trace to let mock trace work
+        producer.setEnableTrace(false);
 
         producer.setNamesrvAddr("127.0.0.1:9876");
         message = new Message(topic, new byte[] {'a', 'b', 'c'});
diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java
index 8cf87444c0c..9f6036153bc 100644
--- a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java
@@ -111,11 +111,12 @@ public LocalTransactionState checkLocalTransaction(MessageExt msg) {
 
         producer.setNamesrvAddr("127.0.0.1:9876");
         message = new Message(topic, new byte[] {'a', 'b', 'c'});
-        asyncTraceDispatcher = (AsyncTraceDispatcher) producer.getTraceDispatcher();
-        traceProducer = asyncTraceDispatcher.getTraceProducer();
 
         producer.start();
 
+        asyncTraceDispatcher = (AsyncTraceDispatcher) producer.getTraceDispatcher();
+        traceProducer = asyncTraceDispatcher.getTraceProducer();
+
         Field field = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory");
         field.setAccessible(true);
         field.set(producer.getDefaultMQProducerImpl(), mQClientFactory);
diff --git a/acl/src/test/resources/both_acl_file_folder_conf/conf/plain_acl.yml b/client/src/test/resources/acl_hook/plain_acl.yml
similarity index 89%
rename from acl/src/test/resources/both_acl_file_folder_conf/conf/plain_acl.yml
rename to client/src/test/resources/acl_hook/plain_acl.yml
index 41afea097ae..66bf5762279 100644
--- a/acl/src/test/resources/both_acl_file_folder_conf/conf/plain_acl.yml
+++ b/client/src/test/resources/acl_hook/plain_acl.yml
@@ -15,7 +15,7 @@
 
 ## suggested format
 
-globalWhiteRemoteAddresses:
-  - 10.10.103.*
-  - 192.168.0.*
-
+- accessKey: rocketmq2
+  secretKey: 12345678
+  whiteRemoteAddress: 192.168.1.*
+  admin: true
diff --git a/acl/src/test/resources/conf/plain_acl_incomplete.yml b/client/src/test/resources/conf/plain_acl_incomplete.yml
similarity index 98%
rename from acl/src/test/resources/conf/plain_acl_incomplete.yml
rename to client/src/test/resources/conf/plain_acl_incomplete.yml
index 0a6bdde7072..9ac39c809f0 100644
--- a/acl/src/test/resources/conf/plain_acl_incomplete.yml
+++ b/client/src/test/resources/conf/plain_acl_incomplete.yml
@@ -16,7 +16,7 @@
 ## suggested format
 
 - accessKey: rocketmq2
-  secretKey: 
+  secretKey:
   whiteRemoteAddress: 192.168.1.*
   # if it is admin, it could access all resources
   admin: true
\ No newline at end of file
diff --git a/common/BUILD.bazel b/common/BUILD.bazel
index 9a0c31e772f..10c5d19fbe8 100644
--- a/common/BUILD.bazel
+++ b/common/BUILD.bazel
@@ -22,6 +22,7 @@ java_library(
     visibility = ["//visibility:public"],
     deps = [
         "@maven//:com_alibaba_fastjson",
+        "@maven//:com_alibaba_fastjson2_fastjson2",
         "@maven//:com_github_luben_zstd_jni",
         "@maven//:com_google_guava_guava",
         "@maven//:commons_collections_commons_collections",
@@ -36,6 +37,8 @@ java_library(
         "@maven//:io_opentelemetry_opentelemetry_sdk_common",
         "@maven//:io_opentelemetry_opentelemetry_sdk_metrics",
         "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp",
+        "@maven//:io_grpc_grpc_api",
+        "@maven//:io_grpc_grpc_context",
         "@maven//:org_apache_commons_commons_lang3",
         "@maven//:org_lz4_lz4_java",
         "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
@@ -53,6 +56,8 @@ java_library(
         "//:test_deps",
         "@maven//:com_google_guava_guava",
         "@maven//:com_alibaba_fastjson",
+        "@maven//:com_alibaba_fastjson2_fastjson2",
+        "@maven//:commons_codec_commons_codec",
         "@maven//:io_netty_netty_all",
         "@maven//:io_opentelemetry_opentelemetry_api",
         "@maven//:io_opentelemetry_opentelemetry_context",
@@ -61,6 +66,8 @@ java_library(
         "@maven//:io_opentelemetry_opentelemetry_sdk",
         "@maven//:io_opentelemetry_opentelemetry_sdk_common",
         "@maven//:io_opentelemetry_opentelemetry_sdk_metrics",
+        "@maven//:io_grpc_grpc_api",
+        "@maven//:io_grpc_grpc_context",
         "@maven//:org_apache_commons_commons_lang3",
     ],
     resources = glob(["src/test/resources/certs/*.pem"]) + glob(["src/test/resources/certs/*.key"])
diff --git a/common/pom.xml b/common/pom.xml
index a28ed228fd4..18b18043dc2 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -36,6 +36,10 @@
             <groupId>com.alibaba</groupId>
             <artifactId>fastjson</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba.fastjson2</groupId>
+            <artifactId>fastjson2</artifactId>
+        </dependency>
         <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-all</artifactId>
diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
index d859f965e40..a49bd004731 100644
--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
+++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java
@@ -17,6 +17,7 @@
 package org.apache.rocketmq.common;
 
 import org.apache.rocketmq.common.annotation.ImportantField;
+import org.apache.rocketmq.common.config.ConfigManagerVersion;
 import org.apache.rocketmq.common.constant.PermName;
 import org.apache.rocketmq.common.message.MessageRequestMode;
 import org.apache.rocketmq.common.metrics.MetricsExporterType;
@@ -70,7 +71,7 @@ public class BrokerConfig extends BrokerIdentity {
     private int putMessageFutureThreadPoolNums = Math.min(PROCESSOR_NUMBER, 4);
     private int pullMessageThreadPoolNums = 16 + PROCESSOR_NUMBER * 2;
     private int litePullMessageThreadPoolNums = 16 + PROCESSOR_NUMBER * 2;
-    private int ackMessageThreadPoolNums = 3;
+    private int ackMessageThreadPoolNums = 16;
     private int processReplyMessageThreadPoolNums = 16 + PROCESSOR_NUMBER * 2;
     private int queryMessageThreadPoolNums = 8 + PROCESSOR_NUMBER;
 
@@ -129,6 +130,8 @@ public class BrokerConfig extends BrokerIdentity {
     private boolean accountStatsEnable = true;
     private boolean accountStatsPrintZeroValues = true;
 
+    private int maxStatsIdleTimeInMinutes = -1;
+
     private boolean transferMsgByHeap = true;
 
     private String regionId = MixAll.DEFAULT_TRACE_REGION_ID;
@@ -148,7 +151,7 @@ public class BrokerConfig extends BrokerIdentity {
     private long waitTimeMillsInHeartbeatQueue = 31 * 1000;
     private long waitTimeMillsInTransactionQueue = 3 * 1000;
     private long waitTimeMillsInAckQueue = 3000;
-
+    private long waitTimeMillsInAdminBrokerQueue = 5 * 1000;
     private long startAcceptSendRequestTimeStamp = 0L;
 
     private boolean traceOn = true;
@@ -185,6 +188,11 @@ public class BrokerConfig extends BrokerIdentity {
      */
     private int registerNameServerPeriod = 1000 * 30;
 
+    /**
+     * This configurable item defines interval of update name server address. Default: 120 * 1000 milliseconds
+     */
+    private int updateNameServerAddrPeriod = 1000 * 120;
+
     /**
      * the interval to send heartbeat to name server for liveness detection.
      */
@@ -221,11 +229,21 @@ public class BrokerConfig extends BrokerIdentity {
     private int popCkMaxBufferSize = 200000;
     private int popCkOffsetMaxQueueSize = 20000;
     private boolean enablePopBatchAck = false;
+    // set the interval to the maxFilterMessageSize in MessageStoreConfig divided by the cq unit size
+    private long popLongPollingForceNotifyInterval = 800;
+    private boolean enableNotifyBeforePopCalculateLag = true;
     private boolean enableNotifyAfterPopOrderLockRelease = true;
     private boolean initPopOffsetByCheckMsgInMem = true;
     // read message from pop retry topic v1, for the compatibility, will be removed in the future version
     private boolean retrieveMessageFromPopRetryTopicV1 = true;
     private boolean enableRetryTopicV2 = false;
+    private int popFromRetryProbability = 20;
+    private boolean popConsumerFSServiceInit = true;
+    private boolean popConsumerKVServiceLog = false;
+    private boolean popConsumerKVServiceInit = false;
+    private boolean popConsumerKVServiceEnable = false;
+    private int popReviveMaxReturnSizePerRead = 16 * 1024;
+    private int popReviveMaxAttemptTimes = 16;
 
     private boolean realTimeNotifyConsumerChange = true;
 
@@ -288,7 +306,7 @@ public class BrokerConfig extends BrokerIdentity {
 
     private boolean enableDetailStat = true;
 
-    private boolean autoDeleteUnusedStats = false;
+    private boolean autoDeleteUnusedStats = true;
 
     /**
      * Whether to distinguish log paths when multiple brokers are deployed on the same machine
@@ -410,6 +428,10 @@ public class BrokerConfig extends BrokerIdentity {
     private long popInflightMessageThreshold = 10000;
     private boolean enablePopMessageThreshold = false;
 
+    private boolean enableFastChannelEventProcess = false;
+    private boolean printChannelGroups = false;
+    private int printChannelGroupsMinNum = 5;
+
     private int splitRegistrationSize = 800;
 
     /**
@@ -419,6 +441,30 @@ public class BrokerConfig extends BrokerIdentity {
      */
     private String configBlackList = "configBlackList;brokerConfigPath";
 
+    // if false, will still rewrite ck after max times 17
+    private boolean skipWhenCKRePutReachMaxTimes = false;
+
+    private boolean appendAckAsync = false;
+
+    private boolean appendCkAsync = false;
+
+    private boolean clearRetryTopicWhenDeleteTopic = true;
+
+    private boolean enableLmqStats = false;
+
+    /**
+     * V2 is recommended in cases where LMQ feature is extensively used.
+     */
+    private String configManagerVersion = ConfigManagerVersion.V1.getVersion();
+
+    private boolean allowRecallWhenBrokerNotWriteable = true;
+
+    private boolean recallMessageEnable = false;
+
+    private boolean enableRegisterProducer = true;
+
+    private boolean enableCreateSysGroup = true;
+
     public String getConfigBlackList() {
         return configBlackList;
     }
@@ -555,6 +601,62 @@ public void setEnablePopLog(boolean enablePopLog) {
         this.enablePopLog = enablePopLog;
     }
 
+    public int getPopFromRetryProbability() {
+        return popFromRetryProbability;
+    }
+
+    public void setPopFromRetryProbability(int popFromRetryProbability) {
+        this.popFromRetryProbability = popFromRetryProbability;
+    }
+
+    public boolean isPopConsumerFSServiceInit() {
+        return popConsumerFSServiceInit;
+    }
+
+    public void setPopConsumerFSServiceInit(boolean popConsumerFSServiceInit) {
+        this.popConsumerFSServiceInit = popConsumerFSServiceInit;
+    }
+
+    public boolean isPopConsumerKVServiceLog() {
+        return popConsumerKVServiceLog;
+    }
+
+    public void setPopConsumerKVServiceLog(boolean popConsumerKVServiceLog) {
+        this.popConsumerKVServiceLog = popConsumerKVServiceLog;
+    }
+
+    public boolean isPopConsumerKVServiceInit() {
+        return popConsumerKVServiceInit;
+    }
+
+    public void setPopConsumerKVServiceInit(boolean popConsumerKVServiceInit) {
+        this.popConsumerKVServiceInit = popConsumerKVServiceInit;
+    }
+
+    public boolean isPopConsumerKVServiceEnable() {
+        return popConsumerKVServiceEnable;
+    }
+
+    public void setPopConsumerKVServiceEnable(boolean popConsumerKVServiceEnable) {
+        this.popConsumerKVServiceEnable = popConsumerKVServiceEnable;
+    }
+
+    public int getPopReviveMaxReturnSizePerRead() {
+        return popReviveMaxReturnSizePerRead;
+    }
+
+    public void setPopReviveMaxReturnSizePerRead(int popReviveMaxReturnSizePerRead) {
+        this.popReviveMaxReturnSizePerRead = popReviveMaxReturnSizePerRead;
+    }
+
+    public int getPopReviveMaxAttemptTimes() {
+        return popReviveMaxAttemptTimes;
+    }
+
+    public void setPopReviveMaxAttemptTimes(int popReviveMaxAttemptTimes) {
+        this.popReviveMaxAttemptTimes = popReviveMaxAttemptTimes;
+    }
+
     public boolean isTraceOn() {
         return traceOn;
     }
@@ -1167,6 +1269,14 @@ public String getMsgTraceTopicName() {
         return msgTraceTopicName;
     }
 
+    public long getWaitTimeMillsInAdminBrokerQueue() {
+        return waitTimeMillsInAdminBrokerQueue;
+    }
+
+    public void setWaitTimeMillsInAdminBrokerQueue(long waitTimeMillsInAdminBrokerQueue) {
+        this.waitTimeMillsInAdminBrokerQueue = waitTimeMillsInAdminBrokerQueue;
+    }
+
     public void setMsgTraceTopicName(String msgTraceTopicName) {
         this.msgTraceTopicName = msgTraceTopicName;
     }
@@ -1179,10 +1289,6 @@ public void setTraceTopicEnable(boolean traceTopicEnable) {
         this.traceTopicEnable = traceTopicEnable;
     }
 
-    public boolean isAclEnable() {
-        return aclEnable;
-    }
-
     public void setAclEnable(boolean aclEnable) {
         this.aclEnable = aclEnable;
     }
@@ -1291,6 +1397,22 @@ public void setEnableNetWorkFlowControl(boolean enableNetWorkFlowControl) {
         this.enableNetWorkFlowControl = enableNetWorkFlowControl;
     }
 
+    public long getPopLongPollingForceNotifyInterval() {
+        return popLongPollingForceNotifyInterval;
+    }
+
+    public void setPopLongPollingForceNotifyInterval(long popLongPollingForceNotifyInterval) {
+        this.popLongPollingForceNotifyInterval = popLongPollingForceNotifyInterval;
+    }
+
+    public boolean isEnableNotifyBeforePopCalculateLag() {
+        return enableNotifyBeforePopCalculateLag;
+    }
+
+    public void setEnableNotifyBeforePopCalculateLag(boolean enableNotifyBeforePopCalculateLag) {
+        this.enableNotifyBeforePopCalculateLag = enableNotifyBeforePopCalculateLag;
+    }
+
     public boolean isEnableNotifyAfterPopOrderLockRelease() {
         return enableNotifyAfterPopOrderLockRelease;
     }
@@ -1419,6 +1541,14 @@ public void setAccountStatsPrintZeroValues(boolean accountStatsPrintZeroValues)
         this.accountStatsPrintZeroValues = accountStatsPrintZeroValues;
     }
 
+    public int getMaxStatsIdleTimeInMinutes() {
+        return maxStatsIdleTimeInMinutes;
+    }
+
+    public void setMaxStatsIdleTimeInMinutes(int maxStatsIdleTimeInMinutes) {
+        this.maxStatsIdleTimeInMinutes = maxStatsIdleTimeInMinutes;
+    }
+
     public boolean isLockInStrictMode() {
         return lockInStrictMode;
     }
@@ -1787,6 +1917,30 @@ public void setEnableSplitRegistration(boolean enableSplitRegistration) {
         this.enableSplitRegistration = enableSplitRegistration;
     }
 
+    public boolean isEnableFastChannelEventProcess() {
+        return enableFastChannelEventProcess;
+    }
+
+    public void setEnableFastChannelEventProcess(boolean enableFastChannelEventProcess) {
+        this.enableFastChannelEventProcess = enableFastChannelEventProcess;
+    }
+
+    public boolean isPrintChannelGroups() {
+        return printChannelGroups;
+    }
+
+    public void setPrintChannelGroups(boolean printChannelGroups) {
+        this.printChannelGroups = printChannelGroups;
+    }
+
+    public int getPrintChannelGroupsMinNum() {
+        return printChannelGroupsMinNum;
+    }
+
+    public void setPrintChannelGroupsMinNum(int printChannelGroupsMinNum) {
+        this.printChannelGroupsMinNum = printChannelGroupsMinNum;
+    }
+
     public int getSplitRegistrationSize() {
         return splitRegistrationSize;
     }
@@ -1818,4 +1972,92 @@ public boolean isEnablePopMessageThreshold() {
     public void setEnablePopMessageThreshold(boolean enablePopMessageThreshold) {
         this.enablePopMessageThreshold = enablePopMessageThreshold;
     }
+
+    public boolean isSkipWhenCKRePutReachMaxTimes() {
+        return skipWhenCKRePutReachMaxTimes;
+    }
+
+    public void setSkipWhenCKRePutReachMaxTimes(boolean skipWhenCKRePutReachMaxTimes) {
+        this.skipWhenCKRePutReachMaxTimes = skipWhenCKRePutReachMaxTimes;
+    }
+
+    public int getUpdateNameServerAddrPeriod() {
+        return updateNameServerAddrPeriod;
+    }
+
+    public void setUpdateNameServerAddrPeriod(int updateNameServerAddrPeriod) {
+        this.updateNameServerAddrPeriod = updateNameServerAddrPeriod;
+    }
+
+    public boolean isAppendAckAsync() {
+        return appendAckAsync;
+    }
+
+    public void setAppendAckAsync(boolean appendAckAsync) {
+        this.appendAckAsync = appendAckAsync;
+    }
+
+    public boolean isAppendCkAsync() {
+        return appendCkAsync;
+    }
+
+    public void setAppendCkAsync(boolean appendCkAsync) {
+        this.appendCkAsync = appendCkAsync;
+    }
+
+    public boolean isClearRetryTopicWhenDeleteTopic() {
+        return clearRetryTopicWhenDeleteTopic;
+    }
+
+    public void setClearRetryTopicWhenDeleteTopic(boolean clearRetryTopicWhenDeleteTopic) {
+        this.clearRetryTopicWhenDeleteTopic = clearRetryTopicWhenDeleteTopic;
+    }
+
+    public boolean isEnableLmqStats() {
+        return enableLmqStats;
+    }
+
+    public void setEnableLmqStats(boolean enableLmqStats) {
+        this.enableLmqStats = enableLmqStats;
+    }
+
+    public String getConfigManagerVersion() {
+        return configManagerVersion;
+    }
+
+    public void setConfigManagerVersion(String configManagerVersion) {
+        this.configManagerVersion = configManagerVersion;
+    }
+
+    public boolean isAllowRecallWhenBrokerNotWriteable() {
+        return allowRecallWhenBrokerNotWriteable;
+    }
+
+    public void setAllowRecallWhenBrokerNotWriteable(boolean allowRecallWhenBrokerNotWriteable) {
+        this.allowRecallWhenBrokerNotWriteable = allowRecallWhenBrokerNotWriteable;
+    }
+
+    public boolean isRecallMessageEnable() {
+        return recallMessageEnable;
+    }
+
+    public void setRecallMessageEnable(boolean recallMessageEnable) {
+        this.recallMessageEnable = recallMessageEnable;
+    }
+
+    public boolean isEnableRegisterProducer() {
+        return enableRegisterProducer;
+    }
+
+    public void setEnableRegisterProducer(boolean enableRegisterProducer) {
+        this.enableRegisterProducer = enableRegisterProducer;
+    }
+
+    public boolean isEnableCreateSysGroup() {
+        return enableCreateSysGroup;
+    }
+
+    public void setEnableCreateSysGroup(boolean enableCreateSysGroup) {
+        this.enableCreateSysGroup = enableCreateSysGroup;
+    }
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/CheckRocksdbCqWriteResult.java b/common/src/main/java/org/apache/rocketmq/common/CheckRocksdbCqWriteResult.java
new file mode 100644
index 00000000000..fc67df86c2f
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/CheckRocksdbCqWriteResult.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common;
+
+public class CheckRocksdbCqWriteResult {
+    String checkResult;
+
+    int checkStatus;
+
+    public enum CheckStatus {
+        CHECK_OK(0),
+        CHECK_NOT_OK(1),
+        CHECK_IN_PROGRESS(2),
+        CHECK_ERROR(3);
+
+        private int value;
+
+        CheckStatus(int value) {
+            this.value = value;
+        }
+
+        public int getValue() {
+            return value;
+        }
+    }
+
+    public String getCheckResult() {
+        return checkResult;
+    }
+
+    public void setCheckResult(String checkResult) {
+        this.checkResult = checkResult;
+    }
+
+    public int getCheckStatus() {
+        return checkStatus;
+    }
+
+    public void setCheckStatus(int checkStatus) {
+        this.checkStatus = checkStatus;
+    }
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java b/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java
index 5e997596194..3fcf466fd77 100644
--- a/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java
+++ b/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java
@@ -16,11 +16,9 @@
  */
 package org.apache.rocketmq.common;
 
-import org.apache.rocketmq.common.config.RocksDBConfigManager;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.rocksdb.Statistics;
 
 import java.io.IOException;
 import java.util.Map;
@@ -28,8 +26,6 @@
 public abstract class ConfigManager {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
 
-    protected RocksDBConfigManager rocksDBConfigManager;
-
     public boolean load() {
         String fileName = null;
         try {
@@ -52,8 +48,8 @@ public boolean load() {
     private boolean loadBak() {
         String fileName = null;
         try {
-            fileName = this.configFilePath();
-            String jsonString = MixAll.file2String(fileName + ".bak");
+            fileName = this.configFilePath() + ".bak";
+            String jsonString = MixAll.file2String(fileName);
             if (jsonString != null && jsonString.length() > 0) {
                 this.decode(jsonString);
                 log.info("load " + fileName + " OK");
@@ -89,10 +85,6 @@ public synchronized void persist() {
         }
     }
 
-    protected void decode0(final byte[] key, final byte[] body) {
-
-    }
-
     public boolean stop() {
         return true;
     }
@@ -104,8 +96,4 @@ public boolean stop() {
     public abstract String encode(final boolean prettyFormat);
 
     public abstract void decode(final String jsonString);
-
-    public Statistics getStatistics() {
-        return rocksDBConfigManager == null ? null : rocksDBConfigManager.getStatistics();
-    }
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java
index 5eb9dc06ac9..20218f51964 100644
--- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java
+++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java
@@ -18,7 +18,7 @@
 
 public class MQVersion {
 
-    public static final int CURRENT_VERSION = Version.V5_2_0.ordinal();
+    public static final int CURRENT_VERSION = Version.V5_3_3.ordinal();
 
     public static String getVersionDesc(int value) {
         int length = Version.values().length;
diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java
index 47b4aac34a4..aca9bd4ed7b 100644
--- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java
+++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java
@@ -44,10 +44,12 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Predicate;
 
+import com.google.common.collect.ImmutableSet;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.annotation.ImportantField;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.help.FAQUrl;
+import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.common.utils.IOTinyUtils;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
@@ -99,8 +101,8 @@ public class MixAll {
     public static final String ACL_CONF_TOOLS_FILE = "/conf/tools.yml";
     public static final String REPLY_MESSAGE_FLAG = "reply";
     public static final String LMQ_PREFIX = "%LMQ%";
-    public static final long LMQ_QUEUE_ID = 0;
-    public static final String MULTI_DISPATCH_QUEUE_SPLITTER = ",";
+    public static final int LMQ_QUEUE_ID = 0;
+    public static final String LMQ_DISPATCH_SEPARATOR = ",";
     public static final String REQ_T = "ReqT";
     public static final String ROCKETMQ_ZONE_ENV = "ROCKETMQ_ZONE";
     public static final String ROCKETMQ_ZONE_PROPERTY = "rocketmq.zone";
@@ -119,22 +121,39 @@ public class MixAll {
 
     private static final String OS = System.getProperty("os.name").toLowerCase();
 
+    private static final Set<String> PREDEFINE_GROUP_SET = ImmutableSet.of(
+        DEFAULT_CONSUMER_GROUP,
+        DEFAULT_PRODUCER_GROUP,
+        TOOLS_CONSUMER_GROUP,
+        SCHEDULE_CONSUMER_GROUP,
+        FILTERSRV_CONSUMER_GROUP,
+        MONITOR_CONSUMER_GROUP,
+        CLIENT_INNER_PRODUCER_GROUP,
+        SELF_TEST_PRODUCER_GROUP,
+        SELF_TEST_CONSUMER_GROUP,
+        ONS_HTTP_PROXY_GROUP,
+        CID_ONSAPI_PERMISSION_GROUP,
+        CID_ONSAPI_OWNER_GROUP,
+        CID_ONSAPI_PULL_GROUP,
+        CID_SYS_RMQ_TRANS
+    );
+
     public static boolean isWindows() {
-        return OS.indexOf("win") >= 0;
+        return OS.contains("win");
     }
 
     public static boolean isMac() {
-        return OS.indexOf("mac") >= 0;
+        return OS.contains("mac");
     }
 
     public static boolean isUnix() {
-        return OS.indexOf("nix") >= 0
-                || OS.indexOf("nux") >= 0
-                || OS.indexOf("aix") > 0;
+        return OS.contains("nix")
+            || OS.contains("nux")
+            || OS.contains("aix");
     }
 
     public static boolean isSolaris() {
-        return OS.indexOf("sunos") >= 0;
+        return OS.contains("sunos");
     }
 
     public static String getWSAddr() {
@@ -159,6 +178,14 @@ public static boolean isSysConsumerGroup(final String consumerGroup) {
         return consumerGroup.startsWith(CID_RMQ_SYS_PREFIX);
     }
 
+    public static boolean isSysConsumerGroupAndEnableCreate(final String consumerGroup, final boolean isEnableCreateSysGroup) {
+        return isEnableCreateSysGroup && isSysConsumerGroup(consumerGroup);
+    }
+
+    public static boolean isPredefinedGroup(final String consumerGroup) {
+        return PREDEFINE_GROUP_SET.contains(consumerGroup);
+    }
+
     public static String getDLQTopic(final String consumerGroup) {
         return DLQ_GROUP_TOPIC_PREFIX + consumerGroup;
     }
@@ -205,7 +232,7 @@ public static void string2FileNotSafe(final String str, final String fileName) t
         if (fileParent != null) {
             fileParent.mkdirs();
         }
-        IOTinyUtils.writeStringToFile(file, str, "UTF-8");
+        IOTinyUtils.writeStringToFile(file, str, DEFAULT_CHARSET);
     }
 
     public static String file2String(final String fileName) throws IOException {
@@ -224,7 +251,7 @@ public static String file2String(final File file) throws IOException {
             }
 
             if (result) {
-                return new String(data, "UTF-8");
+                return new String(data, DEFAULT_CHARSET);
             }
         }
         return null;
@@ -364,9 +391,9 @@ public static void properties2Object(final Properties p, final Object object) {
                     String property = p.getProperty(key);
                     if (property != null) {
                         Class<?>[] pt = method.getParameterTypes();
-                        if (pt != null && pt.length > 0) {
+                        if (pt.length > 0) {
                             String cn = pt[0].getSimpleName();
-                            Object arg = null;
+                            Object arg;
                             if (cn.equals("int") || cn.equals("Integer")) {
                                 arg = Integer.parseInt(property);
                             } else if (cn.equals("long") || cn.equals("Long")) {
@@ -464,7 +491,7 @@ public static String getLocalhostByNetworkInterface() throws SocketException {
             return candidatesHost.get(0);
         }
 
-        // Fallback to loopback 
+        // Fallback to loopback
         return localhost();
     }
 
@@ -507,7 +534,7 @@ public static String dealFilePath(String aclFilePath) {
         return path.normalize().toString();
     }
 
-    public static boolean isSysConsumerGroupForNoColdReadLimit(String consumerGroup) {
+    public static boolean isSysConsumerGroupPullMessage(String consumerGroup) {
         if (DEFAULT_CONSUMER_GROUP.equals(consumerGroup)
             || TOOLS_CONSUMER_GROUP.equals(consumerGroup)
             || SCHEDULE_CONSUMER_GROUP.equals(consumerGroup)
@@ -524,4 +551,10 @@ public static boolean isSysConsumerGroupForNoColdReadLimit(String consumerGroup)
         }
         return false;
     }
+
+    public static boolean topicAllowsLMQ(String topic) {
+        return !topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)
+            && !topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX)
+            && !topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC);
+    }
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/ObjectCreator.java b/common/src/main/java/org/apache/rocketmq/common/ObjectCreator.java
new file mode 100644
index 00000000000..14c645424f3
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/ObjectCreator.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common;
+
+public interface ObjectCreator<T> {
+    T create(Object... args);
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/Pair.java b/common/src/main/java/org/apache/rocketmq/common/Pair.java
index 10213757cae..138ab509264 100644
--- a/common/src/main/java/org/apache/rocketmq/common/Pair.java
+++ b/common/src/main/java/org/apache/rocketmq/common/Pair.java
@@ -27,6 +27,10 @@ public Pair(T1 object1, T2 object2) {
         this.object2 = object2;
     }
 
+    public static <T1, T2> Pair<T1, T2> of(T1 object1, T2 object2) {
+        return new Pair<>(object1, object2);
+    }
+
     public T1 getObject1() {
         return object1;
     }
diff --git a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java
index 95dc8b9800b..96195d53090 100644
--- a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java
+++ b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java
@@ -24,7 +24,7 @@
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 
 public abstract class ServiceThread implements Runnable {
-    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
+    protected static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
 
     private static final long JOIN_TIME = 90 * 1000;
 
diff --git a/common/src/main/java/org/apache/rocketmq/common/TopicAttributes.java b/common/src/main/java/org/apache/rocketmq/common/TopicAttributes.java
index 1f26866e5b3..c507748c677 100644
--- a/common/src/main/java/org/apache/rocketmq/common/TopicAttributes.java
+++ b/common/src/main/java/org/apache/rocketmq/common/TopicAttributes.java
@@ -20,6 +20,7 @@
 import java.util.Map;
 import org.apache.rocketmq.common.attribute.Attribute;
 import org.apache.rocketmq.common.attribute.EnumAttribute;
+import org.apache.rocketmq.common.attribute.LongRangeAttribute;
 import org.apache.rocketmq.common.attribute.TopicMessageType;
 
 import static com.google.common.collect.Sets.newHashSet;
@@ -43,6 +44,13 @@ public class TopicAttributes {
         TopicMessageType.topicMessageTypeSet(),
         TopicMessageType.NORMAL.getValue()
     );
+    public static final LongRangeAttribute TOPIC_RESERVE_TIME_ATTRIBUTE = new LongRangeAttribute(
+        "reserve.time",
+        true,
+        -1,
+        Long.MAX_VALUE,
+        -1
+    );
 
     public static final Map<String, Attribute> ALL;
 
@@ -51,5 +59,6 @@ public class TopicAttributes {
         ALL.put(QUEUE_TYPE_ATTRIBUTE.getName(), QUEUE_TYPE_ATTRIBUTE);
         ALL.put(CLEANUP_POLICY_ATTRIBUTE.getName(), CLEANUP_POLICY_ATTRIBUTE);
         ALL.put(TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(), TOPIC_MESSAGE_TYPE_ATTRIBUTE);
+        ALL.put(TOPIC_RESERVE_TIME_ATTRIBUTE.getName(), TOPIC_RESERVE_TIME_ATTRIBUTE);
     }
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java
index 3629ae648bb..a42ac3f364d 100644
--- a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java
+++ b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java
@@ -326,16 +326,14 @@ public static String bytes2string(byte[] src) {
     }
 
     public static void writeInt(char[] buffer, int pos, int value) {
-        char[] hexArray = HEX_ARRAY;
         for (int moveBits = 28; moveBits >= 0; moveBits -= 4) {
-            buffer[pos++] = hexArray[(value >>> moveBits) & 0x0F];
+            buffer[pos++] = HEX_ARRAY[(value >>> moveBits) & 0x0F];
         }
     }
 
     public static void writeShort(char[] buffer, int pos, int value) {
-        char[] hexArray = HEX_ARRAY;
         for (int moveBits = 12; moveBits >= 0; moveBits -= 4) {
-            buffer[pos++] = hexArray[(value >>> moveBits) & 0x0F];
+            buffer[pos++] = HEX_ARRAY[(value >>> moveBits) & 0x0F];
         }
     }
 
@@ -536,25 +534,18 @@ public static boolean isInternalIP(byte[] ip) {
         } else if (ip[0] == (byte) 127) {
             return true;
         } else if (ip[0] == (byte) 172) {
-            if (ip[1] >= (byte) 16 && ip[1] <= (byte) 31) {
-                return true;
-            }
+            return ip[1] >= (byte) 16 && ip[1] <= (byte) 31;
         } else if (ip[0] == (byte) 192) {
-            if (ip[1] == (byte) 168) {
-                return true;
-            }
+            return ip[1] == (byte) 168;
         }
         return false;
     }
 
     public static boolean isInternalV6IP(InetAddress inetAddr) {
-        if (inetAddr.isAnyLocalAddress() // Wild card ipv6
+        return inetAddr.isAnyLocalAddress() // Wild card ipv6
             || inetAddr.isLinkLocalAddress() // Single broadcast ipv6 address: fe80:xx:xx...
             || inetAddr.isLoopbackAddress() //Loopback ipv6 address
-            || inetAddr.isSiteLocalAddress()) { // Site local ipv6 address: fec0:xx:xx...
-            return true;
-        }
-        return false;
+            || inetAddr.isSiteLocalAddress();// Site local ipv6 address: fec0:xx:xx...
     }
 
     private static boolean ipCheck(byte[] ip) {
@@ -605,15 +596,15 @@ public static String ipToIPv6Str(byte[] ip) {
 
     public static byte[] getIP() {
         try {
-            Enumeration allNetInterfaces = NetworkInterface.getNetworkInterfaces();
-            InetAddress ip = null;
+            Enumeration<NetworkInterface> allNetInterfaces = NetworkInterface.getNetworkInterfaces();
+            InetAddress ip;
             byte[] internalIP = null;
             while (allNetInterfaces.hasMoreElements()) {
-                NetworkInterface netInterface = (NetworkInterface) allNetInterfaces.nextElement();
-                Enumeration addresses = netInterface.getInetAddresses();
+                NetworkInterface netInterface = allNetInterfaces.nextElement();
+                Enumeration<InetAddress> addresses = netInterface.getInetAddresses();
                 while (addresses.hasMoreElements()) {
-                    ip = (InetAddress) addresses.nextElement();
-                    if (ip != null && ip instanceof Inet4Address) {
+                    ip =  addresses.nextElement();
+                    if (ip instanceof Inet4Address) {
                         byte[] ipByte = ip.getAddress();
                         if (ipByte.length == 4) {
                             if (ipCheck(ipByte)) {
@@ -624,7 +615,7 @@ public static byte[] getIP() {
                                 }
                             }
                         }
-                    } else if (ip != null && ip instanceof Inet6Address) {
+                    } else if (ip instanceof Inet6Address) {
                         byte[] ipByte = ip.getAddress();
                         if (ipByte.length == 16) {
                             if (ipV6Check(ipByte)) {
diff --git a/common/src/main/java/org/apache/rocketmq/common/action/Action.java b/common/src/main/java/org/apache/rocketmq/common/action/Action.java
new file mode 100644
index 00000000000..7e123239525
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/action/Action.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.action;
+
+import com.alibaba.fastjson2.annotation.JSONField;
+import org.apache.commons.lang3.StringUtils;
+
+public enum Action {
+
+    UNKNOWN((byte) 0, "Unknown"),
+
+    ALL((byte) 1, "All"),
+
+    ANY((byte) 2, "Any"),
+
+    PUB((byte) 3, "Pub"),
+
+    SUB((byte) 4, "Sub"),
+
+    CREATE((byte) 5, "Create"),
+
+    UPDATE((byte) 6, "Update"),
+
+    DELETE((byte) 7, "Delete"),
+
+    GET((byte) 8, "Get"),
+
+    LIST((byte) 9, "List");
+
+    @JSONField(value = true)
+    private final byte code;
+    private final String name;
+
+    Action(byte code, String name) {
+        this.code = code;
+        this.name = name;
+    }
+
+    public static Action getByName(String name) {
+        for (Action action : Action.values()) {
+            if (StringUtils.equalsIgnoreCase(action.getName(), name)) {
+                return action;
+            }
+        }
+        return null;
+    }
+
+    public byte getCode() {
+        return code;
+    }
+
+    public String getName() {
+        return name;
+    }
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/action/RocketMQAction.java b/common/src/main/java/org/apache/rocketmq/common/action/RocketMQAction.java
new file mode 100644
index 00000000000..251d9d5d85f
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/action/RocketMQAction.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.action;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import org.apache.rocketmq.common.resource.ResourceType;
+
+@Retention(RetentionPolicy.RUNTIME)
+public @interface RocketMQAction {
+
+    int value();
+
+    ResourceType resource() default ResourceType.UNKNOWN;
+
+    Action[] action();
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java b/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java
index 9680acec74d..5e581a34eec 100644
--- a/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java
+++ b/common/src/main/java/org/apache/rocketmq/common/attribute/TopicMessageType.java
@@ -50,7 +50,8 @@ public static TopicMessageType parseFromMessageProperty(Map<String, String> mess
             return TopicMessageType.TRANSACTION;
         } else if (messageProperty.get(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null
             || messageProperty.get(MessageConst.PROPERTY_TIMER_DELIVER_MS) != null
-            || messageProperty.get(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null) {
+            || messageProperty.get(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null
+            || messageProperty.get(MessageConst.PROPERTY_TIMER_DELAY_MS) != null) {
             return TopicMessageType.DELAY;
         } else if (messageProperty.get(MessageConst.PROPERTY_SHARDING_KEY) != null) {
             return TopicMessageType.FIFO;
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategy.java b/common/src/main/java/org/apache/rocketmq/common/chain/Handler.java
similarity index 85%
rename from acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategy.java
rename to common/src/main/java/org/apache/rocketmq/common/chain/Handler.java
index 8eab40c954b..0e2b4a42ae9 100644
--- a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategy.java
+++ b/common/src/main/java/org/apache/rocketmq/common/chain/Handler.java
@@ -14,9 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.acl.plain;
+package org.apache.rocketmq.common.chain;
 
-public interface RemoteAddressStrategy {
+public interface Handler<T, R> {
 
-    boolean match(PlainAccessResource plainAccessResource);
+    R handle(T t, HandlerChain<T, R> chain);
 }
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java b/common/src/main/java/org/apache/rocketmq/common/chain/HandlerChain.java
similarity index 53%
rename from tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java
rename to common/src/main/java/org/apache/rocketmq/common/chain/HandlerChain.java
index 3bbe41dd4b6..220689e36e3 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java
+++ b/common/src/main/java/org/apache/rocketmq/common/chain/HandlerChain.java
@@ -14,41 +14,37 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+package org.apache.rocketmq.common.chain;
 
-package org.apache.rocketmq.tieredstore.provider;
-
-import java.io.InputStream;
-import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
-import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
 
-public class MockFileSegmentInputStream extends FileSegmentInputStream {
+public class HandlerChain<T, R> {
 
-    private final InputStream inputStream;
+    private List<Handler<T, R>> handlers;
+    private Iterator<Handler<T, R>> iterator;
 
-    public MockFileSegmentInputStream(InputStream inputStream) {
-        super(null, null, Integer.MAX_VALUE);
-        this.inputStream = inputStream;
+    public static <T, R> HandlerChain<T, R> create() {
+        return new HandlerChain<>();
     }
 
-    @Override
-    public int read() {
-        int res = -1;
-        try {
-            res = inputStream.read();
-        } catch (Exception e) {
-            return -1;
+    public HandlerChain<T, R> addNext(Handler<T, R> handler) {
+        if (this.handlers == null) {
+            this.handlers = new ArrayList<>();
         }
-        return res;
-    }
-
-    @Override
-    public List<ByteBuffer> getBufferList() {
-        return null;
+        this.handlers.add(handler);
+        return this;
     }
 
-    @Override
-    public ByteBuffer getCodaBuffer() {
+    public R handle(T t) {
+        if (iterator == null) {
+            iterator = handlers.iterator();
+        }
+        if (iterator.hasNext()) {
+            Handler<T, R> handler = iterator.next();
+            return handler.handle(t, this);
+        }
         return null;
     }
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java
index 20319abba3d..6c0bce5929a 100644
--- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java
+++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java
@@ -16,7 +16,10 @@
  */
 package org.apache.rocketmq.common.config;
 
+import com.google.common.collect.Maps;
+import io.netty.buffer.PooledByteBufAllocator;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -25,12 +28,8 @@
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-
-import com.google.common.collect.Maps;
-
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.common.utils.DataConverter;
 import org.apache.rocketmq.common.utils.ThreadUtils;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
@@ -39,7 +38,9 @@
 import org.rocksdb.ColumnFamilyOptions;
 import org.rocksdb.CompactRangeOptions;
 import org.rocksdb.CompactionOptions;
+import org.rocksdb.CompressionType;
 import org.rocksdb.DBOptions;
+import org.rocksdb.Env;
 import org.rocksdb.FlushOptions;
 import org.rocksdb.LiveFileMetaData;
 import org.rocksdb.Priority;
@@ -51,14 +52,21 @@
 import org.rocksdb.WriteBatch;
 import org.rocksdb.WriteOptions;
 
-import static org.rocksdb.RocksDB.NOT_FOUND;
-
 public abstract class AbstractRocksDBStorage {
     protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);
 
+    /**
+     * Direct Jemalloc allocator
+     */
+    public static final PooledByteBufAllocator POOLED_ALLOCATOR = new PooledByteBufAllocator(true);
+
+    public static final byte CTRL_0 = '\u0000';
+    public static final byte CTRL_1 = '\u0001';
+    public static final byte CTRL_2 = '\u0002';
+
     private static final String SPACE = " | ";
 
-    protected String dbPath;
+    protected final String dbPath;
     protected boolean readOnly;
     protected RocksDB db;
     protected DBOptions options;
@@ -73,23 +81,88 @@ public abstract class AbstractRocksDBStorage {
     protected CompactRangeOptions compactRangeOptions;
 
     protected ColumnFamilyHandle defaultCFHandle;
-    protected final List<ColumnFamilyOptions> cfOptions = new ArrayList();
+    protected final List<ColumnFamilyOptions> cfOptions = new ArrayList<>();
+    protected final List<ColumnFamilyHandle> cfHandles = new ArrayList<>();
 
     protected volatile boolean loaded;
+    protected CompressionType compressionType = CompressionType.LZ4_COMPRESSION;
     private volatile boolean closed;
 
     private final Semaphore reloadPermit = new Semaphore(1);
     private final ScheduledExecutorService reloadScheduler = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("RocksDBStorageReloadService_"));
     private final ThreadPoolExecutor manualCompactionThread = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor(
-            1, 1, 1000 * 60, TimeUnit.MILLISECONDS,
-            new ArrayBlockingQueue(1),
-            new ThreadFactoryImpl("RocksDBManualCompactionService_"),
-            new ThreadPoolExecutor.DiscardOldestPolicy());
+        1, 1, 1000 * 60, TimeUnit.MILLISECONDS,
+        new ArrayBlockingQueue<>(1),
+        new ThreadFactoryImpl("RocksDBManualCompactionService_"),
+        new ThreadPoolExecutor.DiscardOldestPolicy());
 
     static {
         RocksDB.loadLibrary();
     }
 
+    public AbstractRocksDBStorage(String dbPath) {
+        this.dbPath = dbPath;
+    }
+
+    protected void initOptions() {
+        initWriteOptions();
+        initAbleWalWriteOptions();
+        initReadOptions();
+        initTotalOrderReadOptions();
+        initCompactRangeOptions();
+        initCompactionOptions();
+    }
+
+    /**
+     * Write options for <a href="https://github.com/facebook/rocksdb/wiki/Atomic-flush">Atomic Flush</a>
+     */
+    protected void initWriteOptions() {
+        this.writeOptions = new WriteOptions();
+        this.writeOptions.setSync(false);
+        this.writeOptions.setDisableWAL(true);
+        // https://github.com/facebook/rocksdb/wiki/Write-Stalls
+        this.writeOptions.setNoSlowdown(false);
+    }
+
+    protected void initAbleWalWriteOptions() {
+        this.ableWalWriteOptions = new WriteOptions();
+        this.ableWalWriteOptions.setSync(false);
+        this.ableWalWriteOptions.setDisableWAL(false);
+        // https://github.com/facebook/rocksdb/wiki/Write-Stalls
+        this.ableWalWriteOptions.setNoSlowdown(false);
+    }
+
+    protected void initReadOptions() {
+        this.readOptions = new ReadOptions();
+        this.readOptions.setPrefixSameAsStart(true);
+        this.readOptions.setTotalOrderSeek(false);
+        this.readOptions.setTailing(false);
+    }
+
+    protected void initTotalOrderReadOptions() {
+        this.totalOrderReadOptions = new ReadOptions();
+        this.totalOrderReadOptions.setPrefixSameAsStart(false);
+        this.totalOrderReadOptions.setTotalOrderSeek(true);
+        this.totalOrderReadOptions.setTailing(false);
+    }
+
+    protected void initCompactRangeOptions() {
+        this.compactRangeOptions = new CompactRangeOptions();
+        this.compactRangeOptions.setBottommostLevelCompaction(CompactRangeOptions.BottommostLevelCompaction.kForce);
+        this.compactRangeOptions.setAllowWriteStall(true);
+        this.compactRangeOptions.setExclusiveManualCompaction(false);
+        this.compactRangeOptions.setChangeLevel(true);
+        this.compactRangeOptions.setTargetLevel(-1);
+        this.compactRangeOptions.setMaxSubcompactions(4);
+    }
+
+    protected void initCompactionOptions() {
+        this.compactionOptions = new CompactionOptions();
+        this.compactionOptions.setCompression(compressionType);
+        this.compactionOptions.setMaxSubcompactions(4);
+        this.compactionOptions.setOutputFileSizeLimit(4 * 1024 * 1024 * 1024L);
+    }
+
     public boolean hold() {
         if (!this.loaded || this.db == null || this.closed) {
             LOGGER.error("hold rocksdb Failed. {}", this.dbPath);
@@ -103,8 +176,8 @@ public void release() {
     }
 
     protected void put(ColumnFamilyHandle cfHandle, WriteOptions writeOptions,
-                       final byte[] keyBytes, final int keyLen,
-                       final byte[] valueBytes, final int valueLen) throws RocksDBException {
+        final byte[] keyBytes, final int keyLen,
+        final byte[] valueBytes, final int valueLen) throws RocksDBException {
         if (!hold()) {
             throw new IllegalStateException("rocksDB:" + this + " is not ready");
         }
@@ -120,7 +193,7 @@ protected void put(ColumnFamilyHandle cfHandle, WriteOptions writeOptions,
     }
 
     protected void put(ColumnFamilyHandle cfHandle, WriteOptions writeOptions,
-                       final ByteBuffer keyBB, final ByteBuffer valueBB) throws RocksDBException {
+        final ByteBuffer keyBB, final ByteBuffer valueBB) throws RocksDBException {
         if (!hold()) {
             throw new IllegalStateException("rocksDB:" + this + " is not ready");
         }
@@ -161,13 +234,13 @@ protected byte[] get(ColumnFamilyHandle cfHandle, ReadOptions readOptions, byte[
         }
     }
 
-    protected boolean get(ColumnFamilyHandle cfHandle, ReadOptions readOptions,
-                          final ByteBuffer keyBB, final ByteBuffer valueBB) throws RocksDBException {
+    protected int get(ColumnFamilyHandle cfHandle, ReadOptions readOptions, final ByteBuffer keyBB,
+        final ByteBuffer valueBB) throws RocksDBException {
         if (!hold()) {
             throw new IllegalStateException("rocksDB:" + this + " is not ready");
         }
         try {
-            return this.db.get(cfHandle, readOptions, keyBB, valueBB) != NOT_FOUND;
+            return this.db.get(cfHandle, readOptions, keyBB, valueBB);
         } catch (RocksDBException e) {
             LOGGER.error("get Failed. {}, {}", this.dbPath, getStatusError(e));
             throw e;
@@ -177,8 +250,8 @@ protected boolean get(ColumnFamilyHandle cfHandle, ReadOptions readOptions,
     }
 
     protected List<byte[]> multiGet(final ReadOptions readOptions,
-                                    final List<ColumnFamilyHandle> columnFamilyHandleList,
-                                    final List<byte[]> keys) throws RocksDBException {
+        final List<ColumnFamilyHandle> columnFamilyHandleList,
+        final List<byte[]> keys) throws RocksDBException {
         if (!hold()) {
             throw new IllegalStateException("rocksDB:" + this + " is not ready");
         }
@@ -192,7 +265,8 @@ protected List<byte[]> multiGet(final ReadOptions readOptions,
         }
     }
 
-    protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, byte[] keyBytes) throws RocksDBException {
+    protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions,
+        byte[] keyBytes) throws RocksDBException {
         if (!hold()) {
             throw new IllegalStateException("rocksDB:" + this + " is not ready");
         }
@@ -206,7 +280,8 @@ protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, by
         }
     }
 
-    protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, ByteBuffer keyBB) throws RocksDBException {
+    protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, ByteBuffer keyBB)
+        throws RocksDBException {
         if (!hold()) {
             throw new IllegalStateException("rocksDB:" + this + " is not ready");
         }
@@ -220,8 +295,8 @@ protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, By
         }
     }
 
-    protected void rangeDelete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions,
-                               final byte[] startKey, final byte[] endKey) throws RocksDBException {
+    protected void rangeDelete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, final byte[] startKey,
+        final byte[] endKey) throws RocksDBException {
         if (!hold()) {
             throw new IllegalStateException("rocksDB:" + this + " is not ready");
         }
@@ -264,19 +339,21 @@ public void run() {
         });
     }
 
-    protected void open(final List<ColumnFamilyDescriptor> cfDescriptors,
-                        final List<ColumnFamilyHandle> cfHandles) throws RocksDBException {
+    protected void open(final List<ColumnFamilyDescriptor> cfDescriptors) throws RocksDBException {
+        this.cfHandles.clear();
         if (this.readOnly) {
             this.db = RocksDB.openReadOnly(this.options, this.dbPath, cfDescriptors, cfHandles);
         } else {
             this.db = RocksDB.open(this.options, this.dbPath, cfDescriptors, cfHandles);
         }
-        this.db.getEnv().setBackgroundThreads(8, Priority.HIGH);
-        this.db.getEnv().setBackgroundThreads(8, Priority.LOW);
+        assert cfDescriptors.size() == cfHandles.size();
 
         if (this.db == null) {
             throw new RocksDBException("open rocksdb null");
         }
+        try (Env env = this.db.getEnv()) {
+            env.setBackgroundThreads(8, Priority.LOW);
+        }
     }
 
     protected abstract boolean postLoad();
@@ -287,7 +364,7 @@ public synchronized boolean start() {
         }
         if (postLoad()) {
             this.loaded = true;
-            LOGGER.info("start OK. {}", this.dbPath);
+            LOGGER.info("RocksDB [{}] starts OK", this.dbPath);
             this.closed = false;
             return true;
         } else {
@@ -295,11 +372,15 @@ public synchronized boolean start() {
         }
     }
 
+    /**
+     * Close column family handles except the default column family
+     */
     protected abstract void preShutdown();
 
     public synchronized boolean shutdown() {
         try {
             if (!this.loaded) {
+                LOGGER.info("RocksDBStorage is not loaded, shutdown OK. dbPath={}, readOnly={}", this.dbPath, this.readOnly);
                 return true;
             }
 
@@ -312,11 +393,12 @@ public synchronized boolean shutdown() {
             }
             this.db.cancelAllBackgroundWork(true);
             this.db.pauseBackgroundWork();
-            //The close order is matter.
+            //The close order matters.
             //1. close column family handles
             preShutdown();
 
             this.defaultCFHandle.close();
+
             //2. close column family options.
             for (final ColumnFamilyOptions opt : this.cfOptions) {
                 opt.close();
@@ -334,9 +416,6 @@ public synchronized boolean shutdown() {
             if (this.totalOrderReadOptions != null) {
                 this.totalOrderReadOptions.close();
             }
-            if (this.options != null) {
-                this.options.close();
-            }
             //4. close db.
             if (db != null && !this.readOnly) {
                 this.db.syncWal();
@@ -344,6 +423,10 @@ public synchronized boolean shutdown() {
             if (db != null) {
                 this.db.closeE();
             }
+            // Close DBOptions after RocksDB instance is closed.
+            if (this.options != null) {
+                this.options.close();
+            }
             //5. help gc.
             this.cfOptions.clear();
             this.db = null;
@@ -354,29 +437,41 @@ public synchronized boolean shutdown() {
             this.options = null;
 
             this.loaded = false;
-            LOGGER.info("shutdown OK. {}", this.dbPath);
+            LOGGER.info("RocksDB shutdown OK. {}", this.dbPath);
         } catch (Exception e) {
-            LOGGER.error("shutdown Failed. {}", this.dbPath, e);
+            LOGGER.error("RocksDB shutdown failed. {}", this.dbPath, e);
             return false;
         }
         return true;
     }
 
-    public void flush(final FlushOptions flushOptions) {
+    public void flush(final FlushOptions flushOptions) throws RocksDBException {
+        flush(flushOptions, this.cfHandles);
+    }
+
+    public void flush(final FlushOptions flushOptions, List<ColumnFamilyHandle> columnFamilyHandles) throws RocksDBException {
         if (!this.loaded || this.readOnly || closed) {
             return;
         }
 
         try {
             if (db != null) {
-                this.db.flush(flushOptions);
+                // For atomic-flush, we have to explicitly specify column family handles
+                // See https://github.com/rust-rocksdb/rust-rocksdb/pull/793
+                // and https://github.com/facebook/rocksdb/blob/8ad4c7efc48d301f5e85467105d7019a49984dc8/include/rocksdb/db.h#L1667
+                this.db.flush(flushOptions, columnFamilyHandles);
             }
         } catch (RocksDBException e) {
             scheduleReloadRocksdb(e);
             LOGGER.error("flush Failed. {}, {}", this.dbPath, getStatusError(e));
+            throw e;
         }
     }
 
+    public void flushWAL() throws RocksDBException {
+        this.db.flushWal(true);
+    }
+
     public Statistics getStatistics() {
         return this.options.statistics();
     }
@@ -443,10 +538,6 @@ private void reloadRocksdb() throws Exception {
         LOGGER.info("reload rocksdb OK. {}", this.dbPath);
     }
 
-    public void flushWAL() throws RocksDBException {
-        this.db.flushWal(true);
-    }
-
     private String getStatusError(RocksDBException e) {
         if (e == null || e.getStatus() == null) {
             return "null";
@@ -471,37 +562,32 @@ private String getStatusError(RocksDBException e) {
 
     public void statRocksdb(Logger logger) {
         try {
+            // Log Memory Usage
+            String blockCacheMemUsage = this.db.getProperty("rocksdb.block-cache-usage");
+            String indexesAndFilterBlockMemUsage = this.db.getProperty("rocksdb.estimate-table-readers-mem");
+            String memTableMemUsage = this.db.getProperty("rocksdb.cur-size-all-mem-tables");
+            String blocksPinnedByIteratorMemUsage = this.db.getProperty("rocksdb.block-cache-pinned-usage");
+            logger.info("RocksDB Memory Usage: BlockCache: {}, IndexesAndFilterBlock: {}, MemTable: {}, BlocksPinnedByIterator: {}",
+                blockCacheMemUsage, indexesAndFilterBlockMemUsage, memTableMemUsage, blocksPinnedByIteratorMemUsage);
 
+            // Log file metadata by level
             List<LiveFileMetaData> liveFileMetaDataList = this.getCompactionStatus();
             if (liveFileMetaDataList == null || liveFileMetaDataList.isEmpty()) {
                 return;
             }
             Map<Integer, StringBuilder> map = Maps.newHashMap();
             for (LiveFileMetaData metaData : liveFileMetaDataList) {
-                StringBuilder sb = map.get(metaData.level());
-                if (sb == null) {
-                    sb = new StringBuilder(256);
-                    map.put(metaData.level(), sb);
-                }
-                sb.append(new String(metaData.columnFamilyName(), DataConverter.CHARSET_UTF8)).append(SPACE).
-                        append(metaData.fileName()).append(SPACE).
-                        append("s: ").append(metaData.size()).append(SPACE).
-                        append("a: ").append(metaData.numEntries()).append(SPACE).
-                        append("r: ").append(metaData.numReadsSampled()).append(SPACE).
-                        append("d: ").append(metaData.numDeletions()).append(SPACE).
-                        append(metaData.beingCompacted()).append("\n");
+                StringBuilder sb = map.computeIfAbsent(metaData.level(), k -> new StringBuilder(256));
+                sb.append(new String(metaData.columnFamilyName(), StandardCharsets.UTF_8)).append(SPACE).
+                    append(metaData.fileName()).append(SPACE).
+                    append("file-size: ").append(metaData.size()).append(SPACE).
+                    append("number-of-entries: ").append(metaData.numEntries()).append(SPACE).
+                    append("file-read-times: ").append(metaData.numReadsSampled()).append(SPACE).
+                    append("deletions: ").append(metaData.numDeletions()).append(SPACE).
+                    append("being-compacted: ").append(metaData.beingCompacted()).append("\n");
             }
-            for (Map.Entry<Integer, StringBuilder> entry : map.entrySet()) {
-                logger.info("level: {}\n{}", entry.getKey(), entry.getValue().toString());
-            }
-
-            String blockCacheMemUsage = this.db.getProperty("rocksdb.block-cache-usage");
-            String indexesAndFilterBlockMemUsage = this.db.getProperty("rocksdb.estimate-table-readers-mem");
-            String memTableMemUsage = this.db.getProperty("rocksdb.cur-size-all-mem-tables");
-            String blocksPinnedByIteratorMemUsage = this.db.getProperty("rocksdb.block-cache-pinned-usage");
-            logger.info("MemUsage. blockCache: {}, indexesAndFilterBlock: {}, memtable: {}, blocksPinnedByIterator: {}",
-                    blockCacheMemUsage, indexesAndFilterBlockMemUsage, memTableMemUsage, blocksPinnedByIteratorMemUsage);
+            map.forEach((key, value) -> logger.info("level: {}\n{}", key, value.toString()));
         } catch (Exception ignored) {
         }
     }
-}
\ No newline at end of file
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java
new file mode 100644
index 00000000000..e3f6f22002e
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.config;
+
+import com.google.common.base.Strings;
+import java.io.File;
+import org.apache.rocketmq.common.UtilAll;
+import org.rocksdb.BlockBasedTableConfig;
+import org.rocksdb.BloomFilter;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.CompactionStyle;
+import org.rocksdb.CompressionType;
+import org.rocksdb.DBOptions;
+import org.rocksdb.DataBlockIndexType;
+import org.rocksdb.IndexType;
+import org.rocksdb.InfoLogLevel;
+import org.rocksdb.LRUCache;
+import org.rocksdb.RateLimiter;
+import org.rocksdb.SkipListMemTableConfig;
+import org.rocksdb.Statistics;
+import org.rocksdb.StatsLevel;
+import org.rocksdb.StringAppendOperator;
+import org.rocksdb.WALRecoveryMode;
+import org.rocksdb.util.SizeUnit;
+
+public class ConfigHelper {
+    public static ColumnFamilyOptions createConfigColumnFamilyOptions() {
+        BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig().
+            setFormatVersion(5).
+            setIndexType(IndexType.kBinarySearch).
+            setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch).
+            setBlockSize(32 * SizeUnit.KB).
+            setFilterPolicy(new BloomFilter(16, false)).
+            // Indicating if we'd put index/filter blocks to the block cache.
+            setCacheIndexAndFilterBlocks(true).
+            setCacheIndexAndFilterBlocksWithHighPriority(true).
+            setPinL0FilterAndIndexBlocksInCache(false).
+            setPinTopLevelIndexAndFilter(true).
+            setBlockCache(new LRUCache(4 * SizeUnit.MB, 8, false)).
+            setWholeKeyFiltering(true);
+
+        ColumnFamilyOptions options = new ColumnFamilyOptions();
+        return options.setMaxWriteBufferNumber(4).
+            setWriteBufferSize(64 * SizeUnit.MB).
+            setMinWriteBufferNumberToMerge(1).
+            setTableFormatConfig(blockBasedTableConfig).
+            setMemTableConfig(new SkipListMemTableConfig()).
+            setCompressionType(CompressionType.NO_COMPRESSION).
+            setNumLevels(7).
+            setCompactionStyle(CompactionStyle.LEVEL).
+            setLevel0FileNumCompactionTrigger(4).
+            setLevel0SlowdownWritesTrigger(8).
+            setLevel0StopWritesTrigger(12).
+            // The target file size for compaction.
+            setTargetFileSizeBase(64 * SizeUnit.MB).
+            setTargetFileSizeMultiplier(2).
+            // The upper-bound of the total size of L1 files in bytes
+            setMaxBytesForLevelBase(256 * SizeUnit.MB).
+            setMaxBytesForLevelMultiplier(2).
+            setMergeOperator(new StringAppendOperator()).
+            setInplaceUpdateSupport(true);
+    }
+
+    public static DBOptions createConfigDBOptions() {
+        // Tune based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide
+        // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java
+        DBOptions options = new DBOptions();
+        Statistics statistics = new Statistics();
+        statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS);
+        return options.
+            setDbLogDir(getDBLogDir()).
+            setInfoLogLevel(InfoLogLevel.INFO_LEVEL).
+            setWalRecoveryMode(WALRecoveryMode.SkipAnyCorruptedRecords).
+            /*
+             * We use manual flush to achieve desired balance between reliability and performance:
+             * for metadata that matters, including {topic, subscription}-config changes, each write incurs a
+             * flush-and-sync to ensure reliability; for {commit, pull}-offset advancements, group-flush are offered for
+             * every N(configurable, 1024 by default) writes or aging of writes, similar to OS page-cache flush
+             * mechanism.
+             */
+            setManualWalFlush(true).
+            // This option takes effect only when we have multiple column families
+            // https://github.com/facebook/rocksdb/issues/4180
+            // setMaxTotalWalSize(1024 * SizeUnit.MB).
+            setDbWriteBufferSize(128 * SizeUnit.MB).
+            setBytesPerSync(SizeUnit.MB).
+            setWalBytesPerSync(SizeUnit.MB).
+            setCreateIfMissing(true).
+            setCreateMissingColumnFamilies(true).
+            setMaxOpenFiles(-1).
+            setMaxLogFileSize(SizeUnit.GB).
+            setKeepLogFileNum(5).
+            setMaxManifestFileSize(SizeUnit.GB).
+            setAllowConcurrentMemtableWrite(false).
+            setStatistics(statistics).
+            setStatsDumpPeriodSec(600).
+            setMaxBackgroundJobs(32).
+            setMaxSubcompactions(4).
+            setParanoidChecks(true).
+            setDelayedWriteRate(16 * SizeUnit.MB).
+            setRateLimiter(new RateLimiter(100 * SizeUnit.MB)).
+            setUseDirectIoForFlushAndCompaction(true).
+            setUseDirectReads(true);
+    }
+
+    public static String getDBLogDir() {
+        String[] rootPaths = new String[] {
+            System.getProperty("user.home"),
+            System.getProperty("java.io.tmpdir"),
+            File.separator + "data"
+        };
+        for (String rootPath : rootPaths) {
+            // Refer bazel test encyclopedia: https://bazel.build/reference/test-encyclopedia
+            // Not all directories is available
+            if (Strings.isNullOrEmpty(rootPath)) {
+                continue;
+            }
+            File rootPathFile = new File(rootPath);
+            if (!rootPathFile.exists() || !rootPathFile.canWrite()) {
+                continue;
+            }
+            String logDirectory = rootPath + File.separator + "logs" + File.separator + "rocketmqlogs";
+            // Create directories recursively.
+            UtilAll.ensureDirOK(logDirectory);
+            return logDirectory;
+        }
+        throw new RuntimeException("Failed to get log directory");
+    }
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigManagerVersion.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigManagerVersion.java
new file mode 100644
index 00000000000..0d5dd6940a1
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigManagerVersion.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.config;
+
+public enum ConfigManagerVersion {
+    V1("v1"),
+    V2("v2"),
+    ;
+    private final String version;
+
+    ConfigManagerVersion(String version) {
+        this.version = version;
+    }
+
+    public String getVersion() {
+        return version;
+    }
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java
index b40f8046e84..5fd9bab2d77 100644
--- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java
+++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java
@@ -16,91 +16,49 @@
  */
 package org.apache.rocketmq.common.config;
 
-import java.io.File;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.UtilAll;
-import org.rocksdb.BlockBasedTableConfig;
-import org.rocksdb.BloomFilter;
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.ColumnFamilyOptions;
-import org.rocksdb.CompactRangeOptions;
-import org.rocksdb.CompactRangeOptions.BottommostLevelCompaction;
-import org.rocksdb.CompactionOptions;
-import org.rocksdb.CompactionStyle;
 import org.rocksdb.CompressionType;
-import org.rocksdb.DBOptions;
-import org.rocksdb.DataBlockIndexType;
-import org.rocksdb.IndexType;
-import org.rocksdb.InfoLogLevel;
-import org.rocksdb.LRUCache;
-import org.rocksdb.RateLimiter;
 import org.rocksdb.ReadOptions;
 import org.rocksdb.RocksDB;
 import org.rocksdb.RocksDBException;
 import org.rocksdb.RocksIterator;
-import org.rocksdb.SkipListMemTableConfig;
-import org.rocksdb.Statistics;
-import org.rocksdb.StatsLevel;
-import org.rocksdb.StringAppendOperator;
-import org.rocksdb.WALRecoveryMode;
 import org.rocksdb.WriteBatch;
-import org.rocksdb.WriteOptions;
-import org.rocksdb.util.SizeUnit;
 
 public class ConfigRocksDBStorage extends AbstractRocksDBStorage {
+    public static final byte[] KV_DATA_VERSION_COLUMN_FAMILY_NAME = "kvDataVersion".getBytes(StandardCharsets.UTF_8);
+    public static final byte[] FORBIDDEN_COLUMN_FAMILY_NAME = "forbidden".getBytes(StandardCharsets.UTF_8);
+
+    protected ColumnFamilyHandle kvDataVersionFamilyHandle;
+    protected ColumnFamilyHandle forbiddenFamilyHandle;
+    public static final byte[] KV_DATA_VERSION_KEY = "kvDataVersionKey".getBytes(StandardCharsets.UTF_8);
+
+
 
     public ConfigRocksDBStorage(final String dbPath) {
-        super();
-        this.dbPath = dbPath;
-        this.readOnly = false;
+        this(dbPath, false);
+    }
+
+    public ConfigRocksDBStorage(final String dbPath, CompressionType compressionType) {
+        this(dbPath, false);
+        this.compressionType = compressionType;
     }
 
     public ConfigRocksDBStorage(final String dbPath, boolean readOnly) {
-        super();
-        this.dbPath = dbPath;
+        super(dbPath);
         this.readOnly = readOnly;
     }
 
-    private void initOptions() {
-        this.options = createConfigDBOptions();
-
-        this.writeOptions = new WriteOptions();
-        this.writeOptions.setSync(false);
-        this.writeOptions.setDisableWAL(true);
-        this.writeOptions.setNoSlowdown(true);
-
-        this.ableWalWriteOptions = new WriteOptions();
-        this.ableWalWriteOptions.setSync(false);
-        this.ableWalWriteOptions.setDisableWAL(false);
-        this.ableWalWriteOptions.setNoSlowdown(true);
-
-        this.readOptions = new ReadOptions();
-        this.readOptions.setPrefixSameAsStart(true);
-        this.readOptions.setTotalOrderSeek(false);
-        this.readOptions.setTailing(false);
-
-        this.totalOrderReadOptions = new ReadOptions();
-        this.totalOrderReadOptions.setPrefixSameAsStart(false);
-        this.totalOrderReadOptions.setTotalOrderSeek(false);
-        this.totalOrderReadOptions.setTailing(false);
-
-        this.compactRangeOptions = new CompactRangeOptions();
-        this.compactRangeOptions.setBottommostLevelCompaction(BottommostLevelCompaction.kForce);
-        this.compactRangeOptions.setAllowWriteStall(true);
-        this.compactRangeOptions.setExclusiveManualCompaction(false);
-        this.compactRangeOptions.setChangeLevel(true);
-        this.compactRangeOptions.setTargetLevel(-1);
-        this.compactRangeOptions.setMaxSubcompactions(4);
-
-        this.compactionOptions = new CompactionOptions();
-        this.compactionOptions.setCompression(CompressionType.LZ4_COMPRESSION);
-        this.compactionOptions.setMaxSubcompactions(4);
-        this.compactionOptions.setOutputFileSizeLimit(4 * 1024 * 1024 * 1024L);
+    protected void initOptions() {
+        this.options = ConfigHelper.createConfigDBOptions();
+        super.initOptions();
     }
 
     @Override
@@ -110,16 +68,19 @@ protected boolean postLoad() {
 
             initOptions();
 
-            final List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList();
+            final List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList<>();
 
-            ColumnFamilyOptions defaultOptions = createConfigOptions();
+            ColumnFamilyOptions defaultOptions = ConfigHelper.createConfigColumnFamilyOptions();
             this.cfOptions.add(defaultOptions);
             cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions));
-
-            final List<ColumnFamilyHandle> cfHandles = new ArrayList();
-            open(cfDescriptors, cfHandles);
+            cfDescriptors.add(new ColumnFamilyDescriptor(KV_DATA_VERSION_COLUMN_FAMILY_NAME, defaultOptions));
+            cfDescriptors.add(new ColumnFamilyDescriptor(FORBIDDEN_COLUMN_FAMILY_NAME, defaultOptions));
+            open(cfDescriptors);
 
             this.defaultCFHandle = cfHandles.get(0);
+            this.kvDataVersionFamilyHandle = cfHandles.get(1);
+            this.forbiddenFamilyHandle = cfHandles.get(2);
+
         } catch (final Exception e) {
             AbstractRocksDBStorage.LOGGER.error("postLoad Failed. {}", this.dbPath, e);
             return false;
@@ -129,100 +90,36 @@ protected boolean postLoad() {
 
     @Override
     protected void preShutdown() {
-
+        this.kvDataVersionFamilyHandle.close();
+        this.forbiddenFamilyHandle.close();
     }
 
-    private ColumnFamilyOptions createConfigOptions() {
-        BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig().
-            setFormatVersion(5).
-            setIndexType(IndexType.kBinarySearch).
-            setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch).
-            setBlockSize(32 * SizeUnit.KB).
-            setFilterPolicy(new BloomFilter(16, false)).
-            // Indicating if we'd put index/filter blocks to the block cache.
-            setCacheIndexAndFilterBlocks(false).
-            setCacheIndexAndFilterBlocksWithHighPriority(true).
-            setPinL0FilterAndIndexBlocksInCache(false).
-            setPinTopLevelIndexAndFilter(true).
-            setBlockCache(new LRUCache(4 * SizeUnit.MB, 8, false)).
-            setWholeKeyFiltering(true);
+    public void put(final byte[] keyBytes, final int keyLen, final byte[] valueBytes) throws Exception {
+        put(this.defaultCFHandle, this.ableWalWriteOptions, keyBytes, keyLen, valueBytes, valueBytes.length);
+    }
 
-        ColumnFamilyOptions options = new ColumnFamilyOptions();
-        return options.setMaxWriteBufferNumber(2).
-            // MemTable size, memtable(cache) -> immutable memtable(cache) -> sst(disk)
-            setWriteBufferSize(8 * SizeUnit.MB).
-            setMinWriteBufferNumberToMerge(1).
-            setTableFormatConfig(blockBasedTableConfig).
-            setMemTableConfig(new SkipListMemTableConfig()).
-            setCompressionType(CompressionType.NO_COMPRESSION).
-            setNumLevels(7).
-            setCompactionStyle(CompactionStyle.LEVEL).
-            setLevel0FileNumCompactionTrigger(4).
-            setLevel0SlowdownWritesTrigger(8).
-            setLevel0StopWritesTrigger(12).
-            // The target file size for compaction.
-            setTargetFileSizeBase(64 * SizeUnit.MB).
-            setTargetFileSizeMultiplier(2).
-            // The upper-bound of the total size of L1 files in bytes
-            setMaxBytesForLevelBase(256 * SizeUnit.MB).
-            setMaxBytesForLevelMultiplier(2).
-            setMergeOperator(new StringAppendOperator()).
-            setInplaceUpdateSupport(true);
+    public void put(final ByteBuffer keyBB, final ByteBuffer valueBB) throws Exception {
+        put(this.defaultCFHandle, this.ableWalWriteOptions, keyBB, valueBB);
     }
 
-    private DBOptions createConfigDBOptions() {
-        //Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide
-        // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java
-        DBOptions options = new DBOptions();
-        Statistics statistics = new Statistics();
-        statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS);
-        return options.
-            setDbLogDir(getDBLogDir()).
-            setInfoLogLevel(InfoLogLevel.INFO_LEVEL).
-            setWalRecoveryMode(WALRecoveryMode.SkipAnyCorruptedRecords).
-            setManualWalFlush(true).
-            setMaxTotalWalSize(500 * SizeUnit.MB).
-            setWalSizeLimitMB(0).
-            setWalTtlSeconds(0).
-            setCreateIfMissing(true).
-            setCreateMissingColumnFamilies(true).
-            setMaxOpenFiles(-1).
-            setMaxLogFileSize(1 * SizeUnit.GB).
-            setKeepLogFileNum(5).
-            setMaxManifestFileSize(1 * SizeUnit.GB).
-            setAllowConcurrentMemtableWrite(false).
-            setStatistics(statistics).
-            setStatsDumpPeriodSec(600).
-            setAtomicFlush(true).
-            setMaxBackgroundJobs(32).
-            setMaxSubcompactions(4).
-            setParanoidChecks(true).
-            setDelayedWriteRate(16 * SizeUnit.MB).
-            setRateLimiter(new RateLimiter(100 * SizeUnit.MB)).
-            setUseDirectIoForFlushAndCompaction(true).
-            setUseDirectReads(true);
+    public byte[] get(final byte[] keyBytes) throws Exception {
+        return get(this.defaultCFHandle, this.totalOrderReadOptions, keyBytes);
     }
 
-    public static String getDBLogDir() {
-        String rootPath = System.getProperty("user.home");
-        if (StringUtils.isEmpty(rootPath)) {
-            return "";
-        }
-        rootPath = rootPath + File.separator + "logs";
-        UtilAll.ensureDirOK(rootPath);
-        return rootPath + File.separator + "rocketmqlogs" + File.separator;
+    public void updateKvDataVersion(final byte[] valueBytes) throws Exception {
+        put(this.kvDataVersionFamilyHandle, this.ableWalWriteOptions, KV_DATA_VERSION_KEY, KV_DATA_VERSION_KEY.length, valueBytes, valueBytes.length);
     }
 
-    public void put(final byte[] keyBytes, final int keyLen, final byte[] valueBytes) throws Exception {
-        put(this.defaultCFHandle, this.ableWalWriteOptions, keyBytes, keyLen, valueBytes, valueBytes.length);
+    public byte[] getKvDataVersion() throws Exception {
+        return get(this.kvDataVersionFamilyHandle, this.totalOrderReadOptions, KV_DATA_VERSION_KEY);
     }
 
-    public void put(final ByteBuffer keyBB, final ByteBuffer valueBB) throws Exception {
-        put(this.defaultCFHandle, this.ableWalWriteOptions, keyBB, valueBB);
+    public void updateForbidden(final byte[] keyBytes, final byte[] valueBytes) throws Exception {
+        put(this.forbiddenFamilyHandle, this.ableWalWriteOptions, keyBytes, keyBytes.length, valueBytes, valueBytes.length);
     }
 
-    public byte[] get(final byte[] keyBytes) throws Exception {
-        return get(this.defaultCFHandle, this.totalOrderReadOptions, keyBytes);
+    public byte[] getForbidden(final byte[] keyBytes) throws Exception {
+        return get(this.forbiddenFamilyHandle, this.totalOrderReadOptions, keyBytes);
     }
 
     public void delete(final byte[] keyBytes) throws Exception {
@@ -246,8 +143,8 @@ public RocksIterator iterator() {
         return this.db.newIterator(this.defaultCFHandle, this.totalOrderReadOptions);
     }
 
-    public void rangeDelete(final byte[] startKey, final byte[] endKey) throws RocksDBException {
-        rangeDelete(this.defaultCFHandle, this.writeOptions, startKey, endKey);
+    public RocksIterator forbiddenIterator() {
+        return this.db.newIterator(this.forbiddenFamilyHandle, this.totalOrderReadOptions);
     }
 
     public RocksIterator iterator(ReadOptions readOptions) {
diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/CommonConstants.java b/common/src/main/java/org/apache/rocketmq/common/constant/CommonConstants.java
new file mode 100644
index 00000000000..9fabcb36583
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/constant/CommonConstants.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.constant;
+
+public class CommonConstants {
+
+    public static final String COLON = ":";
+
+    public static final String ASTERISK = "*";
+
+    public static final String COMMA = ",";
+
+    public static final String EQUAL = "=";
+
+    public static final String SLASH = "/";
+
+    public static final String SPACE = " ";
+
+    public static final String HYPHEN = "-";
+
+    public static final String POUND = "#";
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/InterceptorConstants.java b/common/src/main/java/org/apache/rocketmq/common/constant/GrpcConstants.java
similarity index 93%
rename from proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/InterceptorConstants.java
rename to common/src/main/java/org/apache/rocketmq/common/constant/GrpcConstants.java
index 768f3d96abc..96293e72ecf 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/InterceptorConstants.java
+++ b/common/src/main/java/org/apache/rocketmq/common/constant/GrpcConstants.java
@@ -15,12 +15,12 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.proxy.grpc.interceptor;
+package org.apache.rocketmq.common.constant;
 
 import io.grpc.Context;
 import io.grpc.Metadata;
 
-public class InterceptorConstants {
+public class GrpcConstants {
     public static final Context.Key<Metadata> METADATA = Context.key("rpc-metadata");
 
     /**
@@ -70,4 +70,7 @@ public class InterceptorConstants {
 
     public static final Metadata.Key<String> AUTHORIZATION_AK
         = Metadata.Key.of("x-mq-authorization-ak", Metadata.ASCII_STRING_MARSHALLER);
+
+    public static final Metadata.Key<String> CHANNEL_ID
+        = Metadata.Key.of("x-mq-channel-id", Metadata.ASCII_STRING_MARSHALLER);
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/HAProxyConstants.java b/common/src/main/java/org/apache/rocketmq/common/constant/HAProxyConstants.java
index c1ae0cca184..cd61cea0c16 100644
--- a/common/src/main/java/org/apache/rocketmq/common/constant/HAProxyConstants.java
+++ b/common/src/main/java/org/apache/rocketmq/common/constant/HAProxyConstants.java
@@ -19,6 +19,7 @@
 
 public class HAProxyConstants {
 
+    public static final String CHANNEL_ID = "channel_id";
     public static final String PROXY_PROTOCOL_PREFIX = "proxy_protocol_";
     public static final String PROXY_PROTOCOL_ADDR = PROXY_PROTOCOL_PREFIX + "addr";
     public static final String PROXY_PROTOCOL_PORT = PROXY_PROTOCOL_PREFIX + "port";
diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
index 61310893f43..4a8d307987b 100644
--- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
+++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java
@@ -53,4 +53,6 @@ public class LoggerName {
     public static final String PROXY_WATER_MARK_LOGGER_NAME = "RocketmqProxyWatermark";
     public static final String ROCKETMQ_COLDCTR_LOGGER_NAME = "RocketmqColdCtr";
     public static final String ROCKSDB_LOGGER_NAME = "RocketmqRocksDB";
+
+    public static final String ROCKETMQ_AUTH_AUDIT_LOGGER_NAME = "RocketmqAuthAudit";
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java b/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java
index d87461d7f5a..d9a26a2be17 100644
--- a/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java
+++ b/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java
@@ -68,4 +68,8 @@ public static boolean isValid(final int perm) {
     public static boolean isPriority(final int perm) {
         return (perm & PERM_PRIORITY) == PERM_PRIORITY;
     }
+
+    public static boolean isAccessible(final int perm) {
+        return isReadable(perm) || isWriteable(perm);
+    }
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/message/Message.java b/common/src/main/java/org/apache/rocketmq/common/message/Message.java
index c7997c47318..acd4df96d28 100644
--- a/common/src/main/java/org/apache/rocketmq/common/message/Message.java
+++ b/common/src/main/java/org/apache/rocketmq/common/message/Message.java
@@ -108,6 +108,13 @@ public String getProperty(final String name) {
         return this.properties.get(name);
     }
 
+    public boolean hasProperty(final String name) {
+        if (null == this.properties) {
+            return false;
+        }
+        return this.properties.containsKey(name);
+    }
+
     public String getTopic() {
         return topic;
     }
diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
index b053f827597..713f9405ea9 100644
--- a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
+++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java
@@ -516,13 +516,15 @@ public static MessageExt decode(
                         }
                     }
 
-                    // uncompress body
+                    // inflate body
                     if (deCompressBody && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) {
                         Compressor compressor = CompressorFactory.getCompressor(MessageSysFlag.getCompressionType(sysFlag));
                         body = compressor.decompress(body);
+                        sysFlag &= ~MessageSysFlag.COMPRESSED_FLAG;
                     }
 
                     msgExt.setBody(body);
+                    msgExt.setSysFlag(sysFlag);
                 } else {
                     byteBuffer.position(byteBuffer.position() + bodyLen);
                 }
@@ -666,7 +668,6 @@ public static byte[] encodeMessage(Message message) {
         byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8);
         //note properties length must not more than Short.MAX
         short propertiesLength = (short) propertiesBytes.length;
-        int sysFlag = message.getFlag();
         int storeSize = 4 // 1 TOTALSIZE
             + 4 // 2 MAGICCOD
             + 4 // 3 BODYCRC
diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java
index 147f23f1234..e4f02e2c9b4 100644
--- a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java
+++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java
@@ -16,8 +16,11 @@
  */
 package org.apache.rocketmq.common.message;
 
+import com.google.common.base.Strings;
 import java.nio.ByteBuffer;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.TopicFilterType;
 import org.apache.rocketmq.common.utils.MessageUtils;
 
@@ -41,7 +44,7 @@ public void setEncodedBuff(ByteBuffer encodedBuff) {
     }
 
     public static long tagsString2tagsCode(final TopicFilterType filter, final String tags) {
-        if (null == tags || tags.length() == 0) { return 0; }
+        if (Strings.isNullOrEmpty(tags)) { return 0; }
 
         return tags.hashCode();
     }
@@ -102,4 +105,9 @@ public boolean isEncodeCompleted() {
     public void setEncodeCompleted(boolean encodeCompleted) {
         this.encodeCompleted = encodeCompleted;
     }
+
+    public boolean needDispatchLMQ() {
+        return StringUtils.isNoneBlank(getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))
+            && MixAll.topicAllowsLMQ(getTopic());
+    }
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/namesrv/DefaultTopAddressing.java b/common/src/main/java/org/apache/rocketmq/common/namesrv/DefaultTopAddressing.java
index 179e200ae91..0636e30564a 100644
--- a/common/src/main/java/org/apache/rocketmq/common/namesrv/DefaultTopAddressing.java
+++ b/common/src/main/java/org/apache/rocketmq/common/namesrv/DefaultTopAddressing.java
@@ -107,27 +107,27 @@ public void registerChangeCallBack(NameServerUpdateCallback changeCallBack) {
     }
 
     public final String fetchNSAddr(boolean verbose, long timeoutMills) {
-        String url = this.wsAddr;
+        StringBuilder url = new StringBuilder(this.wsAddr);
         try {
             if (null != para && para.size() > 0) {
                 if (!UtilAll.isBlank(this.unitName)) {
-                    url = url + "-" + this.unitName + "?nofix=1&";
+                    url.append("-").append(this.unitName).append("?nofix=1&");
                 }
                 else {
-                    url = url + "?";
+                    url.append("?");
                 }
                 for (Map.Entry<String, String> entry : this.para.entrySet()) {
-                    url += entry.getKey() + "=" + entry.getValue() + "&";
+                    url.append(entry.getKey()).append("=").append(entry.getValue()).append("&");
                 }
-                url = url.substring(0, url.length() - 1);
+                url = new StringBuilder(url.substring(0, url.length() - 1));
             }
             else {
                 if (!UtilAll.isBlank(this.unitName)) {
-                    url = url + "-" + this.unitName + "?nofix=1";
+                    url.append("-").append(this.unitName).append("?nofix=1");
                 }
             }
 
-            HttpTinyClient.HttpResult result = HttpTinyClient.httpGet(url, null, null, "UTF-8", timeoutMills);
+            HttpTinyClient.HttpResult result = HttpTinyClient.httpGet(url.toString(), null, null, "UTF-8", timeoutMills);
             if (200 == result.code) {
                 String responseStr = result.content;
                 if (responseStr != null) {
diff --git a/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java b/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java
new file mode 100644
index 00000000000..b00b15bd863
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.producer;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Base64;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * handle to recall a message, only support delay message for now
+ * v1 pattern like this:
+ * version topic brokerName timestamp messageId
+ * use Base64 to encode it
+ */
+public class RecallMessageHandle {
+    private static final String SEPARATOR = " ";
+    private static final String VERSION_1 = "v1";
+
+    public static class HandleV1 extends RecallMessageHandle {
+        private String version;
+        private String topic;
+        private String brokerName;
+        private String timestampStr;
+        private String messageId; // id of unique key
+
+        public HandleV1(String topic, String brokerName, String timestamp, String messageId) {
+            this.version = VERSION_1;
+            this.topic = topic;
+            this.brokerName = brokerName;
+            this.timestampStr = timestamp;
+            this.messageId = messageId;
+        }
+
+        // no param check
+        public static String buildHandle(String topic, String brokerName, String timestampStr, String messageId) {
+            String rawString = String.join(SEPARATOR, VERSION_1, topic, brokerName, timestampStr, messageId);
+            return Base64.getUrlEncoder().encodeToString(rawString.getBytes(UTF_8));
+        }
+
+        public String getTopic() {
+            return topic;
+        }
+
+        public String getBrokerName() {
+            return brokerName;
+        }
+
+        public String getTimestampStr() {
+            return timestampStr;
+        }
+
+        public String getMessageId() {
+            return messageId;
+        }
+
+        public String getVersion() {
+            return version;
+        }
+    }
+
+    public static RecallMessageHandle decodeHandle(String handle) throws DecoderException {
+        if (StringUtils.isEmpty(handle)) {
+            throw new DecoderException("recall handle is invalid");
+        }
+        String rawString;
+        try {
+            rawString =
+                new String(Base64.getUrlDecoder().decode(handle.getBytes(UTF_8)), UTF_8);
+        } catch (IllegalArgumentException e) {
+            throw new DecoderException("recall handle is invalid");
+        }
+        String[] items = rawString.split(SEPARATOR);
+        if (!VERSION_1.equals(items[0]) || items.length < 5) {
+            throw new DecoderException("recall handle is invalid");
+        }
+        return new HandleV1(items[1], items[2], items[3], items[4]);
+    }
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/resource/ResourcePattern.java b/common/src/main/java/org/apache/rocketmq/common/resource/ResourcePattern.java
new file mode 100644
index 00000000000..6fac4b74785
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/resource/ResourcePattern.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.resource;
+
+import com.alibaba.fastjson2.annotation.JSONField;
+
+public enum ResourcePattern {
+
+    ANY((byte) 1, "ANY"),
+
+    LITERAL((byte) 2, "LITERAL"),
+
+    PREFIXED((byte) 3, "PREFIXED");
+
+    @JSONField(value = true)
+    private final byte code;
+    private final String name;
+
+    ResourcePattern(byte code, String name) {
+        this.code = code;
+        this.name = name;
+    }
+
+    public byte getCode() {
+        return code;
+    }
+
+    public String getName() {
+        return name;
+    }
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/resource/ResourceType.java b/common/src/main/java/org/apache/rocketmq/common/resource/ResourceType.java
new file mode 100644
index 00000000000..479b8e59b65
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/resource/ResourceType.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.resource;
+
+import com.alibaba.fastjson2.annotation.JSONField;
+import org.apache.commons.lang3.StringUtils;
+
+public enum ResourceType {
+
+    UNKNOWN((byte) 0, "Unknown"),
+
+    ANY((byte) 1, "Any"),
+
+    CLUSTER((byte) 2, "Cluster"),
+
+    NAMESPACE((byte) 3, "Namespace"),
+
+    TOPIC((byte) 4, "Topic"),
+
+    GROUP((byte) 5, "Group");
+
+    @JSONField(value = true)
+    private final byte code;
+    private final String name;
+
+    ResourceType(byte code, String name) {
+        this.code = code;
+        this.name = name;
+    }
+
+    public static ResourceType getByName(String name) {
+        for (ResourceType resourceType : ResourceType.values()) {
+            if (StringUtils.equalsIgnoreCase(resourceType.getName(), name)) {
+                return resourceType;
+            }
+        }
+        return null;
+    }
+
+    public byte getCode() {
+        return code;
+    }
+
+    public String getName() {
+        return name;
+    }
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/resource/RocketMQResource.java b/common/src/main/java/org/apache/rocketmq/common/resource/RocketMQResource.java
new file mode 100644
index 00000000000..f3df6700628
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/resource/RocketMQResource.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.resource;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+@Retention(RetentionPolicy.RUNTIME)
+public @interface RocketMQResource {
+
+    ResourceType value();
+
+    String splitter() default "";
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsBriefInterceptor.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsBriefInterceptor.java
index b0b69378442..3ec295f54cf 100644
--- a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsBriefInterceptor.java
+++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsBriefInterceptor.java
@@ -34,7 +34,7 @@ public StatisticsBriefInterceptor(StatisticsItem item, Pair<String, long[][]>[]
             String name = briefMetas[i].getKey();
             int index = ArrayUtils.indexOf(item.getItemNames(), name);
             if (index < 0) {
-                throw new IllegalArgumentException("illegal breifItemName: " + name);
+                throw new IllegalArgumentException("illegal briefItemName: " + name);
             }
             indexOfItems[i] = index;
             statisticsBriefs[i] = new StatisticsBrief(briefMetas[i].getValue());
diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledIncrementPrinter.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledIncrementPrinter.java
index 2890e6e15cd..e1998473bf2 100644
--- a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledIncrementPrinter.java
+++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledIncrementPrinter.java
@@ -65,15 +65,15 @@ public void run() {
                     item.getStatObject());
                 StatisticsItem increment = snapshot.subtract(lastSnapshot);
 
-                Interceptor inteceptor = item.getInterceptor();
-                String inteceptorStr = formatInterceptor(inteceptor);
-                if (inteceptor != null) {
-                    inteceptor.reset();
+                Interceptor interceptor = item.getInterceptor();
+                String interceptorStr = formatInterceptor(interceptor);
+                if (interceptor != null) {
+                    interceptor.reset();
                 }
 
                 StatisticsItemSampleBrief brief = getSampleBrief(item.getStatKind(), item.getStatObject());
                 if (brief != null && (!increment.allZeros() || printZeroLine())) {
-                    printer.print(name, increment, inteceptorStr, brief.toString());
+                    printer.print(name, increment, interceptorStr, brief.toString());
                 }
 
                 setItemSnapshot(lastItemSnapshots, snapshot);
@@ -85,7 +85,7 @@ public void run() {
         }, getInitialDelay(), interval, TimeUnit.MILLISECONDS);
         addFuture(item, future);
 
-        // sample every TPS_INTREVAL
+        // sample every TPS_INTERVAL
         ScheduledFuture futureSample = executor.scheduleAtFixedRate(new Runnable() {
             @Override
             public void run() {
diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItem.java b/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItem.java
index d38281bf83a..559bb779536 100644
--- a/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItem.java
+++ b/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItem.java
@@ -31,6 +31,7 @@ public class MomentStatsItem {
     private final String statsKey;
     private final ScheduledExecutorService scheduledExecutorService;
     private final Logger log;
+    private long lastUpdateTimestamp = System.currentTimeMillis();
 
     public MomentStatsItem(String statsName, String statsKey,
         ScheduledExecutorService scheduledExecutorService, Logger log) {
@@ -55,10 +56,10 @@ public void run() {
     }
 
     public void printAtMinutes() {
-        log.info(String.format("[%s] [%s] Stats Every 5 Minutes, Value: %d",
+        log.info("[{}] [{}] Stats Every 5 Minutes, Value: {}",
             this.statsName,
             this.statsKey,
-            this.value.get()));
+            this.value.get());
     }
 
     public AtomicLong getValue() {
@@ -72,4 +73,12 @@ public String getStatsKey() {
     public String getStatsName() {
         return statsName;
     }
+
+    public long getLastUpdateTimestamp() {
+        return lastUpdateTimestamp;
+    }
+
+    public void setLastUpdateTimestamp(long lastUpdateTimestamp) {
+        this.lastUpdateTimestamp = lastUpdateTimestamp;
+    }
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java b/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java
index a4571d7b8ad..fd65351a548 100644
--- a/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java
+++ b/common/src/main/java/org/apache/rocketmq/common/stats/MomentStatsItemSet.java
@@ -24,9 +24,12 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 
 public class MomentStatsItemSet {
+    private static final Logger COMMERCIAL_LOG = LoggerFactory.getLogger(LoggerName.COMMERCIAL_LOGGER_NAME);
     private final ConcurrentMap<String/* key */, MomentStatsItem> statsItemTable =
         new ConcurrentHashMap<>(128);
     private final String statsName;
@@ -72,6 +75,13 @@ private void printAtMinutes() {
     public void setValue(final String statsKey, final int value) {
         MomentStatsItem statsItem = this.getAndCreateStatsItem(statsKey);
         statsItem.getValue().set(value);
+        statsItem.setLastUpdateTimestamp(System.currentTimeMillis());
+    }
+
+    public void setValue(final String statsKey, final long value) {
+        MomentStatsItem statsItem = this.getAndCreateStatsItem(statsKey);
+        statsItem.getValue().set(value);
+        statsItem.setLastUpdateTimestamp(System.currentTimeMillis());
     }
 
     public void delValueByInfixKey(final String statsKey, String separator) {
@@ -109,4 +119,17 @@ public MomentStatsItem getAndCreateStatsItem(final String statsKey) {
 
         return statsItem;
     }
+
+    public void cleanResource(int maxStatsIdleTimeInMinutes) {
+        COMMERCIAL_LOG.info("CleanStatisticItem: kind:{}, size:{}", statsName, this.statsItemTable.size());
+        Iterator<Entry<String, MomentStatsItem>> it = this.statsItemTable.entrySet().iterator();
+        while (it.hasNext()) {
+            Entry<String, MomentStatsItem> next = it.next();
+            MomentStatsItem statsItem = next.getValue();
+            if (System.currentTimeMillis() - statsItem.getLastUpdateTimestamp() > maxStatsIdleTimeInMinutes * 60 * 1000L) {
+                it.remove();
+                COMMERCIAL_LOG.info("CleanStatisticItem: removeKind:{}, removeKey:{}", statsName, statsItem.getStatsKey());
+            }
+        }
+    }
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/Stats.java b/common/src/main/java/org/apache/rocketmq/common/stats/Stats.java
index b70f96e412e..f67ccf9ae90 100644
--- a/common/src/main/java/org/apache/rocketmq/common/stats/Stats.java
+++ b/common/src/main/java/org/apache/rocketmq/common/stats/Stats.java
@@ -44,4 +44,7 @@ public class Stats {
     public static final String GROUP_GET_FALL_SIZE = "GROUP_GET_FALL_SIZE";
     public static final String GROUP_GET_FALL_TIME = "GROUP_GET_FALL_TIME";
     public static final String GROUP_GET_LATENCY = "GROUP_GET_LATENCY";
+    public static final String TOPIC_PUT_LATENCY = "TOPIC_PUT_LATENCY";
+    public static final String GROUP_ACK_NUMS = "GROUP_ACK_NUMS";
+    public static final String GROUP_CK_NUMS = "GROUP_CK_NUMS";
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java
index 8307c20aa68..cc5de160954 100644
--- a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java
+++ b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java
@@ -38,6 +38,7 @@ public class StatsItem {
 
     private final String statsName;
     private final String statsKey;
+    private long lastUpdateTimestamp = System.currentTimeMillis();
     private final ScheduledExecutorService scheduledExecutorService;
 
     private final Logger logger;
@@ -229,6 +230,14 @@ public String getStatsName() {
     public LongAdder getTimes() {
         return times;
     }
+
+    public long getLastUpdateTimestamp() {
+        return lastUpdateTimestamp;
+    }
+
+    public void setLastUpdateTimestamp(long lastUpdateTimestamp) {
+        this.lastUpdateTimestamp = lastUpdateTimestamp;
+    }
 }
 
 class CallSnapshot {
diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java
index c5b140b5cc1..8ed1486e9f1 100644
--- a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java
+++ b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java
@@ -24,9 +24,12 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 
 public class StatsItemSet {
+    private static final Logger COMMERCIAL_LOG = LoggerFactory.getLogger(LoggerName.COMMERCIAL_LOGGER_NAME);
     private final ConcurrentMap<String/* key */, StatsItem> statsItemTable =
         new ConcurrentHashMap<>(128);
 
@@ -157,12 +160,14 @@ public void addValue(final String statsKey, final int incValue, final int incTim
         StatsItem statsItem = this.getAndCreateStatsItem(statsKey);
         statsItem.getValue().add(incValue);
         statsItem.getTimes().add(incTimes);
+        statsItem.setLastUpdateTimestamp(System.currentTimeMillis());
     }
 
     public void addRTValue(final String statsKey, final int incValue, final int incTimes) {
         StatsItem statsItem = this.getAndCreateRTStatsItem(statsKey);
         statsItem.getValue().add(incValue);
         statsItem.getTimes().add(incTimes);
+        statsItem.setLastUpdateTimestamp(System.currentTimeMillis());
     }
 
     public void delValue(final String statsKey) {
@@ -256,4 +261,18 @@ public StatsSnapshot getStatsDataInDay(final String statsKey) {
     public StatsItem getStatsItem(final String statsKey) {
         return this.statsItemTable.get(statsKey);
     }
+
+
+    public void cleanResource(int maxStatsIdleTimeInMinutes) {
+        COMMERCIAL_LOG.info("CleanStatisticItemOld: kind:{}, size:{}", statsName, this.statsItemTable.size());
+        Iterator<Entry<String, StatsItem>> it = this.statsItemTable.entrySet().iterator();
+        while (it.hasNext()) {
+            Entry<String, StatsItem> next = it.next();
+            StatsItem statsItem = next.getValue();
+            if (System.currentTimeMillis() - statsItem.getLastUpdateTimestamp() > maxStatsIdleTimeInMinutes * 60 * 1000L) {
+                it.remove();
+                COMMERCIAL_LOG.info("CleanStatisticItemOld: removeKind:{}, removeKey:{}", statsName, statsItem.getStatsKey());
+            }
+        }
+    }
 }
diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java
index 1bfabbffedd..746128d296c 100644
--- a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java
+++ b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java
@@ -105,10 +105,9 @@ public static void logThreadPoolStatus() {
             List<ThreadPoolStatusMonitor> monitors = threadPoolWrapper.getStatusPrinters();
             for (ThreadPoolStatusMonitor monitor : monitors) {
                 double value = monitor.value(threadPoolWrapper.getThreadPoolExecutor());
-                waterMarkLogger.info("\t{}\t{}\t{}", threadPoolWrapper.getName(),
-                    monitor.describe(),
-                    value);
-
+                String nameFormatted = String.format("%-40s", threadPoolWrapper.getName());
+                String descFormatted = String.format("%-12s", monitor.describe());
+                waterMarkLogger.info("{}{}{}", nameFormatted, descFormatted, value);
                 if (enablePrintJstack) {
                     if (monitor.needPrintJstack(threadPoolWrapper.getThreadPoolExecutor(), value) &&
                         System.currentTimeMillis() - jstackTime > jstackPeriodTime) {
diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/AsyncShutdownHelper.java b/common/src/main/java/org/apache/rocketmq/common/utils/AsyncShutdownHelper.java
new file mode 100644
index 00000000000..da765d5e749
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/utils/AsyncShutdownHelper.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.utils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class AsyncShutdownHelper {
+    private final AtomicBoolean shutdown;
+    private final List<Shutdown> targetList;
+
+    private CountDownLatch countDownLatch;
+
+    public AsyncShutdownHelper() {
+        this.targetList = new ArrayList<>();
+        this.shutdown = new AtomicBoolean(false);
+    }
+
+    public void addTarget(Shutdown target) {
+        if (shutdown.get()) {
+            return;
+        }
+        targetList.add(target);
+    }
+
+    public AsyncShutdownHelper shutdown() {
+        if (shutdown.get()) {
+            return this;
+        }
+        if (targetList.isEmpty()) {
+            return this;
+        }
+        this.countDownLatch = new CountDownLatch(targetList.size());
+        for (Shutdown target : targetList) {
+            Runnable runnable = () -> {
+                try {
+                    target.shutdown();
+                } catch (Exception ignored) {
+
+                } finally {
+                    countDownLatch.countDown();
+                }
+            };
+            new Thread(runnable).start();
+        }
+        return this;
+    }
+
+    public boolean await(long time, TimeUnit unit) throws InterruptedException {
+        if (shutdown.get()) {
+            return false;
+        }
+        try {
+            return this.countDownLatch.await(time, unit);
+        } finally {
+            shutdown.compareAndSet(false, true);
+        }
+    }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/ExceptionUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/ExceptionUtils.java
similarity index 97%
rename from proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/ExceptionUtils.java
rename to common/src/main/java/org/apache/rocketmq/common/utils/ExceptionUtils.java
index e85360a5daf..74acc8f660f 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/ExceptionUtils.java
+++ b/common/src/main/java/org/apache/rocketmq/common/utils/ExceptionUtils.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.proxy.common.utils;
+package org.apache.rocketmq.common.utils;
 
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FutureUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/FutureUtils.java
similarity index 97%
rename from proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FutureUtils.java
rename to common/src/main/java/org/apache/rocketmq/common/utils/FutureUtils.java
index ea50e64eeaa..fb88b0a391f 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FutureUtils.java
+++ b/common/src/main/java/org/apache/rocketmq/common/utils/FutureUtils.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.proxy.common.utils;
+package org.apache.rocketmq.common.utils;
 
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/IPAddressUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/IPAddressUtils.java
new file mode 100644
index 00000000000..5133219d9cd
--- /dev/null
+++ b/common/src/main/java/org/apache/rocketmq/common/utils/IPAddressUtils.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.utils;
+
+import java.math.BigInteger;
+import java.net.InetAddress;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.validator.routines.InetAddressValidator;
+
+public class IPAddressUtils {
+
+    private static final String SLASH = "/";
+
+    private static final InetAddressValidator VALIDATOR = InetAddressValidator.getInstance();
+
+    public static boolean isValidIPOrCidr(String ipOrCidr) {
+        return isValidIp(ipOrCidr) || isValidCidr(ipOrCidr);
+    }
+
+    public static boolean isValidIp(String ip) {
+        return VALIDATOR.isValid(ip);
+    }
+
+    public static boolean isValidIPv4(String ip) {
+        return VALIDATOR.isValidInet4Address(ip);
+    }
+
+    public static boolean isValidIPv6(String ip) {
+        return VALIDATOR.isValidInet6Address(ip);
+    }
+
+    public static boolean isValidCidr(String cidr) {
+        return isValidIPv4Cidr(cidr) || isValidIPv6Cidr(cidr);
+    }
+
+    public static boolean isValidIPv4Cidr(String cidr) {
+        try {
+            String[] parts = cidr.split(SLASH);
+            if (parts.length != 2) {
+                return false;
+            }
+            InetAddress ip = InetAddress.getByName(parts[0]);
+            if (ip.getAddress().length != 4) {
+                return false;
+            }
+            int prefix = Integer.parseInt(parts[1]);
+            return prefix >= 0 && prefix <= 32;
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    public static boolean isValidIPv6Cidr(String cidr) {
+        try {
+            String[] parts = cidr.split(SLASH);
+            if (parts.length != 2) {
+                return false;
+            }
+            InetAddress ip = InetAddress.getByName(parts[0]);
+            if (ip.getAddress().length != 16) {
+                return false;
+            }
+            int prefix = Integer.parseInt(parts[1]);
+            return prefix >= 0 && prefix <= 128;
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    public static boolean isIPInRange(String ip, String cidr) {
+        try {
+            String[] parts = cidr.split(SLASH);
+            if (parts.length == 1) {
+                return StringUtils.equals(ip, cidr);
+            }
+            if (parts.length != 2) {
+                return false;
+            }
+            InetAddress cidrIp = InetAddress.getByName(parts[0]);
+            int prefixLength = Integer.parseInt(parts[1]);
+
+            BigInteger cidrIpBigInt = new BigInteger(1, cidrIp.getAddress());
+            BigInteger ipBigInt = new BigInteger(1, InetAddress.getByName(ip).getAddress());
+
+            BigInteger mask = BigInteger.valueOf(-1).shiftLeft(cidrIp.getAddress().length * 8 - prefixLength);
+            BigInteger cidrIpLower = cidrIpBigInt.and(mask);
+            BigInteger cidrIpUpper = cidrIpLower.add(mask.not());
+
+            return ipBigInt.compareTo(cidrIpLower) >= 0 && ipBigInt.compareTo(cidrIpUpper) <= 0;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java
index 7dd83e61799..2dc2a890e77 100644
--- a/common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java
+++ b/common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java
@@ -19,15 +19,21 @@
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Method;
+import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.NetworkInterface;
 import java.net.SocketAddress;
+import java.net.SocketException;
 import java.nio.channels.Selector;
 import java.nio.channels.spi.SelectorProvider;
 import java.util.ArrayList;
 import java.util.Enumeration;
+import java.util.List;
+
+import org.apache.commons.validator.routines.InetAddressValidator;
+import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
@@ -59,18 +65,14 @@ public static Selector openSelector() throws IOException {
         if (isLinuxPlatform()) {
             try {
                 final Class<?> providerClazz = Class.forName("sun.nio.ch.EPollSelectorProvider");
-                if (providerClazz != null) {
-                    try {
-                        final Method method = providerClazz.getMethod("provider");
-                        if (method != null) {
-                            final SelectorProvider selectorProvider = (SelectorProvider) method.invoke(null);
-                            if (selectorProvider != null) {
-                                result = selectorProvider.openSelector();
-                            }
-                        }
-                    } catch (final Exception e) {
-                        log.warn("Open ePoll Selector for linux platform exception", e);
+                try {
+                    final Method method = providerClazz.getMethod("provider");
+                    final SelectorProvider selectorProvider = (SelectorProvider) method.invoke(null);
+                    if (selectorProvider != null) {
+                        result = selectorProvider.openSelector();
                     }
+                } catch (final Exception e) {
+                    log.warn("Open ePoll Selector for linux platform exception", e);
                 }
             } catch (final Exception e) {
                 // ignore
@@ -88,48 +90,75 @@ public static boolean isLinuxPlatform() {
         return isLinuxPlatform;
     }
 
-    public static String getLocalAddress() {
-        try {
-            // Traversal Network interface to get the first non-loopback and non-private address
-            Enumeration<NetworkInterface> enumeration = NetworkInterface.getNetworkInterfaces();
-            ArrayList<String> ipv4Result = new ArrayList<>();
-            ArrayList<String> ipv6Result = new ArrayList<>();
-            while (enumeration.hasMoreElements()) {
-                final NetworkInterface nif = enumeration.nextElement();
-                if (isBridge(nif) || nif.isVirtual() || nif.isPointToPoint() || !nif.isUp()) {
-                    continue;
-                }
-
-                final Enumeration<InetAddress> en = nif.getInetAddresses();
-                while (en.hasMoreElements()) {
-                    final InetAddress address = en.nextElement();
-                    if (!address.isLoopbackAddress()) {
-                        if (address instanceof Inet6Address) {
-                            ipv6Result.add(normalizeHostAddress(address));
-                        } else {
-                            ipv4Result.add(normalizeHostAddress(address));
+    public static List<InetAddress> getLocalInetAddressList() throws SocketException {
+        Enumeration<NetworkInterface> enumeration = NetworkInterface.getNetworkInterfaces();
+        List<InetAddress> inetAddressList = new ArrayList<>();
+        // Traversal Network interface to get the non-bridge and non-virtual and non-ppp and up address
+        while (enumeration.hasMoreElements()) {
+            final NetworkInterface nif = enumeration.nextElement();
+            if (isBridge(nif) || nif.isVirtual() || nif.isPointToPoint() || !nif.isUp()) {
+                continue;
+            }
+            InetAddressValidator validator = InetAddressValidator.getInstance();
+            final Enumeration<InetAddress> en = nif.getInetAddresses();
+            while (en.hasMoreElements()) {
+                final InetAddress address = en.nextElement();
+                if (address instanceof Inet4Address) {
+                    byte[] ipByte = address.getAddress();
+                    if (ipByte.length == 4) {
+                        if (validator.isValidInet4Address(UtilAll.ipToIPv4Str(ipByte))) {
+                            inetAddressList.add(address);
+                        }
+                    }
+                } else if (address instanceof Inet6Address) {
+                    byte[] ipByte = address.getAddress();
+                    if (ipByte.length == 16) {
+                        if (validator.isValidInet6Address(UtilAll.ipToIPv6Str(ipByte))) {
+                            inetAddressList.add(address);
                         }
                     }
                 }
             }
+        }
+        return inetAddressList;
+    }
 
-            // prefer ipv4
+    public static InetAddress getLocalInetAddress() {
+        try {
+            ArrayList<InetAddress> ipv4Result = new ArrayList<>();
+            ArrayList<InetAddress> ipv6Result = new ArrayList<>();
+            List<InetAddress> localInetAddressList = getLocalInetAddressList();
+            for (InetAddress inetAddress : localInetAddressList) {
+                // Skip loopback addresses
+                if (inetAddress.isLoopbackAddress()) {
+                    continue;
+                }
+                if (inetAddress instanceof Inet6Address) {
+                    ipv6Result.add(inetAddress);
+                } else {
+                    ipv4Result.add(inetAddress);
+                }
+            }
+            // prefer ipv4 and prefer external ip
             if (!ipv4Result.isEmpty()) {
-                for (String ip : ipv4Result) {
-                    if (ip.startsWith("127.0") || ip.startsWith("192.168") || ip.startsWith("0.")) {
+                for (InetAddress ip : ipv4Result) {
+                    if (UtilAll.isInternalIP(ip.getAddress())) {
                         continue;
                     }
-
                     return ip;
                 }
-
                 return ipv4Result.get(ipv4Result.size() - 1);
             } else if (!ipv6Result.isEmpty()) {
+                for (InetAddress ip : ipv6Result) {
+                    if (UtilAll.isInternalV6IP(ip)) {
+                        continue;
+                    }
+                    return ip;
+                }
                 return ipv6Result.get(0);
             }
             //If failed to find,fall back to localhost
-            final InetAddress localHost = InetAddress.getLocalHost();
-            return normalizeHostAddress(localHost);
+            return InetAddress.getLocalHost();
         } catch (Exception e) {
             log.error("Failed to obtain local address", e);
         }
@@ -137,6 +166,11 @@ public static String getLocalAddress() {
         return null;
     }
 
+    public static String getLocalAddress() {
+        InetAddress localHost = getLocalInetAddress();
+        return normalizeHostAddress(localHost);
+    }
+
     public static String normalizeHostAddress(final InetAddress localHost) {
         if (localHost instanceof Inet6Address) {
             return "[" + localHost.getHostAddress() + "]";
@@ -149,8 +183,7 @@ public static SocketAddress string2SocketAddress(final String addr) {
         int split = addr.lastIndexOf(":");
         String host = addr.substring(0, split);
         String port = addr.substring(split + 1);
-        InetSocketAddress isa = new InetSocketAddress(host, Integer.parseInt(port));
-        return isa;
+        return new InetSocketAddress(host, Integer.parseInt(port));
     }
 
     public static String socketAddress2String(final SocketAddress addr) {
diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java b/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java
index 65dea47b5ea..49e2c442b23 100644
--- a/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java
+++ b/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java
@@ -50,7 +50,7 @@ public class ServiceProvider {
      * Returns a string that uniquely identifies the specified object, including its class.
      * <p>
      * The returned string is of form "classname@hashcode", ie is the same as the return value of the Object.toString()
-     * method, but works even when the specified object's class has overidden the toString method.
+     * method, but works even when the specified object's class has overridden the toString method.
      *
      * @param o may be null.
      * @return a string of form classname@hashcode, or "null" if param o is null.
diff --git a/common/src/test/java/org/apache/rocketmq/common/BrokerConfigSingletonTest.java b/common/src/test/java/org/apache/rocketmq/common/BrokerConfigSingletonTest.java
new file mode 100644
index 00000000000..b98a6e37e69
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/BrokerConfigSingletonTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class BrokerConfigSingletonTest {
+
+    /**
+     * Tests the behavior of getting the broker configuration when it has not been initialized.
+     * Expects an IllegalArgumentException to be thrown, ensuring that the configuration cannot be obtained without initialization.
+     */
+    @Test(expected = IllegalArgumentException.class)
+    public void getBrokerConfig_NullConfiguration_ThrowsException() {
+        BrokerConfigSingleton.getBrokerConfig();
+    }
+
+    /**
+     * Tests the behavior of setting the broker configuration after it has already been initialized.
+     * Expects an IllegalArgumentException to be thrown, ensuring that the configuration cannot be reset once set.
+     * Also asserts that the returned brokerConfig instance is the same as the one set, confirming the singleton property.
+     */
+    @Test(expected = IllegalArgumentException.class)
+    public void setBrokerConfig_AlreadyInitialized_ThrowsException() {
+        BrokerConfig config = new BrokerConfig();
+        BrokerConfigSingleton.setBrokerConfig(config);
+        Assert.assertSame("Expected brokerConfig instance is not returned", config, BrokerConfigSingleton.getBrokerConfig());
+        BrokerConfigSingleton.setBrokerConfig(config);
+    }
+
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/action/ActionTest.java b/common/src/test/java/org/apache/rocketmq/common/action/ActionTest.java
new file mode 100644
index 00000000000..e3c1b9a3fcb
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/action/ActionTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.action;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class ActionTest {
+
+    @Test
+    public void getByName_NullName_ReturnsNull() {
+        Action result = Action.getByName("null");
+        assertNull(result);
+    }
+
+    @Test
+    public void getByName_UnknownName_ReturnsUnknown() {
+        Action result = Action.getByName("unknown");
+        assertEquals(Action.UNKNOWN, result);
+    }
+
+    @Test
+    public void getByName_AllName_ReturnsAll() {
+        Action result = Action.getByName("All");
+        assertEquals(Action.ALL, result);
+    }
+
+    @Test
+    public void getByName_AnyName_ReturnsAny() {
+        Action result = Action.getByName("Any");
+        assertEquals(Action.ANY, result);
+    }
+
+    @Test
+    public void getByName_PubName_ReturnsPub() {
+        Action result = Action.getByName("Pub");
+        assertEquals(Action.PUB, result);
+    }
+
+    @Test
+    public void getByName_SubName_ReturnsSub() {
+        Action result = Action.getByName("Sub");
+        assertEquals(Action.SUB, result);
+    }
+
+    @Test
+    public void getByName_CreateName_ReturnsCreate() {
+        Action result = Action.getByName("Create");
+        assertEquals(Action.CREATE, result);
+    }
+
+    @Test
+    public void getByName_UpdateName_ReturnsUpdate() {
+        Action result = Action.getByName("Update");
+        assertEquals(Action.UPDATE, result);
+    }
+
+    @Test
+    public void getByName_DeleteName_ReturnsDelete() {
+        Action result = Action.getByName("Delete");
+        assertEquals(Action.DELETE, result);
+    }
+
+    @Test
+    public void getByName_GetName_ReturnsGet() {
+        Action result = Action.getByName("Get");
+        assertEquals(Action.GET, result);
+    }
+
+    @Test
+    public void getByName_ListName_ReturnsList() {
+        Action result = Action.getByName("List");
+        assertEquals(Action.LIST, result);
+    }
+
+    @Test
+    public void getCode_ReturnsCorrectCode() {
+        assertEquals((byte) 1, Action.ALL.getCode());
+        assertEquals((byte) 2, Action.ANY.getCode());
+        assertEquals((byte) 3, Action.PUB.getCode());
+        assertEquals((byte) 4, Action.SUB.getCode());
+        assertEquals((byte) 5, Action.CREATE.getCode());
+        assertEquals((byte) 6, Action.UPDATE.getCode());
+        assertEquals((byte) 7, Action.DELETE.getCode());
+        assertEquals((byte) 8, Action.GET.getCode());
+        assertEquals((byte) 9, Action.LIST.getCode());
+    }
+
+    @Test
+    public void getName_ReturnsCorrectName() {
+        assertEquals("All", Action.ALL.getName());
+        assertEquals("Any", Action.ANY.getName());
+        assertEquals("Pub", Action.PUB.getName());
+        assertEquals("Sub", Action.SUB.getName());
+        assertEquals("Create", Action.CREATE.getName());
+        assertEquals("Update", Action.UPDATE.getName());
+        assertEquals("Delete", Action.DELETE.getName());
+        assertEquals("Get", Action.GET.getName());
+        assertEquals("List", Action.LIST.getName());
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/action/RocketMQActionTest.java b/common/src/test/java/org/apache/rocketmq/common/action/RocketMQActionTest.java
new file mode 100644
index 00000000000..373a1f620c4
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/action/RocketMQActionTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.action;
+
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class RocketMQActionTest {
+
+    @Test
+    public void testRocketMQAction_DefaultResourceType_CustomisedValueAndActionArray() {
+        RocketMQAction annotation = DemoClass.class.getAnnotation(RocketMQAction.class);
+        assertEquals(0, annotation.value());
+        assertEquals(ResourceType.UNKNOWN, annotation.resource());
+        assertArrayEquals(new Action[] {}, annotation.action());
+    }
+
+    @Test
+    public void testRocketMQAction_CustomisedValueAndResourceTypeAndActionArray() {
+        RocketMQAction annotation = CustomisedDemoClass.class.getAnnotation(RocketMQAction.class);
+        assertEquals(1, annotation.value());
+        assertEquals(ResourceType.TOPIC, annotation.resource());
+        assertArrayEquals(new Action[] {Action.CREATE, Action.DELETE}, annotation.action());
+    }
+
+    @RocketMQAction(value = 0, resource = ResourceType.UNKNOWN, action = {})
+    private static class DemoClass {
+    }
+
+    @RocketMQAction(value = 1, resource = ResourceType.TOPIC, action = {Action.CREATE, Action.DELETE})
+    private static class CustomisedDemoClass {
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeParserTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeParserTest.java
index 12398100bec..a89587354b9 100644
--- a/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeParserTest.java
+++ b/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeParserTest.java
@@ -17,18 +17,84 @@
 package org.apache.rocketmq.common.attribute;
 
 import com.google.common.collect.Maps;
-import org.junit.Assert;
-import org.junit.Test;
-
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import org.junit.Assert;
+import org.junit.Test;
 
 import static com.google.common.collect.Maps.newHashMap;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 public class AttributeParserTest {
+
+    @Test
+    public void parseToMap_EmptyString_ReturnsEmptyMap() {
+        String attributesModification = "";
+        Map<String, String> result = AttributeParser.parseToMap(attributesModification);
+        assertTrue(result.isEmpty());
+    }
+
+    @Test
+    public void parseToMap_NullString_ReturnsEmptyMap() {
+        String attributesModification = null;
+        Map<String, String> result = AttributeParser.parseToMap(attributesModification);
+        assertTrue(result.isEmpty());
+    }
+
+    @Test
+    public void parseToMap_ValidAttributesModification_ReturnsExpectedMap() {
+        String attributesModification = "+key1=value1,+key2=value2,-key3,+key4=value4";
+        Map<String, String> result = AttributeParser.parseToMap(attributesModification);
+
+        Map<String, String> expectedMap = new HashMap<>();
+        expectedMap.put("+key1", "value1");
+        expectedMap.put("+key2", "value2");
+        expectedMap.put("-key3", "");
+        expectedMap.put("+key4", "value4");
+
+        assertEquals(expectedMap, result);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void parseToMap_InvalidAddAttributeFormat_ThrowsRuntimeException() {
+        String attributesModification = "+key1=value1,key2=value2,-key3,+key4=value4";
+        AttributeParser.parseToMap(attributesModification);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void parseToMap_InvalidDeleteAttributeFormat_ThrowsRuntimeException() {
+        String attributesModification = "+key1=value1,+key2=value2,key3,+key4=value4";
+        AttributeParser.parseToMap(attributesModification);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void parseToMap_DuplicateKey_ThrowsRuntimeException() {
+        String attributesModification = "+key1=value1,+key1=value2";
+        AttributeParser.parseToMap(attributesModification);
+    }
+
+    @Test
+    public void parseToString_EmptyMap_ReturnsEmptyString() {
+        Map<String, String> attributes = new HashMap<>();
+        String result = AttributeParser.parseToString(attributes);
+        assertEquals("", result);
+    }
+
+    @Test
+    public void parseToString_ValidAttributes_ReturnsExpectedString() {
+        Map<String, String> attributes = new HashMap<>();
+        attributes.put("key1", "value1");
+        attributes.put("key2", "value2");
+        attributes.put("key3", "");
+
+        String result = AttributeParser.parseToString(attributes);
+        String expectedString = "key1=value1,key2=value2,key3";
+        assertEquals(expectedString, result);
+    }
+
     @Test
     public void testParseToMap() {
         Assert.assertEquals(0, AttributeParser.parseToMap(null).size());
diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeTest.java
index 39a12b97ef4..9be0f31f06a 100644
--- a/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeTest.java
+++ b/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeTest.java
@@ -17,12 +17,55 @@
 package org.apache.rocketmq.common.attribute;
 
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
 import static com.google.common.collect.Sets.newHashSet;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 public class AttributeTest {
 
+    private Attribute attribute;
+
+    @Before
+    public void setUp() {
+        attribute = new Attribute("testAttribute", true) {
+            @Override
+            public void verify(String value) {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+
+    @Test
+    public void testGetName_ShouldReturnCorrectName() {
+        assertEquals("testAttribute", attribute.getName());
+    }
+
+    @Test
+    public void testSetName_ShouldSetCorrectName() {
+        attribute.setName("newTestAttribute");
+        assertEquals("newTestAttribute", attribute.getName());
+    }
+
+    @Test
+    public void testIsChangeable_ShouldReturnCorrectChangeableStatus() {
+        assertTrue(attribute.isChangeable());
+    }
+
+    @Test
+    public void testSetChangeable_ShouldSetCorrectChangeableStatus() {
+        attribute.setChangeable(false);
+        assertFalse(attribute.isChangeable());
+    }
+
+    @Test(expected = UnsupportedOperationException.class)
+    public void testVerify_ShouldThrowUnsupportedOperationException() {
+        attribute.verify("testValue");
+    }
+
     @Test
     public void testEnumAttribute() {
         EnumAttribute enumAttribute = new EnumAttribute("enum.key", true, newHashSet("enum-1", "enum-2", "enum-3"), "enum-1");
diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeUtilTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeUtilTest.java
new file mode 100644
index 00000000000..aef46c16680
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeUtilTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.attribute;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class AttributeUtilTest {
+
+    private Map<String, Attribute> allAttributes;
+    private ImmutableMap<String, String> currentAttributes;
+
+    @Before
+    public void setUp() {
+        allAttributes = new HashMap<>();
+        allAttributes.put("attr1", new TestAttribute("value1", true, value -> true));
+        allAttributes.put("attr2", new TestAttribute("value2", true, value -> true));
+        allAttributes.put("attr3", new TestAttribute("value3", true, value -> value.equals("valid")));
+
+        currentAttributes = ImmutableMap.of("attr1", "value1", "attr2", "value2");
+    }
+
+    @Test
+    public void alterCurrentAttributes_CreateMode_ShouldReturnOnlyAddedAttributes() {
+        ImmutableMap<String, String> newAttributes = ImmutableMap.of("+attr1", "new_value1", "+attr2", "value2", "+attr3", "value3");
+
+        Map<String, String> result = AttributeUtil.alterCurrentAttributes(true, allAttributes, currentAttributes, newAttributes);
+
+        assertEquals(3, result.size());
+        assertTrue(result.containsKey("attr1"));
+        assertEquals("new_value1", result.get("attr1"));
+        assertTrue(result.containsKey("attr3"));
+        assertEquals("value3", result.get("attr3"));
+        assertTrue(result.containsKey("attr2"));
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void alterCurrentAttributes_CreateMode_AddNonAddableAttribute_ShouldThrowException() {
+        ImmutableMap<String, String> newAttributes = ImmutableMap.of("attr1", "value1");
+        AttributeUtil.alterCurrentAttributes(true, allAttributes, currentAttributes, newAttributes);
+    }
+
+    @Test
+    public void alterCurrentAttributes_UpdateMode_ShouldReturnUpdatedAndAddedAttributes() {
+        ImmutableMap<String, String> newAttributes = ImmutableMap.of("+attr1", "new_value1", "-attr2", "value2", "+attr3", "value3");
+
+        Map<String, String> result = AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes);
+
+        assertEquals(2, result.size());
+        assertTrue(result.containsKey("attr1"));
+        assertEquals("new_value1", result.get("attr1"));
+        assertTrue(result.containsKey("attr3"));
+        assertEquals("value3", result.get("attr3"));
+        assertFalse(result.containsKey("attr2"));
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void alterCurrentAttributes_UpdateMode_DeleteNonExistentAttribute_ShouldThrowException() {
+        ImmutableMap<String, String> newAttributes = ImmutableMap.of("-attr4", "value4");
+        AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void alterCurrentAttributes_UpdateMode_WrongFormatKey_ShouldThrowException() {
+        ImmutableMap<String, String> newAttributes = ImmutableMap.of("attr1", "+value1");
+        AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void alterCurrentAttributes_UnsupportedKey_ShouldThrowException() {
+        ImmutableMap<String, String> newAttributes = ImmutableMap.of("unsupported_attr", "value");
+        AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void alterCurrentAttributes_AttemptToUpdateUnchangeableAttribute_ShouldThrowException() {
+        ImmutableMap<String, String> newAttributes = ImmutableMap.of("attr2", "new_value2");
+        AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes);
+    }
+
+    private static class TestAttribute extends Attribute {
+        private final AttributeValidator validator;
+
+        public TestAttribute(String name, boolean changeable, AttributeValidator validator) {
+            super(name, changeable);
+            this.validator = validator;
+        }
+
+        @Override
+        public void verify(String value) {
+            validator.validate(value);
+        }
+    }
+
+    private interface AttributeValidator {
+        boolean validate(String value);
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/BooleanAttributeTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/BooleanAttributeTest.java
new file mode 100644
index 00000000000..6bed6ffac69
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/attribute/BooleanAttributeTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.attribute;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+
+public class BooleanAttributeTest {
+
+    private BooleanAttribute booleanAttribute;
+
+    @Before
+    public void setUp() {
+        booleanAttribute = new BooleanAttribute("testAttribute", true, false);
+    }
+
+    @Test
+    public void testVerify_ValidValue_NoExceptionThrown() {
+        booleanAttribute.verify("true");
+        booleanAttribute.verify("false");
+    }
+
+    @Test
+    public void testVerify_InvalidValue_ExceptionThrown() {
+        assertThrows(RuntimeException.class, () -> booleanAttribute.verify("invalid"));
+        assertThrows(RuntimeException.class, () -> booleanAttribute.verify("1"));
+        assertThrows(RuntimeException.class, () -> booleanAttribute.verify("0"));
+        assertThrows(RuntimeException.class, () -> booleanAttribute.verify(""));
+    }
+
+    @Test
+    public void testGetDefaultValue() {
+        assertFalse(booleanAttribute.getDefaultValue());
+    }
+}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/CQItemBufferUtilTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/CQTypeTest.java
similarity index 51%
rename from tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/CQItemBufferUtilTest.java
rename to common/src/test/java/org/apache/rocketmq/common/attribute/CQTypeTest.java
index 7f8caea2053..41aa98ba864 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/CQItemBufferUtilTest.java
+++ b/common/src/test/java/org/apache/rocketmq/common/attribute/CQTypeTest.java
@@ -14,38 +14,32 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.tieredstore.util;
+package org.apache.rocketmq.common.attribute;
 
-import java.nio.ByteBuffer;
-import org.apache.rocketmq.store.ConsumeQueue;
-import org.junit.Assert;
-import org.junit.BeforeClass;
 import org.junit.Test;
 
-public class CQItemBufferUtilTest {
-    private static ByteBuffer cqItem;
+import static org.junit.Assert.assertEquals;
 
-    @BeforeClass
-    public static void setUp() {
-        cqItem = ByteBuffer.allocate(ConsumeQueue.CQ_STORE_UNIT_SIZE);
-        cqItem.putLong(1);
-        cqItem.putInt(2);
-        cqItem.putLong(3);
-        cqItem.flip();
-    }
+public class CQTypeTest {
 
     @Test
-    public void testGetCommitLogOffset() {
-        Assert.assertEquals(1, CQItemBufferUtil.getCommitLogOffset(cqItem));
+    public void testValues() {
+        CQType[] values = CQType.values();
+        assertEquals(3, values.length);
+        assertEquals(CQType.SimpleCQ, values[0]);
+        assertEquals(CQType.BatchCQ, values[1]);
+        assertEquals(CQType.RocksDBCQ, values[2]);
     }
 
     @Test
-    public void testGetSize() {
-        Assert.assertEquals(2, CQItemBufferUtil.getSize(cqItem));
+    public void testValueOf() {
+        assertEquals(CQType.SimpleCQ, CQType.valueOf("SimpleCQ"));
+        assertEquals(CQType.BatchCQ, CQType.valueOf("BatchCQ"));
+        assertEquals(CQType.RocksDBCQ, CQType.valueOf("RocksDBCQ"));
     }
 
-    @Test
-    public void testGetTagCode() {
-        Assert.assertEquals(3, CQItemBufferUtil.getTagCode(cqItem));
+    @Test(expected = IllegalArgumentException.class)
+    public void testValueOf_InvalidName() {
+        CQType.valueOf("InvalidCQ");
     }
 }
diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/CleanupPolicyTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/CleanupPolicyTest.java
new file mode 100644
index 00000000000..584de2b7e42
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/attribute/CleanupPolicyTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.attribute;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class CleanupPolicyTest {
+
+    @Test
+    public void testCleanupPolicy_Delete() {
+        CleanupPolicy cleanupPolicy = CleanupPolicy.DELETE;
+        assertEquals("DELETE", cleanupPolicy.toString());
+    }
+
+    @Test
+    public void testCleanupPolicy_Compaction() {
+        CleanupPolicy cleanupPolicy = CleanupPolicy.COMPACTION;
+        assertEquals("COMPACTION", cleanupPolicy.toString());
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/EnumAttributeTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/EnumAttributeTest.java
new file mode 100644
index 00000000000..637dc302f52
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/attribute/EnumAttributeTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.attribute;
+
+import org.junit.Before;
+import org.junit.Test;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+public class EnumAttributeTest {
+
+    private EnumAttribute enumAttribute;
+
+    @Before
+    public void setUp() {
+        Set<String> universe = new HashSet<>();
+        universe.add("value1");
+        universe.add("value2");
+        universe.add("value3");
+
+        enumAttribute = new EnumAttribute("testAttribute", true, universe, "value1");
+    }
+
+    @Test
+    public void verify_ValidValue_NoExceptionThrown() {
+        enumAttribute.verify("value1");
+        enumAttribute.verify("value2");
+        enumAttribute.verify("value3");
+    }
+
+    @Test
+    public void verify_InvalidValue_ExceptionThrown() {
+        RuntimeException exception = assertThrows(RuntimeException.class, () -> {
+            enumAttribute.verify("invalidValue");
+        });
+
+        assertTrue(exception.getMessage().startsWith("value is not in set:"));
+    }
+
+    @Test
+    public void getDefaultValue_ReturnsDefaultValue() {
+        assertEquals("value1", enumAttribute.getDefaultValue());
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/LongRangeAttributeTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/LongRangeAttributeTest.java
new file mode 100644
index 00000000000..222f9092d54
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/attribute/LongRangeAttributeTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.attribute;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+
+public class LongRangeAttributeTest {
+
+    private LongRangeAttribute longRangeAttribute;
+
+    @Before
+    public void setUp() {
+        longRangeAttribute = new LongRangeAttribute("testAttribute", true, 0, 100, 50);
+    }
+
+    @Test
+    public void verify_ValidValue_NoExceptionThrown() {
+        longRangeAttribute.verify("50");
+    }
+
+    @Test
+    public void verify_MinValue_NoExceptionThrown() {
+        longRangeAttribute.verify("0");
+    }
+
+    @Test
+    public void verify_MaxValue_NoExceptionThrown() {
+        longRangeAttribute.verify("100");
+    }
+
+    @Test
+    public void verify_ValueLessThanMin_ThrowsRuntimeException() {
+        RuntimeException exception = assertThrows(RuntimeException.class, () -> longRangeAttribute.verify("-1"));
+        assertEquals("value is not in range(0, 100)", exception.getMessage());
+    }
+
+    @Test
+    public void verify_ValueGreaterThanMax_ThrowsRuntimeException() {
+        RuntimeException exception = assertThrows(RuntimeException.class, () -> longRangeAttribute.verify("101"));
+        assertEquals("value is not in range(0, 100)", exception.getMessage());
+    }
+
+    @Test
+    public void getDefaultValue_ReturnsDefaultValue() {
+        assertEquals(50, longRangeAttribute.getDefaultValue());
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/TopicMessageTypeTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/TopicMessageTypeTest.java
new file mode 100644
index 00000000000..0321679ccc0
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/attribute/TopicMessageTypeTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.attribute;
+
+import com.google.common.collect.Sets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TopicMessageTypeTest {
+
+    private Map<String, String> normalMessageProperty;
+    private Map<String, String> transactionMessageProperty;
+    private Map<String, String> delayMessageProperty;
+    private Map<String, String> fifoMessageProperty;
+
+    @Before
+    public void setUp() {
+        normalMessageProperty = new HashMap<>();
+        transactionMessageProperty = new HashMap<>();
+        delayMessageProperty = new HashMap<>();
+        fifoMessageProperty = new HashMap<>();
+
+        transactionMessageProperty.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, "true");
+        delayMessageProperty.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, "1");
+        fifoMessageProperty.put(MessageConst.PROPERTY_SHARDING_KEY, "shardingKey");
+    }
+
+    @Test
+    public void testTopicMessageTypeSet() {
+        Set<String> expectedSet = Sets.newHashSet("UNSPECIFIED", "NORMAL", "FIFO", "DELAY", "TRANSACTION", "MIXED");
+        Set<String> actualSet = TopicMessageType.topicMessageTypeSet();
+        assertEquals(expectedSet, actualSet);
+    }
+
+    @Test
+    public void testParseFromMessageProperty_Normal() {
+        TopicMessageType actual = TopicMessageType.parseFromMessageProperty(normalMessageProperty);
+        assertEquals(TopicMessageType.NORMAL, actual);
+    }
+
+    @Test
+    public void testParseFromMessageProperty_Transaction() {
+        TopicMessageType actual = TopicMessageType.parseFromMessageProperty(transactionMessageProperty);
+        assertEquals(TopicMessageType.TRANSACTION, actual);
+    }
+
+    @Test
+    public void testParseFromMessageProperty_Delay() {
+        TopicMessageType actual = TopicMessageType.parseFromMessageProperty(delayMessageProperty);
+        assertEquals(TopicMessageType.DELAY, actual);
+    }
+
+    @Test
+    public void testParseFromMessageProperty_Fifo() {
+        TopicMessageType actual = TopicMessageType.parseFromMessageProperty(fifoMessageProperty);
+        assertEquals(TopicMessageType.FIFO, actual);
+    }
+
+    @Test
+    public void testGetMetricsValue() {
+        for (TopicMessageType type : TopicMessageType.values()) {
+            String expected = type.getValue().toLowerCase();
+            String actual = type.getMetricsValue();
+            assertEquals(expected, actual);
+        }
+    }
+
+    @Test
+    public void testParseFromMessageProperty() {
+        Map<String, String> properties = new HashMap<>();
+
+        // TRANSACTION
+        properties.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, "true");
+        Assert.assertEquals(TopicMessageType.TRANSACTION, TopicMessageType.parseFromMessageProperty(properties));
+
+        // DELAY
+        properties.clear();
+        properties.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, "3");
+        Assert.assertEquals(TopicMessageType.DELAY, TopicMessageType.parseFromMessageProperty(properties));
+
+        properties.clear();
+        properties.put(MessageConst.PROPERTY_TIMER_DELIVER_MS, System.currentTimeMillis() + 10000 + "");
+        Assert.assertEquals(TopicMessageType.DELAY, TopicMessageType.parseFromMessageProperty(properties));
+
+        properties.clear();
+        properties.put(MessageConst.PROPERTY_TIMER_DELAY_SEC, 10 + "");
+        Assert.assertEquals(TopicMessageType.DELAY, TopicMessageType.parseFromMessageProperty(properties));
+
+        properties.clear();
+        properties.put(MessageConst.PROPERTY_TIMER_DELAY_MS, 10000 + "");
+        Assert.assertEquals(TopicMessageType.DELAY, TopicMessageType.parseFromMessageProperty(properties));
+
+        // FIFO
+        properties.clear();
+        properties.put(MessageConst.PROPERTY_SHARDING_KEY, "sharding_key");
+        Assert.assertEquals(TopicMessageType.FIFO, TopicMessageType.parseFromMessageProperty(properties));
+
+        // NORMAL
+        properties.clear();
+        Assert.assertEquals(TopicMessageType.NORMAL, TopicMessageType.parseFromMessageProperty(properties));
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/chain/HandlerChainTest.java b/common/src/test/java/org/apache/rocketmq/common/chain/HandlerChainTest.java
new file mode 100644
index 00000000000..3a8499ebad2
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/chain/HandlerChainTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.chain;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class HandlerChainTest {
+
+    private HandlerChain<Integer, String> handlerChain;
+    private Handler<Integer, String> handler1;
+    private Handler<Integer, String> handler2;
+
+    @Before
+    public void setUp() {
+        handlerChain = HandlerChain.create();
+        handler1 = (t, chain) -> "Handler1";
+        handler2 = (t, chain) -> null;
+    }
+
+    @Test
+    public void testHandle_withEmptyChain() {
+        handlerChain.addNext(handler1);
+        handlerChain.handle(1);
+        assertNull("Expected null since the handler chain is empty", handlerChain.handle(2));
+    }
+
+    @Test
+    public void testHandle_withNonEmptyChain() {
+        handlerChain.addNext(handler1);
+
+        String result = handlerChain.handle(1);
+
+        assertEquals("Handler1", result);
+    }
+
+    @Test
+    public void testHandle_withMultipleHandlers() {
+        handlerChain.addNext(handler1);
+        handlerChain.addNext(handler2);
+
+        String result1 = handlerChain.handle(1);
+        String result2 = handlerChain.handle(2);
+
+        assertEquals("Handler1", result1);
+        assertNull("Expected null since there are no more handlers", result2);
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/coldctr/AccAndTimeStampTest.java b/common/src/test/java/org/apache/rocketmq/common/coldctr/AccAndTimeStampTest.java
new file mode 100644
index 00000000000..01bb4ae3701
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/coldctr/AccAndTimeStampTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.coldctr;
+
+import java.util.concurrent.atomic.AtomicLong;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class AccAndTimeStampTest {
+
+    private AccAndTimeStamp accAndTimeStamp;
+
+    @Before
+    public void setUp() {
+        accAndTimeStamp = new AccAndTimeStamp(new AtomicLong());
+    }
+
+    @Test
+    public void testInitialValues() {
+        assertEquals("Cold accumulator should be initialized to 0", 0, accAndTimeStamp.getColdAcc().get());
+        assertTrue("Last cold read time should be initialized to current time", accAndTimeStamp.getLastColdReadTimeMills() >= System.currentTimeMillis() - 1000);
+        assertTrue("Create time should be initialized to current time", accAndTimeStamp.getCreateTimeMills() >= System.currentTimeMillis() - 1000);
+    }
+
+    @Test
+    public void testSetColdAcc() {
+        AtomicLong newColdAcc = new AtomicLong(100L);
+        accAndTimeStamp.setColdAcc(newColdAcc);
+        assertEquals("Cold accumulator should be set to new value", newColdAcc, accAndTimeStamp.getColdAcc());
+    }
+
+    @Test
+    public void testSetLastColdReadTimeMills() {
+        long newLastColdReadTimeMills = System.currentTimeMillis() + 1000;
+        accAndTimeStamp.setLastColdReadTimeMills(newLastColdReadTimeMills);
+        assertEquals("Last cold read time should be set to new value", newLastColdReadTimeMills, accAndTimeStamp.getLastColdReadTimeMills().longValue());
+    }
+
+    @Test
+    public void testSetCreateTimeMills() {
+        long newCreateTimeMills = System.currentTimeMillis() + 2000;
+        accAndTimeStamp.setCreateTimeMills(newCreateTimeMills);
+        assertEquals("Create time should be set to new value", newCreateTimeMills, accAndTimeStamp.getCreateTimeMills().longValue());
+    }
+
+    @Test
+    public void testToStringContainsCorrectInformation() {
+        String toStringOutput = accAndTimeStamp.toString();
+        assertTrue("ToString should contain cold accumulator value", toStringOutput.contains("coldAcc=" + accAndTimeStamp.getColdAcc()));
+        assertTrue("ToString should contain last cold read time", toStringOutput.contains("lastColdReadTimeMills=" + accAndTimeStamp.getLastColdReadTimeMills()));
+        assertTrue("ToString should contain create time", toStringOutput.contains("createTimeMills=" + accAndTimeStamp.getCreateTimeMills()));
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/compression/CompressionTypeTest.java b/common/src/test/java/org/apache/rocketmq/common/compression/CompressionTypeTest.java
new file mode 100644
index 00000000000..f9586bd2da8
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/compression/CompressionTypeTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.compression;
+
+import org.apache.rocketmq.common.sysflag.MessageSysFlag;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+
+public class CompressionTypeTest {
+
+    @Test
+    public void testCompressionTypeValues() {
+        assertEquals(1, CompressionType.LZ4.getValue());
+        assertEquals(2, CompressionType.ZSTD.getValue());
+        assertEquals(3, CompressionType.ZLIB.getValue());
+    }
+
+    @Test
+    public void testCompressionTypeOf() {
+        assertEquals(CompressionType.LZ4, CompressionType.of("LZ4"));
+        assertEquals(CompressionType.ZSTD, CompressionType.of("ZSTD"));
+        assertEquals(CompressionType.ZLIB, CompressionType.of("ZLIB"));
+        assertThrows(RuntimeException.class, () -> CompressionType.of("UNKNOWN"));
+    }
+
+    @Test
+    public void testCompressionTypeFindByValue() {
+        assertEquals(CompressionType.LZ4, CompressionType.findByValue(1));
+        assertEquals(CompressionType.ZSTD, CompressionType.findByValue(2));
+        assertEquals(CompressionType.ZLIB, CompressionType.findByValue(3));
+        assertEquals(CompressionType.ZLIB, CompressionType.findByValue(0));
+        assertThrows(RuntimeException.class, () -> CompressionType.findByValue(99));
+    }
+
+    @Test
+    public void testCompressionFlag() {
+        assertEquals(MessageSysFlag.COMPRESSION_LZ4_TYPE, CompressionType.LZ4.getCompressionFlag());
+        assertEquals(MessageSysFlag.COMPRESSION_ZSTD_TYPE, CompressionType.ZSTD.getCompressionFlag());
+        assertEquals(MessageSysFlag.COMPRESSION_ZLIB_TYPE, CompressionType.ZLIB.getCompressionFlag());
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/compression/CompressorFactoryTest.java b/common/src/test/java/org/apache/rocketmq/common/compression/CompressorFactoryTest.java
new file mode 100644
index 00000000000..e150fb2f7aa
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/compression/CompressorFactoryTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.compression;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class CompressorFactoryTest {
+
+    @Test
+    public void testGetCompressor_ReturnsNonNull() {
+        for (CompressionType type : CompressionType.values()) {
+            Compressor compressor = CompressorFactory.getCompressor(type);
+            Assert.assertNotNull("Compressor should not be null for type " + type, compressor);
+        }
+    }
+
+    @Test
+    public void testGetCompressor_ReturnsCorrectType() {
+        for (CompressionType type : CompressionType.values()) {
+            Compressor compressor = CompressorFactory.getCompressor(type);
+            Assert.assertTrue("Compressor type mismatch for " + type,
+                compressor instanceof Lz4Compressor && type == CompressionType.LZ4 ||
+                    compressor instanceof ZstdCompressor && type == CompressionType.ZSTD ||
+                    compressor instanceof ZlibCompressor && type == CompressionType.ZLIB);
+        }
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/compression/Lz4CompressorTest.java b/common/src/test/java/org/apache/rocketmq/common/compression/Lz4CompressorTest.java
new file mode 100644
index 00000000000..ca59025c133
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/compression/Lz4CompressorTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.compression;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import org.junit.Test;
+
+public class Lz4CompressorTest {
+
+    private static final String TEST_STRING = "The quick brown fox jumps over the lazy dog";
+
+    @Test
+    public void testCompressAndDecompress() throws Exception {
+        byte[] originalData = TEST_STRING.getBytes();
+        Compressor compressor = new Lz4Compressor();
+        byte[] compressedData = compressor.compress(originalData, 1);
+        assertTrue("Compressed data should be bigger than original", compressedData.length > originalData.length);
+
+        byte[] decompressedData = compressor.decompress(compressedData);
+        assertArrayEquals("Decompressed data should match original data", originalData, decompressedData);
+    }
+
+    @Test
+    public void testCompressWithIOException() throws Exception {
+        byte[] originalData = new byte[] {1, 2, 3};
+        Compressor compressor = new Lz4Compressor();
+        compressor.compress(originalData, 1);
+    }
+
+    @Test(expected = IOException.class)
+    public void testDecompressWithIOException() throws Exception {
+        byte[] compressedData = new byte[] {1, 2, 3};
+        Compressor compressor = new Lz4Compressor();
+        compressor.decompress(compressedData);
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/compression/ZlibCompressorTest.java b/common/src/test/java/org/apache/rocketmq/common/compression/ZlibCompressorTest.java
new file mode 100644
index 00000000000..f46ac7c6691
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/compression/ZlibCompressorTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.compression;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import org.junit.Test;
+
+public class ZlibCompressorTest {
+
+    private static final String TEST_STRING = "The quick brown fox jumps over the lazy dog";
+
+    @Test
+    public void testCompressionAndDecompression() throws Exception {
+        byte[] originalData = TEST_STRING.getBytes();
+        ZlibCompressor compressor = new ZlibCompressor();
+        byte[] compressedData = compressor.compress(originalData, 0);
+        assertTrue("Compressed data should be bigger than original", compressedData.length > originalData.length);
+
+        byte[] decompressedData = compressor.decompress(compressedData);
+        assertArrayEquals("Decompressed data should match original", originalData, decompressedData);
+    }
+
+    @Test
+    public void testCompressionFailureWithInvalidData() throws Exception {
+        byte[] originalData = new byte[] {0, 1, 2, 3, 4};
+        ZlibCompressor compressor = new ZlibCompressor();
+        compressor.compress(originalData, 0);
+    }
+
+    @Test(expected = IOException.class)
+    public void testDecompressionFailureWithInvalidData() throws Exception {
+        byte[] compressedData = new byte[] {0, 1, 2, 3, 4};
+        ZlibCompressor compressor = new ZlibCompressor();
+        compressor.decompress(compressedData); // Invalid compressed data
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/compression/ZstdCompressorTest.java b/common/src/test/java/org/apache/rocketmq/common/compression/ZstdCompressorTest.java
new file mode 100644
index 00000000000..574e1281811
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/compression/ZstdCompressorTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.compression;
+
+import java.io.IOException;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ZstdCompressorTest {
+
+    @Test
+    public void testCompressAndDecompress() throws IOException {
+        byte[] originalData = "RocketMQ is awesome!".getBytes();
+        ZstdCompressor compressor = new ZstdCompressor();
+        byte[] compressedData = compressor.compress(originalData, 1);
+        assertTrue("Compressed data should be bigger than original", compressedData.length > originalData.length);
+
+        byte[] decompressedData = compressor.decompress(compressedData);
+        assertArrayEquals("Decompressed data should match original data", originalData, decompressedData);
+    }
+
+    @Test
+    public void testCompressWithInvalidData() throws IOException {
+        byte[] invalidData = new byte[] {-1, -1, -1, -1};
+        ZstdCompressor compressor = new ZstdCompressor();
+        compressor.compress(invalidData, 1);
+    }
+
+    @Test(expected = IOException.class)
+    public void testDecompressWithInvalidData() throws IOException {
+        byte[] invalidData = new byte[] {-1, -1, -1, -1};
+        ZstdCompressor compressor = new ZstdCompressor();
+        compressor.decompress(invalidData);
+    }
+
+    @Test
+    public void testCompressAndDecompressEmptyString() throws IOException {
+        byte[] originalData = "".getBytes();
+        ZstdCompressor compressor = new ZstdCompressor();
+        byte[] compressedData = compressor.compress(originalData, 1);
+        assertTrue("Compressed data for empty string should not be empty", compressedData.length > 0);
+
+        byte[] decompressedData = compressor.decompress(compressedData);
+        assertArrayEquals("Decompressed data for empty string should match original", originalData, decompressedData);
+    }
+
+    @Test
+    public void testCompressAndDecompressLargeData() throws IOException {
+        StringBuilder largeStringBuilder = new StringBuilder();
+        for (int i = 0; i < 10000; i++) {
+            largeStringBuilder.append("RocketMQ is awesome! ");
+        }
+        byte[] originalData = largeStringBuilder.toString().getBytes();
+
+        ZstdCompressor compressor = new ZstdCompressor();
+        byte[] compressedData = compressor.compress(originalData, 1);
+        assertTrue("Compressed data for large data should be smaller than original", compressedData.length < originalData.length);
+
+        byte[] decompressedData = compressor.decompress(compressedData);
+        assertArrayEquals("Decompressed data for large data should match original", originalData, decompressedData);
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/config/ConfigHelperTest.java b/common/src/test/java/org/apache/rocketmq/common/config/ConfigHelperTest.java
new file mode 100644
index 00000000000..1fcc967d677
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/config/ConfigHelperTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.config;
+
+import org.junit.Test;
+
+public class ConfigHelperTest {
+
+    @Test
+    public void testGetDBLogDir() {
+        // Should not raise exception.
+        ConfigHelper.getDBLogDir();
+    }
+
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/consumer/ReceiptHandleTest.java b/common/src/test/java/org/apache/rocketmq/common/consumer/ReceiptHandleTest.java
new file mode 100644
index 00000000000..54741817e12
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/consumer/ReceiptHandleTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.consumer;
+
+import org.apache.rocketmq.common.KeyBuilder;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ReceiptHandleTest {
+
+    @Test
+    public void testEncodeAndDecode() {
+        long startOffset = 1000L;
+        long retrieveTime = System.currentTimeMillis();
+        long invisibleTime = 1000L;
+        int reviveQueueId = 1;
+        String topicType = "NORMAL";
+        String brokerName = "BrokerA";
+        int queueId = 2;
+        long offset = 2000L;
+        long commitLogOffset = 3000L;
+        ReceiptHandle receiptHandle = ReceiptHandle.builder()
+            .startOffset(startOffset)
+            .retrieveTime(retrieveTime)
+            .invisibleTime(invisibleTime)
+            .reviveQueueId(reviveQueueId)
+            .topicType(topicType)
+            .brokerName(brokerName)
+            .queueId(queueId)
+            .offset(offset)
+            .commitLogOffset(commitLogOffset)
+            .build();
+
+        String encoded = receiptHandle.encode();
+        ReceiptHandle decoded = ReceiptHandle.decode(encoded);
+
+        assertEquals(receiptHandle.getStartOffset(), decoded.getStartOffset());
+        assertEquals(receiptHandle.getRetrieveTime(), decoded.getRetrieveTime());
+        assertEquals(receiptHandle.getInvisibleTime(), decoded.getInvisibleTime());
+        assertEquals(receiptHandle.getReviveQueueId(), decoded.getReviveQueueId());
+        assertEquals(receiptHandle.getTopicType(), decoded.getTopicType());
+        assertEquals(receiptHandle.getBrokerName(), decoded.getBrokerName());
+        assertEquals(receiptHandle.getQueueId(), decoded.getQueueId());
+        assertEquals(receiptHandle.getOffset(), decoded.getOffset());
+        assertEquals(receiptHandle.getCommitLogOffset(), decoded.getCommitLogOffset());
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testDecodeWithInvalidString() {
+        String invalidReceiptHandle = "invalid_data";
+
+        ReceiptHandle.decode(invalidReceiptHandle);
+    }
+
+    @Test
+    public void testIsExpired() {
+        long startOffset = 1000L;
+        long retrieveTime = System.currentTimeMillis();
+        long invisibleTime = 1000L;
+        int reviveQueueId = 1;
+        String topicType = "NORMAL";
+        String brokerName = "BrokerA";
+        int queueId = 2;
+        long offset = 2000L;
+        long commitLogOffset = 3000L;
+        long pastTime = System.currentTimeMillis() - 1000L;
+        ReceiptHandle receiptHandle = new ReceiptHandle(startOffset, retrieveTime, invisibleTime, pastTime, reviveQueueId, topicType, brokerName, queueId, offset, commitLogOffset, "");
+
+        boolean isExpired = receiptHandle.isExpired();
+
+        assertTrue(isExpired);
+    }
+
+    @Test
+    public void testGetRealTopic() {
+        // Arrange
+        String topic = "TestTopic";
+        String groupName = "TestGroup";
+        ReceiptHandle receiptHandle = ReceiptHandle.builder()
+            .topicType(ReceiptHandle.RETRY_TOPIC)
+            .build();
+
+        String realTopic = receiptHandle.getRealTopic(topic, groupName);
+
+        assertEquals(KeyBuilder.buildPopRetryTopicV1(topic, groupName), realTopic);
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/producer/RecallMessageHandleTest.java b/common/src/test/java/org/apache/rocketmq/common/producer/RecallMessageHandleTest.java
new file mode 100644
index 00000000000..56608227693
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/producer/RecallMessageHandleTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.producer;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.rocketmq.common.message.MessageClientIDSetter;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class RecallMessageHandleTest {
+    @Test
+    public void testHandleInvalid() {
+        Assert.assertThrows(DecoderException.class, () -> {
+            RecallMessageHandle.decodeHandle("");
+        });
+        Assert.assertThrows(DecoderException.class, () -> {
+            RecallMessageHandle.decodeHandle(null);
+        });
+
+        Assert.assertThrows(DecoderException.class, () -> {
+            String invalidHandle = Base64.getUrlEncoder().encodeToString("v1 a b c".getBytes(StandardCharsets.UTF_8));
+            RecallMessageHandle.decodeHandle(invalidHandle);
+        });
+        Assert.assertThrows(DecoderException.class, () -> {
+            String invalidHandle = Base64.getUrlEncoder().encodeToString("v2 a b c d".getBytes(StandardCharsets.UTF_8));
+            RecallMessageHandle.decodeHandle(invalidHandle);
+        });
+        Assert.assertThrows(DecoderException.class, () -> {
+            String invalidHandle = "v1 a b c d";
+            RecallMessageHandle.decodeHandle(invalidHandle);
+        });
+    }
+
+    @Test
+    public void testEncodeAndDecodeV1() throws DecoderException {
+        String topic = "topic";
+        String brokerName = "broker-0";
+        String timestampStr = String.valueOf(System.currentTimeMillis());
+        String messageId = MessageClientIDSetter.createUniqID();
+        String handle = RecallMessageHandle.HandleV1.buildHandle(topic, brokerName, timestampStr, messageId);
+        RecallMessageHandle handleEntity = RecallMessageHandle.decodeHandle(handle);
+        Assert.assertTrue(handleEntity instanceof RecallMessageHandle.HandleV1);
+        RecallMessageHandle.HandleV1 handleV1 = (RecallMessageHandle.HandleV1) handleEntity;
+        Assert.assertEquals(handleV1.getVersion(), "v1");
+        Assert.assertEquals(handleV1.getTopic(), topic);
+        Assert.assertEquals(handleV1.getBrokerName(), brokerName);
+        Assert.assertEquals(handleV1.getTimestampStr(), timestampStr);
+        Assert.assertEquals(handleV1.getMessageId(), messageId);
+    }
+}
diff --git a/common/src/test/java/org/apache/rocketmq/common/utils/IPAddressUtilsTest.java b/common/src/test/java/org/apache/rocketmq/common/utils/IPAddressUtilsTest.java
new file mode 100644
index 00000000000..404250e6c0e
--- /dev/null
+++ b/common/src/test/java/org/apache/rocketmq/common/utils/IPAddressUtilsTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.utils;
+
+import org.junit.Test;
+
+public class IPAddressUtilsTest {
+
+    @Test
+    public void isIPInRange() {
+
+        // IPv4 test
+        String ipv4Address = "192.168.1.10";
+        String ipv4Cidr = "192.168.1.0/24";
+        assert IPAddressUtils.isIPInRange(ipv4Address, ipv4Cidr);
+
+        ipv4Address = "192.168.2.10";
+        assert !IPAddressUtils.isIPInRange(ipv4Address, ipv4Cidr);
+
+        // IPv6 test
+        String ipv6Address = "2001:0db8:85a3:0000:0000:8a2e:0370:7334";
+        String ipv6Cidr = "2001:0db8:85a3::/48";
+        assert IPAddressUtils.isIPInRange(ipv6Address, ipv6Cidr);
+    }
+
+    @Test
+    public void isValidCidr() {
+        String ipv4Cidr = "192.168.1.0/24";
+        String ipv6Cidr = "2001:0db8:1234:5678::/64";
+        String invalidCidr = "192.168.1.0";
+
+        assert IPAddressUtils.isValidCidr(ipv4Cidr);
+        assert IPAddressUtils.isValidCidr(ipv6Cidr);
+        assert !IPAddressUtils.isValidCidr(invalidCidr);
+    }
+
+    @Test
+    public void isValidIp() {
+        String ipv4 = "192.168.1.0";
+        String ipv6 = "2001:0db8:85a3:0000:0000:8a2e:0370:7334";
+        String invalidIp = "192.168.1.256";
+        String ipv4Cidr = "192.168.1.0/24";
+
+        assert IPAddressUtils.isValidIp(ipv4);
+        assert IPAddressUtils.isValidIp(ipv6);
+        assert !IPAddressUtils.isValidIp(invalidIp);
+        assert !IPAddressUtils.isValidIp(ipv4Cidr);
+    }
+
+    @Test
+    public void isValidIPOrCidr() {
+        String ipv4 = "192.168.1.0";
+        String ipv6 = "2001:0db8:85a3:0000:0000:8a2e:0370:7334";
+        String ipv4Cidr = "192.168.1.0/24";
+        String ipv6Cidr = "2001:0db8:1234:5678::/64";
+        assert IPAddressUtils.isValidIPOrCidr(ipv4);
+        assert IPAddressUtils.isValidIPOrCidr(ipv6);
+        assert IPAddressUtils.isValidIPOrCidr(ipv4Cidr);
+        assert IPAddressUtils.isValidIPOrCidr(ipv6Cidr);
+    }
+}
\ No newline at end of file
diff --git a/container/pom.xml b/container/pom.xml
index 8af231e013e..5fa66e65d82 100644
--- a/container/pom.xml
+++ b/container/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java b/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java
index d0a550be635..b8b7f7e1023 100644
--- a/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java
+++ b/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java
@@ -83,7 +83,7 @@ public BrokerContainer(
         this.nettyServerConfig = nettyServerConfig;
         this.nettyClientConfig = nettyClientConfig;
 
-        this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig);
+        this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig, null);
 
         this.brokerContainerProcessor = new BrokerContainerProcessor(this);
         this.brokerContainerProcessor.registerBrokerBootHook(this.brokerBootHookList);
diff --git a/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java b/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java
index 5ced0825761..80dd6ccb13b 100644
--- a/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java
+++ b/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java
@@ -86,7 +86,7 @@ public boolean rejectRequest() {
     }
 
     private synchronized RemotingCommand addBroker(ChannelHandlerContext ctx,
-        RemotingCommand request) throws Exception {
+                                                   RemotingCommand request) throws Exception {
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
         final AddBrokerRequestHeader requestHeader = (AddBrokerRequestHeader) request.decodeCommandCustomHeader(AddBrokerRequestHeader.class);
 
@@ -124,10 +124,7 @@ private synchronized RemotingCommand addBroker(ChannelHandlerContext ctx,
         MixAll.properties2Object(brokerProperties, messageStoreConfig);
 
         messageStoreConfig.setHaListenPort(brokerConfig.getListenPort() + 1);
-
-        if (configPath != null && !configPath.isEmpty()) {
-            brokerConfig.setBrokerConfigPath(configPath);
-        }
+        brokerConfig.setBrokerConfigPath(configPath);
 
         if (!messageStoreConfig.isEnableDLegerCommitLog()) {
             if (!brokerConfig.isEnableControllerMode()) {
diff --git a/container/src/main/java/org/apache/rocketmq/container/InnerBrokerController.java b/container/src/main/java/org/apache/rocketmq/container/InnerBrokerController.java
index a1c1eecf590..616188e52d1 100644
--- a/container/src/main/java/org/apache/rocketmq/container/InnerBrokerController.java
+++ b/container/src/main/java/org/apache/rocketmq/container/InnerBrokerController.java
@@ -23,6 +23,7 @@
 import org.apache.rocketmq.common.AbstractBrokerRunnable;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.remoting.RemotingServer;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
 import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.store.MessageStore;
@@ -43,8 +44,11 @@ public InnerBrokerController(
 
     @Override
     protected void initializeRemotingServer() {
-        this.remotingServer = this.brokerContainer.getRemotingServer().newRemotingServer(brokerConfig.getListenPort());
-        this.fastRemotingServer = this.brokerContainer.getRemotingServer().newRemotingServer(brokerConfig.getListenPort() - 2);
+        RemotingServer remotingServer = this.brokerContainer.getRemotingServer().newRemotingServer(brokerConfig.getListenPort());
+        RemotingServer fastRemotingServer = this.brokerContainer.getRemotingServer().newRemotingServer(brokerConfig.getListenPort() - 2);
+
+        setRemotingServer(remotingServer);
+        setFastRemotingServer(fastRemotingServer);
     }
 
     @Override
@@ -119,11 +123,11 @@ public void shutdown() {
             scheduledFuture.cancel(true);
         }
 
-        if (this.remotingServer != null) {
+        if (getRemotingServer() != null) {
             this.brokerContainer.getRemotingServer().removeRemotingServer(brokerConfig.getListenPort());
         }
 
-        if (this.fastRemotingServer != null) {
+        if (getFastRemotingServer() != null) {
             this.brokerContainer.getRemotingServer().removeRemotingServer(brokerConfig.getListenPort() - 2);
         }
     }
diff --git a/container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineTest.java b/container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineServiceTest.java
similarity index 63%
rename from container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineTest.java
rename to container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineServiceTest.java
index 2158b8d9aca..6a46ed1f6ff 100644
--- a/container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineTest.java
+++ b/container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineServiceTest.java
@@ -17,15 +17,12 @@
 
 package org.apache.rocketmq.container;
 
-import java.lang.reflect.Field;
-import java.time.Duration;
-import java.util.HashMap;
-import java.util.Map;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.BrokerPreOnlineService;
 import org.apache.rocketmq.broker.out.BrokerOuterAPI;
 import org.apache.rocketmq.broker.transaction.TransactionalMessageCheckService;
 import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.remoting.protocol.BrokerSyncInfo;
 import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;
 import org.apache.rocketmq.store.DefaultMessageStore;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
@@ -34,12 +31,21 @@
 import org.mockito.Mock;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import java.lang.reflect.Field;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
 import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
-public class BrokerPreOnlineTest {
+public class BrokerPreOnlineServiceTest {
     @Mock
     private BrokerContainer brokerContainer;
 
@@ -48,25 +54,27 @@ public class BrokerPreOnlineTest {
     @Mock
     private BrokerOuterAPI brokerOuterAPI;
 
-    public void init() throws Exception {
+    public void init(final long brokerId) throws Exception {
         when(brokerContainer.getBrokerOuterAPI()).thenReturn(brokerOuterAPI);
 
         BrokerMemberGroup brokerMemberGroup1 = new BrokerMemberGroup();
         Map<Long, String> brokerAddrMap = new HashMap<>();
-        brokerAddrMap.put(1L, "127.0.0.1:20911");
+        brokerAddrMap.put(0L, "127.0.0.1:10911");
         brokerMemberGroup1.setBrokerAddrs(brokerAddrMap);
 
         BrokerMemberGroup brokerMemberGroup2 = new BrokerMemberGroup();
         brokerMemberGroup2.setBrokerAddrs(new HashMap<>());
-
-//        when(brokerOuterAPI.syncBrokerMemberGroup(anyString(), anyString()))
-//            .thenReturn(brokerMemberGroup1)
-//            .thenReturn(brokerMemberGroup2);
-//        doNothing().when(brokerOuterAPI).sendBrokerHaInfo(anyString(), anyString(), anyLong(), anyString());
+        when(brokerOuterAPI.syncBrokerMemberGroup(anyString(), anyString(), anyBoolean()))
+                .thenReturn(brokerMemberGroup1)
+                .thenReturn(brokerMemberGroup2);
+        BrokerSyncInfo brokerSyncInfo = mock(BrokerSyncInfo.class);
+        when(brokerOuterAPI.retrieveBrokerHaInfo(anyString())).thenReturn(brokerSyncInfo);
 
         DefaultMessageStore defaultMessageStore = mock(DefaultMessageStore.class);
         when(defaultMessageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());
-        when(defaultMessageStore.getBrokerConfig()).thenReturn(new BrokerConfig());
+        BrokerConfig brokerConfig = new BrokerConfig();
+        brokerConfig.setBrokerId(brokerId);
+        when(defaultMessageStore.getBrokerConfig()).thenReturn(brokerConfig);
 
 //        HAService haService = new DefaultHAService();
 //        haService.init(defaultMessageStore);
@@ -91,11 +99,38 @@ public void init() throws Exception {
 
     @Test
     public void testMasterOnlineConnTimeout() throws Exception {
-        init();
+        init(0);
         BrokerPreOnlineService brokerPreOnlineService = new BrokerPreOnlineService(innerBrokerController);
 
         brokerPreOnlineService.start();
 
         await().atMost(Duration.ofSeconds(30)).until(() -> !innerBrokerController.isIsolated());
     }
+
+    @Test
+    public void testMasterOnlineNormal() throws Exception {
+        init(0);
+        BrokerPreOnlineService brokerPreOnlineService = new BrokerPreOnlineService(innerBrokerController);
+        brokerPreOnlineService.start();
+        TimeUnit.SECONDS.sleep(1);
+        brokerPreOnlineService.shutdown();
+        await().atMost(Duration.ofSeconds(30)).until(brokerPreOnlineService::isStopped);
+    }
+
+    @Test
+    public void testSlaveOnlineNormal() throws Exception {
+        init(1);
+        BrokerPreOnlineService brokerPreOnlineService = new BrokerPreOnlineService(innerBrokerController);
+        brokerPreOnlineService.start();
+        TimeUnit.SECONDS.sleep(1);
+        brokerPreOnlineService.shutdown();
+        await().atMost(Duration.ofSeconds(30)).until(brokerPreOnlineService::isStopped);
+    }
+
+    @Test
+    public void testGetServiceName() throws Exception {
+        init(1);
+        BrokerPreOnlineService brokerPreOnlineService = new BrokerPreOnlineService(innerBrokerController);
+        assertEquals(BrokerPreOnlineService.class.getSimpleName(), brokerPreOnlineService.getServiceName());
+    }
 }
diff --git a/controller/pom.xml b/controller/pom.xml
index 996197f9b6b..e5547ba0ffe 100644
--- a/controller/pom.xml
+++ b/controller/pom.xml
@@ -15,12 +15,11 @@
   limitations under the License.
   -->
 
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <packaging>jar</packaging>
diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java
index a032b7b6211..3421010340a 100644
--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java
+++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java
@@ -101,7 +101,7 @@ public class DLedgerController implements Controller {
 
     private final List<BrokerLifecycleListener> brokerLifecycleListeners;
 
-    // Usr for checking whether the broker is alive
+    // use for checking whether the broker is alive
     private BrokerValidPredicate brokerAlivePredicate;
     // use for elect a master
     private ElectPolicy electPolicy;
@@ -569,7 +569,7 @@ public void handle(long term, MemberState.Role role) {
                                 break;
                             }
                             tryTimes++;
-                            log.error(String.format("Controller leader append initial log failed, try %d times", tryTimes));
+                            log.error("Controller leader append initial log failed, try {} times", tryTimes);
                             if (tryTimes % 3 == 0) {
                                 log.warn("Controller leader append initial log failed too many times, please wait a while");
                             }
diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java
index 5ec298a383e..05d742fb7b0 100644
--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java
+++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java
@@ -184,7 +184,7 @@ public Map<String, Map<String, Integer>> getActiveBrokersNum() {
             .forEach(id -> {
                 map.computeIfAbsent(id.getClusterName(), k -> new HashMap<>());
                 map.get(id.getClusterName()).compute(id.getBrokerName(), (broker, num) ->
-                    num == null ? 0 : num + 1
+                    num == null ? 1 : num + 1
                 );
             });
         return map;
diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/RaftBrokerHeartBeatManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/RaftBrokerHeartBeatManager.java
index 99f7b34d4a4..d981ff430c9 100644
--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/RaftBrokerHeartBeatManager.java
+++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/RaftBrokerHeartBeatManager.java
@@ -263,7 +263,7 @@ public Map<String, Map<String, Integer>> getActiveBrokersNum() {
             .forEach(id -> {
                 map.computeIfAbsent(id.getClusterName(), k -> new HashMap<>());
                 map.get(id.getClusterName()).compute(id.getBrokerName(), (broker, num) ->
-                    num == null ? 0 : num + 1
+                    num == null ? 1 : num + 1
                 );
             });
         return map;
diff --git a/distribution/bin/runbroker.cmd b/distribution/bin/runbroker.cmd
index 0ea87f876db..f9a710985a0 100644
--- a/distribution/bin/runbroker.cmd
+++ b/distribution/bin/runbroker.cmd
@@ -23,7 +23,7 @@ set BASE_DIR=%~dp0
 set BASE_DIR=%BASE_DIR:~0,-1%
 for %%d in (%BASE_DIR%) do set BASE_DIR=%%~dpd
 
-set CLASSPATH=.;%BASE_DIR%conf;%BASE_DIR%lib\*;%CLASSPATH%
+set CLASSPATH=.;%BASE_DIR%conf;%BASE_DIR%lib\*;"%CLASSPATH%"
 
 rem ===========================================================================================
 rem  JVM Configuration
@@ -53,8 +53,8 @@ if %JAVA_MAJOR_VERSION% lss 17 (
    set "JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow"
    set "JAVA_OPT=%JAVA_OPT% -XX:+AlwaysPreTouch"
    set "JAVA_OPT=%JAVA_OPT% -XX:MaxDirectMemorySize=15g"
-   set "JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages -XX:-UseBiasedLocking"
+   set "JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages -XX:-UseBiasedLocking -XX:+IgnoreUnrecognizedVMOptions"
    set "JAVA_OPT=%JAVA_OPT% %JAVA_OPT_EXT% -cp "%CLASSPATH%""
 )
 
-"%JAVA%" %JAVA_OPT% %*
\ No newline at end of file
+"%JAVA%" %JAVA_OPT% %*
diff --git a/distribution/bin/runbroker.sh b/distribution/bin/runbroker.sh
index e6e2132aba3..e701e6c3200 100644
--- a/distribution/bin/runbroker.sh
+++ b/distribution/bin/runbroker.sh
@@ -105,7 +105,7 @@ choose_gc_options
 JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow"
 JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch"
 JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=15g"
-JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking"
+JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking -XX:+IgnoreUnrecognizedVMOptions"
 #JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n"
 JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}"
 JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}"
diff --git a/distribution/conf/plain_acl.yml b/distribution/conf/plain_acl.yml
deleted file mode 100644
index 2435380d856..00000000000
--- a/distribution/conf/plain_acl.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-globalWhiteRemoteAddresses:
-  - 10.10.103.*
-  - 192.168.0.*
-
-accounts:
-  - accessKey: RocketMQ
-    secretKey: 12345678
-    whiteRemoteAddress:
-    admin: false
-    defaultTopicPerm: DENY
-    defaultGroupPerm: SUB
-    topicPerms:
-      - topicA=DENY
-      - topicB=PUB|SUB
-      - topicC=SUB
-    groupPerms:
-      # the group should convert to retry topic
-      - groupA=DENY
-      - groupB=PUB|SUB
-      - groupC=SUB
-
-  - accessKey: rocketmq2
-    secretKey: 12345678
-    whiteRemoteAddress: 192.168.1.*
-    # if it is admin, it could access all resources
-    admin: true
-    
diff --git a/distribution/pom.xml b/distribution/pom.xml
index c3c504a139a..c325b9fbcbd 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
     <artifactId>rocketmq-distribution</artifactId>
     <name>rocketmq-distribution ${project.version}</name>
diff --git a/docs/cn/BrokerContainer.md b/docs/cn/BrokerContainer.md
index 236439284be..a4de9889f27 100644
--- a/docs/cn/BrokerContainer.md
+++ b/docs/cn/BrokerContainer.md
@@ -72,7 +72,7 @@ sh mqbrokercontainer -c broker-container.conf
 ```
 mqbrokercontainer脚本路径为distribution/bin/mqbrokercontainer。
 
-## 运行时增加或较少Broker
+## 运行时增加或减少Broker
 
 当BrokerContainer进程启动后，也可以通过Admin命令来增加或减少Broker。
 
diff --git a/docs/cn/Debug_In_Idea.md b/docs/cn/Debug_In_Idea.md
index fd01751ee99..92cb8010efc 100644
--- a/docs/cn/Debug_In_Idea.md
+++ b/docs/cn/Debug_In_Idea.md
@@ -6,7 +6,7 @@
 
 ### Step1: 启动NameServer
 1. NamerServer的启动类在`org.apache.rocketmq.namesrv.NamesrvStartup`
-2. `Idea-Edit Configurations`中添加运行参数 `ROCKETMQ_HOME=<rocketmq仓库目录>`
+2. `Idea-Edit Configurations`中添加环境变量 `ROCKETMQ_HOME=<rocketmq仓库目录>`
 ![Idea_config_nameserver.png](image/Idea_config_nameserver.png)
 3. 运行NameServer，观察到如下日志输出则启动成功
 ```shell
@@ -26,9 +26,9 @@ deleteWhen = 04
 fileReservedTime = 48
 brokerRole = ASYNC_MASTER
 flushDiskType = ASYNC_FLUSH
-namesrvAddr = 127.0.0.1:9876 # name server地址
+namesrvAddr = 127.0.0.1:9876
 ```
-3. `Idea-Edit Configurations`中添加运行参数 `ROCKETMQ_HOME=<rocketmq仓库目录>` 以及环境变量`-c /Users/xxx/rocketmq/conf/broker.conf`
+3. `Idea-Edit Configurations`中添加运行参数 `-c /Users/xxx/rocketmq/conf/broker.conf` 以及环境变量 `ROCKETMQ_HOME=<rocketmq仓库目录>`
 ![Idea_config_broker.png](image/Idea_config_broker.png)
 4. 运行Broker，观察到如下日志则启动成功
 ```shell
@@ -40,7 +40,7 @@ The broker[broker-a,192.169.1.2:10911] boot success...
 
 ### 补充：本地启动Proxy
 1. RocketMQ5.x支持了Proxy模式，使用`LOCAL`模式可以免去`Step2`，启动类在`org.apache.rocketmq.proxy.ProxyStartup`
-2. `Idea-Edit Configurations`中添加运行参数 `ROCKETMQ_HOME=<rocketmq仓库目录>`
+2. `Idea-Edit Configurations`中添加环境变量 `ROCKETMQ_HOME=<rocketmq仓库目录>`
 3. 在`/conf/`下新建配置文件`rmq-proxy.json`
 ```json
 {
diff --git a/docs/cn/Deployment.md b/docs/cn/Deployment.md
index 14529d111b0..c13f3280350 100644
--- a/docs/cn/Deployment.md
+++ b/docs/cn/Deployment.md
@@ -69,7 +69,7 @@ $ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker
 ...
 ```
 
-上面显示的启动命令用于单个NameServer的情况。对于多个NameServer的集群，broker 启动命令中-n参数后面的地址列表用分号隔开，例如 192.168.1.1:9876;192.161.2:9876
+上面显示的启动命令用于单个NameServer的情况。对于多个NameServer的集群，broker 启动命令中-n参数后面的地址列表用分号隔开，例如 192.168.1.1:9876;192.168.1.2:9876
 
 ### 3 多Master多Slave模式-异步复制
 
@@ -168,4 +168,3 @@ RocketMQ 5.0 开始支持自动主从切换的模式，可参考以下文档
 
 [设计思想](controller/design.md)
 
-
diff --git a/docs/cn/RocketMQ_Example.md b/docs/cn/RocketMQ_Example.md
index ece3a56f229..77c1bd7b270 100644
--- a/docs/cn/RocketMQ_Example.md
+++ b/docs/cn/RocketMQ_Example.md
@@ -645,7 +645,7 @@ RocketMQ只定义了一些基本语法来支持这个特性。你也可以很容
 
 只有使用push模式的消费者才能用使用SQL92标准的sql语句，接口如下：
 ```
-public void subscribe(finalString topic, final MessageSelector messageSelector)
+public void subscribe(final String topic, final MessageSelector messageSelector)
 ```
 
 ### 5.2 使用样例
@@ -785,7 +785,7 @@ public class TransactionListenerImpl implements TransactionListener {
 3. 事务消息将在 Broker 配置文件中的参数 transactionTimeout 这样的特定时间长度之后被检查。当发送事务消息时，用户还可以通过设置用户属性 CHECK_IMMUNITY_TIME_IN_SECONDS 来改变这个限制，该参数优先于 `transactionTimeout` 参数。
 4. 事务性消息可能不止一次被检查或消费。
 5. 提交给用户的目标主题消息可能会失败，目前这依日志的记录而定。它的高可用性通过 RocketMQ 本身的高可用性机制来保证，如果希望确保事务消息不丢失、并且事务完整性得到保证，建议使用同步的双重写入机制。
-6. 事务消息的生产者 ID 不能与其他类型消息的生产者 ID 共享。与其他类型的消息不同，事务消息允许反向查询、MQ服务器能通过它们的生产者 ID 查询到消费者。
+6. 事务消息的生产者 GroupName 不能与其他类型消息的生产者 GroupName 共享。与其他类型的消息不同，事务消息允许反向查询、MQ服务器能通过它们的生产者 GroupName 查询到生产者。
 
 7 Logappender样例
 -----------------
diff --git a/docs/cn/SlaveActingMasterMode.md b/docs/cn/SlaveActingMasterMode.md
index b1e266f2b42..b64adc61204 100644
--- a/docs/cn/SlaveActingMasterMode.md
+++ b/docs/cn/SlaveActingMasterMode.md
@@ -70,9 +70,9 @@ public void changeSpecialServiceStatus(boolean shouldStart) {
 
 2. Broker能及时感知到同组Broker的上下线情况。
 
-针对1，Nameserver原本就存在判活机制，定时会扫描不活跃的broker使其下线，而原本broker与nameserver的“心跳”则依赖于registerBroker操作，而这个操作涉及到topic信息上报，过于“重”，而且注册间隔过于长，因此需要一个轻量级的心跳机制，RoccketMQ 5.0在nameserver和broker间新增BrokerHeartbeat请求，broker会定时向nameserver发送心跳，若nameserver定时任务扫描发现超过心跳超时时间仍未收到该broker的心跳，将unregister该broker。registerBroker时会完成心跳超时时间的设置，并且注册时如果发现broker组内最小brokerId发生变化，将反向通知该组所有broker，并在路由获取时将最小brokerId的Slave路由替换使其充当只读模式的Master的角色
+针对1，Nameserver原本就存在判活机制，定时会扫描不活跃的broker使其下线，而原本broker与nameserver的“心跳”则依赖于registerBroker操作，而这个操作涉及到topic信息上报，过于“重”，而且注册间隔过于长，因此需要一个轻量级的心跳机制，RocketMQ 5.0在nameserver和broker间新增BrokerHeartbeat请求，broker会定时向nameserver发送心跳，若nameserver定时任务扫描发现超过心跳超时时间仍未收到该broker的心跳，将unregister该broker。registerBroker时会完成心跳超时时间的设置，并且注册时如果发现broker组内最小brokerId发生变化，将反向通知该组所有broker，并在路由获取时将最小brokerId的Slave路由替换使其充当只读模式的Master的角色
 
-针对2，通过两个机制来及时感知同组broker上下线情况，1是上文中介绍的当nameserver发现该broker组内最小brokerId发生变化，反向通知该组所有broker。2是broker自身会有定时任务，向nameserver同步本broker组存活broker的信息，RoccketMQ 5.0会新增GetBrokerMemberGroup请求来完成该工作。
+针对2，通过两个机制来及时感知同组broker上下线情况，1是上文中介绍的当nameserver发现该broker组内最小brokerId发生变化，反向通知该组所有broker。2是broker自身会有定时任务，向nameserver同步本broker组存活broker的信息，RocketMQ 5.0会新增GetBrokerMemberGroup请求来完成该工作。
 
 Slave Broker发现自己是该组中最小的brokerId，将会开启代理模式，而一旦Master Broker重新上线，Slave Broker同样会通过Nameserver反向通知或自身定时任务同步同组broker的信息感知到，并自动结束代理模式。
 
@@ -80,7 +80,7 @@ Slave Broker发现自己是该组中最小的brokerId，将会开启代理模式
 
 代理模式开启后，brokerId最小的Slave会承担起二级消息的扫描和重新投递功能。
 
-二级消息一般分为两个阶段，发送或者消费时会发送到一个特殊topic中，后台会有线程会扫描，最终的满足要求的消息会被重新投递到Commitlog中。我们可以让brokerId最小的Slave进行扫描，但如果扫描之后的消息重新投递到本Commitlog，那将会破坏Slave不可写的语义，造成Commitlog分叉。因此RoccketMQ 5.0提出一种逃逸机制，将重放的二级消息远程或本地投放到其他Master的Commitlog中。
+二级消息一般分为两个阶段，发送或者消费时会发送到一个特殊topic中，后台会有线程会扫描，最终的满足要求的消息会被重新投递到Commitlog中。我们可以让brokerId最小的Slave进行扫描，但如果扫描之后的消息重新投递到本Commitlog，那将会破坏Slave不可写的语义，造成Commitlog分叉。因此RocketMQ 5.0提出一种逃逸机制，将重放的二级消息远程或本地投放到其他Master的Commitlog中。
 
 - 远程逃逸
 
diff --git a/docs/cn/best_practice.md b/docs/cn/best_practice.md
index 5cc5b37643f..36d6acff6bd 100755
--- a/docs/cn/best_practice.md
+++ b/docs/cn/best_practice.md
@@ -253,7 +253,7 @@ DefaultMQProducer、TransactionMQProducer、DefaultMQPushConsumer、DefaultMQPul
 | clientIP                      | 本机IP  | 客户端本机IP地址，某些机器会发生无法识别客户端IP地址情况，需要应用在代码中强制指定 |
 | instanceName                  | DEFAULT | 客户端实例名称，客户端创建的多个Producer、Consumer实际是共用一个内部实例（这个实例包含网络连接、线程资源等） |
 | clientCallbackExecutorThreads | 4       | 通信层异步回调线程数                                         |
-| pollNameServerInteval         | 30000   | 轮询Name Server间隔时间，单位毫秒                            |
+| pollNameServerInterval        | 30000   | 轮询Name Server间隔时间，单位毫秒                            |
 | heartbeatBrokerInterval       | 30000   | 向Broker发送心跳间隔时间，单位毫秒                           |
 | persistConsumerOffsetInterval | 5000    | 持久化Consumer消费进度间隔时间，单位毫秒                     |
 
diff --git a/docs/cn/controller/deploy.md b/docs/cn/controller/deploy.md
index fa599f3dccc..78816e090d2 100644
--- a/docs/cn/controller/deploy.md
+++ b/docs/cn/controller/deploy.md
@@ -26,14 +26,33 @@ notifyBrokerRoleChanged = true
 
 - enableControllerInNamesrv：Nameserver中是否开启controller，默认false。
 - controllerDLegerGroup：DLedger Raft Group的名字，同一个DLedger Raft Group保持一致即可。
-- controllerDLegerPeers：DLedger Group 内各节点的端口信息，同一个 Group 内的各个节点配置必须要保证一致。
+- controllerDLegerPeers：DLedger Group 内各节点的地址信息，同一个 Group 内的各个节点配置必须要保证一致。
 - controllerDLegerSelfId：节点 id，必须属于 controllerDLegerPeers 中的一个；同 Group 内各个节点要唯一。
 - controllerStorePath：controller日志存储位置。controller是有状态的，controller重启或宕机需要依靠日志来恢复数据，该目录非常重要，不可以轻易删除。
 - enableElectUncleanMaster：是否可以从SyncStateSet以外选举Master，若为true，可能会选取数据落后的副本作为Master而丢失消息，默认为false。
 - notifyBrokerRoleChanged：当broker副本组上角色发生变化时是否主动通知，默认为true。
 - scanNotActiveBrokerInterval：扫描 Broker是否存活的时间间隔。
 
-其他一些参数可以参考ControllerConfig代码。
+- 其他一些参数可以参考ControllerConfig代码。
+
+> 5.2.0 Controller 开始支持 jRaft 内核启动，不支持 DLedger 内核到 jRaft 内核原地升级
+
+```
+controllerType = jRaft
+jRaftGroupId = jRaft-Controller
+jRaftServerId = localhost:9880
+jRaftInitConf = localhost:9880,localhost:9881,localhost:9882
+jRaftControllerRPCAddr = localhost:9770,localhost:9771,localhost:9772
+jRaftSnapshotIntervalSecs = 3600
+```
+
+jRaft 版本相关参数
+- controllerType：controllerType=jRaft的时候内核启动使用jRaft，默认为DLedger。
+- jRaftGroupId：jRaft Group的名字，同一个jRaft Group保持一致即可。
+- jRaftServerId：标志自己节点的ServerId，必须出现在 jRaftInitConf 中。
+- jRaftInitConf：jRaft Group 内部通信各节点的地址信息，用逗号分隔，是 jRaft 内部通信来做选举和复制所用的地址。
+- jRaftControllerRPCAddr：Controller 外部通信的各节点的地址信息，用逗号分隔，比如 Controller 与 Broker 通信会使用该地址。
+- jRaftSnapshotIntervalSecs：Raft Snapshot 持久化时间间隔。
 
 参数设置完成后，指定配置文件启动Nameserver即可。
 
diff --git a/docs/cn/controller/design.md b/docs/cn/controller/design.md
index 563a624eddc..13eba7764a6 100644
--- a/docs/cn/controller/design.md
+++ b/docs/cn/controller/design.md
@@ -121,13 +121,13 @@ nextTransferFromWhere + size > currentTransferEpochEndOffset，则将 selectMapp
 
 ![示意图](../image/controller/controller_design_3.png)
 
-`current state(4byte) + Two flags(4byte) + slaveAddressLength(4byte) + slaveAddress(50byte)`
+`current state(4byte) + Two flags(4byte) + slaveBrokerId(8byte)`
 
 - Current state 代表当前的 HAConnectionState，也即 HANDSHAKE。
 
 - Two flags 是两个状态标志位，其中，isSyncFromLastFile 代表是否要从 Master 的最后一个文件开始复制，isAsyncLearner 代表该 Slave 是否是异步复制，并以 Learner 的形式接入 Master。
 
-- slaveAddressLength 与 slaveAddress 代表了该 Slave 的地址，用于后续加入 SyncStateSet 。
+- slaveBrokerId 代表了该 Slave 的 brokerId，用于后续加入 SyncStateSet 。
 
 2.AutoSwitchHaConnection (Master) 会向 Slave 回送 HandShake 包，如下：
 
diff --git a/docs/cn/image/controller/controller_design_3.png b/docs/cn/image/controller/controller_design_3.png
index 8c475bcecf1..0379c231d46 100644
Binary files a/docs/cn/image/controller/controller_design_3.png and b/docs/cn/image/controller/controller_design_3.png differ
diff --git a/docs/cn/msg_trace/user_guide.md b/docs/cn/msg_trace/user_guide.md
index 9cf139fd347..a04c2601f48 100644
--- a/docs/cn/msg_trace/user_guide.md
+++ b/docs/cn/msg_trace/user_guide.md
@@ -103,7 +103,7 @@ RocketMQ的消息轨迹特性支持两种存储轨迹数据的方式：
 ### 4.4 使用mqadmin命令发送和查看轨迹
 - 发送消息
 ```shell
-./mqadmin sendMessage -m true --topic some-topic-name -n 127.0.0.1:9876 -p "your meesgae content"
+./mqadmin sendMessage -m true --topic some-topic-name -n 127.0.0.1:9876 -p "your message content"
 ```
 - 查询轨迹
 ```shell
diff --git a/docs/en/Configuration_Client.md b/docs/en/Configuration_Client.md
index 4d999b2feda..af83ffcb400 100644
--- a/docs/en/Configuration_Client.md
+++ b/docs/en/Configuration_Client.md
@@ -1,119 +1,107 @@
 ## Client Configuration
 
-  Relative to RocketMQ's Broker cluster, producers and consumers are client. In this section, it mainly describes the common behavior configuration of producers and consumers.
-​ 
-### 1 Client Addressing mode
+Relative to RocketMQ's Broker cluster, producers and consumers are clients. This section describes the common behavior configuration of producers and consumers, including updates for **RocketMQ 5.x**.
 
-```RocketMQ``` can let client find the ```Name Server```, and then find the ```Broker```by the ```Name Server```. Followings show a variety of configurations, and priority level from highly to lower, the highly priority configurations can override the lower priority configurations.
+### 1 Client Addressing Mode
 
--  Specified ```Name Server``` address in the code, and multiple ```Name Server``` addresses are separated by semicolons
+`RocketMQ` allows clients to locate the `Name Server`, which then helps them find the `Broker`. Below are various configurations, prioritized from highest to lowest. Higher priority configurations override lower ones.
+
+- Specified `Name Server` address in the code (multiple addresses separated by semicolons):
 
 ```java
 producer.setNamesrvAddr("192.168.0.1:9876;192.168.0.2:9876");  
-
 consumer.setNamesrvAddr("192.168.0.1:9876;192.168.0.2:9876");
 ```
-- Specified ```Name Server``` address in the Java setup parameters
+
+- Specified `Name Server` address in Java setup parameters:
 
 ```text
 -Drocketmq.namesrv.addr=192.168.0.1:9876;192.168.0.2:9876  
 ```
-- Specified ```Name Server``` address in the environment variables
+
+- Specified `Name Server` address in environment variables:
 
 ```text
-export   NAMESRV_ADDR=192.168.0.1:9876;192.168.0.2:9876   
+export NAMESRV_ADDR=192.168.0.1:9876;192.168.0.2:9876   
 ```
-- HTTP static server addressing(default)
 
-After client started, it will access the http static server address, as: <http://jmenv.tbsite.net:8080/rocketmq/nsaddr>, this URL return the following contents:
+- HTTP static server addressing (default):
 
+Clients retrieve `Name Server` addresses from a static HTTP server:
 ```text
-192.168.0.1:9876;192.168.0.2:9876   
+http://jmenv.tbsite.net:8080/rocketmq/nsaddr
 ```
-By default, the client accesses the HTTP server every 2 minutes, and update the local Name Server address.The URL is hardcoded in the code, you can change the target server by updating ```/etc/hosts``` file, such as add following configuration at the ```/etc/hosts```:
-```text
-10.232.22.67    jmenv.tbsite.net   
-```
-HTTP static server addressing is recommended, because it is simple client deployment, and the Name Server cluster can be upgraded hot.
+
+- **New in RocketMQ 5.x:**
+  - Improved service discovery mechanism, allowing dynamic Name Server registration.
+  - Introduces `VIP_CHANNEL_ENABLED` for better failover:
+
+  ```java
+  producer.setVipChannelEnabled(false);
+  consumer.setVipChannelEnabled(false);
+  ```
 
 ### 2 Client Configuration
 
-```DefaultMQProducer```,```TransactionMQProducer```,```DefaultMQPushConsumer```,```DefaultMQPullConsumer``` all extends the ```ClientConfig``` Class, ```ClientConfig``` as the client common configuration class. Client configuration style like getXXX,setXXX, each of the parameters can config by spring and also config their in the code. Such as the ```namesrvAddr``` parameter: ```producer.setNamesrvAddr("192.168.0.1:9876")```, same with the other parameters.
+`DefaultMQProducer`, `TransactionMQProducer`, `DefaultMQPushConsumer`, and `DefaultMQPullConsumer` all extend the `ClientConfig` class, which provides common client configurations.
 
 #### 2.1 Client Common Configuration
 
-| Parameter Name                        | Default Value  | Description                                                         |
-| ----------------------------- | ------- | ------------------------------------------------------------ |
-| namesrvAddr                   |         | Name Server address list, multiple NameServer addresses are separated by semicolons           |
-| clientIP                      | local IP  | Client local ip address, some machines will fail to recognize the client IP address, which needs to be enforced in the code |
-| instanceName                  | DEFAULT | Name of the client instance, Multiple producers and consumers created by the client actually share one internal instance (this instance contains network connection, thread resources, etc.). |
-| clientCallbackExecutorThreads | 4       | Number of communication layer asynchronous callback threads                                        |
-| pollNameServerInteval         | 30000   | Polling the Name Server interval in milliseconds                          |
-| heartbeatBrokerInterval       | 30000   | The heartbeat interval, in milliseconds, is sent to the Broker                         |
-| persistConsumerOffsetInterval | 5000    | The persistent Consumer consumes the progress interval in milliseconds         |
+| Parameter Name                | Default Value  | Description  |
+|-------------------------------|---------------|--------------|
+| namesrvAddr                   |               | Name Server address list (semicolon-separated) |
+| clientIP                      | Local IP      | Client's local IP address (useful if automatic detection fails) |
+| instanceName                  | DEFAULT       | Unique name for the client instance |
+| clientCallbackExecutorThreads | 4             | Number of communication layer asynchronous callback threads |
+| pollNameServerInterval        | 30000         | Interval (ms) to poll Name Server |
+| heartbeatBrokerInterval       | 30000         | Interval (ms) for sending heartbeats to Broker |
+| persistConsumerOffsetInterval | 5000          | Interval (ms) for persisting consumer offsets |
+| **autoUpdateNameServer** (5.x) | true          | Automatically update Name Server addresses from registry |
+| **instanceId** (5.x)          |               | Unique identifier for each client instance |
 
 #### 2.2 Producer Configuration
 
-| Parameter Name                       | Default Value          | Description                                                        |
-| -------------------------------- | ---------------- | ------------------------------------------------------------ |
-| producerGroup                    | DEFAULT_PRODUCER | The name of the Producer group. If multiple producers belong to one application and send the same message, they should be grouped into the same group |
-| createTopicKey                   | TBW102           | When a message is sent, topics that do not exist on the server are automatically created and a Key is specified that can be used to configure the default route to the topic where the message is sent.|
-| defaultTopicQueueNums            | 4                | The number of default queue when sending messages and auto created topic which not exists the server|
-| sendMsgTimeout                   | 3000             | Timeout time of sending message in milliseconds                           |
-| compressMsgBodyOverHowmuch       | 4096             | The message Body begins to compress beyond the size(the Consumer gets the message automatically unzipped.), unit of byte|
-| retryAnotherBrokerWhenNotStoreOK | FALSE            | If send message and return sendResult but sendStatus!=SEND_OK, Whether to resend |
-| retryTimesWhenSendFailed         | 2                | If send message failed, maximum number of retries, this parameter only works for synchronous send mode|
-| maxMessageSize                   | 4MB              | Client limit message body size, over it may error. Server also limit so need to work with server |
-| transactionCheckListener         |                  | The transaction message looks back to the listener, if you want send transaction message, you must setup this
-| checkThreadPoolMinSize           | 1                | Minimum of thread in thread pool when Broker look back Producer transaction status                     |
-| checkThreadPoolMaxSize           | 1                | Maximum of thread in thread pool when Broker look back Producer transaction status                     |
-| checkRequestHoldMax              | 2000             | Producer local buffer request queue size when Broker look back Producer transaction status                     |
-| RPCHook                          | null             | This parameter is passed in when the Producer is creating, including the pre-processing before the message sending and the processing after the message response. The user can do some security control or other operations in the first interface.|
+| Parameter Name                 | Default Value         | Description  |
+|--------------------------------|----------------------|--------------|
+| producerGroup                  | DEFAULT_PRODUCER     | Producer group name |
+| sendMsgTimeout                 | 3000                 | Timeout (ms) for sending messages |
+| retryTimesWhenSendFailed       | 2                    | Max retries for failed messages |
+| **enableBatchSend** (5.x)      | true                 | Enables batch message sending |
+| **enableBackPressure** (5.x)   | true                 | Prevents overload in high-traffic scenarios |
 
 #### 2.3 PushConsumer Configuration
 
-| Parameter Name                         | Default Value                      | Description                                                         |
-| ---------------------------- | ----------------------------- | ------------------------------------------------------------ |
-| consumerGroup                | DEFAULT_CONSUMER              | Consumer group name. If multi Consumer belong to an application, subscribe the same message and consume logic as the same, they should be gathered together |
-| messageModel                 | CLUSTERING                    | Message support two mode: cluster consumption and broadcast consumption                          |
-| consumeFromWhere             | CONSUME_FROM_LAST_OFFSET      | After Consumer started, default consumption from last location, it include two situation: One is last consumption location is not expired, and consumption start at last location; The other is last location expired, start consumption at current queue's first message |
-| consumeTimestamp             | Half an hour ago              | Only consumeFromWhere=CONSUME_FROM_TIMESTAMP, this can work |
-| allocateMessageQueueStrategy | AllocateMessageQueueAveragely | Implements strategy of Rebalance algorithms                                        |
-| subscription                 |                               | subscription relation                                                    |
-| messageListener              |                               | message listener                                                  |
-| offsetStore                  |                               | Consumption progress store                                                 |
-| consumeThreadMin             | 20                            | Minimum of thread in consumption thread pool                                               |
-| consumeThreadMax             | 20                            | Maximum of thread in consumption thread pool                                               |
-|                              |                               |                                                              |
-| consumeConcurrentlyMaxSpan   | 2000                          | Maximum span allowed for single queue parallel consumption                                 |
-| pullThresholdForQueue        | 1000                          | Pull message local queue cache maximum number of messages                               |
-| pullInterval                 | 0                             | Pull message interval, because long polling it is 0, but for flow control, you can set value which greater than 0 in milliseconds |
-| consumeMessageBatchMaxSize   | 1                             | Batch consume message                                 |
-| pullBatchSize                | 32                            | Batch pull message                                 |
+| Parameter Name                      | Default Value                      | Description |
+|--------------------------------------|------------------------------------|-------------|
+| consumerGroup                        | DEFAULT_CONSUMER                  | Consumer group name |
+| messageModel                         | CLUSTERING                        | Message consumption mode (CLUSTERING or BROADCAST) |
+| consumeFromWhere                     | CONSUME_FROM_LAST_OFFSET          | Default consumption position |
+| consumeThreadMin                     | 20                                | Min consumption thread count |
+| consumeThreadMax                     | 20                                | Max consumption thread count |
+| **Rebalance Strategies (5.x)**        | AllocateMessageQueueAveragelyByCircle | New rebalance strategy for better distribution |
 
 #### 2.4 PullConsumer Configuration
 
-| Parameter Name                     | Default Value                 | Description                                                         |
-| -------------------------------- | ----------------------------- | ------------------------------------------------------------ |
-| consumerGroup                    | DEFAULT_CONSUMER              | Consumer group name. If multi Consumer belong to an application, subscribe the same message and consume logic as the same, they should be gathered together |
-| brokerSuspendMaxTimeMillis       | 20000                         | Long polling, Consumer pull message request suspended for the longest time in the Broker in milliseconds     |
-| consumerTimeoutMillisWhenSuspend | 30000                         | Long polling, Consumer pull message request suspend in the Broker over this time value, client think timeout. Unit is milliseconds |
-| consumerPullTimeoutMillis        | 10000                         | Not long polling, timeout time of pull message in milliseconds                            |
-| messageModel                     | CLUSTERING                    | Message support two mode: cluster consumption and broadcast consumption           |
-| messageQueueListener             |                               | Listening changing of queue                                                 |
-| offsetStore                      |                               | Consumption schedule store                                              |
-| registerTopics                   |                               | Collection of registered topics                                              |
-| allocateMessageQueueStrategy     | AllocateMessageQueueAveragely | Implements strategy about Rebalance algorithm                                     |
+| Parameter Name                     | Default Value                 | Description |
+|------------------------------------|------------------------------|-------------|
+| consumerGroup                      | DEFAULT_CONSUMER             | Consumer group name |
+| brokerSuspendMaxTimeMillis         | 20000                        | Max suspension time (ms) for long polling |
+| consumerPullTimeoutMillis          | 10000                        | Timeout (ms) for pull requests |
+| **messageGroup** (5.x)             |                              | Enables orderly consumption based on groups |
 
 #### 2.5 Message Data Structure
 
-| Field Name         | Default Value  | Description                                                         |
-| -------------- | ------ | ------------------------------------------------------------ |
-| Topic          | null   | Required, the name of the topic to which the message belongs                                       |
-| Body           | null   | Required, message body                                                |
-| Tags           | null   | Optional, message tag, convenient for server filtering. Currently only one tag per message is supported |
-| Keys           | null   | Optional, represent this message's business keys, server create hash indexes based keys. After setting, you can find message by ```Topics```,```Keys``` in Console system. Because of hash indexes, please make key as unique as possible, such as order number, goods Id and so on.|
-| Flag           | 0      | Optional, it is entirely up to the application, and RocketMQ does not intervene                     |
-| DelayTimeLevel | 0      | Optional, message delay level, 0 represent no delay, greater tan 0 can consume |
-| WaitStoreMsgOK | TRUE   | Optional, indicates whether the message is not answered until the server is down.                |
+| Field Name        | Default Value  | Description |
+|-------------------|---------------|-------------|
+| Topic            | null          | Required: Name of the message topic |
+| Body             | null          | Required: Message content |
+| Tags             | null          | Optional: Tag for filtering |
+| Keys             | null          | Optional: Business keys (e.g., order IDs) |
+| Flag             | 0             | Optional: Custom flag |
+| DelayTimeLevel   | 0             | Optional: Message delay level |
+| WaitStoreMsgOK   | TRUE          | Optional: Acknowledgment before storing |
+| **maxReconsumeTimes** (5.x) |   | Max retries before moving to dead-letter queue |
+| **messageGroup** (5.x) |   | Group-based message ordering |
+
+---
 
diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md
index e263b0c4c28..95589d95fbe 100644
--- a/docs/en/Configuration_System.md
+++ b/docs/en/Configuration_System.md
@@ -1,6 +1,6 @@
-# The system configuration
+# The System Configuration (RocketMQ 5.x)
 
-This section focuses on the configuration of the system (JVM/OS)
+This section focuses on the configuration of the system (JVM/OS) for **RocketMQ 5.x**.
 
 ## **1 JVM Options** ##
 
@@ -16,56 +16,42 @@ If you don’t care about the boot time of RocketMQ broker, pre-touch the Java h
     
     -XX:+AlwaysPreTouch
 
-Disable biased locking maybe reduce JVM pauses:
+Disable biased locking to potentially reduce JVM pauses:
 
     -XX:-UseBiasedLocking
 
-As for garbage collection, G1 collector with JDK 1.8 is recommended:
+As for garbage collection, the G1 collector with JDK 1.8 is recommended:
 
-    -XX:+UseG1GC -XX:G1HeapRegionSize=16m 
+    -XX:+UseG1GC -XX:G1HeapRegionSize=16m
     -XX:G1ReservePercent=25
     -XX:InitiatingHeapOccupancyPercent=30
 
-These GC options looks a little aggressive, but it’s proved to have good performance in our production environment
+These GC options may seem a bit aggressive, but they have been proven to provide good performance in our production environment.
 
-Don’t set a too small value for -XX:MaxGCPauseMillis, otherwise JVM will use a small young generation to achieve this goal which will cause very frequent minor GC.So use rolling GC log file is recommended:
+Do not set a too small value for -XX:MaxGCPauseMillis, as it may cause the JVM to use a small young generation, leading to frequent minor GCs. Using a rolling GC log file is recommended:
     
-    -XX:+UseGCLogFileRotation 
-    -XX:NumberOfGCLogFiles=5 
+    -XX:+UseGCLogFileRotation
+    -XX:NumberOfGCLogFiles=5
     -XX:GCLogFileSize=30m
     
-If write GC file will increase latency of broker, consider redirect GC log file to a memory file system:
+If writing GC logs to disk increases broker latency, consider redirecting the GC log file to a memory file system:
     
     -Xloggc:/dev/shm/mq_gc_%p.log123
 
-## 2 Linux Kernel Parameters ##
+## **2 Linux Kernel Parameters** ##
 
-There is a os.sh script that lists a lot of kernel parameters in folder bin which can be used for production use with minor changes. Below parameters need attention, and more details please refer to documentation for /proc/sys/vm/*.
+There is an `os.sh` script in the `bin` folder that lists several kernel parameters that can be used for production with minor modifications. Below parameters require attention. For more details, refer to the documentation for `/proc/sys/vm/*`.
 
+- **vm.extra_free_kbytes**: Tells the VM to keep extra free memory between the threshold where background reclaim (kswapd) kicks in and the threshold where direct reclaim (by allocating processes) kicks in. RocketMQ uses this parameter to avoid high latency in memory allocation. (Kernel version-specific)
 
+- **vm.min_free_kbytes**: If set lower than 1024KB, the system may become subtly broken and prone to deadlocks under high loads.
 
+- **vm.max_map_count**: Limits the maximum number of memory map areas a process may have. RocketMQ uses `mmap` to load `CommitLog` and `ConsumeQueue`, so setting a higher value is recommended.
 
-- **vm.extra_free_kbytes**, tells the VM to keep extra free memory between the threshold where background reclaim (kswapd) kicks in, and the threshold where direct reclaim (by allocating processes) kicks in. RocketMQ uses this parameter to avoid high latency in memory allocation. (It is specific to the kernel version）
+- **vm.swappiness**: Defines how aggressively the kernel swaps memory pages. Higher values increase aggressiveness, while lower values decrease the amount of swap. A value of **10** is recommended to avoid swap latency.
 
+- **File descriptor limits**: RocketMQ requires open file descriptors for files (`CommitLog` and `ConsumeQueue`) and network connections. It is recommended to set this to **655350**.
 
+- **Disk scheduler**: The **deadline I/O scheduler** is recommended for RocketMQ as it attempts to provide a guaranteed latency for requests.
 
-- **vm.min_free_kbytes**, if you set this to lower than 1024KB, your system will become subtly broken, and prone to deadlock under high loads.
-
-
-
-
-
-- **vm.max_map_count**, limits the maximum number of memory map areas a process may have. RocketMQ will use mmap to load CommitLog and ConsumeQueue, so set a bigger value for this parameter is recommended.
-
-
-
-- **vm.swappiness**, define how aggressive the kernel will swap memory pages. Higher values will increase aggressiveness, lower values decrease the amount of swap. 10 is recommended for this value to avoid swap latency.
-
-
-
-- **File descriptor limits**, RocketMQ needs open file descriptors for files(CommitLog and ConsumeQueue) and network connections. We recommend setting  655350 for file descriptors.
-
-
-
-- **Disk scheduler**, the deadline I/O scheduler is recommended for RocketMQ, which attempts to provide a guaranteed latency for requests.
-
+---
diff --git a/docs/en/Debug_In_Idea.md b/docs/en/Debug_In_Idea.md
index 9967980671f..0dee9039c40 100644
--- a/docs/en/Debug_In_Idea.md
+++ b/docs/en/Debug_In_Idea.md
@@ -6,7 +6,7 @@
 
 ### Step1: Start NameServer
 1. The startup class for NameServer is located in `org.apache.rocketmq.namesrv.NamesrvStartup`.
-2. Add runtime `ROCKETMQ_HOME=<rocketmq repository directory>` parameters in `Idea-Edit Configurations`.
+2. Add environment variable `ROCKETMQ_HOME=<rocketmq repository directory>` in `Idea-Edit Configurations`.
 ![Idea_config_nameserver.png](../cn/image/Idea_config_nameserver.png)
 3. Run NameServer and if the following log output is observed, it indicates successful startup.
 ```shell
@@ -26,9 +26,9 @@ deleteWhen = 04
 fileReservedTime = 48
 brokerRole = ASYNC_MASTER
 flushDiskType = ASYNC_FLUSH
-namesrvAddr = 127.0.0.1:9876 # name server地址
+namesrvAddr = 127.0.0.1:9876
 ```
-3. Add the runtime parameter `ROCKETMQ_HOME=<rocketmq repository directory>` and the environment variable `-c /Users/xxx/rocketmq/conf/broker.conf` in `Idea-Edit Configurations`.
+3. Add the runtime parameter `-c /Users/xxx/rocketmq/conf/broker.conf` and the environment variable `ROCKETMQ_HOME=<rocketmq repository directory>` in `Idea-Edit Configurations`.
 ![Idea_config_broker.png](../cn/image/Idea_config_broker.png)
 4. Run the Broker and if the following log is observed, it indicates successful startup.
 ```shell
@@ -40,7 +40,7 @@ RocketMQ startup is now complete. You can use the examples provided in `/example
 
 ### Additional: Start the Proxy locally.
 1. RocketMQ 5.x introduced the Proxy mode. Using the `LOCAL` mode eliminates the need for `Step2`. The startup class is located at `org.apache.rocketmq.proxy.ProxyStartup`.
-2. Add the runtime parameter `ROCKETMQ_HOME=<rocketmq repository directory>` in `Idea-Edit Configurations`.
+2. Add the environment variable `ROCKETMQ_HOME=<rocketmq repository directory>` in `Idea-Edit Configurations`.
 3. Create a new configuration file named `rmq-proxy.json` in the `/conf/` directory.
 ```json
 {
diff --git a/docs/en/Deployment.md b/docs/en/Deployment.md
index 5dc93488a77..8f5c8f1e388 100644
--- a/docs/en/Deployment.md
+++ b/docs/en/Deployment.md
@@ -69,7 +69,7 @@ $ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker
 ...
 ```
 
-The boot command shown above is used in the case of a single NameServer.For clusters of multiple NameServer, the address list after the -n argument in the broker boot command is separated by semicolons, for example, 192.168.1.1: 9876;192.161.2: 9876.
+The boot command shown above is used in the case of a single NameServer.For clusters of multiple NameServer, the address list after the -n argument in the broker boot command is separated by semicolons, for example, 192.168.1.1: 9876;192.168.1.2: 9876.
 
 ### 3 Multiple Master And Multiple Slave Mode-Asynchronous replication
 
diff --git a/docs/en/controller/design.md b/docs/en/controller/design.md
index ba2de58af14..af4958a4d3e 100644
--- a/docs/en/controller/design.md
+++ b/docs/en/controller/design.md
@@ -112,13 +112,13 @@ According to the above, we can know the AutoSwitchHaService protocol divides log
 
 ![示意图](../image/controller/controller_design_3.png)
 
-`current state(4byte) + Two flags(4byte) + slaveAddressLength(4byte) + slaveAddress(50byte)`
+`current state(4byte) + Two flags(4byte) + slaveBrokerId(8byte)`
 
 - `Current state` represents the current HAConnectionState, which is HANDSHAKE.
 
 - Two flags are two status flags, where `isSyncFromLastFile` indicates whether to start copying from the Master's last file, and `isAsyncLearner` indicates whether the Slave is an asynchronous copy and joins the Master as a Learner.
 
-- `slaveAddressLength` and `slaveAddress` represent the address of the Slave, which will be used later to join the SyncStateSet.
+- `slaveBrokerId` represent the brokerId of the Slave, which will be used later to join the SyncStateSet.
 
 2.AutoSwitchHaConnection (Master) will send a HandShake packet back to the Slave as follows:
 
diff --git a/docs/en/image/controller/controller_design_3.png b/docs/en/image/controller/controller_design_3.png
index 8c475bcecf1..0379c231d46 100644
Binary files a/docs/en/image/controller/controller_design_3.png and b/docs/en/image/controller/controller_design_3.png differ
diff --git a/example/pom.xml b/example/pom.xml
index a8c7f538224..43bc9d90ad4 100644
--- a/example/pom.xml
+++ b/example/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -54,7 +54,7 @@
         </dependency>
         <dependency>
             <groupId>${project.groupId}</groupId>
-            <artifactId>rocketmq-acl</artifactId>
+            <artifactId>rocketmq-auth</artifactId>
         </dependency>
         <dependency>
             <groupId>org.javassist</groupId>
diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java
index c4a6162a5f5..21a4b3b7e77 100644
--- a/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java
@@ -102,8 +102,8 @@ public static void main(String[] args) throws MQClientException {
             String compressType = commandLine.hasOption("ct") ? commandLine.getOptionValue("ct").trim() : "ZLIB";
             int compressLevel = commandLine.hasOption("cl") ? Integer.parseInt(commandLine.getOptionValue("cl")) : 5;
             int compressOverHowMuch = commandLine.hasOption("ch") ? Integer.parseInt(commandLine.getOptionValue("ch")) : 4096;
-            producer.getDefaultMQProducerImpl().setCompressType(CompressionType.of(compressType));
-            producer.getDefaultMQProducerImpl().setCompressLevel(compressLevel);
+            producer.setCompressType(CompressionType.of(compressType));
+            producer.setCompressLevel(compressLevel);
             producer.setCompressMsgBodyOverHowmuch(compressOverHowMuch);
             System.out.printf("compressType: %s compressLevel: %s%n", compressType, compressLevel);
         } else {
diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java
index 480d16b7581..a945283f576 100644
--- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java
@@ -160,8 +160,8 @@ public void run() {
             String compressType = commandLine.hasOption("ct") ? commandLine.getOptionValue("ct").trim() : "ZLIB";
             int compressLevel = commandLine.hasOption("cl") ? Integer.parseInt(commandLine.getOptionValue("cl")) : 5;
             int compressOverHowMuch = commandLine.hasOption("ch") ? Integer.parseInt(commandLine.getOptionValue("ch")) : 4096;
-            producer.getDefaultMQProducerImpl().setCompressType(CompressionType.of(compressType));
-            producer.getDefaultMQProducerImpl().setCompressLevel(compressLevel);
+            producer.setCompressType(CompressionType.of(compressType));
+            producer.setCompressLevel(compressLevel);
             producer.setCompressMsgBodyOverHowmuch(compressOverHowMuch);
             System.out.printf("compressType: %s compressLevel: %s%n", compressType, compressLevel);
         } else {
diff --git a/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java
new file mode 100644
index 00000000000..da6ad920f30
--- /dev/null
+++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.example.lmq;
+
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+
+public class LMQProducer {
+    public static final String PRODUCER_GROUP = "ProducerGroupName";
+
+    public static final String DEFAULT_NAMESRVADDR = "127.0.0.1:9876";
+
+    public static final String TOPIC = "TopicLMQParent";
+
+    public static final String TAG = "TagA";
+
+    public static final String LMQ_TOPIC_1 = MixAll.LMQ_PREFIX + "123";
+
+    public static final String LMQ_TOPIC_2 = MixAll.LMQ_PREFIX + "456";
+
+    public static void main(String[] args) throws MQClientException, InterruptedException {
+        DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP);
+
+        // Uncomment the following line while debugging, namesrvAddr should be set to your local address
+        producer.setNamesrvAddr(DEFAULT_NAMESRVADDR);
+
+        producer.start();
+        for (int i = 0; i < 128; i++) {
+            try {
+                Message msg = new Message(TOPIC, TAG, ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET));
+                msg.setKeys("Key" + i);
+                msg.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH /* "INNER_MULTI_DISPATCH" */,
+                    String.join(MixAll.LMQ_DISPATCH_SEPARATOR, LMQ_TOPIC_1, LMQ_TOPIC_2) /* "%LMQ%123,%LMQ%456" */);
+                SendResult sendResult = producer.send(msg);
+                System.out.printf("%s%n", sendResult);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
+
+        producer.shutdown();
+    }
+}
diff --git a/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPullConsumer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPullConsumer.java
new file mode 100644
index 00000000000..931dd96b48f
--- /dev/null
+++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPullConsumer.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.example.lmq;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+import org.apache.rocketmq.client.consumer.PullCallback;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+
+@SuppressWarnings("deprecation")
+public class LMQPullConsumer {
+    public static final String BROKER_NAME = "broker-a";
+
+    public static final String CONSUMER_GROUP = "CID_LMQ_PULL_1";
+
+    public static final String TOPIC = "TopicLMQParent";
+
+    public static final String LMQ_TOPIC = MixAll.LMQ_PREFIX + "123";
+
+    public static final String NAMESRV_ADDR = "127.0.0.1:9876";
+
+    public static void main(String[] args) throws MQClientException, RemotingException, InterruptedException {
+
+        DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(CONSUMER_GROUP);
+        consumer.setNamesrvAddr(NAMESRV_ADDR);
+        consumer.setRegisterTopics(new HashSet<>(Arrays.asList(TOPIC)));
+        consumer.start();
+
+        // use parent topic to fill up broker addr table
+        consumer.getDefaultMQPullConsumerImpl().getRebalanceImpl().getmQClientFactory()
+            .updateTopicRouteInfoFromNameServer(TOPIC);
+
+        final MessageQueue lmq = new MessageQueue(LMQ_TOPIC, BROKER_NAME, (int) MixAll.LMQ_QUEUE_ID);
+        long offset = consumer.minOffset(lmq);
+
+        consumer.pullBlockIfNotFound(lmq, "*", offset, 32, new PullCallback() {
+            @Override
+            public void onSuccess(PullResult pullResult) {
+                List<MessageExt> list = pullResult.getMsgFoundList();
+                if (list == null || list.isEmpty()) {
+                    return;
+                }
+
+                for (MessageExt msg : list) {
+                    System.out.printf("%s Pull New Messages: %s %n", Thread.currentThread().getName(), msg);
+                }
+            }
+
+            @Override
+            public void onException(Throwable e) {
+
+            }
+        });
+    }
+}
diff --git a/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushConsumer.java
new file mode 100644
index 00000000000..f8926a05dfd
--- /dev/null
+++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushConsumer.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.example.lmq;
+
+import com.google.common.collect.Lists;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
+
+public class LMQPushConsumer {
+    public static final String CLUSTER_NAME = "DefaultCluster";
+
+    public static final String BROKER_NAME = "broker-a";
+
+    public static final String TOPIC = "TopicLMQParent";
+
+    public static final String LMQ_TOPIC = MixAll.LMQ_PREFIX + "123";
+
+    public static final String CONSUMER_GROUP = "CID_LMQ_1";
+
+    public static final String NAMESRV_ADDR = "127.0.0.1:9876";
+
+    public static final HashMap<Long, String> BROKER_ADDR_MAP = new HashMap<Long, String>() {
+        {
+            put(MixAll.MASTER_ID, "127.0.0.1:10911");
+        }
+    };
+
+    public static void main(String[] args) throws InterruptedException, MQClientException {
+        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP);
+        consumer.setNamesrvAddr(NAMESRV_ADDR);
+        consumer.subscribe(LMQ_TOPIC, "*");
+        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
+        consumer.registerMessageListener(new MessageListenerConcurrently() {
+
+            @Override
+            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
+                System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
+                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+            }
+        });
+        consumer.start();
+
+        // use parent topic to fill up broker addr table
+        consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(TOPIC);
+
+        final TopicRouteData topicRouteData = new TopicRouteData();
+        final BrokerData brokerData = new BrokerData();
+        brokerData.setCluster(CLUSTER_NAME);
+        brokerData.setBrokerName(BROKER_NAME);
+        brokerData.setBrokerAddrs(BROKER_ADDR_MAP);
+        topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData));
+        // compensate LMQ topic route for MQClientInstance#findBrokerAddrByTopic
+        consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().getTopicRouteTable().put(LMQ_TOPIC, topicRouteData);
+        // compensate for RebalanceImpl#topicSubscribeInfoTable
+        consumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(LMQ_TOPIC,
+            new HashSet<>(Arrays.asList(new MessageQueue(LMQ_TOPIC, BROKER_NAME, (int) MixAll.LMQ_QUEUE_ID))));
+        // re-balance immediately to start pulling messages
+        consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().doRebalance();
+
+        System.out.printf("Consumer Started.%n");
+    }
+}
diff --git a/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushPopConsumer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushPopConsumer.java
new file mode 100644
index 00000000000..517eb12b7d2
--- /dev/null
+++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushPopConsumer.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.example.lmq;
+
+import com.google.common.collect.Lists;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
+import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageRequestMode;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+
+public class LMQPushPopConsumer {
+    public static final String CLUSTER_NAME = "DefaultCluster";
+
+    public static final String BROKER_NAME = "broker-a";
+
+    public static final String TOPIC = "TopicLMQParent";
+
+    public static final String LMQ_TOPIC = MixAll.LMQ_PREFIX + "456";
+
+    public static final String NAMESRV_ADDR = "127.0.0.1:9876";
+
+    public static final String CONSUMER_GROUP = "CID_LMQ_POP_1";
+
+    public static final HashMap<Long, String> BROKER_ADDR_MAP = new HashMap<Long, String>() {
+        {
+            put(MixAll.MASTER_ID, "127.0.0.1:10911");
+        }
+    };
+
+    public static void main(String[] args) throws Exception {
+        switchPop();
+
+        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP);
+        consumer.setNamesrvAddr(NAMESRV_ADDR);
+        consumer.subscribe(LMQ_TOPIC, "*");
+        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
+        consumer.registerMessageListener(new MessageListenerConcurrently() {
+            @Override
+            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
+                System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
+                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
+            }
+        });
+        // use server side rebalance
+        consumer.setClientRebalance(false);
+        consumer.start();
+
+        // use parent topic to fill up broker addr table
+        consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(TOPIC);
+
+        final TopicRouteData topicRouteData = new TopicRouteData();
+        final BrokerData brokerData = new BrokerData();
+        brokerData.setCluster(CLUSTER_NAME);
+        brokerData.setBrokerName(BROKER_NAME);
+        brokerData.setBrokerAddrs(BROKER_ADDR_MAP);
+        topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData));
+        // compensate LMQ topic route for MQClientInstance#findBrokerAddrByTopic
+        consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().getTopicRouteTable().put(LMQ_TOPIC, topicRouteData);
+        // re-balance immediately to start pulling messages
+        consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().doRebalance();
+
+        System.out.printf("Consumer Started.%n");
+    }
+
+    private static void switchPop() throws Exception {
+        DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt();
+        mqAdminExt.setNamesrvAddr(NAMESRV_ADDR);
+        mqAdminExt.start();
+        List<BrokerData> brokerDatas = mqAdminExt.examineTopicRouteInfo(TOPIC).getBrokerDatas();
+        for (BrokerData brokerData : brokerDatas) {
+            Set<String> brokerAddrs = new HashSet<>(brokerData.getBrokerAddrs().values());
+            for (String brokerAddr : brokerAddrs) {
+                mqAdminExt.setMessageRequestMode(brokerAddr, LMQ_TOPIC, CONSUMER_GROUP, MessageRequestMode.POP, 8,
+                    3_000);
+            }
+        }
+    }
+}
diff --git a/example/src/main/java/org/apache/rocketmq/example/operation/Consumer.java b/example/src/main/java/org/apache/rocketmq/example/operation/Consumer.java
index 90f2e133a1c..378a9769975 100644
--- a/example/src/main/java/org/apache/rocketmq/example/operation/Consumer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/operation/Consumer.java
@@ -36,15 +36,15 @@ public class Consumer {
     public static void main(String[] args) throws MQClientException {
         CommandLine commandLine = buildCommandline(args);
         if (commandLine != null) {
-            String group = commandLine.getOptionValue('g');
+            String subGroup = commandLine.getOptionValue('g');
             String topic = commandLine.getOptionValue('t');
-            String subscription = commandLine.getOptionValue('s');
+            String subExpression = commandLine.getOptionValue('s');
             final String returnFailedHalf = commandLine.getOptionValue('f');
 
-            DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group);
+            DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(subGroup);
             consumer.setInstanceName(Long.toString(System.currentTimeMillis()));
 
-            consumer.subscribe(topic, subscription);
+            consumer.subscribe(topic, subExpression);
 
             consumer.registerMessageListener(new MessageListenerConcurrently() {
                 AtomicLong consumeTimes = new AtomicLong(0);
diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/PopConsumer.java b/example/src/main/java/org/apache/rocketmq/example/simple/PopConsumer.java
index 6321e36d9ac..f3edc5c1a0c 100644
--- a/example/src/main/java/org/apache/rocketmq/example/simple/PopConsumer.java
+++ b/example/src/main/java/org/apache/rocketmq/example/simple/PopConsumer.java
@@ -32,6 +32,7 @@
 public class PopConsumer {
     public static final String TOPIC = "TopicTest";
     public static final String CONSUMER_GROUP = "CID_JODIE_1";
+    public static final String NAMESRV_ADDR = "127.0.0.1:9876";
     public static void main(String[] args) throws Exception {
         switchPop();
         DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP);
@@ -44,12 +45,15 @@ public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeCo
                 return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
             }
         });
+        // Uncomment the following line while debugging, namesrvAddr should be set to your local address
+        // consumer.setNamesrvAddr(NAMESRV_ADDR);
         consumer.setClientRebalance(false);
         consumer.start();
         System.out.printf("Consumer Started.%n");
     }
     private static void switchPop() throws Exception {
         DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt();
+        // mqAdminExt.setNamesrvAddr(NAMESRV_ADDR);
         mqAdminExt.start();
         List<BrokerData> brokerDatas = mqAdminExt.examineTopicRouteInfo(TOPIC).getBrokerDatas();
         for (BrokerData brokerData : brokerDatas) {
diff --git a/filter/pom.xml b/filter/pom.xml
index 892f46e9d16..8707af8b7ed 100644
--- a/filter/pom.xml
+++ b/filter/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.java b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.java
index 0aaf2bc01a0..7b44aa2efba 100644
--- a/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.java
+++ b/filter/src/main/java/org/apache/rocketmq/filter/parser/SelectorParser.java
@@ -41,25 +41,14 @@
 public class SelectorParser implements SelectorParserConstants {
 
     private static final Cache<String, Object> PARSE_CACHE = CacheBuilder.newBuilder().maximumSize(100).build();
-//    private static final String CONVERT_STRING_EXPRESSIONS_PREFIX = "convert_string_expressions:";
 
     public static BooleanExpression parse(String sql) throws MQFilterException {
-//        sql = "("+sql+")";
         Object result = PARSE_CACHE.getIfPresent(sql);
         if (result instanceof MQFilterException) {
             throw (MQFilterException) result;
         } else if (result instanceof BooleanExpression) {
             return (BooleanExpression) result;
         } else {
-
-//            boolean convertStringExpressions = false;
-//            if( sql.startsWith(CONVERT_STRING_EXPRESSIONS_PREFIX)) {
-//                convertStringExpressions = true;
-//                sql = sql.substring(CONVERT_STRING_EXPRESSIONS_PREFIX.length());
-//            }
-//            if( convertStringExpressions ) {
-//                ComparisonExpression.CONVERT_STRING_EXPRESSIONS.set(true);
-//            }
             ComparisonExpression.CONVERT_STRING_EXPRESSIONS.set(true);
             try {
 
@@ -71,9 +60,6 @@ public static BooleanExpression parse(String sql) throws MQFilterException {
                 throw t;
             } finally {
                 ComparisonExpression.CONVERT_STRING_EXPRESSIONS.remove();
-//                if( convertStringExpressions ) {
-//                    ComparisonExpression.CONVERT_STRING_EXPRESSIONS.remove();
-//                }
             }
         }
     }
@@ -111,12 +97,8 @@ private BooleanExpression asBooleanExpression(Expression value) throws ParseExce
     // Grammar
     // ----------------------------------------------------------------------------
     final public BooleanExpression JmsSelector() throws ParseException {
-        Expression left = null;
-        left = orExpression();
-        {
-            if (true) return asBooleanExpression(left);
-        }
-        throw new Error("Missing return statement in function");
+        Expression left = orExpression();
+        return asBooleanExpression(left);
     }
 
     final public Expression orExpression() throws ParseException {
@@ -136,10 +118,7 @@ final public Expression orExpression() throws ParseException {
             right = andExpression();
             left = LogicExpression.createOR(asBooleanExpression(left), asBooleanExpression(right));
         }
-        {
-            if (true) return left;
-        }
-        throw new Error("Missing return statement in function");
+        return left;
     }
 
     final public Expression andExpression() throws ParseException {
@@ -159,10 +138,7 @@ final public Expression andExpression() throws ParseException {
             right = equalityExpression();
             left = LogicExpression.createAND(asBooleanExpression(left), asBooleanExpression(right));
         }
-        {
-            if (true) return left;
-        }
-        throw new Error("Missing return statement in function");
+        return left;
     }
 
     final public Expression equalityExpression() throws ParseException {
@@ -213,10 +189,7 @@ final public Expression equalityExpression() throws ParseException {
                     }
             }
         }
-        {
-            if (true) return left;
-        }
-        throw new Error("Missing return statement in function");
+        return left;
     }
 
     final public Expression comparisonExpression() throws ParseException {
@@ -387,10 +360,7 @@ final public Expression comparisonExpression() throws ParseException {
                     }
             }
         }
-        {
-            if (true) return left;
-        }
-        throw new Error("Missing return statement in function");
+        return left;
     }
 
     final public Expression unaryExpr() throws ParseException {
@@ -427,10 +397,7 @@ final public Expression unaryExpr() throws ParseException {
                     throw new ParseException();
             }
         }
-        {
-            if (true) return left;
-        }
-        throw new Error("Missing return statement in function");
+        return left;
     }
 
     final public Expression primaryExpr() throws ParseException {
@@ -457,10 +424,7 @@ final public Expression primaryExpr() throws ParseException {
                 jj_consume_token(-1);
                 throw new ParseException();
         }
-        {
-            if (true) return left;
-        }
-        throw new Error("Missing return statement in function");
+        return left;
     }
 
     final public ConstantExpression literal() throws ParseException {
@@ -497,10 +461,7 @@ final public ConstantExpression literal() throws ParseException {
                 jj_consume_token(-1);
                 throw new ParseException();
         }
-        {
-            if (true) return left;
-        }
-        throw new Error("Missing return statement in function");
+        return left;
     }
 
     final public String stringLitteral() throws ParseException {
@@ -516,10 +477,7 @@ final public String stringLitteral() throws ParseException {
                 i++;
             rc.append(c);
         }
-        {
-            if (true) return rc.toString();
-        }
-        throw new Error("Missing return statement in function");
+        return rc.toString();
     }
 
     final public PropertyExpression variable() throws ParseException {
@@ -527,10 +485,7 @@ final public PropertyExpression variable() throws ParseException {
         PropertyExpression left = null;
         t = jj_consume_token(ID);
         left = new PropertyExpression(t.image);
-        {
-            if (true) return left;
-        }
-        throw new Error("Missing return statement in function");
+        return left;
     }
 
     private boolean jj_2_1(int xla) {
diff --git a/filter/src/test/java/org/apache/rocketmq/filter/FilterSpiTest.java b/filter/src/test/java/org/apache/rocketmq/filter/FilterSpiTest.java
index bfbddf49135..25f8cfdecb8 100644
--- a/filter/src/test/java/org/apache/rocketmq/filter/FilterSpiTest.java
+++ b/filter/src/test/java/org/apache/rocketmq/filter/FilterSpiTest.java
@@ -68,6 +68,7 @@ public void testRegister() {
             e.printStackTrace();
             assertThat(Boolean.FALSE).isTrue();
         }
+        FilterFactory.INSTANCE.unRegister("Nothing");
     }
 
     @Test
diff --git a/namesrv/pom.xml b/namesrv/pom.xml
index e320ed5732f..3968d833f0a 100644
--- a/namesrv/pom.xml
+++ b/namesrv/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java
index 179d3bb0d63..23d09062165 100644
--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java
+++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java
@@ -43,8 +43,8 @@
 
 public class NamesrvStartup {
 
-    private final static Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);
-    private final static Logger logConsole = LoggerFactory.getLogger(LoggerName.NAMESRV_CONSOLE_LOGGER_NAME);
+    private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);
+    private static final Logger logConsole = LoggerFactory.getLogger(LoggerName.NAMESRV_CONSOLE_LOGGER_NAME);
     private static Properties properties = null;
     private static NamesrvConfig namesrvConfig = null;
     private static NettyServerConfig nettyServerConfig = null;
diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClientRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClientRequestProcessor.java
index 17a070c7f07..1ef6beadd3d 100644
--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClientRequestProcessor.java
+++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClientRequestProcessor.java
@@ -21,7 +21,6 @@
 import io.netty.channel.ChannelHandlerContext;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.rocketmq.common.MQVersion;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.help.FAQUrl;
@@ -43,8 +42,6 @@ public class ClientRequestProcessor implements NettyRequestProcessor {
     protected NamesrvController namesrvController;
     private long startupTimeMillis;
 
-    private AtomicBoolean needCheckNamesrvReady = new AtomicBoolean(true);
-
     public ClientRequestProcessor(final NamesrvController namesrvController) {
         this.namesrvController = namesrvController;
         this.startupTimeMillis = System.currentTimeMillis();
@@ -62,7 +59,7 @@ public RemotingCommand getRouteInfoByTopic(ChannelHandlerContext ctx,
         final GetRouteInfoRequestHeader requestHeader =
             (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class);
 
-        boolean namesrvReady = needCheckNamesrvReady.get() && System.currentTimeMillis() - startupTimeMillis >= TimeUnit.SECONDS.toMillis(namesrvController.getNamesrvConfig().getWaitSecondsForService());
+        boolean namesrvReady = System.currentTimeMillis() - startupTimeMillis >= TimeUnit.SECONDS.toMillis(namesrvController.getNamesrvConfig().getWaitSecondsForService());
 
         if (namesrvController.getNamesrvConfig().isNeedWaitForService() && !namesrvReady) {
             log.warn("name server not ready. request code {} ", request.getCode());
@@ -74,11 +71,6 @@ public RemotingCommand getRouteInfoByTopic(ChannelHandlerContext ctx,
         TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic());
 
         if (topicRouteData != null) {
-            //topic route info register success ,so disable namesrvReady check
-            if (needCheckNamesrvReady.get()) {
-                needCheckNamesrvReady.set(false);
-            }
-
             if (this.namesrvController.getNamesrvConfig().isOrderMessageEnable()) {
                 String orderTopicConf =
                     this.namesrvController.getKvConfigManager().getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG,
diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHook.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHook.java
index 4983c88c8af..a740a0f1b4e 100644
--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHook.java
+++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHook.java
@@ -56,7 +56,6 @@ public void doAfterResponse(String remoteAddr, RemotingCommand request, Remoting
             return;
         }
         TopicRouteData topicRouteData = RemotingSerializable.decode(response.getBody(), TopicRouteData.class);
-
         response.setBody(filterByZoneName(topicRouteData, zoneName).encode());
     }
 
@@ -64,6 +63,9 @@ private TopicRouteData filterByZoneName(TopicRouteData topicRouteData, String zo
         List<BrokerData> brokerDataReserved = new ArrayList<>();
         Map<String, BrokerData> brokerDataRemoved = new HashMap<>();
         for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {
+            if (brokerData.getBrokerAddrs() == null) {
+                continue;
+            }
             //master down, consume from slave. break nearby route rule.
             if (brokerData.getBrokerAddrs().get(MixAll.MASTER_ID) == null
                 || StringUtils.equalsIgnoreCase(brokerData.getZoneName(), zoneName)) {
@@ -85,9 +87,6 @@ private TopicRouteData filterByZoneName(TopicRouteData topicRouteData, String zo
         if (topicRouteData.getFilterServerTable() != null && !topicRouteData.getFilterServerTable().isEmpty()) {
             for (Entry<String, BrokerData> entry : brokerDataRemoved.entrySet()) {
                 BrokerData brokerData = entry.getValue();
-                if (brokerData.getBrokerAddrs() == null) {
-                    continue;
-                }
                 brokerData.getBrokerAddrs().values()
                     .forEach(brokerAddr -> topicRouteData.getFilterServerTable().remove(brokerAddr));
             }
diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
index ce311d392aa..e3f9d0de9b4 100644
--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
+++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
@@ -67,7 +67,7 @@
 
 public class RouteInfoManager {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME);
-    private final static long DEFAULT_BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2;
+    private static final long DEFAULT_BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2;
     private final ReadWriteLock lock = new ReentrantReadWriteLock();
     private final Map<String/* topic */, Map<String, QueueData>> topicQueueTable;
     private final Map<String/* brokerName */, BrokerData> brokerAddrTable;
@@ -281,7 +281,7 @@ public RegisterBrokerResult registerBroker(
                     long oldStateVersion = oldBrokerInfo.getDataVersion().getStateVersion();
                     long newStateVersion = topicConfigWrapper.getDataVersion().getStateVersion();
                     if (oldStateVersion > newStateVersion) {
-                        log.warn("Registered Broker conflicts with the existed one, just ignore.: Cluster:{}, BrokerName:{}, BrokerId:{}, " +
+                        log.warn("Registering Broker conflicts with the existed one, just ignore.: Cluster:{}, BrokerName:{}, BrokerId:{}, " +
                                 "Old BrokerAddr:{}, Old Version:{}, New BrokerAddr:{}, New Version:{}.",
                             clusterName, brokerName, brokerId, oldBrokerAddr, oldStateVersion, brokerAddr, newStateVersion);
                         //Remove the rejected brokerAddr from brokerLiveTable.
diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/RequestProcessorTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/RequestProcessorTest.java
index 2b2cf629494..831558a0f68 100644
--- a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/RequestProcessorTest.java
+++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/RequestProcessorTest.java
@@ -448,6 +448,42 @@ public void testGetBrokerClusterInfo() throws RemotingCommandException {
         assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);
     }
 
+    @Test
+    public void testQueryDataVersion()throws RemotingCommandException {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        when(ctx.channel()).thenReturn(null);
+        RemotingCommand request = getRemotingCommand(RequestCode.QUERY_DATA_VERSION);
+        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);
+        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testGetBrokerMemberBroker() throws RemotingCommandException {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        when(ctx.channel()).thenReturn(null);
+        RemotingCommand request = getRemotingCommand(RequestCode.GET_BROKER_MEMBER_GROUP);
+        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);
+        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testBrokerHeartBeat() throws RemotingCommandException {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        when(ctx.channel()).thenReturn(null);
+        RemotingCommand request = getRemotingCommand(RequestCode.BROKER_HEARTBEAT);
+        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);
+        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testAddWritePermOfBroker() throws RemotingCommandException {
+        ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+        when(ctx.channel()).thenReturn(null);
+        RemotingCommand request = getRemotingCommand(RequestCode.ADD_WRITE_PERM_OF_BROKER);
+        RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request);
+        assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
     @Test
     public void testWipeWritePermOfBroker() throws RemotingCommandException {
         ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookMoreTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookMoreTest.java
new file mode 100644
index 00000000000..ed42becdf53
--- /dev/null
+++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookMoreTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.namesrv.route;
+
+
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.QueueData;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class ZoneRouteRPCHookMoreTest {
+
+    private ZoneRouteRPCHook zoneRouteRPCHook;
+
+    @Before
+    public void setUp() {
+        zoneRouteRPCHook = new ZoneRouteRPCHook();
+    }
+
+    @Test
+    public void testFilterByZoneName_ValidInput_ShouldFilterCorrectly() {
+        // Arrange
+        TopicRouteData topicRouteData = new TopicRouteData();
+        topicRouteData.setBrokerDatas(generateBrokerDataList());
+        topicRouteData.setQueueDatas(generateQueueDataList());
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC,null);
+        request.setExtFields(createExtFields("true","ZoneA"));
+
+        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "remark");
+
+        // Act
+        zoneRouteRPCHook.doAfterResponse("127.0.0.1", request, response);
+        TopicRouteData decodedResponse = RemotingSerializable.decode(response.getBody(), TopicRouteData.class);
+
+        // Assert
+        assertNull(decodedResponse);
+    }
+
+    @Test
+    public void testFilterByZoneName_NoZoneName_ShouldNotFilter() {
+        // Arrange
+        TopicRouteData topicRouteData = new TopicRouteData();
+        topicRouteData.setBrokerDatas(generateBrokerDataList());
+        topicRouteData.setQueueDatas(generateQueueDataList());
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC,null);
+        HashMap<String, String> extFields = new HashMap<>();
+        extFields.put(MixAll.ZONE_MODE, "true");
+        request.setExtFields(extFields);
+
+        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);
+
+        // Act
+        zoneRouteRPCHook.doAfterResponse("127.0.0.1", request, response);
+        TopicRouteData decodedResponse = RemotingSerializable.decode(response.getBody(), TopicRouteData.class);
+
+        // Assert
+        assertEquals(topicRouteData.getBrokerDatas().size(), 2);
+        assertEquals(topicRouteData.getQueueDatas().size(), 2);
+    }
+
+    @Test
+    public void testFilterByZoneName_ZoneModeFalse_ShouldNotFilter() {
+        // Arrange
+        TopicRouteData topicRouteData = new TopicRouteData();
+        topicRouteData.setBrokerDatas(generateBrokerDataList());
+        topicRouteData.setQueueDatas(generateQueueDataList());
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC,null);
+        request.setExtFields(createExtFields("false","ZoneA"));
+
+        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS ,null);
+
+        // Act
+        zoneRouteRPCHook.doAfterResponse("127.0.0.1", request, response);
+        TopicRouteData decodedResponse = RemotingSerializable.decode(response.getBody(), TopicRouteData.class);
+
+        // Assert
+        assertEquals(topicRouteData.getBrokerDatas().size(), 2);
+        assertEquals(topicRouteData.getQueueDatas().size(), 2);
+    }
+
+    private List<BrokerData> generateBrokerDataList() {
+        List<BrokerData> brokerDataList = new ArrayList<>();
+        BrokerData brokerData1 = new BrokerData();
+        brokerData1.setBrokerName("BrokerA");
+        brokerData1.setZoneName("ZoneA");
+        Map<Long, String> brokerAddrs = new HashMap<>();
+        brokerAddrs.put(MixAll.MASTER_ID, "127.0.0.1:10911");
+        brokerData1.setBrokerAddrs((HashMap<Long, String>) brokerAddrs);
+        brokerDataList.add(brokerData1);
+
+        BrokerData brokerData2 = new BrokerData();
+        brokerData2.setBrokerName("BrokerB");
+        brokerData2.setZoneName("ZoneB");
+        brokerAddrs = new HashMap<>();
+        brokerAddrs.put(MixAll.MASTER_ID, "127.0.0.1:10912");
+        brokerData2.setBrokerAddrs((HashMap<Long, String>) brokerAddrs);
+        brokerDataList.add(brokerData2);
+
+        return brokerDataList;
+    }
+
+    private List<QueueData> generateQueueDataList() {
+        List<QueueData> queueDataList = new ArrayList<>();
+        QueueData queueData1 = new QueueData();
+        queueData1.setBrokerName("BrokerA");
+        queueDataList.add(queueData1);
+
+        QueueData queueData2 = new QueueData();
+        queueData2.setBrokerName("BrokerB");
+        queueDataList.add(queueData2);
+
+        return queueDataList;
+    }
+
+    private HashMap<String, String> createExtFields(String zoneMode, String zoneName) {
+        HashMap<String, String> extFields = new HashMap<>();
+        extFields.put(MixAll.ZONE_MODE, zoneMode);
+        extFields.put(MixAll.ZONE_NAME, zoneName);
+        return extFields;
+    }
+
+}
\ No newline at end of file
diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookTest.java
new file mode 100644
index 00000000000..1bf4a6c677f
--- /dev/null
+++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookTest.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.namesrv.route;
+
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.QueueData;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+
+public class ZoneRouteRPCHookTest {
+
+    private ZoneRouteRPCHook zoneRouteRPCHook;
+
+    @Before
+    public void setup() {
+        zoneRouteRPCHook = new ZoneRouteRPCHook();
+    }
+
+    @Test
+    public void testDoAfterResponseWithNoZoneMode() {
+        RemotingCommand request1 = RemotingCommand.createRequestCommand(106,null);
+        zoneRouteRPCHook.doAfterResponse("", request1, null);
+
+        HashMap<String, String> extFields = new HashMap<>();
+        extFields.put(MixAll.ZONE_MODE, "false");
+        RemotingCommand request = RemotingCommand.createRequestCommand(105,null);
+        request.setExtFields(extFields);
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(ResponseCode.SUCCESS);
+        response.setBody(RemotingSerializable.encode(createSampleTopicRouteData()));
+        zoneRouteRPCHook.doAfterResponse("", request, response);
+    }
+
+    @Test
+    public void testDoAfterResponseWithNoZoneName() {
+        HashMap<String, String> extFields = new HashMap<>();
+        extFields.put(MixAll.ZONE_MODE, "true");
+        RemotingCommand request = RemotingCommand.createRequestCommand(105,null);
+        request.setExtFields(extFields);
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(ResponseCode.SUCCESS);
+        response.setBody(RemotingSerializable.encode(createSampleTopicRouteData()));
+        zoneRouteRPCHook.doAfterResponse("", request, response);
+    }
+
+    @Test
+    public void testDoAfterResponseWithNoResponse() {
+        HashMap<String, String> extFields = new HashMap<>();
+        extFields.put(MixAll.ZONE_MODE, "true");
+        RemotingCommand request = RemotingCommand.createRequestCommand(105,null);
+        request.setExtFields(extFields);
+        zoneRouteRPCHook.doAfterResponse("", request, null);
+
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(ResponseCode.SUCCESS);
+        zoneRouteRPCHook.doAfterResponse("", request, response);
+
+        response.setBody(RemotingSerializable.encode(createSampleTopicRouteData()));
+        response.setCode(ResponseCode.NO_PERMISSION);
+        zoneRouteRPCHook.doAfterResponse("", request, response);
+    }
+
+
+    @Test
+    public void testDoAfterResponseWithValidZoneFiltering() throws Exception {
+        HashMap<String, String> extFields = new HashMap<>();
+        extFields.put(MixAll.ZONE_MODE, "true");
+        extFields.put(MixAll.ZONE_NAME,"zone1");
+        RemotingCommand request = RemotingCommand.createRequestCommand(105,null);
+        request.setExtFields(extFields);
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(ResponseCode.SUCCESS);
+        TopicRouteData topicRouteData = createSampleTopicRouteData();
+        response.setBody(RemotingSerializable.encode(topicRouteData));
+        zoneRouteRPCHook.doAfterResponse("", request, response);
+
+        HashMap<Long,String> brokeraddrs = new HashMap<>();
+        brokeraddrs.put(MixAll.MASTER_ID,"127.0.0.1:10911");
+        topicRouteData.getBrokerDatas().get(0).setBrokerAddrs(brokeraddrs);
+        response.setBody(RemotingSerializable.encode(topicRouteData));
+        zoneRouteRPCHook.doAfterResponse("", request, response);
+
+        topicRouteData.getQueueDatas().add(createQueueData("BrokerB"));
+        HashMap<Long,String> brokeraddrsB = new HashMap<>();
+        brokeraddrsB.put(MixAll.MASTER_ID,"127.0.0.1:10912");
+        BrokerData brokerData1 = createBrokerData("BrokerB","zone2",brokeraddrsB);
+        BrokerData brokerData2 = createBrokerData("BrokerC","zone1",null);
+        topicRouteData.getBrokerDatas().add(brokerData1);
+        topicRouteData.getBrokerDatas().add(brokerData2);
+        response.setBody(RemotingSerializable.encode(topicRouteData));
+        zoneRouteRPCHook.doAfterResponse("", request, response);
+
+        topicRouteData.getFilterServerTable().put("127.0.0.1:10911",new ArrayList<>());
+        response.setBody(RemotingSerializable.encode(topicRouteData));
+        zoneRouteRPCHook.doAfterResponse("", request, response);
+        Assert.assertEquals(1,RemotingSerializable
+                .decode(response.getBody(), TopicRouteData.class)
+                .getFilterServerTable()
+                .size());
+
+        topicRouteData.getFilterServerTable().put("127.0.0.1:10912",new ArrayList<>());
+        response.setBody(RemotingSerializable.encode(topicRouteData));
+        zoneRouteRPCHook.doAfterResponse("", request, response);
+        Assert.assertEquals(1,RemotingSerializable
+                .decode(response.getBody(), TopicRouteData.class)
+                .getFilterServerTable()
+                .size());
+    }
+
+    private TopicRouteData createSampleTopicRouteData() {
+        TopicRouteData topicRouteData = new TopicRouteData();
+        List<BrokerData> brokerDatas = new ArrayList<>();
+        BrokerData brokerData = createBrokerData("BrokerA","zone1",new HashMap<>());
+        List<QueueData> queueDatas = new ArrayList<>();
+        QueueData queueData = createQueueData("BrokerA");
+        queueDatas.add(queueData);
+        brokerDatas.add(brokerData);
+        topicRouteData.setBrokerDatas(brokerDatas);
+        topicRouteData.setQueueDatas(queueDatas);
+        return topicRouteData;
+    }
+
+    private BrokerData createBrokerData(String brokerName,String zoneName,HashMap<Long,String> brokerAddrs) {
+        BrokerData brokerData = new BrokerData();
+        brokerData.setBrokerName(brokerName);
+        brokerData.setZoneName(zoneName);
+        brokerData.setBrokerAddrs(brokerAddrs);
+        return  brokerData;
+    }
+
+    private QueueData createQueueData(String brokerName) {
+        QueueData queueData = new QueueData();
+        queueData.setBrokerName(brokerName);
+        queueData.setReadQueueNums(8);
+        queueData.setWriteQueueNums(8);
+        queueData.setPerm(6);
+        return queueData;
+    }
+}
diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerNewTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerNewTest.java
index b52cf50740a..5e58cfc124e 100644
--- a/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerNewTest.java
+++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerNewTest.java
@@ -416,9 +416,12 @@ public void pickupTopicRouteDataWithSlave() {
     }
 
     @Test
-    public void scanNotActiveBroker() {
+    public void scanNotActiveBroker() throws InterruptedException {
         registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic");
         routeInfoManager.scanNotActiveBroker();
+        registerBrokerWithNormalTopicAndExpire(BrokerBasicInfo.defaultBroker(),"TestTopic");
+        Thread.sleep(30000);
+        routeInfoManager.scanNotActiveBroker();
     }
 
     @Test
@@ -589,6 +592,16 @@ public void onChannelDestroy() {
         assertThat(routeInfoManager.pickupTopicRouteData("TestTopic")).isNull();
     }
 
+    @Test
+    public void onChannelDestroyByBrokerInfo() {
+        registerBroker(BrokerBasicInfo.defaultBroker(), mock(Channel.class), null, "TestTopic", "TestTopic1");
+        BrokerAddrInfo brokerAddrInfo = new BrokerAddrInfo(DEFAULT_CLUSTER, DEFAULT_ADDR);
+        routeInfoManager.onChannelDestroy(brokerAddrInfo);
+        await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0);
+        assertThat(routeInfoManager.pickupTopicRouteData("TestTopic")).isNull();
+        assertThat(routeInfoManager.pickupTopicRouteData("TestTopic1")).isNull();
+    }
+
     @Test
     public void switchBrokerRole_ChannelDestroy() {
         final BrokerBasicInfo masterBroker = BrokerBasicInfo.defaultBroker().enableActingMaster(false);
@@ -728,6 +741,23 @@ private RegisterBrokerResult registerBrokerWithNormalTopic(BrokerBasicInfo broke
         return registerBroker(brokerInfo, mock(Channel.class), topicConfigConcurrentHashMap, topics);
     }
 
+    private RegisterBrokerResult registerBrokerWithNormalTopicAndExpire(BrokerBasicInfo brokerInfo, String... topics) {
+        ConcurrentHashMap<String, TopicConfig> topicConfigConcurrentHashMap = new ConcurrentHashMap<>();
+        TopicConfig baseTopic = new TopicConfig("baseTopic");
+        topicConfigConcurrentHashMap.put(baseTopic.getTopicName(), baseTopic);
+        for (final String topic : topics) {
+            TopicConfig topicConfig = new TopicConfig();
+            topicConfig.setWriteQueueNums(8);
+            topicConfig.setTopicName(topic);
+            topicConfig.setPerm(6);
+            topicConfig.setReadQueueNums(8);
+            topicConfig.setOrder(false);
+            topicConfigConcurrentHashMap.put(topic, topicConfig);
+        }
+
+        return registerBrokerWithExpiredTime(brokerInfo, mock(Channel.class), topicConfigConcurrentHashMap, topics);
+    }
+
     private RegisterBrokerResult registerBrokerWithOrderTopic(BrokerBasicInfo brokerBasicInfo, String... topics) {
         ConcurrentHashMap<String, TopicConfig> topicConfigConcurrentHashMap = new ConcurrentHashMap<>();
 
@@ -785,7 +815,7 @@ private RegisterBrokerResult registerBroker(BrokerBasicInfo brokerInfo, Channel
         topicConfigSerializeWrapper.setDataVersion(brokerInfo.dataVersion);
         topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap);
 
-        RegisterBrokerResult registerBrokerResult = routeInfoManager.registerBroker(
+        return routeInfoManager.registerBroker(
             brokerInfo.clusterName,
             brokerInfo.brokerAddr,
             brokerInfo.brokerName,
@@ -795,7 +825,40 @@ private RegisterBrokerResult registerBroker(BrokerBasicInfo brokerInfo, Channel
             null,
             brokerInfo.enableActingMaster,
             topicConfigSerializeWrapper, new ArrayList<>(), channel);
-        return registerBrokerResult;
+    }
+
+    private RegisterBrokerResult registerBrokerWithExpiredTime(BrokerBasicInfo brokerInfo, Channel channel,
+                                                               ConcurrentMap<String, TopicConfig> topicConfigConcurrentHashMap, String... topics) {
+
+        if (topicConfigConcurrentHashMap == null) {
+            topicConfigConcurrentHashMap = new ConcurrentHashMap<>();
+            TopicConfig baseTopic = new TopicConfig("baseTopic");
+            topicConfigConcurrentHashMap.put(baseTopic.getTopicName(), baseTopic);
+            for (final String topic : topics) {
+                TopicConfig topicConfig = new TopicConfig();
+                topicConfig.setWriteQueueNums(8);
+                topicConfig.setTopicName(topic);
+                topicConfig.setPerm(6);
+                topicConfig.setReadQueueNums(8);
+                topicConfig.setOrder(false);
+                topicConfigConcurrentHashMap.put(topic, topicConfig);
+            }
+        }
+
+        TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();
+        topicConfigSerializeWrapper.setDataVersion(brokerInfo.dataVersion);
+        topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap);
+
+        return routeInfoManager.registerBroker(
+                brokerInfo.clusterName,
+                brokerInfo.brokerAddr,
+                brokerInfo.brokerName,
+                brokerInfo.brokerId,
+                brokerInfo.haAddr,
+                "",
+                30000L,
+                brokerInfo.enableActingMaster,
+                topicConfigSerializeWrapper, new ArrayList<>(), channel);
     }
 
     private void registerSingleTopicWithBrokerName(String brokerName, String... topics) {
diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml
index f10c8af6f0e..8e68fd09995 100644
--- a/openmessaging/pom.xml
+++ b/openmessaging/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java
index 36ac27f417a..46e607a5802 100644
--- a/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java
+++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java
@@ -82,7 +82,7 @@ public V get(final long timeout) {
                         try {
                             lock.wait(waitTime);
                         } catch (InterruptedException e) {
-                            LOG.error("promise get value interrupted,excepiton:{}", e.getMessage());
+                            LOG.error("promise get value interrupted,exception:{}", e.getMessage());
                         }
 
                         if (!isDoing()) {
diff --git a/pom.xml b/pom.xml
index 342671712d9..641cf63dab3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@
     <inceptionYear>2012</inceptionYear>
     <groupId>org.apache.rocketmq</groupId>
     <artifactId>rocketmq-all</artifactId>
-    <version>5.1.5-SNAPSHOT</version>
+    <version>5.3.3</version>
     <packaging>pom</packaging>
     <name>Apache RocketMQ ${project.version}</name>
     <url>http://rocketmq.apache.org/</url>
@@ -37,7 +37,7 @@
         <url>git@github.com:apache/rocketmq.git</url>
         <connection>scm:git:git@github.com:apache/rocketmq.git</connection>
         <developerConnection>scm:git:git@github.com:apache/rocketmq.git</developerConnection>
-        <tag>HEAD</tag>
+        <tag>rocketmq-all-5.3.3</tag>
     </scm>
 
     <mailingLists>
@@ -100,14 +100,15 @@
         <maven.compiler.target>1.8</maven.compiler.target>
 
         <commons-cli.version>1.5.0</commons-cli.version>
-        <netty.version>4.1.65.Final</netty.version>
+        <netty.version>4.1.114.Final</netty.version>
         <netty.tcnative.version>2.0.53.Final</netty.tcnative.version>
         <bcpkix-jdk15on.version>1.69</bcpkix-jdk15on.version>
         <fastjson.version>1.2.83</fastjson.version>
+        <fastjson2.version>2.0.43</fastjson2.version>
         <javassist.version>3.20.0-GA</javassist.version>
         <jna.version>4.2.2</jna.version>
         <commons-lang3.version>3.12.0</commons-lang3.version>
-        <commons-io.version>2.7</commons-io.version>
+        <commons-io.version>2.14.0</commons-io.version>
         <guava.version>32.0.1-jre</guava.version>
         <gson.version>2.9.0</gson.version>
         <openmessaging.version>0.3.1-alpha</openmessaging.version>
@@ -120,12 +121,12 @@
         <zstd-jni.version>1.5.2-2</zstd-jni.version>
         <lz4-java.version>1.8.0</lz4-java.version>
         <opentracing.version>0.33.0</opentracing.version>
-        <jaeger.version>1.6.0</jaeger.version>
+        <jaeger.version>1.8.1</jaeger.version>
         <dleger.version>0.3.1.2</dleger.version>
         <annotations-api.version>6.0.53</annotations-api.version>
         <extra-enforcer-rules.version>1.0-beta-4</extra-enforcer-rules.version>
         <concurrentlinkedhashmap-lru.version>1.4.2</concurrentlinkedhashmap-lru.version>
-        <rocketmq-proto.version>2.0.3</rocketmq-proto.version>
+        <rocketmq-proto.version>2.0.4</rocketmq-proto.version>
         <grpc.version>1.53.0</grpc.version>
         <protobuf.version>3.20.1</protobuf.version>
         <disruptor.version>1.2.10</disruptor.version>
@@ -145,6 +146,7 @@
         <junit.version>4.13.2</junit.version>
         <assertj-core.version>3.22.0</assertj-core.version>
         <mockito-core.version>3.10.0</mockito-core.version>
+        <mockito-junit-jupiter.version>4.11.0</mockito-junit-jupiter.version>
         <powermock-version>2.0.9</powermock-version>
         <awaitility.version>4.1.0</awaitility.version>
         <truth.version>0.30</truth.version>
@@ -191,7 +193,7 @@
         <module>test</module>
         <module>distribution</module>
         <module>openmessaging</module>
-        <module>acl</module>
+        <module>auth</module>
         <module>example</module>
         <module>container</module>
         <module>controller</module>
@@ -314,13 +316,14 @@
                         <exclude>src/test/resources/**</exclude>
                         <exclude>src/test/resources/certs/*</exclude>
                         <exclude>src/test/**/*.log</exclude>
-                        <exclude>src/test/resources/META-INF/service/*</exclude>
-                        <exclude>src/main/resources/META-INF/service/*</exclude>
+                        <exclude>src/test/resources/META-INF/services/*</exclude>
+                        <exclude>src/main/resources/META-INF/services/*</exclude>
                         <exclude>*/target/**</exclude>
                         <exclude>*/*.iml</exclude>
                         <exclude>docs/**</exclude>
                         <exclude>localbin/**</exclude>
                         <exclude>conf/rmq-proxy.json</exclude>
+                        <exclude>.bazelversion</exclude>
                     </excludes>
                 </configuration>
             </plugin>
@@ -522,13 +525,28 @@
                 <sonar.host.url>https://builds.apache.org/analysis</sonar.host.url>
             </properties>
         </profile>
+        <profile>
+            <id>skip-unit-tests</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <version>${maven-surefire-plugin.version}</version>
+                        <configuration>
+                            <skipTests>true</skipTests>
+                            <skip>true</skip>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
     </profiles>
 
     <dependencyManagement>
         <dependencies>
             <dependency>
                 <groupId>org.apache.rocketmq</groupId>
-                <artifactId>rocketmq-acl</artifactId>
+                <artifactId>rocketmq-auth</artifactId>
                 <version>${project.version}</version>
             </dependency>
             <dependency>
@@ -617,16 +635,8 @@
                 <version>${rocketmq-proto.version}</version>
                 <exclusions>
                     <exclusion>
-                        <groupId>io.grpc</groupId>
-                        <artifactId>grpc-protobuf</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>io.grpc</groupId>
-                        <artifactId>grpc-stub</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>io.grpc</groupId>
-                        <artifactId>grpc-netty-shaded</artifactId>
+                        <groupId>*</groupId>
+                        <artifactId>*</artifactId>
                     </exclusion>
                 </exclusions>
             </dependency>
@@ -652,6 +662,11 @@
                 <artifactId>fastjson</artifactId>
                 <version>${fastjson.version}</version>
             </dependency>
+            <dependency>
+                <groupId>com.alibaba.fastjson2</groupId>
+                <artifactId>fastjson2</artifactId>
+                <version>${fastjson2.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.javassist</groupId>
                 <artifactId>javassist</artifactId>
@@ -827,6 +842,12 @@
                 <version>${mockito-core.version}</version>
                 <scope>test</scope>
             </dependency>
+            <dependency>
+                <groupId>org.mockito</groupId>
+                <artifactId>mockito-junit-jupiter</artifactId>
+                <version>${mockito-junit-jupiter.version}</version>
+                <scope>test</scope>
+            </dependency>
             <dependency>
                 <groupId>org.awaitility</groupId>
                 <artifactId>awaitility</artifactId>
@@ -1062,6 +1083,12 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+
+            <dependency>
+                <groupId>jakarta.annotation</groupId>
+                <artifactId>jakarta.annotation-api</artifactId>
+                <version>1.3.5</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -1084,6 +1111,12 @@
             <version>${mockito-core.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-junit-jupiter</artifactId>
+            <version>${mockito-junit-jupiter.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.awaitility</groupId>
             <artifactId>awaitility</artifactId>
diff --git a/proxy/BUILD.bazel b/proxy/BUILD.bazel
index eb069528ea2..b996e8b39c5 100644
--- a/proxy/BUILD.bazel
+++ b/proxy/BUILD.bazel
@@ -21,7 +21,7 @@ java_library(
     srcs = glob(["src/main/java/**/*.java"]),
     visibility = ["//visibility:public"],
     deps = [
-        "//acl",
+        "//auth",
         "//broker",
         "//client",
         "//common",
@@ -30,6 +30,7 @@ java_library(
         "@maven//:ch_qos_logback_logback_classic",
         "@maven//:ch_qos_logback_logback_core",
         "@maven//:com_alibaba_fastjson",
+        "@maven//:com_alibaba_fastjson2_fastjson2",
         "@maven//:com_github_ben_manes_caffeine_caffeine",
         "@maven//:com_github_luben_zstd_jni",
         "@maven//:com_google_code_findbugs_jsr305",
@@ -78,7 +79,7 @@ java_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
-	"//acl",
+	    "//auth",
         ":proxy",
         "//:test_deps",
         "//broker",
@@ -87,6 +88,7 @@ java_library(
         "//remoting",
         "@maven//:ch_qos_logback_logback_core",
         "@maven//:com_alibaba_fastjson",
+        "@maven//:com_alibaba_fastjson2_fastjson2",
         "@maven//:com_github_ben_manes_caffeine_caffeine",
         "@maven//:com_google_guava_guava",
         "@maven//:com_google_protobuf_protobuf_java",
diff --git a/proxy/pom.xml b/proxy/pom.xml
index 5c5349a8c13..58a0698638f 100644
--- a/proxy/pom.xml
+++ b/proxy/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -53,7 +53,7 @@
         </dependency>
         <dependency>
             <groupId>org.apache.rocketmq</groupId>
-            <artifactId>rocketmq-acl</artifactId>
+            <artifactId>rocketmq-auth</artifactId>
         </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java
index 3b2ca99bfd0..e37ba975fec 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java
@@ -20,27 +20,20 @@
 import com.google.common.collect.Lists;
 import io.grpc.protobuf.services.ChannelzService;
 import io.grpc.protobuf.services.ProtoReflectionService;
-import java.util.Date;
-import java.util.List;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.DefaultParser;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.acl.AccessValidator;
-import org.apache.rocketmq.acl.plain.PlainAccessValidator;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.BrokerStartup;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
-import org.apache.rocketmq.common.utils.ServiceProvider;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.common.utils.AbstractStartAndShutdown;
 import org.apache.rocketmq.common.utils.StartAndShutdown;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.proxy.config.Configuration;
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
 import org.apache.rocketmq.proxy.config.ProxyConfig;
@@ -54,6 +47,11 @@
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.srvutil.ServerUtil;
 
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
 public class ProxyStartup {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
     private static final ProxyStartAndShutdown PROXY_START_AND_SHUTDOWN = new ProxyStartAndShutdown();
@@ -78,18 +76,17 @@ public static void main(String[] args) {
 
             MessagingProcessor messagingProcessor = createMessagingProcessor();
 
-            List<AccessValidator> accessValidators = loadAccessValidators();
             // create grpcServer
             GrpcServer grpcServer = GrpcServerBuilder.newBuilder(executor, ConfigurationManager.getProxyConfig().getGrpcServerPort())
                 .addService(createServiceProcessor(messagingProcessor))
                 .addService(ChannelzService.newInstance(100))
                 .addService(ProtoReflectionService.newInstance())
-                .configInterceptor(accessValidators)
+                .configInterceptor()
                 .shutdownTime(ConfigurationManager.getProxyConfig().getGrpcShutdownTimeSeconds(), TimeUnit.SECONDS)
                 .build();
             PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(grpcServer);
 
-            RemotingProtocolServer remotingServer = new RemotingProtocolServer(messagingProcessor, accessValidators);
+            RemotingProtocolServer remotingServer = new RemotingProtocolServer(messagingProcessor);
             PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(remotingServer);
 
             // start servers one by one.
@@ -114,15 +111,6 @@ public static void main(String[] args) {
         log.info(new Date() + " rocketmq-proxy startup successfully");
     }
 
-    protected static List<AccessValidator> loadAccessValidators() {
-        List<AccessValidator> accessValidators = ServiceProvider.load(AccessValidator.class);
-        if (accessValidators.isEmpty()) {
-            log.info("ServiceProvider loaded no AccessValidator, using default org.apache.rocketmq.acl.plain.PlainAccessValidator");
-            accessValidators.add(new PlainAccessValidator());
-        }
-        return accessValidators;
-    }
-
     protected static void initConfiguration(CommandLineArgument commandLineArgument) throws Exception {
         if (StringUtils.isNotBlank(commandLineArgument.getProxyConfigPath())) {
             System.setProperty(Configuration.CONFIG_PATH_PROPERTY, commandLineArgument.getProxyConfigPath());
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/auth/ProxyAuthenticationMetadataProvider.java b/proxy/src/main/java/org/apache/rocketmq/proxy/auth/ProxyAuthenticationMetadataProvider.java
new file mode 100644
index 00000000000..dd084fad0d0
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/auth/ProxyAuthenticationMetadataProvider.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.auth;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import org.apache.rocketmq.auth.authentication.provider.AuthenticationMetadataProvider;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.proxy.service.metadata.MetadataService;
+
+public class ProxyAuthenticationMetadataProvider implements AuthenticationMetadataProvider {
+
+    protected AuthConfig authConfig;
+    protected MetadataService metadataService;
+
+    @Override
+    public void initialize(AuthConfig authConfig, Supplier<?> metadataService) {
+        this.authConfig = authConfig;
+        if (metadataService != null) {
+            this.metadataService = (MetadataService) metadataService.get();
+        }
+    }
+
+    @Override
+    public void shutdown() {
+
+    }
+
+    @Override
+    public CompletableFuture<Void> createUser(User user) {
+        return null;
+    }
+
+    @Override
+    public CompletableFuture<Void> deleteUser(String username) {
+        return null;
+    }
+
+    @Override
+    public CompletableFuture<Void> updateUser(User user) {
+        return null;
+    }
+
+    @Override
+    public CompletableFuture<User> getUser(String username) {
+        return this.metadataService.getUser(null, username);
+    }
+
+    @Override
+    public CompletableFuture<List<User>> listUser(String filter) {
+        return null;
+    }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/auth/ProxyAuthorizationMetadataProvider.java b/proxy/src/main/java/org/apache/rocketmq/proxy/auth/ProxyAuthorizationMetadataProvider.java
new file mode 100644
index 00000000000..54fa7e436ef
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/auth/ProxyAuthorizationMetadataProvider.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.auth;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authorization.provider.AuthorizationMetadataProvider;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.proxy.service.metadata.MetadataService;
+
+public class ProxyAuthorizationMetadataProvider implements AuthorizationMetadataProvider {
+
+    protected AuthConfig authConfig;
+
+    protected MetadataService metadataService;
+
+    @Override
+    public void initialize(AuthConfig authConfig, Supplier<?> metadataService) {
+        this.authConfig = authConfig;
+        if (metadataService != null) {
+            this.metadataService = (MetadataService) metadataService.get();
+        }
+    }
+
+    @Override
+    public void shutdown() {
+
+    }
+
+    @Override
+    public CompletableFuture<Void> createAcl(Acl acl) {
+        return null;
+    }
+
+    @Override
+    public CompletableFuture<Void> deleteAcl(Subject subject) {
+        return null;
+    }
+
+    @Override
+    public CompletableFuture<Void> updateAcl(Acl acl) {
+        return null;
+    }
+
+    @Override
+    public CompletableFuture<Acl> getAcl(Subject subject) {
+        return this.metadataService.getAcl(null, subject);
+    }
+
+    @Override
+    public CompletableFuture<List<Acl>> listAcl(String subjectFilter, String resourceFilter) {
+        return null;
+    }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java
index 2fc1dab40ed..1f247194e2a 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java
@@ -18,6 +18,7 @@
 
 import com.google.common.net.HostAndPort;
 import java.util.Objects;
+import org.apache.rocketmq.common.utils.IPAddressUtils;
 
 public class Address {
 
@@ -31,6 +32,11 @@ public enum AddressScheme {
     private AddressScheme addressScheme;
     private HostAndPort hostAndPort;
 
+    public Address(HostAndPort hostAndPort) {
+        this.addressScheme = buildScheme(hostAndPort);
+        this.hostAndPort = hostAndPort;
+    }
+
     public Address(AddressScheme addressScheme, HostAndPort hostAndPort) {
         this.addressScheme = addressScheme;
         this.hostAndPort = hostAndPort;
@@ -52,6 +58,20 @@ public void setHostAndPort(HostAndPort hostAndPort) {
         this.hostAndPort = hostAndPort;
     }
 
+    private AddressScheme buildScheme(HostAndPort hostAndPort) {
+        if (hostAndPort == null) {
+            return AddressScheme.UNRECOGNIZED;
+        }
+        String address = hostAndPort.getHost();
+        if (IPAddressUtils.isValidIPv4(address)) {
+            return AddressScheme.IPv4;
+        }
+        if (IPAddressUtils.isValidIPv6(address)) {
+            return AddressScheme.IPv6;
+        }
+        return AddressScheme.DOMAIN_NAME;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java
index 6fee38d117b..15da628dc3c 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java
@@ -25,14 +25,19 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.consumer.ReceiptHandle;
 import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
 
 public class ReceiptHandleGroup {
+    protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
 
     // The messages having the same messageId will be deduplicated based on the parameters of broker, queueId, and offset
     protected final Map<String /* msgID */, Map<HandleKey, HandleData>> receiptHandleMap = new ConcurrentHashMap<>();
@@ -98,6 +103,7 @@ public long getOffset() {
 
     public static class HandleData {
         private final Semaphore semaphore = new Semaphore(1);
+        private final AtomicLong lastLockTimeMs = new AtomicLong(-1L);
         private volatile boolean needRemove = false;
         private volatile MessageReceiptHandle messageReceiptHandle;
 
@@ -105,15 +111,40 @@ public HandleData(MessageReceiptHandle messageReceiptHandle) {
             this.messageReceiptHandle = messageReceiptHandle;
         }
 
-        public boolean lock(long timeoutMs) {
+        public Long lock(long timeoutMs) {
             try {
-                return this.semaphore.tryAcquire(timeoutMs, TimeUnit.MILLISECONDS);
+                boolean result = this.semaphore.tryAcquire(timeoutMs, TimeUnit.MILLISECONDS);
+                long currentTimeMs = System.currentTimeMillis();
+                if (result) {
+                    this.lastLockTimeMs.set(currentTimeMs);
+                    return currentTimeMs;
+                } else {
+                    // if the lock is expired, can be acquired again
+                    long expiredTimeMs = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup() * 3;
+                    if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) {
+                        synchronized (this) {
+                            if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) {
+                                log.warn("HandleData lock expired, acquire lock success and reset lock time. " +
+                                    "MessageReceiptHandle={}, lockTime={}", messageReceiptHandle, currentTimeMs);
+                                this.lastLockTimeMs.set(currentTimeMs);
+                                return currentTimeMs;
+                            }
+                        }
+                    }
+                }
+                return null;
             } catch (InterruptedException e) {
-                return false;
+                return null;
             }
         }
 
-        public void unlock() {
+        public void unlock(long lockTimeMs) {
+            // if the lock is expired, we don't need to unlock it
+            if (System.currentTimeMillis() - lockTimeMs > ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup() * 2) {
+                log.warn("HandleData lock expired, unlock fail. MessageReceiptHandle={}, lockTime={}, now={}",
+                    messageReceiptHandle, lockTimeMs, System.currentTimeMillis());
+                return;
+            }
             this.semaphore.release();
         }
 
@@ -149,7 +180,8 @@ public void put(String msgID, MessageReceiptHandle value) {
             if (handleData == null || handleData.needRemove) {
                 return new HandleData(value);
             }
-            if (!handleData.lock(timeout)) {
+            Long lockTimeMs = handleData.lock(timeout);
+            if (lockTimeMs == null) {
                 throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "try to put handle failed");
             }
             try {
@@ -158,7 +190,7 @@ public void put(String msgID, MessageReceiptHandle value) {
                 }
                 handleData.messageReceiptHandle = value;
             } finally {
-                handleData.unlock();
+                handleData.unlock(lockTimeMs);
             }
             return handleData;
         });
@@ -176,7 +208,8 @@ public MessageReceiptHandle get(String msgID, String handle) {
         long timeout = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup();
         AtomicReference<MessageReceiptHandle> res = new AtomicReference<>();
         handleMap.computeIfPresent(new HandleKey(handle), (handleKey, handleData) -> {
-            if (!handleData.lock(timeout)) {
+            Long lockTimeMs = handleData.lock(timeout);
+            if (lockTimeMs == null) {
                 throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "try to get handle failed");
             }
             try {
@@ -185,7 +218,7 @@ public MessageReceiptHandle get(String msgID, String handle) {
                 }
                 res.set(handleData.messageReceiptHandle);
             } finally {
-                handleData.unlock();
+                handleData.unlock(lockTimeMs);
             }
             return handleData;
         });
@@ -200,7 +233,8 @@ public MessageReceiptHandle remove(String msgID, String handle) {
         long timeout = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup();
         AtomicReference<MessageReceiptHandle> res = new AtomicReference<>();
         handleMap.computeIfPresent(new HandleKey(handle), (handleKey, handleData) -> {
-            if (!handleData.lock(timeout)) {
+            Long lockTimeMs = handleData.lock(timeout);
+            if (lockTimeMs == null) {
                 throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "try to remove and get handle failed");
             }
             try {
@@ -210,7 +244,7 @@ public MessageReceiptHandle remove(String msgID, String handle) {
                 }
                 return null;
             } finally {
-                handleData.unlock();
+                handleData.unlock(lockTimeMs);
             }
         });
         removeHandleMapKeyIfNeed(msgID);
@@ -240,7 +274,8 @@ public void computeIfPresent(String msgID, String handle,
         }
         long timeout = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup();
         handleMap.computeIfPresent(new HandleKey(handle), (handleKey, handleData) -> {
-            if (!handleData.lock(timeout)) {
+            Long lockTimeMs = handleData.lock(timeout);
+            if (lockTimeMs == null) {
                 throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "try to compute failed");
             }
             CompletableFuture<MessageReceiptHandle> future = function.apply(handleData.messageReceiptHandle);
@@ -255,7 +290,7 @@ public void computeIfPresent(String msgID, String handle,
                         handleData.messageReceiptHandle = messageReceiptHandle;
                     }
                 } finally {
-                    handleData.unlock();
+                    handleData.unlock(lockTimeMs);
                 }
                 if (handleData.needRemove) {
                     handleMap.remove(handleKey, handleData);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/GrpcUtils.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/GrpcUtils.java
new file mode 100644
index 00000000000..5c50de4426e
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/GrpcUtils.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.common.utils;
+
+import io.grpc.Attributes;
+import io.grpc.Metadata;
+import io.grpc.ServerCall;
+
+public class GrpcUtils {
+
+    private GrpcUtils() {
+    }
+
+    public static <T> void putHeaderIfNotExist(Metadata headers, Metadata.Key<T> key, T value) {
+        if (headers == null) {
+            return;
+        }
+        if (!headers.containsKey(key) && value != null) {
+            headers.put(key, value);
+        }
+    }
+
+    public static <R, W, T> T getAttribute(ServerCall<R, W> call, Attributes.Key<T> key) {
+        Attributes attributes = call.getAttributes();
+        if (attributes == null) {
+            return null;
+        }
+        return attributes.get(key);
+    }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/Configuration.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/Configuration.java
index 2561d44190e..5b7c6c3007d 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/Configuration.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/Configuration.java
@@ -27,6 +27,7 @@
 import java.nio.file.Files;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.config.AuthConfig;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
@@ -34,6 +35,7 @@
 public class Configuration {
     private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
     private final AtomicReference<ProxyConfig> proxyConfigReference = new AtomicReference<>();
+    private final AtomicReference<AuthConfig> authConfigReference = new AtomicReference<>();
     public static final String CONFIG_PATH_PROPERTY = "com.rocketmq.proxy.configPath";
 
     public void init() throws Exception {
@@ -42,6 +44,11 @@ public void init() throws Exception {
         ProxyConfig proxyConfig = JSON.parseObject(proxyConfigData, ProxyConfig.class);
         proxyConfig.initData();
         setProxyConfig(proxyConfig);
+
+        AuthConfig authConfig = JSON.parseObject(proxyConfigData, AuthConfig.class);
+        setAuthConfig(authConfig);
+        authConfig.setConfigName(proxyConfig.getProxyName());
+        authConfig.setClusterName(proxyConfig.getRocketMQClusterName());
     }
 
     public static String loadJsonConfig() throws Exception {
@@ -79,4 +86,12 @@ public ProxyConfig getProxyConfig() {
     public void setProxyConfig(ProxyConfig proxyConfig) {
         proxyConfigReference.set(proxyConfig);
     }
+
+    public AuthConfig getAuthConfig() {
+        return authConfigReference.get();
+    }
+
+    public void setAuthConfig(AuthConfig authConfig) {
+        authConfigReference.set(authConfig);
+    }
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigurationManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigurationManager.java
index 911e1f28f2d..24e1bd44b41 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigurationManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigurationManager.java
@@ -20,6 +20,7 @@
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.serializer.SerializerFeature;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.config.AuthConfig;
 import org.apache.rocketmq.common.MixAll;
 
 public class ConfigurationManager {
@@ -52,6 +53,10 @@ public static ProxyConfig getProxyConfig() {
         return configuration.getProxyConfig();
     }
 
+    public static AuthConfig getAuthConfig() {
+        return configuration.getAuthConfig();
+    }
+
     public static String formatProxyConfig() {
         return JSON.toJSONString(ConfigurationManager.getProxyConfig(),
                 SerializerFeature.PrettyFormat, SerializerFeature.WriteMapNullValue, SerializerFeature.WriteDateUseDateFormat, SerializerFeature.WriteNullListAsEmpty);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
index e907a1ccc3f..e3e60b76bb5 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java
@@ -17,16 +17,6 @@
 
 package org.apache.rocketmq.proxy.config;
 
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.time.Duration;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.constant.LoggerName;
@@ -38,6 +28,17 @@
 import org.apache.rocketmq.proxy.common.ProxyException;
 import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
 
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.time.Duration;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
 public class ProxyConfig implements ConfigFile {
     private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
     public final static String DEFAULT_CONFIG_FILE_NAME = "rmq-proxy.json";
@@ -103,6 +104,10 @@ public class ProxyConfig implements ConfigFile {
      * max message body size, 0 or negative number means no limit for proxy
      */
     private int maxMessageSize = 4 * 1024 * 1024;
+    /**
+     * if true, proxy will check message body size and reject msg if it's body is empty
+     */
+    private boolean enableMessageBodyEmptyCheck = true;
     /**
      * max user property size, 0 or negative number means no limit for proxy
      */
@@ -168,6 +173,12 @@ public class ProxyConfig implements ConfigFile {
     private int subscriptionGroupConfigCacheExpiredSeconds = 300;
     private int subscriptionGroupConfigCacheRefreshSeconds = 20;
     private int subscriptionGroupConfigCacheMaxNum = 20000;
+    private int userCacheExpiredSeconds = 300;
+    private int userCacheRefreshSeconds = 20;
+    private int userCacheMaxNum = 20000;
+    private int aclCacheExpiredSeconds = 300;
+    private int aclCacheRefreshSeconds = 20;
+    private int aclCacheMaxNum = 20000;
     private int metadataThreadPoolNums = 3;
     private int metadataThreadPoolQueueCapacity = 100000;
 
@@ -193,8 +204,6 @@ public class ProxyConfig implements ConfigFile {
     private long renewMaxTimeMillis = TimeUnit.HOURS.toMillis(3);
     private long renewSchedulePeriodMillis = TimeUnit.SECONDS.toMillis(5);
 
-    private boolean enableACL = false;
-
     private boolean enableAclRpcHookForClusterMode = false;
 
     private boolean useDelayLevel = false;
@@ -900,6 +909,54 @@ public void setSubscriptionGroupConfigCacheMaxNum(int subscriptionGroupConfigCac
         this.subscriptionGroupConfigCacheMaxNum = subscriptionGroupConfigCacheMaxNum;
     }
 
+    public int getUserCacheExpiredSeconds() {
+        return userCacheExpiredSeconds;
+    }
+
+    public void setUserCacheExpiredSeconds(int userCacheExpiredSeconds) {
+        this.userCacheExpiredSeconds = userCacheExpiredSeconds;
+    }
+
+    public int getUserCacheRefreshSeconds() {
+        return userCacheRefreshSeconds;
+    }
+
+    public void setUserCacheRefreshSeconds(int userCacheRefreshSeconds) {
+        this.userCacheRefreshSeconds = userCacheRefreshSeconds;
+    }
+
+    public int getUserCacheMaxNum() {
+        return userCacheMaxNum;
+    }
+
+    public void setUserCacheMaxNum(int userCacheMaxNum) {
+        this.userCacheMaxNum = userCacheMaxNum;
+    }
+
+    public int getAclCacheExpiredSeconds() {
+        return aclCacheExpiredSeconds;
+    }
+
+    public void setAclCacheExpiredSeconds(int aclCacheExpiredSeconds) {
+        this.aclCacheExpiredSeconds = aclCacheExpiredSeconds;
+    }
+
+    public int getAclCacheRefreshSeconds() {
+        return aclCacheRefreshSeconds;
+    }
+
+    public void setAclCacheRefreshSeconds(int aclCacheRefreshSeconds) {
+        this.aclCacheRefreshSeconds = aclCacheRefreshSeconds;
+    }
+
+    public int getAclCacheMaxNum() {
+        return aclCacheMaxNum;
+    }
+
+    public void setAclCacheMaxNum(int aclCacheMaxNum) {
+        this.aclCacheMaxNum = aclCacheMaxNum;
+    }
+
     public int getMetadataThreadPoolNums() {
         return metadataThreadPoolNums;
     }
@@ -988,14 +1045,6 @@ public void setLongPollingReserveTimeInMillis(long longPollingReserveTimeInMilli
         this.longPollingReserveTimeInMillis = longPollingReserveTimeInMillis;
     }
 
-    public boolean isEnableACL() {
-        return enableACL;
-    }
-
-    public void setEnableACL(boolean enableACL) {
-        this.enableACL = enableACL;
-    }
-
     public boolean isEnableAclRpcHookForClusterMode() {
         return enableAclRpcHookForClusterMode;
     }
@@ -1471,4 +1520,12 @@ public boolean isEnableBatchAck() {
     public void setEnableBatchAck(boolean enableBatchAck) {
         this.enableBatchAck = enableBatchAck;
     }
+
+    public boolean isEnableMessageBodyEmptyCheck() {
+        return enableMessageBodyEmptyCheck;
+    }
+
+    public void setEnableMessageBodyEmptyCheck(boolean enableMessageBodyEmptyCheck) {
+        this.enableMessageBodyEmptyCheck = enableMessageBodyEmptyCheck;
+    }
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java
index 0e79006f6b3..ab00b967e66 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java
@@ -24,19 +24,17 @@
 import io.grpc.netty.shaded.io.netty.channel.epoll.EpollServerSocketChannel;
 import io.grpc.netty.shaded.io.netty.channel.nio.NioEventLoopGroup;
 import io.grpc.netty.shaded.io.netty.channel.socket.nio.NioServerSocketChannel;
-import java.util.List;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import org.apache.rocketmq.acl.AccessValidator;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
-import org.apache.rocketmq.proxy.grpc.interceptor.AuthenticationInterceptor;
 import org.apache.rocketmq.proxy.grpc.interceptor.ContextInterceptor;
 import org.apache.rocketmq.proxy.grpc.interceptor.GlobalExceptionInterceptor;
 import org.apache.rocketmq.proxy.grpc.interceptor.HeaderInterceptor;
 
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
 public class GrpcServerBuilder {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
     protected NettyServerBuilder serverBuilder;
@@ -75,9 +73,7 @@ protected GrpcServerBuilder(ThreadPoolExecutor executor, int port) {
         serverBuilder.maxInboundMessageSize(maxInboundMessageSize)
                 .maxConnectionIdle(idleTimeMills, TimeUnit.MILLISECONDS);
 
-        log.info(
-            "grpc server has built. port: {}, tlsKeyPath: {}, tlsCertPath: {}, threadPool: {}, queueCapacity: {}, "
-                + "boosLoop: {}, workerLoop: {}, maxInboundMessageSize: {}",
+        log.info("grpc server has built. port: {}, bossLoopNum: {}, workerLoopNum: {}, maxInboundMessageSize: {}",
             port, bossLoopNum, workerLoopNum, maxInboundMessageSize);
     }
 
@@ -106,15 +102,11 @@ public GrpcServer build() {
         return new GrpcServer(this.serverBuilder.build(), time, unit);
     }
 
-    public GrpcServerBuilder configInterceptor(List<AccessValidator> accessValidators) {
-        // grpc interceptors, including acl, logging etc.
-        this.serverBuilder.intercept(new AuthenticationInterceptor(accessValidators));
-
+    public GrpcServerBuilder configInterceptor() {
         this.serverBuilder
             .intercept(new GlobalExceptionInterceptor())
             .intercept(new ContextInterceptor())
             .intercept(new HeaderInterceptor());
-
         return this;
     }
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java
index 7c92866803f..e0a6099ccc6 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java
@@ -126,6 +126,8 @@ private class ProxyAndTlsProtocolHandler extends ByteToMessageDecoder {
 
         private final GrpcHttp2ConnectionHandler grpcHandler;
 
+        private ProtocolNegotiationEvent pne = InternalProtocolNegotiationEvent.getDefault();
+
         public ProxyAndTlsProtocolHandler(GrpcHttp2ConnectionHandler grpcHandler) {
             this.grpcHandler = grpcHandler;
         }
@@ -146,13 +148,25 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) {
                     ctx.pipeline().addAfter(ctx.name(), TLS_MODE_HANDLER, new TlsModeHandler(grpcHandler));
                 }
 
-                ctx.fireUserEventTriggered(InternalProtocolNegotiationEvent.getDefault());
+                Attributes.Builder builder = InternalProtocolNegotiationEvent.getAttributes(pne).toBuilder();
+                builder.set(AttributeKeys.CHANNEL_ID, ctx.channel().id().asLongText());
+
+                ctx.fireUserEventTriggered(InternalProtocolNegotiationEvent.withAttributes(pne, builder.build()));
                 ctx.pipeline().remove(this);
             } catch (Exception e) {
                 log.error("process proxy protocol negotiator failed.", e);
                 throw e;
             }
         }
+
+        @Override
+        public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
+            if (evt instanceof ProtocolNegotiationEvent) {
+                pne = (ProtocolNegotiationEvent) evt;
+            } else {
+                super.userEventTriggered(ctx, evt);
+            }
+        }
     }
 
     private class HAProxyMessageHandler extends ChannelInboundHandlerAdapter {
@@ -200,6 +214,15 @@ private void handleWithMessage(HAProxyMessage msg) {
                 msg.release();
             }
         }
+
+        @Override
+        public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
+            if (evt instanceof ProtocolNegotiationEvent) {
+                pne = (ProtocolNegotiationEvent) evt;
+            } else {
+                super.userEventTriggered(ctx, evt);
+            }
+        }
     }
 
     protected void handleHAProxyTLV(HAProxyTLV tlv, Attributes.Builder builder) {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/constant/AttributeKeys.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/constant/AttributeKeys.java
index 096a5ba3d3d..874b3981959 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/constant/AttributeKeys.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/constant/AttributeKeys.java
@@ -24,6 +24,9 @@
 
 public class AttributeKeys {
 
+    public static final Attributes.Key<String> CHANNEL_ID =
+        Attributes.Key.create(HAProxyConstants.CHANNEL_ID);
+
     public static final Attributes.Key<String> PROXY_PROTOCOL_ADDR =
             Attributes.Key.create(HAProxyConstants.PROXY_PROTOCOL_ADDR);
 
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/AuthenticationInterceptor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/AuthenticationInterceptor.java
deleted file mode 100644
index 951ebf006b5..00000000000
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/AuthenticationInterceptor.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.proxy.grpc.interceptor;
-
-import com.google.protobuf.GeneratedMessageV3;
-import io.grpc.Context;
-import io.grpc.ForwardingServerCallListener;
-import io.grpc.Metadata;
-import io.grpc.ServerCall;
-import io.grpc.ServerCallHandler;
-import io.grpc.ServerInterceptor;
-import io.grpc.Status;
-import io.grpc.StatusRuntimeException;
-import java.util.List;
-import org.apache.rocketmq.acl.AccessResource;
-import org.apache.rocketmq.acl.AccessValidator;
-import org.apache.rocketmq.acl.common.AclException;
-import org.apache.rocketmq.acl.common.AuthenticationHeader;
-import org.apache.rocketmq.acl.plain.PlainAccessResource;
-import org.apache.rocketmq.proxy.config.ConfigurationManager;
-
-public class AuthenticationInterceptor implements ServerInterceptor {
-    protected final List<AccessValidator> accessValidatorList;
-
-    public AuthenticationInterceptor(List<AccessValidator> accessValidatorList) {
-        this.accessValidatorList = accessValidatorList;
-    }
-
-    @Override
-    public <R, W> ServerCall.Listener<R> interceptCall(ServerCall<R, W> call, Metadata headers,
-        ServerCallHandler<R, W> next) {
-        return new ForwardingServerCallListener.SimpleForwardingServerCallListener<R>(next.startCall(call, headers)) {
-            @Override
-            public void onMessage(R message) {
-                GeneratedMessageV3 messageV3 = (GeneratedMessageV3) message;
-                headers.put(InterceptorConstants.RPC_NAME, messageV3.getDescriptorForType().getFullName());
-                headers.put(InterceptorConstants.SIMPLE_RPC_NAME, messageV3.getDescriptorForType().getName());
-                if (ConfigurationManager.getProxyConfig().isEnableACL()) {
-                    try {
-                        AuthenticationHeader authenticationHeader = AuthenticationHeader.builder()
-                            .remoteAddress(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.REMOTE_ADDRESS))
-                            .namespace(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.NAMESPACE_ID))
-                            .authorization(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.AUTHORIZATION))
-                            .datetime(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.DATE_TIME))
-                            .sessionToken(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.SESSION_TOKEN))
-                            .requestId(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.REQUEST_ID))
-                            .language(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.LANGUAGE))
-                            .clientVersion(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.CLIENT_VERSION))
-                            .protocol(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.PROTOCOL_VERSION))
-                            .requestCode(RequestMapping.map(messageV3.getDescriptorForType().getFullName()))
-                            .build();
-
-                        validate(authenticationHeader, headers, messageV3);
-                        super.onMessage(message);
-                    } catch (AclException aclException) {
-                        throw new StatusRuntimeException(Status.PERMISSION_DENIED, headers);
-                    }
-                } else {
-                    super.onMessage(message);
-                }
-            }
-        };
-    }
-
-    protected void validate(AuthenticationHeader authenticationHeader, Metadata headers, GeneratedMessageV3 messageV3) {
-        for (AccessValidator accessValidator : accessValidatorList) {
-            AccessResource accessResource = accessValidator.parse(messageV3, authenticationHeader);
-            accessValidator.validate(accessResource);
-
-            if (accessResource instanceof PlainAccessResource) {
-                PlainAccessResource plainAccessResource = (PlainAccessResource) accessResource;
-                headers.put(InterceptorConstants.AUTHORIZATION_AK, plainAccessResource.getAccessKey());
-            }
-        }
-    }
-}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/ContextInterceptor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/ContextInterceptor.java
index 07d7ab9bf38..112dd263afd 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/ContextInterceptor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/ContextInterceptor.java
@@ -23,6 +23,7 @@
 import io.grpc.ServerCall;
 import io.grpc.ServerCallHandler;
 import io.grpc.ServerInterceptor;
+import org.apache.rocketmq.common.constant.GrpcConstants;
 
 public class ContextInterceptor implements ServerInterceptor {
 
@@ -32,7 +33,7 @@ public <R, W> ServerCall.Listener<R> interceptCall(
         Metadata headers,
         ServerCallHandler<R, W> next
     ) {
-        Context context = Context.current().withValue(InterceptorConstants.METADATA, headers);
+        Context context = Context.current().withValue(GrpcConstants.METADATA, headers);
         return Contexts.interceptCall(context, call, headers, next);
     }
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/HeaderInterceptor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/HeaderInterceptor.java
index 13893e5eda3..e3e78841559 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/HeaderInterceptor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/HeaderInterceptor.java
@@ -26,6 +26,8 @@
 import io.grpc.ServerInterceptor;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.constant.HAProxyConstants;
+import org.apache.rocketmq.common.constant.GrpcConstants;
+import org.apache.rocketmq.proxy.common.utils.GrpcUtils;
 import org.apache.rocketmq.proxy.grpc.constant.AttributeKeys;
 
 import java.net.InetSocketAddress;
@@ -43,11 +45,11 @@ public <R, W> ServerCall.Listener<R> interceptCall(
             SocketAddress remoteSocketAddress = call.getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR);
             remoteAddress = parseSocketAddress(remoteSocketAddress);
         }
-        headers.put(InterceptorConstants.REMOTE_ADDRESS, remoteAddress);
+        GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.REMOTE_ADDRESS, remoteAddress);
 
         SocketAddress localSocketAddress = call.getAttributes().get(Grpc.TRANSPORT_ATTR_LOCAL_ADDR);
         String localAddress = parseSocketAddress(localSocketAddress);
-        headers.put(InterceptorConstants.LOCAL_ADDRESS, localAddress);
+        GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.LOCAL_ADDRESS, localAddress);
 
         for (Attributes.Key<?> key : call.getAttributes().keys()) {
             if (!StringUtils.startsWith(key.toString(), HAProxyConstants.PROXY_PROTOCOL_PREFIX)) {
@@ -56,7 +58,12 @@ public <R, W> ServerCall.Listener<R> interceptCall(
             Metadata.Key<String> headerKey
                     = Metadata.Key.of(key.toString(), Metadata.ASCII_STRING_MARSHALLER);
             String headerValue = String.valueOf(call.getAttributes().get(key));
-            headers.put(headerKey, headerValue);
+            GrpcUtils.putHeaderIfNotExist(headers, headerKey, headerValue);
+        }
+
+        String channelId = call.getAttributes().get(AttributeKeys.CHANNEL_ID);
+        if (StringUtils.isNotBlank(channelId)) {
+            GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.CHANNEL_ID, channelId);
         }
 
         return next.startCall(call, headers);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/RequestMapping.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/RequestMapping.java
index 866124d747c..f5edc03ba4a 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/RequestMapping.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/RequestMapping.java
@@ -25,6 +25,7 @@
 import apache.rocketmq.v2.NotifyClientTerminationRequest;
 import apache.rocketmq.v2.QueryAssignmentRequest;
 import apache.rocketmq.v2.QueryRouteRequest;
+import apache.rocketmq.v2.RecallMessageRequest;
 import apache.rocketmq.v2.ReceiveMessageRequest;
 import apache.rocketmq.v2.SendMessageRequest;
 import java.util.HashMap;
@@ -38,6 +39,7 @@ public class RequestMapping {
             put(QueryRouteRequest.getDescriptor().getFullName(), RequestCode.GET_ROUTEINFO_BY_TOPIC);
             put(HeartbeatRequest.getDescriptor().getFullName(), RequestCode.HEART_BEAT);
             put(SendMessageRequest.getDescriptor().getFullName(), RequestCode.SEND_MESSAGE_V2);
+            put(RecallMessageRequest.getDescriptor().getFullName(), RequestCode.RECALL_MESSAGE);
             put(QueryAssignmentRequest.getDescriptor().getFullName(), RequestCode.GET_ROUTEINFO_BY_TOPIC);
             put(ReceiveMessageRequest.getDescriptor().getFullName(), RequestCode.PULL_MESSAGE);
             put(AckMessageRequest.getDescriptor().getFullName(), RequestCode.UPDATE_CONSUMER_OFFSET);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/AuthenticationPipeline.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/AuthenticationPipeline.java
new file mode 100644
index 00000000000..e317b48f1ed
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/AuthenticationPipeline.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.grpc.pipeline;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Context;
+import io.grpc.Metadata;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.authentication.AuthenticationEvaluator;
+import org.apache.rocketmq.auth.authentication.context.AuthenticationContext;
+import org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.constant.GrpcConstants;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.common.utils.GrpcUtils;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+
+public class AuthenticationPipeline implements RequestPipeline {
+    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+    private final AuthConfig authConfig;
+    private final AuthenticationEvaluator authenticationEvaluator;
+
+    public AuthenticationPipeline(AuthConfig authConfig, MessagingProcessor messagingProcessor) {
+        this.authConfig = authConfig;
+        this.authenticationEvaluator = AuthenticationFactory.getEvaluator(authConfig, messagingProcessor::getMetadataService);
+    }
+
+    @Override
+    public void execute(ProxyContext context, Metadata headers, GeneratedMessageV3 request) {
+        if (!authConfig.isAuthenticationEnabled()) {
+            return;
+        }
+        try {
+            Metadata metadata = GrpcConstants.METADATA.get(Context.current());
+            AuthenticationContext authenticationContext = newContext(context, metadata, request);
+            authenticationEvaluator.evaluate(authenticationContext);
+        } catch (AuthenticationException ex) {
+            throw ex;
+        } catch (Throwable ex) {
+            LOGGER.error("authenticate failed, request:{}", request, ex);
+            throw ex;
+        }
+    }
+
+    /**
+     * Create Context, for extension
+     *
+     * @param context for extension
+     * @param headers gRPC headers
+     * @param request
+     * @return
+     */
+    protected AuthenticationContext newContext(ProxyContext context, Metadata headers, GeneratedMessageV3 request) {
+        AuthenticationContext result = AuthenticationFactory.newContext(authConfig, headers, request);
+        if (result instanceof DefaultAuthenticationContext) {
+            DefaultAuthenticationContext defaultAuthenticationContext = (DefaultAuthenticationContext) result;
+            if (StringUtils.isNotBlank(defaultAuthenticationContext.getUsername())) {
+                GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.AUTHORIZATION_AK, defaultAuthenticationContext.getUsername());
+            }
+        }
+        return result;
+    }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/AuthorizationPipeline.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/AuthorizationPipeline.java
new file mode 100644
index 00000000000..c0b33426da5
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/AuthorizationPipeline.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.grpc.pipeline;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import java.util.List;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authorization.AuthorizationEvaluator;
+import org.apache.rocketmq.auth.authorization.context.AuthorizationContext;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+
+public class AuthorizationPipeline implements RequestPipeline {
+    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+    private final AuthConfig authConfig;
+    private final AuthorizationEvaluator authorizationEvaluator;
+
+    public AuthorizationPipeline(AuthConfig authConfig, MessagingProcessor messagingProcessor) {
+        this.authConfig = authConfig;
+        this.authorizationEvaluator = AuthorizationFactory.getEvaluator(authConfig, messagingProcessor::getMetadataService);
+    }
+
+    @Override
+    public void execute(ProxyContext context, Metadata headers, GeneratedMessageV3 request) {
+        if (!authConfig.isAuthorizationEnabled()) {
+            return;
+        }
+        try {
+            List<AuthorizationContext> contexts = newContexts(context, headers, request);
+            authorizationEvaluator.evaluate(contexts);
+        } catch (AuthorizationException | AuthenticationException ex) {
+            throw ex;
+        }  catch (Throwable ex) {
+            LOGGER.error("authorize failed, request:{}", request, ex);
+            throw ex;
+        }
+    }
+
+    protected List<AuthorizationContext> newContexts(ProxyContext context, Metadata headers, GeneratedMessageV3 request) {
+        return AuthorizationFactory.newContexts(authConfig, headers, request);
+    }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/ContextInitPipeline.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/ContextInitPipeline.java
new file mode 100644
index 00000000000..515b9acae15
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/ContextInitPipeline.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.grpc.pipeline;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Context;
+import io.grpc.Metadata;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.constant.GrpcConstants;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
+
+public class ContextInitPipeline implements RequestPipeline {
+    @Override
+    public void execute(ProxyContext context, Metadata headers, GeneratedMessageV3 request) {
+        Context ctx = Context.current();
+        context.setLocalAddress(getDefaultStringMetadataInfo(headers, GrpcConstants.LOCAL_ADDRESS))
+            .setRemoteAddress(getDefaultStringMetadataInfo(headers, GrpcConstants.REMOTE_ADDRESS))
+            .setClientID(getDefaultStringMetadataInfo(headers, GrpcConstants.CLIENT_ID))
+            .setProtocolType(ChannelProtocolType.GRPC_V2.getName())
+            .setLanguage(getDefaultStringMetadataInfo(headers, GrpcConstants.LANGUAGE))
+            .setClientVersion(getDefaultStringMetadataInfo(headers, GrpcConstants.CLIENT_VERSION))
+            .setAction(getDefaultStringMetadataInfo(headers, GrpcConstants.SIMPLE_RPC_NAME))
+            .setNamespace(getDefaultStringMetadataInfo(headers, GrpcConstants.NAMESPACE_ID));
+        if (ctx.getDeadline() != null) {
+            context.setRemainingMs(ctx.getDeadline().timeRemaining(TimeUnit.MILLISECONDS));
+        }
+    }
+
+    protected String getDefaultStringMetadataInfo(Metadata headers, Metadata.Key<String> key) {
+        return StringUtils.defaultString(headers.get(key));
+    }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/RequestPipeline.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/RequestPipeline.java
new file mode 100644
index 00000000000..342320894c9
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/RequestPipeline.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.grpc.pipeline;
+
+import com.google.protobuf.GeneratedMessageV3;
+import io.grpc.Metadata;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+
+public interface RequestPipeline {
+
+    void execute(ProxyContext context, Metadata headers, GeneratedMessageV3 request);
+
+    default RequestPipeline pipe(RequestPipeline source) {
+        return (ctx, headers, request) -> {
+            source.execute(ctx, headers, request);
+            execute(ctx, headers, request);
+        };
+    }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java
index 091e9086ecc..3c6f120ee58 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java
@@ -32,6 +32,8 @@
 import apache.rocketmq.v2.QueryAssignmentResponse;
 import apache.rocketmq.v2.QueryRouteRequest;
 import apache.rocketmq.v2.QueryRouteResponse;
+import apache.rocketmq.v2.RecallMessageRequest;
+import apache.rocketmq.v2.RecallMessageResponse;
 import apache.rocketmq.v2.ReceiveMessageRequest;
 import apache.rocketmq.v2.ReceiveMessageResponse;
 import apache.rocketmq.v2.SendMessageRequest;
@@ -51,6 +53,7 @@
 import org.apache.rocketmq.proxy.grpc.v2.consumer.ChangeInvisibleDurationActivity;
 import org.apache.rocketmq.proxy.grpc.v2.consumer.ReceiveMessageActivity;
 import org.apache.rocketmq.proxy.grpc.v2.producer.ForwardMessageToDLQActivity;
+import org.apache.rocketmq.proxy.grpc.v2.producer.RecallMessageActivity;
 import org.apache.rocketmq.proxy.grpc.v2.producer.SendMessageActivity;
 import org.apache.rocketmq.proxy.grpc.v2.route.RouteActivity;
 import org.apache.rocketmq.proxy.grpc.v2.transaction.EndTransactionActivity;
@@ -65,6 +68,7 @@ public class DefaultGrpcMessingActivity extends AbstractStartAndShutdown impleme
     protected AckMessageActivity ackMessageActivity;
     protected ChangeInvisibleDurationActivity changeInvisibleDurationActivity;
     protected SendMessageActivity sendMessageActivity;
+    protected RecallMessageActivity recallMessageActivity;
     protected ForwardMessageToDLQActivity forwardMessageToDLQActivity;
     protected EndTransactionActivity endTransactionActivity;
     protected RouteActivity routeActivity;
@@ -82,6 +86,7 @@ protected void init(MessagingProcessor messagingProcessor) {
         this.ackMessageActivity = new AckMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);
         this.changeInvisibleDurationActivity = new ChangeInvisibleDurationActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);
         this.sendMessageActivity = new SendMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);
+        this.recallMessageActivity = new RecallMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);
         this.forwardMessageToDLQActivity = new ForwardMessageToDLQActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);
         this.endTransactionActivity = new EndTransactionActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);
         this.routeActivity = new RouteActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);
@@ -145,6 +150,12 @@ public CompletableFuture<ChangeInvisibleDurationResponse> changeInvisibleDuratio
         return this.changeInvisibleDurationActivity.changeInvisibleDuration(ctx, request);
     }
 
+    @Override
+    public CompletableFuture<RecallMessageResponse> recallMessage(ProxyContext ctx,
+        RecallMessageRequest request) {
+        return this.recallMessageActivity.recallMessage(ctx, request);
+    }
+
     @Override
     public ContextStreamObserver<TelemetryCommand> telemetry(StreamObserver<TelemetryCommand> responseObserver) {
         return this.clientActivity.telemetry(responseObserver);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java
index a344b0590c4..c470eda55ca 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java
@@ -35,14 +35,16 @@
 import apache.rocketmq.v2.QueryAssignmentResponse;
 import apache.rocketmq.v2.QueryRouteRequest;
 import apache.rocketmq.v2.QueryRouteResponse;
+import apache.rocketmq.v2.RecallMessageRequest;
+import apache.rocketmq.v2.RecallMessageResponse;
 import apache.rocketmq.v2.ReceiveMessageRequest;
 import apache.rocketmq.v2.ReceiveMessageResponse;
 import apache.rocketmq.v2.SendMessageRequest;
 import apache.rocketmq.v2.SendMessageResponse;
 import apache.rocketmq.v2.Status;
 import apache.rocketmq.v2.TelemetryCommand;
+import com.google.protobuf.GeneratedMessageV3;
 import io.grpc.Context;
-import io.grpc.Metadata;
 import io.grpc.stub.StreamObserver;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.RejectedExecutionHandler;
@@ -50,34 +52,42 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.constant.GrpcConstants;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
-import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.common.utils.StartAndShutdown;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
 import org.apache.rocketmq.proxy.config.ProxyConfig;
-import org.apache.rocketmq.proxy.grpc.interceptor.InterceptorConstants;
+import org.apache.rocketmq.proxy.grpc.pipeline.AuthenticationPipeline;
+import org.apache.rocketmq.proxy.grpc.pipeline.AuthorizationPipeline;
+import org.apache.rocketmq.proxy.grpc.pipeline.ContextInitPipeline;
+import org.apache.rocketmq.proxy.grpc.pipeline.RequestPipeline;
 import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException;
 import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;
 import org.apache.rocketmq.proxy.grpc.v2.common.ResponseWriter;
 import org.apache.rocketmq.proxy.processor.MessagingProcessor;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
 
 public class GrpcMessagingApplication extends MessagingServiceGrpc.MessagingServiceImplBase implements StartAndShutdown {
     private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
 
     private final GrpcMessingActivity grpcMessingActivity;
 
+    protected final RequestPipeline requestPipeline;
+
     protected ThreadPoolExecutor routeThreadPoolExecutor;
     protected ThreadPoolExecutor producerThreadPoolExecutor;
     protected ThreadPoolExecutor consumerThreadPoolExecutor;
     protected ThreadPoolExecutor clientManagerThreadPoolExecutor;
     protected ThreadPoolExecutor transactionThreadPoolExecutor;
 
-    protected GrpcMessagingApplication(GrpcMessingActivity grpcMessingActivity) {
+
+    protected GrpcMessagingApplication(GrpcMessingActivity grpcMessingActivity, RequestPipeline requestPipeline) {
         this.grpcMessingActivity = grpcMessingActivity;
+        this.requestPipeline = requestPipeline;
 
         ProxyConfig config = ConfigurationManager.getProxyConfig();
         this.routeThreadPoolExecutor = ThreadPoolMonitor.createAndMonitor(
@@ -135,9 +145,18 @@ protected void init() {
     }
 
     public static GrpcMessagingApplication create(MessagingProcessor messagingProcessor) {
-        return new GrpcMessagingApplication(new DefaultGrpcMessingActivity(
-            messagingProcessor
-        ));
+        RequestPipeline pipeline = (context, headers, request) -> {
+        };
+        // add pipeline
+        // the last pipe add will execute at the first
+        AuthConfig authConfig = ConfigurationManager.getAuthConfig();
+        if (authConfig != null) {
+            pipeline = pipeline
+                .pipe(new AuthorizationPipeline(authConfig, messagingProcessor))
+                .pipe(new AuthenticationPipeline(authConfig, messagingProcessor));
+        }
+        pipeline = pipeline.pipe(new ContextInitPipeline());
+        return new GrpcMessagingApplication(new DefaultGrpcMessingActivity(messagingProcessor), pipeline);
     }
 
     protected Status flowLimitStatus() {
@@ -150,6 +169,12 @@ protected Status convertExceptionToStatus(Throwable t) {
 
     protected <V, T> void addExecutor(ExecutorService executor, ProxyContext context, V request, Runnable runnable,
         StreamObserver<T> responseObserver, Function<Status, T> statusResponseCreator) {
+        if (request instanceof GeneratedMessageV3) {
+            requestPipeline.execute(context, GrpcConstants.METADATA.get(Context.current()), (GeneratedMessageV3) request);
+            validateContext(context);
+        } else {
+            log.error("[BUG]grpc request pipe is not been executed");
+        }
         executor.submit(new GrpcTask<>(runnable, context, request, responseObserver, statusResponseCreator.apply(flowLimitStatus())));
     }
 
@@ -166,21 +191,7 @@ protected <V, T> void writeResponse(ProxyContext context, V request, T response,
     }
 
     protected ProxyContext createContext() {
-        Context ctx = Context.current();
-        Metadata headers = InterceptorConstants.METADATA.get(ctx);
-        ProxyContext context = ProxyContext.create()
-            .setLocalAddress(getDefaultStringMetadataInfo(headers, InterceptorConstants.LOCAL_ADDRESS))
-            .setRemoteAddress(getDefaultStringMetadataInfo(headers, InterceptorConstants.REMOTE_ADDRESS))
-            .setClientID(getDefaultStringMetadataInfo(headers, InterceptorConstants.CLIENT_ID))
-            .setProtocolType(ChannelProtocolType.GRPC_V2.getName())
-            .setLanguage(getDefaultStringMetadataInfo(headers, InterceptorConstants.LANGUAGE))
-            .setClientVersion(getDefaultStringMetadataInfo(headers, InterceptorConstants.CLIENT_VERSION))
-            .setAction(getDefaultStringMetadataInfo(headers, InterceptorConstants.SIMPLE_RPC_NAME))
-            .setNamespace(getDefaultStringMetadataInfo(headers, InterceptorConstants.NAMESPACE_ID));
-        if (ctx.getDeadline() != null) {
-            context.setRemainingMs(ctx.getDeadline().timeRemaining(TimeUnit.MILLISECONDS));
-        }
-        return context;
+        return ProxyContext.create();
     }
 
     protected void validateContext(ProxyContext context) {
@@ -189,16 +200,11 @@ protected void validateContext(ProxyContext context) {
         }
     }
 
-    protected String getDefaultStringMetadataInfo(Metadata headers, Metadata.Key<String> key) {
-        return StringUtils.defaultString(headers.get(key));
-    }
-
     @Override
     public void queryRoute(QueryRouteRequest request, StreamObserver<QueryRouteResponse> responseObserver) {
         Function<Status, QueryRouteResponse> statusResponseCreator = status -> QueryRouteResponse.newBuilder().setStatus(status).build();
         ProxyContext context = createContext();
         try {
-            validateContext(context);
             this.addExecutor(this.routeThreadPoolExecutor,
                 context,
                 request,
@@ -216,7 +222,6 @@ public void heartbeat(HeartbeatRequest request, StreamObserver<HeartbeatResponse
         Function<Status, HeartbeatResponse> statusResponseCreator = status -> HeartbeatResponse.newBuilder().setStatus(status).build();
         ProxyContext context = createContext();
         try {
-            validateContext(context);
             this.addExecutor(this.clientManagerThreadPoolExecutor,
                 context,
                 request,
@@ -234,7 +239,6 @@ public void sendMessage(SendMessageRequest request, StreamObserver<SendMessageRe
         Function<Status, SendMessageResponse> statusResponseCreator = status -> SendMessageResponse.newBuilder().setStatus(status).build();
         ProxyContext context = createContext();
         try {
-            validateContext(context);
             this.addExecutor(this.producerThreadPoolExecutor,
                 context,
                 request,
@@ -253,7 +257,6 @@ public void queryAssignment(QueryAssignmentRequest request,
         Function<Status, QueryAssignmentResponse> statusResponseCreator = status -> QueryAssignmentResponse.newBuilder().setStatus(status).build();
         ProxyContext context = createContext();
         try {
-            validateContext(context);
             this.addExecutor(this.routeThreadPoolExecutor,
                 context,
                 request,
@@ -271,7 +274,6 @@ public void receiveMessage(ReceiveMessageRequest request, StreamObserver<Receive
         Function<Status, ReceiveMessageResponse> statusResponseCreator = status -> ReceiveMessageResponse.newBuilder().setStatus(status).build();
         ProxyContext context = createContext();
         try {
-            validateContext(context);
             this.addExecutor(this.consumerThreadPoolExecutor,
                 context,
                 request,
@@ -288,7 +290,6 @@ public void ackMessage(AckMessageRequest request, StreamObserver<AckMessageRespo
         Function<Status, AckMessageResponse> statusResponseCreator = status -> AckMessageResponse.newBuilder().setStatus(status).build();
         ProxyContext context = createContext();
         try {
-            validateContext(context);
             this.addExecutor(this.consumerThreadPoolExecutor,
                 context,
                 request,
@@ -307,7 +308,6 @@ public void forwardMessageToDeadLetterQueue(ForwardMessageToDeadLetterQueueReque
         Function<Status, ForwardMessageToDeadLetterQueueResponse> statusResponseCreator = status -> ForwardMessageToDeadLetterQueueResponse.newBuilder().setStatus(status).build();
         ProxyContext context = createContext();
         try {
-            validateContext(context);
             this.addExecutor(this.producerThreadPoolExecutor,
                 context,
                 request,
@@ -325,7 +325,6 @@ public void endTransaction(EndTransactionRequest request, StreamObserver<EndTran
         Function<Status, EndTransactionResponse> statusResponseCreator = status -> EndTransactionResponse.newBuilder().setStatus(status).build();
         ProxyContext context = createContext();
         try {
-            validateContext(context);
             this.addExecutor(this.transactionThreadPoolExecutor,
                 context,
                 request,
@@ -344,7 +343,6 @@ public void notifyClientTermination(NotifyClientTerminationRequest request,
         Function<Status, NotifyClientTerminationResponse> statusResponseCreator = status -> NotifyClientTerminationResponse.newBuilder().setStatus(status).build();
         ProxyContext context = createContext();
         try {
-            validateContext(context);
             this.addExecutor(this.clientManagerThreadPoolExecutor,
                 context,
                 request,
@@ -363,7 +361,6 @@ public void changeInvisibleDuration(ChangeInvisibleDurationRequest request,
         Function<Status, ChangeInvisibleDurationResponse> statusResponseCreator = status -> ChangeInvisibleDurationResponse.newBuilder().setStatus(status).build();
         ProxyContext context = createContext();
         try {
-            validateContext(context);
             this.addExecutor(this.consumerThreadPoolExecutor,
                 context,
                 request,
@@ -376,6 +373,25 @@ public void changeInvisibleDuration(ChangeInvisibleDurationRequest request,
         }
     }
 
+    @Override
+    public void recallMessage(RecallMessageRequest request, StreamObserver<RecallMessageResponse> responseObserver) {
+        Function<Status, RecallMessageResponse> statusResponseCreator =
+            status -> RecallMessageResponse.newBuilder().setStatus(status).build();
+        ProxyContext context = createContext();
+        try {
+            this.addExecutor(this.producerThreadPoolExecutor, // reuse producer thread pool
+                context,
+                request,
+                () -> grpcMessingActivity.recallMessage(context, request)
+                    .whenComplete((response, throwable) ->
+                        writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)),
+                responseObserver,
+                statusResponseCreator);
+        } catch (Throwable t) {
+            writeResponse(context, request, null, responseObserver, t, statusResponseCreator);
+        }
+    }
+
     @Override
     public StreamObserver<TelemetryCommand> telemetry(StreamObserver<TelemetryCommand> responseObserver) {
         Function<Status, TelemetryCommand> statusResponseCreator = status -> TelemetryCommand.newBuilder().setStatus(status).build();
@@ -385,7 +401,6 @@ public StreamObserver<TelemetryCommand> telemetry(StreamObserver<TelemetryComman
             public void onNext(TelemetryCommand value) {
                 ProxyContext context = createContext();
                 try {
-                    validateContext(context);
                     addExecutor(clientManagerThreadPoolExecutor,
                         context,
                         value,
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessingActivity.java
index 77bd3a88f9d..db15f25f6f7 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessingActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessingActivity.java
@@ -33,6 +33,8 @@
 import apache.rocketmq.v2.QueryAssignmentResponse;
 import apache.rocketmq.v2.QueryRouteRequest;
 import apache.rocketmq.v2.QueryRouteResponse;
+import apache.rocketmq.v2.RecallMessageRequest;
+import apache.rocketmq.v2.RecallMessageResponse;
 import apache.rocketmq.v2.ReceiveMessageRequest;
 import apache.rocketmq.v2.ReceiveMessageResponse;
 import apache.rocketmq.v2.SendMessageRequest;
@@ -69,5 +71,7 @@ CompletableFuture<NotifyClientTerminationResponse> notifyClientTermination(Proxy
     CompletableFuture<ChangeInvisibleDurationResponse> changeInvisibleDuration(ProxyContext ctx,
         ChangeInvisibleDurationRequest request);
 
+    CompletableFuture<RecallMessageResponse> recallMessage(ProxyContext ctx, RecallMessageRequest request);
+
     ContextStreamObserver<TelemetryCommand> telemetry(StreamObserver<TelemetryCommand> responseObserver);
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java
index 714d0bf019e..f05251c58c5 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java
@@ -30,6 +30,7 @@
 import io.grpc.stub.StreamObserver;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelId;
+import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.rocketmq.common.constant.LoggerName;
@@ -210,7 +211,7 @@ protected CompletableFuture<Void> processCheckTransaction(CheckTransactionStateR
     protected CompletableFuture<Void> processGetConsumerRunningInfo(RemotingCommand command,
         GetConsumerRunningInfoRequestHeader header,
         CompletableFuture<ProxyRelayResult<ConsumerRunningInfo>> responseFuture) {
-        if (!header.isJstackEnable()) {
+        if (Objects.isNull(header) || !header.isJstackEnable()) {
             return CompletableFuture.completedFuture(null);
         }
         this.writeTelemetryCommand(TelemetryCommand.newBuilder()
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java
index 59d8432ae88..a46bc99fef8 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java
@@ -138,6 +138,12 @@ public CompletableFuture<NotifyClientTerminationResponse> notifyClientTerminatio
             String clientId = ctx.getClientID();
             LanguageCode languageCode = LanguageCode.valueOf(ctx.getLanguage());
             Settings clientSettings = grpcClientSettingsManager.removeAndGetClientSettings(ctx);
+            if (clientSettings == null) {
+                future.complete(NotifyClientTerminationResponse.newBuilder()
+                    .setStatus(ResponseBuilder.getInstance().buildStatus(Code.UNRECOGNIZED_CLIENT_TYPE, "cannot find client settings for this client"))
+                    .build());
+                return future;
+            }
 
             switch (clientSettings.getClientType()) {
                 case PRODUCER:
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java
index efa879a9cbd..ee5fc019e1a 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java
@@ -21,6 +21,8 @@
 import apache.rocketmq.v2.Status;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
 import org.apache.rocketmq.client.common.ClientErrorCode;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
@@ -28,7 +30,7 @@
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.proxy.common.ProxyException;
-import org.apache.rocketmq.proxy.common.utils.ExceptionUtils;
+import org.apache.rocketmq.common.utils.ExceptionUtils;
 import org.apache.rocketmq.proxy.service.route.TopicRouteHelper;
 import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
@@ -84,6 +86,9 @@ public Status buildStatus(Throwable t) {
         if (t instanceof RemotingTimeoutException) {
             return buildStatus(Code.PROXY_TIMEOUT, t.getMessage());
         }
+        if (t instanceof AuthenticationException || t instanceof AuthorizationException) {
+            return buildStatus(Code.UNAUTHORIZED, t.getMessage());
+        }
 
         log.error("internal server error", t);
         return buildStatus(Code.INTERNAL_SERVER_ERROR, ExceptionUtils.getErrorDetailMessage(t));
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java
index 4a5b9cfcd62..76019a1ca94 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java
@@ -34,6 +34,7 @@
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
 import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity;
 import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;
+import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel;
 import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
 import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;
 import org.apache.rocketmq.proxy.processor.BatchAckResult;
@@ -193,10 +194,12 @@ protected void setAckResponseStatus(AckMessageResponse.Builder responseBuilder,
 
     protected String getHandleString(ProxyContext ctx, String group, AckMessageRequest request, AckMessageEntry ackMessageEntry) {
         String handleString = ackMessageEntry.getReceiptHandle();
-
-        MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle());
-        if (messageReceiptHandle != null) {
-            handleString = messageReceiptHandle.getReceiptHandleStr();
+        GrpcClientChannel channel = grpcChannelManager.getChannel(ctx.getClientID());
+        if (channel != null) {
+            MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, channel, group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle());
+            if (messageReceiptHandle != null) {
+                handleString = messageReceiptHandle.getReceiptHandleStr();
+            }
         }
         return handleString;
     }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivity.java
new file mode 100644
index 00000000000..28ec97dca34
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivity.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.grpc.v2.producer;
+
+import apache.rocketmq.v2.Code;
+import apache.rocketmq.v2.RecallMessageRequest;
+import apache.rocketmq.v2.RecallMessageResponse;
+import apache.rocketmq.v2.Resource;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity;
+import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;
+import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
+import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+
+import java.time.Duration;
+import java.util.concurrent.CompletableFuture;
+
+public class RecallMessageActivity extends AbstractMessingActivity {
+
+    public RecallMessageActivity(MessagingProcessor messagingProcessor,
+                                 GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) {
+        super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);
+    }
+
+    public CompletableFuture<RecallMessageResponse> recallMessage(ProxyContext ctx,
+        RecallMessageRequest request) {
+        CompletableFuture<RecallMessageResponse> future = new CompletableFuture<>();
+
+        try {
+            Resource topic = request.getTopic();
+            validateTopic(topic);
+
+            future = this.messagingProcessor.recallMessage(
+                ctx,
+                topic.getName(),
+                request.getRecallHandle(),
+                Duration.ofSeconds(2).toMillis()
+            ).thenApply(result -> RecallMessageResponse.newBuilder()
+                .setMessageId(result)
+                .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()))
+                .build());
+        } catch (Throwable t) {
+            future.completeExceptionally(t);
+        }
+        return future;
+    }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java
index 8679bfbe388..f7b8014bb99 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java
@@ -132,6 +132,11 @@ protected int buildSysFlag(apache.rocketmq.v2.Message protoMessage) {
     }
 
     protected void validateMessageBodySize(ByteString body) {
+        if (ConfigurationManager.getProxyConfig().isEnableMessageBodyEmptyCheck()) {
+            if (body.isEmpty()) {
+                throw new GrpcProxyException(Code.MESSAGE_BODY_EMPTY, "message body cannot be empty");
+            }
+        }
         int max = ConfigurationManager.getProxyConfig().getMaxMessageSize();
         if (max <= 0) {
             return;
@@ -336,6 +341,7 @@ protected SendMessageResponse convertToSendMessageResponse(ProxyContext ctx, Sen
                         .setOffset(result.getQueueOffset())
                         .setMessageId(StringUtils.defaultString(result.getMsgId()))
                         .setTransactionId(StringUtils.defaultString(result.getTransactionId()))
+                        .setRecallHandle(StringUtils.defaultString(result.getRecallHandle()))
                         .build();
                     break;
                 default:
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivity.java
index fe14fe01c64..20ae3aa6c82 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivity.java
@@ -244,6 +244,7 @@ protected List<MessageQueue> genMessageQueueFromQueueData(QueueData queueData, R
         int r = 0;
         int w = 0;
         int rw = 0;
+        int n = 0;
         if (PermName.isWriteable(queueData.getPerm()) && PermName.isReadable(queueData.getPerm())) {
             rw = Math.min(queueData.getWriteQueueNums(), queueData.getReadQueueNums());
             r = queueData.getReadQueueNums() - rw;
@@ -252,6 +253,8 @@ protected List<MessageQueue> genMessageQueueFromQueueData(QueueData queueData, R
             w = queueData.getWriteQueueNums();
         } else if (PermName.isReadable(queueData.getPerm())) {
             r = queueData.getReadQueueNums();
+        } else if (!PermName.isAccessible(queueData.getPerm())) {
+            n = Math.max(1, Math.max(queueData.getWriteQueueNums(), queueData.getReadQueueNums()));
         }
 
         // r here means readOnly queue nums, w means writeOnly queue nums, while rw means both readable and writable queue nums.
@@ -283,6 +286,15 @@ protected List<MessageQueue> genMessageQueueFromQueueData(QueueData queueData, R
             messageQueueList.add(messageQueue);
         }
 
+        for (int i = 0; i < n; i++) {
+            MessageQueue messageQueue = MessageQueue.newBuilder().setBroker(broker).setTopic(topic)
+                .setId(queueIdIndex++)
+                .setPermission(Permission.NONE)
+                .addAllAcceptMessageTypes(parseTopicMessageType(topicMessageType))
+                .build();
+            messageQueueList.add(messageQueue);
+        }
+
         return messageQueueList;
     }
 
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivity.java
index 1e114865823..ce143d5c7ec 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivity.java
@@ -61,6 +61,7 @@ public CompletableFuture<EndTransactionResponse> endTransaction(ProxyContext ctx
             }
             future = this.messagingProcessor.endTransaction(
                 ctx,
+                request.getTopic().getName(),
                 request.getTransactionId(),
                 request.getMessageId(),
                 request.getTopic().getName(),
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java
index 3ff34237014..ace8af1b994 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java
@@ -40,12 +40,12 @@
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.utils.FutureUtils;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.proxy.common.ProxyException;
 import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
-import org.apache.rocketmq.proxy.common.utils.FutureUtils;
 import org.apache.rocketmq.proxy.common.utils.ProxyUtils;
 import org.apache.rocketmq.proxy.service.ServiceManager;
 import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
@@ -96,7 +96,7 @@ public CompletableFuture<PopResult> popMessage(
             }
             return popMessage(ctx, messageQueue, consumerGroup, topic, maxMsgNums, invisibleTime, pollTime, initMode,
                 subscriptionData, fifo, popMessageResultFilter, attemptId, timeoutMillis);
-        }  catch (Throwable t) {
+        } catch (Throwable t) {
             future.completeExceptionally(t);
         }
         return future;
@@ -137,7 +137,6 @@ public CompletableFuture<PopResult> popMessage(
             requestHeader.setExp(subscriptionData.getSubString());
             requestHeader.setOrder(fifo);
             requestHeader.setAttemptId(attemptId);
-            requestHeader.setBornTime(System.currentTimeMillis());
 
             future = this.serviceManager.getMessageService().popMessage(
                     ctx,
@@ -288,7 +287,8 @@ public CompletableFuture<List<BatchAckResult>> batchAckMessage(
         return FutureUtils.addExecutor(future, this.executor);
     }
 
-    protected CompletableFuture<List<BatchAckResult>> processBrokerHandle(ProxyContext ctx, String consumerGroup, String topic, List<ReceiptHandleMessage> handleMessageList, long timeoutMillis) {
+    protected CompletableFuture<List<BatchAckResult>> processBrokerHandle(ProxyContext ctx, String consumerGroup,
+        String topic, List<ReceiptHandleMessage> handleMessageList, long timeoutMillis) {
         return this.serviceManager.getMessageService().batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis)
             .thenApply(result -> {
                 List<BatchAckResult> results = new ArrayList<>();
@@ -394,6 +394,24 @@ public CompletableFuture<Void> updateConsumerOffset(ProxyContext ctx, MessageQue
         return FutureUtils.addExecutor(future, this.executor);
     }
 
+    public CompletableFuture<Void> updateConsumerOffsetAsync(ProxyContext ctx, MessageQueue messageQueue,
+        String consumerGroup, long commitOffset, long timeoutMillis) {
+        CompletableFuture<Void> future = new CompletableFuture<>();
+        try {
+            AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService()
+                .buildAddressableMessageQueue(ctx, messageQueue);
+            UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader();
+            requestHeader.setConsumerGroup(consumerGroup);
+            requestHeader.setTopic(addressableMessageQueue.getTopic());
+            requestHeader.setQueueId(addressableMessageQueue.getQueueId());
+            requestHeader.setCommitOffset(commitOffset);
+            future = serviceManager.getMessageService().updateConsumerOffsetAsync(ctx, addressableMessageQueue, requestHeader, timeoutMillis);
+        } catch (Throwable t) {
+            future.completeExceptionally(t);
+        }
+        return FutureUtils.addExecutor(future, this.executor);
+    }
+
     public CompletableFuture<Long> queryConsumerOffset(ProxyContext ctx, MessageQueue messageQueue,
         String consumerGroup, long timeoutMillis) {
         CompletableFuture<Long> future = new CompletableFuture<>();
@@ -502,9 +520,9 @@ public CompletableFuture<Long> getMinOffset(ProxyContext ctx, MessageQueue messa
 
     protected Set<AddressableMessageQueue> buildAddressableSet(ProxyContext ctx, Set<MessageQueue> mqSet) {
         Set<AddressableMessageQueue> addressableMessageQueueSet = new HashSet<>(mqSet.size());
-        for (MessageQueue mq:mqSet) {
+        for (MessageQueue mq : mqSet) {
             try {
-                addressableMessageQueueSet.add(serviceManager.getTopicRouteService().buildAddressableMessageQueue(ctx, mq)) ;
+                addressableMessageQueueSet.add(serviceManager.getTopicRouteService().buildAddressableMessageQueue(ctx, mq));
             } catch (Exception e) {
                 log.error("build addressable message queue fail, messageQueue = {}", mq, e);
             }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
index ba150051bc0..d0c0dd6e655 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java
@@ -16,13 +16,18 @@
  */
 package org.apache.rocketmq.proxy.processor;
 
+import com.alibaba.fastjson2.JSON;
 import io.netty.channel.Channel;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
 import org.apache.rocketmq.acl.common.AclUtils;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.auth.config.AuthConfig;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.client.ClientChannelInfo;
 import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
@@ -70,7 +75,7 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen
     protected ThreadPoolExecutor producerProcessorExecutor;
     protected ThreadPoolExecutor consumerProcessorExecutor;
     protected static final String ROCKETMQ_HOME = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY,
-            System.getenv(MixAll.ROCKETMQ_HOME_ENV));
+        System.getenv(MixAll.ROCKETMQ_HOME_ENV));
 
     protected DefaultMessagingProcessor(ServiceManager serviceManager) {
         ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
@@ -112,7 +117,17 @@ public static DefaultMessagingProcessor createForLocalMode(BrokerController brok
 
     public static DefaultMessagingProcessor createForClusterMode() {
         RPCHook rpcHook = null;
-        if (ConfigurationManager.getProxyConfig().isEnableAclRpcHookForClusterMode()) {
+        if (!ConfigurationManager.getProxyConfig().isEnableAclRpcHookForClusterMode()) {
+            return createForClusterMode(rpcHook);
+        }
+        AuthConfig authConfig = ConfigurationManager.getAuthConfig();
+        if (StringUtils.isNotBlank(authConfig.getInnerClientAuthenticationCredentials())) {
+            SessionCredentials sessionCredentials =
+                JSON.parseObject(authConfig.getInnerClientAuthenticationCredentials(), SessionCredentials.class);
+            if (StringUtils.isNotBlank(sessionCredentials.getAccessKey()) && StringUtils.isNotBlank(sessionCredentials.getSecretKey())) {
+                rpcHook = new AclClientRPCHook(sessionCredentials);
+            }
+        } else {
             rpcHook = AclUtils.getAclRPCHook(ROCKETMQ_HOME + MixAll.ACL_CONF_TOOLS_FILE);
         }
         return createForClusterMode(rpcHook);
@@ -152,10 +167,11 @@ public CompletableFuture<RemotingCommand> forwardMessageToDeadLetterQueue(ProxyC
     }
 
     @Override
-    public CompletableFuture<Void> endTransaction(ProxyContext ctx, String transactionId, String messageId, String producerGroup,
+    public CompletableFuture<Void> endTransaction(ProxyContext ctx, String topic, String transactionId,
+        String messageId, String producerGroup,
         TransactionStatus transactionStatus, boolean fromTransactionCheck,
         long timeoutMillis) {
-        return this.transactionProcessor.endTransaction(ctx, transactionId, messageId, producerGroup, transactionStatus, fromTransactionCheck, timeoutMillis);
+        return this.transactionProcessor.endTransaction(ctx, topic, transactionId, messageId, producerGroup, transactionStatus, fromTransactionCheck, timeoutMillis);
     }
 
     @Override
@@ -210,6 +226,12 @@ public CompletableFuture<Void> updateConsumerOffset(ProxyContext ctx, MessageQue
         return this.consumerProcessor.updateConsumerOffset(ctx, messageQueue, consumerGroup, commitOffset, timeoutMillis);
     }
 
+    @Override
+    public CompletableFuture<Void> updateConsumerOffsetAsync(ProxyContext ctx, MessageQueue messageQueue,
+        String consumerGroup, long commitOffset, long timeoutMillis) {
+        return this.consumerProcessor.updateConsumerOffsetAsync(ctx, messageQueue, consumerGroup, commitOffset, timeoutMillis);
+    }
+
     @Override
     public CompletableFuture<Long> queryConsumerOffset(ProxyContext ctx, MessageQueue messageQueue,
         String consumerGroup, long timeoutMillis) {
@@ -239,6 +261,12 @@ public CompletableFuture<Long> getMinOffset(ProxyContext ctx, MessageQueue messa
         return this.consumerProcessor.getMinOffset(ctx, messageQueue, timeoutMillis);
     }
 
+    @Override
+    public CompletableFuture<String> recallMessage(ProxyContext ctx, String topic,
+                                                   String recallHandle, long timeoutMillis) {
+        return this.producerProcessor.recallMessage(ctx, topic, recallHandle, timeoutMillis);
+    }
+
     @Override
     public CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,
         long timeoutMillis) {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
index 2ae7418ba72..fee0465e2bf 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java
@@ -33,10 +33,10 @@
 import org.apache.rocketmq.common.consumer.ReceiptHandle;
 import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.utils.StartAndShutdown;
 import org.apache.rocketmq.proxy.common.Address;
 import org.apache.rocketmq.proxy.common.MessageReceiptHandle;
 import org.apache.rocketmq.proxy.common.ProxyContext;
-import org.apache.rocketmq.common.utils.StartAndShutdown;
 import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
 import org.apache.rocketmq.proxy.service.metadata.MetadataService;
 import org.apache.rocketmq.proxy.service.relay.ProxyRelayService;
@@ -102,17 +102,19 @@ CompletableFuture<RemotingCommand> forwardMessageToDeadLetterQueue(
 
     default CompletableFuture<Void> endTransaction(
         ProxyContext ctx,
+        String topic,
         String transactionId,
         String messageId,
         String producerGroup,
         TransactionStatus transactionStatus,
         boolean fromTransactionCheck
     ) {
-        return endTransaction(ctx, transactionId, messageId, producerGroup, transactionStatus, fromTransactionCheck, DEFAULT_TIMEOUT_MILLS);
+        return endTransaction(ctx, topic, transactionId, messageId, producerGroup, transactionStatus, fromTransactionCheck, DEFAULT_TIMEOUT_MILLS);
     }
 
     CompletableFuture<Void> endTransaction(
         ProxyContext ctx,
+        String topic,
         String transactionId,
         String messageId,
         String producerGroup,
@@ -215,6 +217,14 @@ CompletableFuture<Void> updateConsumerOffset(
         long timeoutMillis
     );
 
+    CompletableFuture<Void> updateConsumerOffsetAsync(
+        ProxyContext ctx,
+        MessageQueue messageQueue,
+        String consumerGroup,
+        long commitOffset,
+        long timeoutMillis
+    );
+
     CompletableFuture<Long> queryConsumerOffset(
         ProxyContext ctx,
         MessageQueue messageQueue,
@@ -250,6 +260,13 @@ CompletableFuture<Long> getMinOffset(
         long timeoutMillis
     );
 
+    CompletableFuture<String> recallMessage(
+        ProxyContext ctx,
+        String topic,
+        String recallHandle,
+        long timeoutMillis
+    );
+
     CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,
         long timeoutMillis);
 
@@ -319,7 +336,9 @@ void addTransactionSubscription(
 
     MetadataService getMetadataService();
 
-    void addReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID, MessageReceiptHandle messageReceiptHandle);
+    void addReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID,
+        MessageReceiptHandle messageReceiptHandle);
 
-    MessageReceiptHandle removeReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID, String receiptHandle);
+    MessageReceiptHandle removeReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID,
+        String receiptHandle);
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java
index a80f6df0b07..17a2f27fa74 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java
@@ -19,7 +19,7 @@
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
-
+import org.apache.commons.codec.DecoderException;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.client.producer.SendResult;
 import org.apache.rocketmq.client.producer.SendStatus;
@@ -33,14 +33,15 @@
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageId;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.common.utils.FutureUtils;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.proxy.common.ProxyException;
 import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
-import org.apache.rocketmq.proxy.common.utils.FutureUtils;
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
 import org.apache.rocketmq.proxy.processor.validator.DefaultTopicMessageTypeValidator;
 import org.apache.rocketmq.proxy.processor.validator.TopicMessageTypeValidator;
@@ -50,6 +51,7 @@
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
 import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
 
 public class ProducerProcessor extends AbstractProcessor {
@@ -72,7 +74,7 @@ public CompletableFuture<List<SendResult>> sendMessage(ProxyContext ctx, QueueSe
         try {
             Message message = messageList.get(0);
             String topic = message.getTopic();
-            if (ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()) {
+            if (isNeedCheckTopicMessageType(message)) {
                 if (topicMessageTypeValidator != null) {
                     // Do not check retry or dlq topic
                     if (!NamespaceUtil.isRetryTopic(topic) && !NamespaceUtil.isDLQTopic(topic)) {
@@ -125,6 +127,33 @@ public CompletableFuture<List<SendResult>> sendMessage(ProxyContext ctx, QueueSe
         return FutureUtils.addExecutor(future, this.executor);
     }
 
+    public CompletableFuture<String> recallMessage(ProxyContext ctx, String topic,
+                                                   String recallHandle, long timeoutMillis) {
+        CompletableFuture<String> future = new CompletableFuture<>();
+        try {
+            if (ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()) {
+                TopicMessageType messageType = serviceManager.getMetadataService().getTopicMessageType(ctx, topic);
+                topicMessageTypeValidator.validate(messageType, TopicMessageType.DELAY);
+            }
+
+            RecallMessageHandle.HandleV1 handleEntity;
+            try {
+                handleEntity = (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(recallHandle);
+            } catch (DecoderException e) {
+                throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, e.getMessage());
+            }
+            String brokerName = handleEntity.getBrokerName();
+            RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();
+            requestHeader.setTopic(topic);
+            requestHeader.setRecallHandle(recallHandle);
+            requestHeader.setBrokerName(brokerName);
+            future = serviceManager.getMessageService().recallMessage(ctx, brokerName, requestHeader, timeoutMillis);
+        } catch (Throwable t) {
+            future.completeExceptionally(t);
+        }
+        return FutureUtils.addExecutor(future, this.executor);
+    }
+
     protected void fillTransactionData(ProxyContext ctx, String producerGroup, AddressableMessageQueue messageQueue, SendResult sendResult, List<Message> messageList) {
         try {
             MessageId id;
@@ -136,6 +165,7 @@ protected void fillTransactionData(ProxyContext ctx, String producerGroup, Addre
             this.serviceManager.getTransactionService().addTransactionDataByBrokerName(
                 ctx,
                 messageQueue.getBrokerName(),
+                messageList.get(0).getTopic(),
                 producerGroup,
                 sendResult.getQueueOffset(),
                 id.getOffset(),
@@ -231,4 +261,8 @@ public CompletableFuture<RemotingCommand> forwardMessageToDeadLetterQueue(ProxyC
         return FutureUtils.addExecutor(future, this.executor);
     }
 
+    private boolean isNeedCheckTopicMessageType(Message message) {
+        return ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()
+            && !message.hasProperty(MessageConst.PROPERTY_TRANSFER_FLAG);
+    }
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/TransactionProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/TransactionProcessor.java
index c0ba255f544..3450bb24c05 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/TransactionProcessor.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/TransactionProcessor.java
@@ -31,12 +31,13 @@ public TransactionProcessor(MessagingProcessor messagingProcessor,
         super(messagingProcessor, serviceManager);
     }
 
-    public CompletableFuture<Void> endTransaction(ProxyContext ctx, String transactionId, String messageId, String producerGroup,
+    public CompletableFuture<Void> endTransaction(ProxyContext ctx, String topic, String transactionId, String messageId, String producerGroup,
         TransactionStatus transactionStatus, boolean fromTransactionCheck, long timeoutMillis) {
         CompletableFuture<Void> future = new CompletableFuture<>();
         try {
             EndTransactionRequestData headerData = serviceManager.getTransactionService().genEndTransactionRequestHeader(
                 ctx,
+                topic,
                 producerGroup,
                 buildCommitOrRollback(transactionStatus),
                 fromTransactionCheck,
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java
index 59342ca3cd2..5a21ec68e5d 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java
@@ -102,8 +102,9 @@ private static ClientAuth parseClientAuthMode(String authMode) {
             return ClientAuth.NONE;
         }
 
+        String authModeUpper = authMode.toUpperCase();
         for (ClientAuth clientAuth : ClientAuth.values()) {
-            if (clientAuth.name().equals(authMode.toUpperCase())) {
+            if (clientAuth.name().equals(authModeUpper)) {
                 return clientAuth;
             }
         }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
index 3227d1e1c63..c9acf728a3b 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
@@ -19,13 +19,7 @@
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import io.netty.channel.Channel;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import org.apache.rocketmq.acl.AccessValidator;
+import org.apache.rocketmq.auth.config.AuthConfig;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.future.FutureTaskExt;
 import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
@@ -44,10 +38,13 @@
 import org.apache.rocketmq.proxy.remoting.activity.GetTopicRouteActivity;
 import org.apache.rocketmq.proxy.remoting.activity.PopMessageActivity;
 import org.apache.rocketmq.proxy.remoting.activity.PullMessageActivity;
+import org.apache.rocketmq.proxy.remoting.activity.RecallMessageActivity;
 import org.apache.rocketmq.proxy.remoting.activity.SendMessageActivity;
 import org.apache.rocketmq.proxy.remoting.activity.TransactionActivity;
 import org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager;
 import org.apache.rocketmq.proxy.remoting.pipeline.AuthenticationPipeline;
+import org.apache.rocketmq.proxy.remoting.pipeline.AuthorizationPipeline;
+import org.apache.rocketmq.proxy.remoting.pipeline.ContextInitPipeline;
 import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
 import org.apache.rocketmq.remoting.ChannelEventListener;
 import org.apache.rocketmq.remoting.InvokeCallback;
@@ -61,6 +58,12 @@
 import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
 
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
 public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOutClient {
     private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
 
@@ -72,6 +75,7 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
     protected final ClientManagerActivity clientManagerActivity;
     protected final ConsumerManagerActivity consumerManagerActivity;
     protected final SendMessageActivity sendMessageActivity;
+    protected final RecallMessageActivity recallMessageActivity;
     protected final TransactionActivity transactionActivity;
     protected final PullMessageActivity pullMessageActivity;
     protected final PopMessageActivity popMessageActivity;
@@ -85,15 +89,16 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
     protected final ThreadPoolExecutor defaultExecutor;
     protected final ScheduledExecutorService timerExecutor;
 
-    public RemotingProtocolServer(MessagingProcessor messagingProcessor, List<AccessValidator> accessValidators) {
+    public RemotingProtocolServer(MessagingProcessor messagingProcessor) {
         this.messagingProcessor = messagingProcessor;
         this.remotingChannelManager = new RemotingChannelManager(this, messagingProcessor.getProxyRelayService());
 
-        RequestPipeline pipeline = createRequestPipeline(accessValidators);
+        RequestPipeline pipeline = createRequestPipeline(messagingProcessor);
         this.getTopicRouteActivity = new GetTopicRouteActivity(pipeline, messagingProcessor);
         this.clientManagerActivity = new ClientManagerActivity(pipeline, messagingProcessor, remotingChannelManager);
         this.consumerManagerActivity = new ConsumerManagerActivity(pipeline, messagingProcessor);
         this.sendMessageActivity = new SendMessageActivity(pipeline, messagingProcessor);
+        this.recallMessageActivity = new RecallMessageActivity(pipeline, messagingProcessor);
         this.transactionActivity = new TransactionActivity(pipeline, messagingProcessor);
         this.pullMessageActivity = new PullMessageActivity(pipeline, messagingProcessor);
         this.popMessageActivity = new PopMessageActivity(pipeline, messagingProcessor);
@@ -192,6 +197,7 @@ protected void registerRemotingServer(RemotingServer remotingServer) {
         remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageActivity, sendMessageExecutor);
 
         remotingServer.registerProcessor(RequestCode.END_TRANSACTION, transactionActivity, sendMessageExecutor);
+        remotingServer.registerProcessor(RequestCode.RECALL_MESSAGE, recallMessageActivity, sendMessageExecutor);
 
         remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManagerActivity, this.heartbeatExecutor);
         remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManagerActivity, this.defaultExecutor);
@@ -262,12 +268,17 @@ public void operationFail(Throwable throwable) {
         return future;
     }
 
-    protected RequestPipeline createRequestPipeline(List<AccessValidator> accessValidators) {
+    protected RequestPipeline createRequestPipeline(MessagingProcessor messagingProcessor) {
         RequestPipeline pipeline = (ctx, request, context) -> {
         };
         // add pipeline
         // the last pipe add will execute at the first
-        return pipeline.pipe(new AuthenticationPipeline(accessValidators));
+        AuthConfig authConfig = ConfigurationManager.getAuthConfig();
+        if (authConfig != null) {
+            pipeline = pipeline.pipe(new AuthorizationPipeline(authConfig, messagingProcessor))
+                .pipe(new AuthenticationPipeline(authConfig, messagingProcessor));
+        }
+        return pipeline.pipe(new ContextInitPipeline());
     }
 
     protected class ThreadPoolHeadSlowTimeMillsMonitor implements ThreadPoolStatusMonitor {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
index ce4a633976f..f3d8fb6e314 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivity.java
@@ -17,37 +17,29 @@
 
 package org.apache.rocketmq.proxy.remoting.activity;
 
-import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.rocketmq.acl.common.AclException;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
-import org.apache.rocketmq.common.MQVersion;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.message.MessageConst;
-import org.apache.rocketmq.common.utils.NetworkUtil;
+import org.apache.rocketmq.common.utils.ExceptionUtils;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.proxy.common.ProxyException;
 import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
-import org.apache.rocketmq.proxy.common.utils.ExceptionUtils;
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
 import org.apache.rocketmq.proxy.config.ProxyConfig;
 import org.apache.rocketmq.proxy.processor.MessagingProcessor;
-import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
 import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
-import org.apache.rocketmq.remoting.common.RemotingHelper;
-import org.apache.rocketmq.remoting.netty.AttributeKeys;
 import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-
 public abstract class AbstractRemotingActivity implements NettyRequestProcessor {
     protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
     protected final MessagingProcessor messagingProcessor;
@@ -71,7 +63,7 @@ public AbstractRemotingActivity(RequestPipeline requestPipeline, MessagingProces
     protected RemotingCommand request(ChannelHandlerContext ctx, RemotingCommand request,
         ProxyContext context, long timeoutMillis) throws Exception {
         String brokerName;
-        if (request.getCode() == RequestCode.SEND_MESSAGE_V2) {
+        if (request.getCode() == RequestCode.SEND_MESSAGE_V2 || request.getCode() == RequestCode.SEND_BATCH_MESSAGE) {
             if (request.getExtFields().get(BROKER_NAME_FIELD_FOR_SEND_MESSAGE_V2) == null) {
                 return RemotingCommand.buildErrorResponse(ResponseCode.VERSION_NOT_SUPPORTED,
                     "Request doesn't have field bname");
@@ -99,7 +91,7 @@ protected RemotingCommand request(ChannelHandlerContext ctx, RemotingCommand req
 
     @Override
     public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {
-        ProxyContext context = createContext(ctx, request);
+        ProxyContext context = createContext();
         try {
             this.requestPipeline.execute(ctx, request, context);
             RemotingCommand response = this.processRequest0(ctx, request, context);
@@ -121,23 +113,8 @@ public boolean rejectRequest() {
     protected abstract RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
         ProxyContext context) throws Exception;
 
-    protected ProxyContext createContext(ChannelHandlerContext ctx, RemotingCommand request) {
-        ProxyContext context = ProxyContext.create();
-        Channel channel = ctx.channel();
-        context.setAction(RemotingHelper.getRequestCodeDesc(request.getCode()))
-            .setProtocolType(ChannelProtocolType.REMOTING.getName())
-            .setChannel(channel)
-            .setLocalAddress(NetworkUtil.socketAddress2String(ctx.channel().localAddress()))
-            .setRemoteAddress(RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
-
-        Optional.ofNullable(RemotingHelper.getAttributeValue(AttributeKeys.LANGUAGE_CODE_KEY, channel))
-            .ifPresent(language -> context.setLanguage(language.name()));
-        Optional.ofNullable(RemotingHelper.getAttributeValue(AttributeKeys.CLIENT_ID_KEY, channel))
-            .ifPresent(context::setClientID);
-        Optional.ofNullable(RemotingHelper.getAttributeValue(AttributeKeys.VERSION_KEY, channel))
-            .ifPresent(version -> context.setClientVersion(MQVersion.getVersionDesc(version)));
-
-        return context;
+    protected ProxyContext createContext() {
+        return ProxyContext.create();
     }
 
     protected void writeErrResponse(ChannelHandlerContext ctx, final ProxyContext context,
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java
index c671593a34b..05d8e5fbe13 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java
@@ -125,22 +125,30 @@ protected RemotingCommand unregisterClient(ChannelHandlerContext ctx, RemotingCo
         final String producerGroup = requestHeader.getProducerGroup();
         if (producerGroup != null) {
             RemotingChannel channel = this.remotingChannelManager.removeProducerChannel(context, producerGroup, ctx.channel());
-            ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
-                channel,
-                requestHeader.getClientID(),
-                request.getLanguage(),
-                request.getVersion());
-            this.messagingProcessor.unRegisterProducer(context, producerGroup, clientChannelInfo);
+            if (channel != null) {
+                ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+                    channel,
+                    requestHeader.getClientID(),
+                    request.getLanguage(),
+                    request.getVersion());
+                this.messagingProcessor.unRegisterProducer(context, producerGroup, clientChannelInfo);
+            } else {
+                log.warn("unregister producer failed, channel not exist, may has been removed, producerGroup={}, channel={}", producerGroup, ctx.channel());
+            }
         }
         final String consumerGroup = requestHeader.getConsumerGroup();
         if (consumerGroup != null) {
             RemotingChannel channel = this.remotingChannelManager.removeConsumerChannel(context, consumerGroup, ctx.channel());
-            ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
-                channel,
-                requestHeader.getClientID(),
-                request.getLanguage(),
-                request.getVersion());
-            this.messagingProcessor.unRegisterConsumer(context, consumerGroup, clientChannelInfo);
+            if (channel != null) {
+                ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+                    channel,
+                    requestHeader.getClientID(),
+                    request.getLanguage(),
+                    request.getVersion());
+                this.messagingProcessor.unRegisterConsumer(context, consumerGroup, clientChannelInfo);
+            } else {
+                log.warn("unregister consumer failed, channel not exist, may has been removed, consumerGroup={}, channel={}", consumerGroup, ctx.channel());
+            }
         }
         response.setCode(ResponseCode.SUCCESS);
         response.setRemark("");
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
index 9972c26c991..56ec34fae6a 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java
@@ -50,7 +50,7 @@ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCom
             (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class);
         List<Address> addressList = new ArrayList<>();
         // AddressScheme is just a placeholder and will not affect topic route result in this case.
-        addressList.add(new Address(Address.AddressScheme.IPv4, HostAndPort.fromParts(proxyConfig.getRemotingAccessAddr(), proxyConfig.getRemotingListenPort())));
+        addressList.add(new Address(HostAndPort.fromParts(proxyConfig.getRemotingAccessAddr(), proxyConfig.getRemotingListenPort())));
         ProxyTopicRouteData proxyTopicRouteData = messagingProcessor.getTopicRouteDataForProxy(context, addressList, requestHeader.getTopic());
         TopicRouteData topicRouteData = proxyTopicRouteData.buildTopicRouteData();
 
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivity.java
new file mode 100644
index 00000000000..6f17745c974
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivity.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.common.attribute.TopicMessageType;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.processor.validator.DefaultTopicMessageTypeValidator;
+import org.apache.rocketmq.proxy.processor.validator.TopicMessageTypeValidator;
+import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+
+import java.time.Duration;
+
+public class RecallMessageActivity extends AbstractRemotingActivity {
+    TopicMessageTypeValidator topicMessageTypeValidator;
+
+    public RecallMessageActivity(RequestPipeline requestPipeline,
+                                 MessagingProcessor messagingProcessor) {
+        super(requestPipeline, messagingProcessor);
+        this.topicMessageTypeValidator = new DefaultTopicMessageTypeValidator();
+    }
+
+    @Override
+    public RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCommand request,
+        ProxyContext context) throws Exception {
+        RecallMessageRequestHeader requestHeader = request.decodeCommandCustomHeader(RecallMessageRequestHeader.class);
+        String topic = requestHeader.getTopic();
+        if (ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()) {
+            TopicMessageType messageType = messagingProcessor.getMetadataService().getTopicMessageType(context, topic);
+            topicMessageTypeValidator.validate(messageType, TopicMessageType.DELAY);
+        }
+        return request(ctx, request, context, Duration.ofSeconds(2).toMillis());
+    }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java
index 17af0fdcb37..22d9efd9347 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/SendMessageActivity.java
@@ -21,17 +21,18 @@
 import java.time.Duration;
 import java.util.Map;
 import org.apache.rocketmq.common.attribute.TopicMessageType;
+import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
-import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
-import org.apache.rocketmq.remoting.protocol.RequestCode;
-import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
 import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
 import org.apache.rocketmq.proxy.processor.MessagingProcessor;
 import org.apache.rocketmq.proxy.processor.validator.DefaultTopicMessageTypeValidator;
 import org.apache.rocketmq.proxy.processor.validator.TopicMessageTypeValidator;
 import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
+import org.apache.rocketmq.remoting.protocol.NamespaceUtil;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
 
 public class SendMessageActivity extends AbstractRemotingActivity {
     TopicMessageTypeValidator topicMessageTypeValidator;
@@ -66,7 +67,7 @@ protected RemotingCommand sendMessage(ChannelHandlerContext ctx, RemotingCommand
         String topic = requestHeader.getTopic();
         Map<String, String> property = MessageDecoder.string2messageProperties(requestHeader.getProperties());
         TopicMessageType messageType = TopicMessageType.parseFromMessageProperty(property);
-        if (ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()) {
+        if (isNeedCheckTopicMessageType(property)) {
             if (topicMessageTypeValidator != null) {
                 // Do not check retry or dlq topic
                 if (!NamespaceUtil.isRetryTopic(topic) && !NamespaceUtil.isDLQTopic(topic)) {
@@ -87,4 +88,9 @@ protected RemotingCommand consumerSendMessage(ChannelHandlerContext ctx, Remotin
         ProxyContext context) throws Exception {
         return request(ctx, request, context, Duration.ofSeconds(3).toMillis());
     }
+
+    private boolean isNeedCheckTopicMessageType(Map<String, String> property) {
+        return ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()
+            && !property.containsKey(MessageConst.PROPERTY_TRANSFER_FLAG);
+    }
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java
index bc5e0ca35bb..b6cd9d07678 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/TransactionActivity.java
@@ -57,6 +57,7 @@ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCom
 
         this.messagingProcessor.endTransaction(
             context,
+            requestHeader.getTopic(),
             requestHeader.getTransactionId(),
             requestHeader.getMsgId(),
             requestHeader.getProducerGroup(),
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
index d755fdcc493..5dbdea1b2e3 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java
@@ -34,8 +34,8 @@
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
-import org.apache.rocketmq.proxy.common.utils.ExceptionUtils;
-import org.apache.rocketmq.proxy.common.utils.FutureUtils;
+import org.apache.rocketmq.common.utils.ExceptionUtils;
+import org.apache.rocketmq.common.utils.FutureUtils;
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
 import org.apache.rocketmq.proxy.processor.channel.ChannelExtendAttributeGetter;
 import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
@@ -123,6 +123,7 @@ protected CompletableFuture<Void> processCheckTransaction(CheckTransactionStateR
         CompletableFuture<Void> writeFuture = new CompletableFuture<>();
         try {
             CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader();
+            requestHeader.setTopic(messageExt.getTopic());
             requestHeader.setCommitLogOffset(transactionData.getCommitLogOffset());
             requestHeader.setTranStateTableOffset(transactionData.getTranStateTableOffset());
             requestHeader.setTransactionId(transactionData.getTransactionId());
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthenticationPipeline.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthenticationPipeline.java
index 4bcc1479dcc..1bfc484bdb1 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthenticationPipeline.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthenticationPipeline.java
@@ -18,29 +18,45 @@
 package org.apache.rocketmq.proxy.remoting.pipeline;
 
 import io.netty.channel.ChannelHandlerContext;
-import java.util.List;
-import org.apache.rocketmq.acl.AccessResource;
-import org.apache.rocketmq.acl.AccessValidator;
+import org.apache.rocketmq.auth.authentication.AuthenticationEvaluator;
+import org.apache.rocketmq.auth.authentication.context.AuthenticationContext;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.proxy.common.ProxyContext;
-import org.apache.rocketmq.proxy.config.ConfigurationManager;
-import org.apache.rocketmq.proxy.config.ProxyConfig;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
 public class AuthenticationPipeline implements RequestPipeline {
-    private final List<AccessValidator> accessValidatorList;
+    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+    private final AuthConfig authConfig;
+    private final AuthenticationEvaluator authenticationEvaluator;
 
-    public AuthenticationPipeline(List<AccessValidator> accessValidatorList) {
-        this.accessValidatorList = accessValidatorList;
+    public AuthenticationPipeline(AuthConfig authConfig, MessagingProcessor messagingProcessor) {
+        this.authConfig = authConfig;
+        this.authenticationEvaluator = AuthenticationFactory.getEvaluator(authConfig, messagingProcessor::getMetadataService);
     }
 
     @Override
     public void execute(ChannelHandlerContext ctx, RemotingCommand request, ProxyContext context) throws Exception {
-        ProxyConfig config = ConfigurationManager.getProxyConfig();
-        if (config.isEnableACL()) {
-            for (AccessValidator accessValidator : accessValidatorList) {
-                AccessResource accessResource = accessValidator.parse(request, context.getRemoteAddress());
-                accessValidator.validate(accessResource);
-            }
+        if (!authConfig.isAuthenticationEnabled()) {
+            return;
         }
+        try {
+            AuthenticationContext authenticationContext = newContext(ctx, request, context);
+            authenticationEvaluator.evaluate(authenticationContext);
+        } catch (AuthenticationException ex) {
+            throw ex;
+        } catch (Throwable ex) {
+            LOGGER.error("authenticate failed, request:{}", request, ex);
+            throw ex;
+        }
+    }
+
+    protected AuthenticationContext newContext(ChannelHandlerContext ctx, RemotingCommand request, ProxyContext context) {
+        return AuthenticationFactory.newContext(authConfig, ctx, request);
     }
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthorizationPipeline.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthorizationPipeline.java
new file mode 100644
index 00000000000..49eb647ea53
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/AuthorizationPipeline.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.pipeline;
+
+import io.netty.channel.ChannelHandlerContext;
+import java.util.List;
+import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
+import org.apache.rocketmq.auth.authorization.AuthorizationEvaluator;
+import org.apache.rocketmq.auth.authorization.context.AuthorizationContext;
+import org.apache.rocketmq.auth.authorization.exception.AuthorizationException;
+import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory;
+import org.apache.rocketmq.auth.config.AuthConfig;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class AuthorizationPipeline implements RequestPipeline {
+    private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+    private final AuthConfig authConfig;
+    private final AuthorizationEvaluator authorizationEvaluator;
+
+    public AuthorizationPipeline(AuthConfig authConfig, MessagingProcessor messagingProcessor) {
+        this.authConfig = authConfig;
+        this.authorizationEvaluator = AuthorizationFactory.getEvaluator(authConfig, messagingProcessor::getMetadataService);
+    }
+
+    @Override
+    public void execute(ChannelHandlerContext ctx, RemotingCommand request, ProxyContext context) throws Exception {
+        if (!authConfig.isAuthorizationEnabled()) {
+            return;
+        }
+        try {
+            List<AuthorizationContext> contexts = newContexts(request, ctx, context);
+            authorizationEvaluator.evaluate(contexts);
+        } catch (AuthorizationException | AuthenticationException ex) {
+            throw ex;
+        }  catch (Throwable ex) {
+            LOGGER.error("authorize failed, request:{}", request, ex);
+            throw ex;
+        }
+    }
+
+    protected List<AuthorizationContext> newContexts(RemotingCommand request, ChannelHandlerContext ctx, ProxyContext context) {
+        return AuthorizationFactory.newContexts(authConfig, ctx, request);
+    }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/ContextInitPipeline.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/ContextInitPipeline.java
new file mode 100644
index 00000000000..0a30f620aed
--- /dev/null
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/pipeline/ContextInitPipeline.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.remoting.pipeline;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.common.MQVersion;
+import org.apache.rocketmq.common.utils.NetworkUtil;
+import org.apache.rocketmq.proxy.common.ProxyContext;
+import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.netty.AttributeKeys;
+import org.apache.rocketmq.remoting.protocol.LanguageCode;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class ContextInitPipeline implements RequestPipeline {
+
+    @Override
+    public void execute(ChannelHandlerContext ctx, RemotingCommand request, ProxyContext context) throws Exception {
+        Channel channel = ctx.channel();
+        LanguageCode languageCode = RemotingHelper.getAttributeValue(AttributeKeys.LANGUAGE_CODE_KEY, channel);
+        String clientId = RemotingHelper.getAttributeValue(AttributeKeys.CLIENT_ID_KEY, channel);
+        Integer version = RemotingHelper.getAttributeValue(AttributeKeys.VERSION_KEY, channel);
+        context.setAction(RemotingHelper.getRequestCodeDesc(request.getCode()))
+            .setProtocolType(ChannelProtocolType.REMOTING.getName())
+            .setChannel(channel)
+            .setLocalAddress(NetworkUtil.socketAddress2String(ctx.channel().localAddress()))
+            .setRemoteAddress(RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+        if (languageCode != null) {
+            context.setLanguage(languageCode.name());
+        }
+        if (clientId != null) {
+            context.setClientID(clientId);
+        }
+        if (version != null) {
+            context.setClientVersion(MQVersion.getVersionDesc(version));
+        }
+    }
+}
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java
index 99cb99d5307..518868831f4 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java
@@ -29,23 +29,24 @@
 import io.netty.handler.codec.haproxy.HAProxyTLV;
 import io.netty.util.Attribute;
 import io.netty.util.DefaultAttributeMap;
+import java.lang.reflect.Field;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.rocketmq.acl.common.AclUtils;
+import org.apache.rocketmq.common.constant.CommonConstants;
 import org.apache.rocketmq.common.constant.HAProxyConstants;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.netty.AttributeKeys;
 
-import java.lang.reflect.Field;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-
 public class HAProxyMessageForwarder extends ChannelInboundHandlerAdapter {
 
     private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);
@@ -73,58 +74,88 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
     }
 
     private void forwardHAProxyMessage(Channel inboundChannel, Channel outboundChannel) throws Exception {
-        if (!inboundChannel.hasAttr(AttributeKeys.PROXY_PROTOCOL_ADDR)) {
-            return;
-        }
-
         if (!(inboundChannel instanceof DefaultAttributeMap)) {
             return;
         }
 
-        Attribute<?>[] attributes = (Attribute<?>[]) FieldUtils.readField(FIELD_ATTRIBUTE, inboundChannel);
-        if (ArrayUtils.isEmpty(attributes)) {
+        HAProxyMessage message = buildHAProxyMessage(inboundChannel);
+        if (message == null) {
             return;
         }
 
+        outboundChannel.writeAndFlush(message).sync();
+    }
+
+    protected HAProxyMessage buildHAProxyMessage(Channel inboundChannel) throws IllegalAccessException, DecoderException {
         String sourceAddress = null, destinationAddress = null;
         int sourcePort = 0, destinationPort = 0;
-        List<HAProxyTLV> haProxyTLVs = new ArrayList<>();
-
-        for (Attribute<?> attribute : attributes) {
-            String attributeKey = attribute.key().name();
-            if (!StringUtils.startsWith(attributeKey, HAProxyConstants.PROXY_PROTOCOL_PREFIX)) {
-                continue;
-            }
-            String attributeValue = (String) attribute.get();
-            if (StringUtils.isEmpty(attributeValue)) {
-                continue;
+        if (inboundChannel.hasAttr(AttributeKeys.PROXY_PROTOCOL_ADDR)) {
+            Attribute<?>[] attributes = (Attribute<?>[]) FieldUtils.readField(FIELD_ATTRIBUTE, inboundChannel);
+            if (ArrayUtils.isEmpty(attributes)) {
+                return null;
             }
-            if (attribute.key() == AttributeKeys.PROXY_PROTOCOL_ADDR) {
-                sourceAddress = attributeValue;
-            }
-            if (attribute.key() == AttributeKeys.PROXY_PROTOCOL_PORT) {
-                sourcePort = Integer.parseInt(attributeValue);
-            }
-            if (attribute.key() == AttributeKeys.PROXY_PROTOCOL_SERVER_ADDR) {
-                destinationAddress = attributeValue;
-            }
-            if (attribute.key() == AttributeKeys.PROXY_PROTOCOL_SERVER_PORT) {
-                destinationPort = Integer.parseInt(attributeValue);
-            }
-            if (StringUtils.startsWith(attributeKey, HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX)) {
-                HAProxyTLV haProxyTLV = buildHAProxyTLV(attributeKey, attributeValue);
-                if (haProxyTLV != null) {
-                    haProxyTLVs.add(haProxyTLV);
+            for (Attribute<?> attribute : attributes) {
+                String attributeKey = attribute.key().name();
+                if (!StringUtils.startsWith(attributeKey, HAProxyConstants.PROXY_PROTOCOL_PREFIX)) {
+                    continue;
+                }
+                String attributeValue = (String) attribute.get();
+                if (StringUtils.isEmpty(attributeValue)) {
+                    continue;
+                }
+                if (attribute.key() == AttributeKeys.PROXY_PROTOCOL_ADDR) {
+                    sourceAddress = attributeValue;
+                }
+                if (attribute.key() == AttributeKeys.PROXY_PROTOCOL_PORT) {
+                    sourcePort = Integer.parseInt(attributeValue);
+                }
+                if (attribute.key() == AttributeKeys.PROXY_PROTOCOL_SERVER_ADDR) {
+                    destinationAddress = attributeValue;
+                }
+                if (attribute.key() == AttributeKeys.PROXY_PROTOCOL_SERVER_PORT) {
+                    destinationPort = Integer.parseInt(attributeValue);
                 }
             }
+        } else {
+            String remoteAddr = RemotingHelper.parseChannelRemoteAddr(inboundChannel);
+            sourceAddress = StringUtils.substringBeforeLast(remoteAddr, CommonConstants.COLON);
+            sourcePort = Integer.parseInt(StringUtils.substringAfterLast(remoteAddr, CommonConstants.COLON));
+
+            String localAddr = RemotingHelper.parseChannelLocalAddr(inboundChannel);
+            destinationAddress = StringUtils.substringBeforeLast(localAddr, CommonConstants.COLON);
+            destinationPort = Integer.parseInt(StringUtils.substringAfterLast(localAddr, CommonConstants.COLON));
         }
 
         HAProxyProxiedProtocol proxiedProtocol = AclUtils.isColon(sourceAddress) ? HAProxyProxiedProtocol.TCP6 :
             HAProxyProxiedProtocol.TCP4;
 
-        HAProxyMessage message = new HAProxyMessage(HAProxyProtocolVersion.V2, HAProxyCommand.PROXY,
+        List<HAProxyTLV> haProxyTLVs = buildHAProxyTLV(inboundChannel);
+
+        return new HAProxyMessage(HAProxyProtocolVersion.V2, HAProxyCommand.PROXY,
             proxiedProtocol, sourceAddress, destinationAddress, sourcePort, destinationPort, haProxyTLVs);
-        outboundChannel.writeAndFlush(message).sync();
+    }
+
+    protected List<HAProxyTLV> buildHAProxyTLV(Channel inboundChannel) throws IllegalAccessException, DecoderException {
+        List<HAProxyTLV> result = new ArrayList<>();
+        if (!inboundChannel.hasAttr(AttributeKeys.PROXY_PROTOCOL_ADDR)) {
+            return result;
+        }
+        Attribute<?>[] attributes = (Attribute<?>[]) FieldUtils.readField(FIELD_ATTRIBUTE, inboundChannel);
+        if (ArrayUtils.isEmpty(attributes)) {
+            return result;
+        }
+        for (Attribute<?> attribute : attributes) {
+            String attributeKey = attribute.key().name();
+            if (!StringUtils.startsWith(attributeKey, HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX)) {
+                continue;
+            }
+            String attributeValue = (String) attribute.get();
+            HAProxyTLV haProxyTLV = buildHAProxyTLV(attributeKey, attributeValue);
+            if (haProxyTLV != null) {
+                result.add(haProxyTLV);
+            }
+        }
+        return result;
     }
 
     protected HAProxyTLV buildHAProxyTLV(String attributeKey, String attributeValue) throws DecoderException {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java
index 7ce563b0305..103b099bbe3 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandler.java
@@ -32,6 +32,7 @@
 import io.netty.handler.ssl.SslHandler;
 import io.netty.handler.ssl.SslProvider;
 import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import javax.net.ssl.SSLException;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
@@ -39,11 +40,8 @@
 import org.apache.rocketmq.proxy.config.ProxyConfig;
 import org.apache.rocketmq.proxy.remoting.protocol.ProtocolHandler;
 import org.apache.rocketmq.remoting.common.TlsMode;
-import org.apache.rocketmq.remoting.netty.AttributeKeys;
 import org.apache.rocketmq.remoting.netty.TlsSystemConfig;
 
-import javax.net.ssl.SSLException;
-
 public class Http2ProtocolProxyHandler implements ProtocolHandler {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);
     private static final String LOCAL_HOST = "127.0.0.1";
@@ -131,9 +129,7 @@ protected void initChannel(Channel ch) throws Exception {
     }
 
     protected void configPipeline(Channel inboundChannel, Channel outboundChannel) {
-        if (inboundChannel.hasAttr(AttributeKeys.PROXY_PROTOCOL_ADDR)) {
-            inboundChannel.pipeline().addLast(new HAProxyMessageForwarder(outboundChannel));
-            outboundChannel.pipeline().addFirst(HAProxyMessageEncoder.INSTANCE);
-        }
+        inboundChannel.pipeline().addLast(new HAProxyMessageForwarder(outboundChannel));
+        outboundChannel.pipeline().addFirst(HAProxyMessageEncoder.INSTANCE);
     }
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java
index 9786cec5577..33b65d2550e 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java
@@ -28,6 +28,7 @@
 import org.apache.rocketmq.client.common.NameserverAccessConfig;
 import org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor;
 import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
+import org.apache.rocketmq.common.ObjectCreator;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.utils.AbstractStartAndShutdown;
 import org.apache.rocketmq.common.utils.ThreadUtils;
@@ -51,6 +52,7 @@
 import org.apache.rocketmq.proxy.service.transaction.ClusterTransactionService;
 import org.apache.rocketmq.proxy.service.transaction.TransactionService;
 import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.RemotingClient;
 
 public class ClusterServiceManager extends AbstractStartAndShutdown implements ServiceManager {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
@@ -70,6 +72,10 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S
     protected MQClientAPIFactory transactionClientAPIFactory;
 
     public ClusterServiceManager(RPCHook rpcHook) {
+        this(rpcHook, null);
+    }
+
+    public ClusterServiceManager(RPCHook rpcHook, ObjectCreator<RemotingClient> remotingClientCreator) {
         ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
         NameserverAccessConfig nameserverAccessConfig = new NameserverAccessConfig(proxyConfig.getNamesrvAddr(),
             proxyConfig.getNamesrvDomain(), proxyConfig.getNamesrvDomainSubgroup());
@@ -81,14 +87,18 @@ public ClusterServiceManager(RPCHook rpcHook) {
             proxyConfig.getRocketmqMQClientNum(),
             new DoNothingClientRemotingProcessor(null),
             rpcHook,
-            scheduledExecutorService);
+            scheduledExecutorService,
+            remotingClientCreator
+        );
+
         this.operationClientAPIFactory = new MQClientAPIFactory(
             nameserverAccessConfig,
             "OperationClient_",
             1,
             new DoNothingClientRemotingProcessor(null),
             rpcHook,
-            this.scheduledExecutorService
+            this.scheduledExecutorService,
+            remotingClientCreator
         );
 
         this.topicRouteService = new ClusterTopicRouteService(operationClientAPIFactory);
@@ -105,7 +115,10 @@ public ClusterServiceManager(RPCHook rpcHook) {
             1,
             new ProxyClientRemotingProcessor(producerManager),
             rpcHook,
-            scheduledExecutorService);
+            scheduledExecutorService,
+            remotingClientCreator
+        );
+
         this.clusterTransactionService = new ClusterTransactionService(this.topicRouteService, this.producerManager,
             this.transactionClientAPIFactory);
         this.proxyRelayService = new ClusterProxyRelayService(this.clusterTransactionService);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManagerFactory.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManagerFactory.java
index c186752788d..e1252fe31fd 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManagerFactory.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManagerFactory.java
@@ -17,7 +17,9 @@
 package org.apache.rocketmq.proxy.service;
 
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.ObjectCreator;
 import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.RemotingClient;
 
 public class ServiceManagerFactory {
     public static ServiceManager createForLocalMode(BrokerController brokerController) {
@@ -29,10 +31,14 @@ public static ServiceManager createForLocalMode(BrokerController brokerControlle
     }
 
     public static ServiceManager createForClusterMode() {
-        return createForClusterMode(null);
+        return createForClusterMode(null, null);
     }
 
     public static ServiceManager createForClusterMode(RPCHook rpcHook) {
-        return new ClusterServiceManager(rpcHook);
+        return createForClusterMode(rpcHook, null);
+    }
+
+    public static ServiceManager createForClusterMode(RPCHook rpcHook, ObjectCreator<RemotingClient> remotingClientCreator) {
+        return new ClusterServiceManager(rpcHook, remotingClientCreator);
     }
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
index 70b72deae18..f6f3406ab4e 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java
@@ -29,10 +29,10 @@
 import org.apache.rocketmq.common.consumer.ReceiptHandle;
 import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.utils.FutureUtils;
 import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.proxy.common.ProxyException;
 import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
-import org.apache.rocketmq.proxy.common.utils.FutureUtils;
 import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
 import org.apache.rocketmq.proxy.service.route.TopicRouteService;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
@@ -47,6 +47,7 @@
 import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
 
@@ -139,7 +140,8 @@ public CompletableFuture<AckResult> ackMessage(ProxyContext ctx, ReceiptHandle h
     }
 
     @Override
-    public CompletableFuture<AckResult> batchAckMessage(ProxyContext ctx, List<ReceiptHandleMessage> handleList, String consumerGroup,
+    public CompletableFuture<AckResult> batchAckMessage(ProxyContext ctx, List<ReceiptHandleMessage> handleList,
+        String consumerGroup,
         String topic, long timeoutMillis) {
         List<String> extraInfoList = handleList.stream().map(message -> message.getReceiptHandle().getReceiptHandle()).collect(Collectors.toList());
         return this.mqClientAPIFactory.getClient().batchAckMessageAsync(
@@ -181,6 +183,16 @@ public CompletableFuture<Void> updateConsumerOffset(ProxyContext ctx, Addressabl
         );
     }
 
+    @Override
+    public CompletableFuture<Void> updateConsumerOffsetAsync(ProxyContext ctx, AddressableMessageQueue messageQueue,
+        UpdateConsumerOffsetRequestHeader requestHeader, long timeoutMillis) {
+        return this.mqClientAPIFactory.getClient().updateConsumerOffsetAsync(
+            messageQueue.getBrokerAddr(),
+            requestHeader,
+            timeoutMillis
+        );
+    }
+
     @Override
     public CompletableFuture<Set<MessageQueue>> lockBatchMQ(ProxyContext ctx, AddressableMessageQueue messageQueue,
         LockBatchRequestBody requestBody, long timeoutMillis) {
@@ -221,6 +233,16 @@ public CompletableFuture<Long> getMinOffset(ProxyContext ctx, AddressableMessage
         );
     }
 
+    @Override
+    public CompletableFuture<String> recallMessage(ProxyContext ctx, String brokerName,
+        RecallMessageRequestHeader requestHeader, long timeoutMillis) {
+        return this.mqClientAPIFactory.getClient().recallMessageAsync(
+            this.resolveBrokerAddr(ctx, brokerName),
+            requestHeader,
+            timeoutMillis
+        );
+    }
+
     @Override
     public CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,
         long timeoutMillis) {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
index 9181f966f4c..cb9b7a4ae00 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java
@@ -71,6 +71,8 @@
 import org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
@@ -153,6 +155,7 @@ public CompletableFuture<List<SendResult>> sendMessage(ProxyContext ctx, Address
             sendResult.setQueueOffset(responseHeader.getQueueOffset());
             sendResult.setTransactionId(responseHeader.getTransactionId());
             sendResult.setOffsetMsgId(responseHeader.getMsgId());
+            sendResult.setRecallHandle(responseHeader.getRecallHandle());
             return Collections.singletonList(sendResult);
         });
     }
@@ -176,7 +179,8 @@ public CompletableFuture<RemotingCommand> sendMessageBack(ProxyContext ctx, Rece
     }
 
     @Override
-    public CompletableFuture<Void> endTransactionOneway(ProxyContext ctx, String brokerName, EndTransactionRequestHeader requestHeader,
+    public CompletableFuture<Void> endTransactionOneway(ProxyContext ctx, String brokerName,
+        EndTransactionRequestHeader requestHeader,
         long timeoutMillis) {
         CompletableFuture<Void> future = new CompletableFuture<>();
         SimpleChannel channel = channelManager.createChannel(ctx);
@@ -195,6 +199,7 @@ public CompletableFuture<Void> endTransactionOneway(ProxyContext ctx, String bro
     @Override
     public CompletableFuture<PopResult> popMessage(ProxyContext ctx, AddressableMessageQueue messageQueue,
         PopMessageRequestHeader requestHeader, long timeoutMillis) {
+        requestHeader.setBornTime(System.currentTimeMillis());
         RemotingCommand request = LocalRemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader, ctx.getLanguage());
         CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
         SimpleChannel channel = channelManager.createInvocationChannel(ctx);
@@ -309,9 +314,8 @@ public CompletableFuture<AckResult> changeInvisibleTime(ProxyContext ctx, Receip
         RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader, ctx.getLanguage());
         CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
         try {
-            RemotingCommand response = brokerController.getChangeInvisibleTimeProcessor()
-                .processRequest(channelHandlerContext, command);
-            future.complete(response);
+            future = brokerController.getChangeInvisibleTimeProcessor()
+                .processRequestAsync(channelHandlerContext.channel(), command, true);
         } catch (Exception e) {
             log.error("Fail to process changeInvisibleTime command", e);
             future.completeExceptionally(e);
@@ -439,6 +443,12 @@ public CompletableFuture<Void> updateConsumerOffset(ProxyContext ctx, Addressabl
         throw new NotImplementedException("updateConsumerOffset is not implemented in LocalMessageService");
     }
 
+    @Override
+    public CompletableFuture<Void> updateConsumerOffsetAsync(ProxyContext ctx, AddressableMessageQueue messageQueue,
+        UpdateConsumerOffsetRequestHeader requestHeader, long timeoutMillis) {
+        throw new NotImplementedException("updateConsumerOffsetAsync is not implemented in LocalMessageService");
+    }
+
     @Override
     public CompletableFuture<Set<MessageQueue>> lockBatchMQ(ProxyContext ctx, AddressableMessageQueue messageQueue,
         LockBatchRequestBody requestBody, long timeoutMillis) {
@@ -463,6 +473,32 @@ public CompletableFuture<Long> getMinOffset(ProxyContext ctx, AddressableMessage
         throw new NotImplementedException("getMinOffset is not implemented in LocalMessageService");
     }
 
+    @Override
+    public CompletableFuture<String> recallMessage(ProxyContext ctx, String brokerName,
+        RecallMessageRequestHeader requestHeader, long timeoutMillis) {
+        SimpleChannel channel = channelManager.createChannel(ctx);
+        ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext();
+        RemotingCommand command =
+            LocalRemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader, ctx.getLanguage());
+        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+        try {
+            RemotingCommand response = brokerController.getRecallMessageProcessor()
+                .processRequest(channelHandlerContext, command);
+            future.complete(response);
+        } catch (Exception e) {
+            log.error("Fail to process recallMessage command", e);
+            future.completeExceptionally(e);
+        }
+        return future.thenApply(r -> {
+            switch (r.getCode()) {
+                case ResponseCode.SUCCESS:
+                    return ((RecallMessageResponseHeader) r.readCustomHeader()).getMsgId();
+                default:
+                    throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, r.getRemark());
+            }
+        });
+    }
+
     @Override
     public CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,
         long timeoutMillis) {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalRemotingCommand.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalRemotingCommand.java
index 7bf4a169820..25066ea5305 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalRemotingCommand.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalRemotingCommand.java
@@ -16,13 +16,11 @@
  */
 package org.apache.rocketmq.proxy.service.message;
 
+import java.util.HashMap;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
-import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.protocol.LanguageCode;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
-import java.util.HashMap;
-
 public class LocalRemotingCommand extends RemotingCommand {
 
     public static LocalRemotingCommand createRequestCommand(int code, CommandCustomHeader customHeader, String language) {
@@ -35,11 +33,4 @@ public static LocalRemotingCommand createRequestCommand(int code, CommandCustomH
         cmd.makeCustomHeaderToNet();
         return cmd;
     }
-
-    @Override
-    public CommandCustomHeader decodeCommandCustomHeader(
-        Class<? extends CommandCustomHeader> classHeader) throws RemotingCommandException {
-        return classHeader.cast(readCustomHeader());
-    }
-    
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java
index 58a835adb46..80f5ae7217c 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java
@@ -40,6 +40,7 @@
 import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
 
@@ -120,6 +121,13 @@ CompletableFuture<Void> updateConsumerOffset(
         long timeoutMillis
     );
 
+    CompletableFuture<Void> updateConsumerOffsetAsync(
+        ProxyContext ctx,
+        AddressableMessageQueue messageQueue,
+        UpdateConsumerOffsetRequestHeader requestHeader,
+        long timeoutMillis
+    );
+
     CompletableFuture<Set<MessageQueue>> lockBatchMQ(
         ProxyContext ctx,
         AddressableMessageQueue messageQueue,
@@ -148,6 +156,13 @@ CompletableFuture<Long> getMinOffset(
         long timeoutMillis
     );
 
+    CompletableFuture<String> recallMessage(
+        ProxyContext ctx,
+        String brokerName,
+        RecallMessageRequestHeader requestHeader,
+        long timeoutMillis
+    );
+
     CompletableFuture<RemotingCommand> request(ProxyContext ctx, String brokerName, RemotingCommand request,
         long timeoutMillis);
 
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java
index d34a0efd9e1..70ce1d3480e 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java
@@ -19,22 +19,32 @@
 
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.LoadingCache;
+import java.util.List;
 import java.util.Optional;
+import java.util.Random;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authorization.model.Acl;
+import org.apache.rocketmq.broker.auth.converter.AclConverter;
+import org.apache.rocketmq.broker.auth.converter.UserConverter;
+import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
 import org.apache.rocketmq.common.attribute.TopicMessageType;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
+import org.apache.rocketmq.common.utils.AbstractStartAndShutdown;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.proxy.common.AbstractCacheLoader;
-import org.apache.rocketmq.common.utils.AbstractStartAndShutdown;
 import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
 import org.apache.rocketmq.proxy.config.ProxyConfig;
-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
 import org.apache.rocketmq.proxy.service.route.TopicRouteHelper;
 import org.apache.rocketmq.proxy.service.route.TopicRouteService;
+import org.apache.rocketmq.remoting.protocol.body.AclInfo;
+import org.apache.rocketmq.remoting.protocol.body.UserInfo;
 import org.apache.rocketmq.remoting.protocol.route.BrokerData;
 import org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;
 import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
@@ -54,6 +64,17 @@ public class ClusterMetadataService extends AbstractStartAndShutdown implements
     protected final LoadingCache<String, SubscriptionGroupConfig> subscriptionGroupConfigCache;
     protected final static SubscriptionGroupConfig EMPTY_SUBSCRIPTION_GROUP_CONFIG = new SubscriptionGroupConfig();
 
+    protected final LoadingCache<String, User> userCache;
+
+    protected final static User EMPTY_USER = new User();
+
+    protected final LoadingCache<String, Acl> aclCache;
+
+    protected final static Acl EMPTY_ACL = new Acl();
+
+    protected final Random random = new Random();
+
+
     public ClusterMetadataService(TopicRouteService topicRouteService, MQClientAPIFactory mqClientAPIFactory) {
         this.topicRouteService = topicRouteService;
         this.mqClientAPIFactory = mqClientAPIFactory;
@@ -77,6 +98,16 @@ public ClusterMetadataService(TopicRouteService topicRouteService, MQClientAPIFa
             .expireAfterAccess(config.getSubscriptionGroupConfigCacheExpiredSeconds(), TimeUnit.SECONDS)
             .refreshAfterWrite(config.getSubscriptionGroupConfigCacheRefreshSeconds(), TimeUnit.SECONDS)
             .build(new ClusterSubscriptionGroupConfigCacheLoader());
+        this.userCache = CacheBuilder.newBuilder()
+            .maximumSize(config.getUserCacheMaxNum())
+            .expireAfterAccess(config.getUserCacheExpiredSeconds(), TimeUnit.SECONDS)
+            .refreshAfterWrite(config.getUserCacheRefreshSeconds(), TimeUnit.SECONDS)
+            .build(new ClusterUserCacheLoader());
+        this.aclCache = CacheBuilder.newBuilder()
+            .maximumSize(config.getAclCacheMaxNum())
+            .expireAfterAccess(config.getAclCacheExpiredSeconds(), TimeUnit.SECONDS)
+            .refreshAfterWrite(config.getAclCacheRefreshSeconds(), TimeUnit.SECONDS)
+            .build(new ClusterAclCacheLoader());
 
         this.init();
     }
@@ -113,6 +144,36 @@ public SubscriptionGroupConfig getSubscriptionGroupConfig(ProxyContext ctx, Stri
         return config;
     }
 
+    @Override
+    public CompletableFuture<User> getUser(ProxyContext ctx, String username) {
+        CompletableFuture<User> result = new CompletableFuture<>();
+        try {
+            User user = this.userCache.get(username);
+            if (user == EMPTY_USER) {
+                user = null;
+            }
+            result.complete(user);
+        } catch (Exception e) {
+            result.completeExceptionally(e);
+        }
+        return result;
+    }
+
+    @Override
+    public CompletableFuture<Acl> getAcl(ProxyContext ctx, Subject subject) {
+        CompletableFuture<Acl> result = new CompletableFuture<>();
+        try {
+            Acl acl = this.aclCache.get(subject.getSubjectKey());
+            if (acl == EMPTY_ACL) {
+                acl = null;
+            }
+            result.complete(acl);
+        } catch (Exception e) {
+            result.completeExceptionally(e);
+        }
+        return result;
+    }
+
     protected class ClusterSubscriptionGroupConfigCacheLoader extends AbstractCacheLoader<String, SubscriptionGroupConfig> {
 
         public ClusterSubscriptionGroupConfigCacheLoader() {
@@ -159,9 +220,67 @@ protected void onErr(String key, Exception e) {
         }
     }
 
+    protected class ClusterUserCacheLoader extends AbstractCacheLoader<String, User> {
+
+        public ClusterUserCacheLoader() {
+            super(cacheRefreshExecutor);
+        }
+
+        @Override
+        protected User getDirectly(String username) throws Exception {
+            ProxyConfig config = ConfigurationManager.getProxyConfig();
+            String clusterName = config.getRocketMQClusterName();
+            Optional<BrokerData> brokerDataOptional = findOneBroker(clusterName);
+            if (brokerDataOptional.isPresent()) {
+                String brokerAddress = brokerDataOptional.get().selectBrokerAddr();
+                UserInfo userInfo = mqClientAPIFactory.getClient().getUser(brokerAddress, username, DEFAULT_TIMEOUT);
+                if (userInfo == null) {
+                    return EMPTY_USER;
+                }
+                return UserConverter.convertUser(userInfo);
+            }
+            return EMPTY_USER;
+        }
+
+        @Override
+        protected void onErr(String key, Exception e) {
+            log.error("load user failed. username:{}", key, e);
+        }
+    }
+
+    protected class ClusterAclCacheLoader extends AbstractCacheLoader<String, Acl> {
+
+        public ClusterAclCacheLoader() {
+            super(cacheRefreshExecutor);
+        }
+
+        @Override
+        protected Acl getDirectly(String subject) throws Exception {
+            ProxyConfig config = ConfigurationManager.getProxyConfig();
+            String clusterName = config.getRocketMQClusterName();
+            Optional<BrokerData> brokerDataOptional = findOneBroker(clusterName);
+            if (brokerDataOptional.isPresent()) {
+                String brokerAddress = brokerDataOptional.get().selectBrokerAddr();
+                AclInfo aclInfo = mqClientAPIFactory.getClient().getAcl(brokerAddress, subject, DEFAULT_TIMEOUT);
+                if (aclInfo == null) {
+                    return EMPTY_ACL;
+                }
+                return AclConverter.convertAcl(aclInfo);
+            }
+            return EMPTY_ACL;
+        }
+
+        @Override
+        protected void onErr(String key, Exception e) {
+            log.error("load user failed. username:{}", key, e);
+        }
+    }
+
     protected Optional<BrokerData> findOneBroker(String topic) throws Exception {
         try {
-            return topicRouteService.getAllMessageQueueView(ProxyContext.createForInner(this.getClass()), topic).getTopicRouteData().getBrokerDatas().stream().findAny();
+            List<BrokerData> brokerDatas = topicRouteService.getAllMessageQueueView(ProxyContext.createForInner(this.getClass()), topic).getTopicRouteData().getBrokerDatas();
+            int skipNum = random.nextInt(brokerDatas.size());
+            return brokerDatas.stream().skip(skipNum).findFirst();
         } catch (Exception e) {
             if (TopicRouteHelper.isTopicNotExistError(e)) {
                 return Optional.empty();
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/LocalMetadataService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/LocalMetadataService.java
index 7f3c041f259..7b43f760d1c 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/LocalMetadataService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/LocalMetadataService.java
@@ -17,6 +17,10 @@
 
 package org.apache.rocketmq.proxy.service.metadata;
 
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authorization.model.Acl;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.attribute.TopicMessageType;
@@ -43,4 +47,14 @@ public TopicMessageType getTopicMessageType(ProxyContext ctx, String topic) {
     public SubscriptionGroupConfig getSubscriptionGroupConfig(ProxyContext ctx, String group) {
         return this.brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().get(group);
     }
+
+    @Override
+    public CompletableFuture<User> getUser(ProxyContext ctx, String username) {
+        return this.brokerController.getAuthenticationMetadataManager().getUser(username);
+    }
+
+    @Override
+    public CompletableFuture<Acl> getAcl(ProxyContext ctx, Subject subject) {
+        return this.brokerController.getAuthorizationMetadataManager().getAcl(subject);
+    }
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/MetadataService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/MetadataService.java
index 3ee0f3eacd3..bc8654d9c61 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/MetadataService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/MetadataService.java
@@ -17,6 +17,10 @@
 
 package org.apache.rocketmq.proxy.service.metadata;
 
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.auth.authentication.model.Subject;
+import org.apache.rocketmq.auth.authentication.model.User;
+import org.apache.rocketmq.auth.authorization.model.Acl;
 import org.apache.rocketmq.common.attribute.TopicMessageType;
 import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
@@ -26,4 +30,8 @@ public interface MetadataService {
     TopicMessageType getTopicMessageType(ProxyContext ctx, String topic);
 
     SubscriptionGroupConfig getSubscriptionGroupConfig(ProxyContext ctx, String group);
+
+    CompletableFuture<User> getUser(ProxyContext ctx, String username);
+
+    CompletableFuture<Acl> getAcl(ProxyContext ctx, Subject subject);
 }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java
index 207603fe811..0cb519306eb 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java
@@ -53,7 +53,7 @@
 import org.apache.rocketmq.proxy.common.RenewEvent;
 import org.apache.rocketmq.proxy.common.RenewStrategyPolicy;
 import org.apache.rocketmq.proxy.common.channel.ChannelHelper;
-import org.apache.rocketmq.proxy.common.utils.ExceptionUtils;
+import org.apache.rocketmq.common.utils.ExceptionUtils;
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
 import org.apache.rocketmq.proxy.config.ProxyConfig;
 import org.apache.rocketmq.proxy.service.metadata.MetadataService;
@@ -159,7 +159,8 @@ protected void scheduleRenewTask() {
                     if (handle.getNextVisibleTime() - current > proxyConfig.getRenewAheadTimeMillis()) {
                         return;
                     }
-                    renewalWorkerService.submit(() -> renewMessage(key, group, msgID, handleStr));
+                    renewalWorkerService.submit(() -> renewMessage(createContext("RenewMessage"), key, group,
+                        msgID, handleStr));
                 });
             }
         } catch (Exception e) {
@@ -169,15 +170,15 @@ protected void scheduleRenewTask() {
         log.debug("scan for renewal done. cost:{}ms", stopwatch.elapsed().toMillis());
     }
 
-    protected void renewMessage(ReceiptHandleGroupKey key, ReceiptHandleGroup group, String msgID, String handleStr) {
+    protected void renewMessage(ProxyContext context, ReceiptHandleGroupKey key, ReceiptHandleGroup group, String msgID, String handleStr) {
         try {
-            group.computeIfPresent(msgID, handleStr, messageReceiptHandle -> startRenewMessage(key, messageReceiptHandle));
+            group.computeIfPresent(msgID, handleStr, messageReceiptHandle -> startRenewMessage(context, key, messageReceiptHandle));
         } catch (Exception e) {
             log.error("error when renew message. msgID:{}, handleStr:{}", msgID, handleStr, e);
         }
     }
 
-    protected CompletableFuture<MessageReceiptHandle> startRenewMessage(ReceiptHandleGroupKey key, MessageReceiptHandle messageReceiptHandle) {
+    protected CompletableFuture<MessageReceiptHandle> startRenewMessage(ProxyContext context, ReceiptHandleGroupKey key, MessageReceiptHandle messageReceiptHandle) {
         CompletableFuture<MessageReceiptHandle> resFuture = new CompletableFuture<>();
         ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
         long current = System.currentTimeMillis();
@@ -209,7 +210,6 @@ protected CompletableFuture<MessageReceiptHandle> startRenewMessage(ReceiptHandl
                     }
                 });
             } else {
-                ProxyContext context = createContext("RenewMessage");
                 SubscriptionGroupConfig subscriptionGroupConfig =
                     metadataService.getSubscriptionGroupConfig(context, messageReceiptHandle.getGroup());
                 if (subscriptionGroupConfig == null) {
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/AbstractProxyRelayService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/AbstractProxyRelayService.java
index 08f00bd83c0..d881a51d48d 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/AbstractProxyRelayService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/AbstractProxyRelayService.java
@@ -45,6 +45,7 @@ public RelayData<TransactionData, Void> processCheckTransactionState(ProxyContex
         TransactionData transactionData = transactionService.addTransactionDataByBrokerAddr(
             context,
             command.getExtFields().get(ProxyUtils.BROKER_ADDR),
+            messageExt.getTopic(),
             group,
             header.getTranStateTableOffset(),
             header.getCommitLogOffset(),
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ClusterTopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ClusterTopicRouteService.java
index 84252f8b8e7..a4df98971cb 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ClusterTopicRouteService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ClusterTopicRouteService.java
@@ -17,11 +17,10 @@
 package org.apache.rocketmq.proxy.service.route;
 
 import java.util.List;
+import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.proxy.common.Address;
-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
 import org.apache.rocketmq.proxy.common.ProxyContext;
-import org.apache.rocketmq.remoting.protocol.route.BrokerData;
 import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
 
 public class ClusterTopicRouteService extends TopicRouteService {
@@ -39,21 +38,7 @@ public MessageQueueView getCurrentMessageQueueView(ProxyContext ctx, String topi
     public ProxyTopicRouteData getTopicRouteForProxy(ProxyContext ctx, List<Address> requestHostAndPortList,
         String topicName) throws Exception {
         TopicRouteData topicRouteData = getAllMessageQueueView(ctx, topicName).getTopicRouteData();
-
-        ProxyTopicRouteData proxyTopicRouteData = new ProxyTopicRouteData();
-        proxyTopicRouteData.setQueueDatas(topicRouteData.getQueueDatas());
-
-        for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {
-            ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData();
-            proxyBrokerData.setCluster(brokerData.getCluster());
-            proxyBrokerData.setBrokerName(brokerData.getBrokerName());
-            for (Long brokerId : brokerData.getBrokerAddrs().keySet()) {
-                proxyBrokerData.getBrokerAddrs().put(brokerId, requestHostAndPortList);
-            }
-            proxyTopicRouteData.getBrokerDatas().add(proxyBrokerData);
-        }
-
-        return proxyTopicRouteData;
+        return new ProxyTopicRouteData(topicRouteData, requestHostAndPortList);
     }
 
     @Override
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java
index aced15cee51..f2a42c0aed9 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java
@@ -17,10 +17,10 @@
 package org.apache.rocketmq.proxy.service.route;
 
 import com.google.common.collect.Lists;
-import com.google.common.net.HostAndPort;
 import java.util.HashMap;
 import java.util.List;
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.TopicConfig;
@@ -28,7 +28,6 @@
 import org.apache.rocketmq.proxy.common.Address;
 import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
 import org.apache.rocketmq.remoting.protocol.route.BrokerData;
 import org.apache.rocketmq.remoting.protocol.route.QueueData;
 import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
@@ -62,25 +61,7 @@ public ProxyTopicRouteData getTopicRouteForProxy(ProxyContext ctx, List<Address>
         String topicName) throws Exception {
         MessageQueueView messageQueueView = getAllMessageQueueView(ctx, topicName);
         TopicRouteData topicRouteData = messageQueueView.getTopicRouteData();
-
-        ProxyTopicRouteData proxyTopicRouteData = new ProxyTopicRouteData();
-        proxyTopicRouteData.setQueueDatas(topicRouteData.getQueueDatas());
-
-        for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {
-            ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData();
-            proxyBrokerData.setCluster(brokerData.getCluster());
-            proxyBrokerData.setBrokerName(brokerData.getBrokerName());
-            for (Long brokerId : brokerData.getBrokerAddrs().keySet()) {
-                String brokerAddr = brokerData.getBrokerAddrs().get(brokerId);
-                HostAndPort brokerHostAndPort = HostAndPort.fromString(brokerAddr);
-                HostAndPort grpcHostAndPort = HostAndPort.fromParts(brokerHostAndPort.getHost(), grpcPort);
-
-                proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(Address.AddressScheme.IPv4, grpcHostAndPort)));
-            }
-            proxyTopicRouteData.getBrokerDatas().add(proxyBrokerData);
-        }
-
-        return proxyTopicRouteData;
+        return new ProxyTopicRouteData(topicRouteData, grpcPort);
     }
 
     @Override
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java
index da8b3f61127..4c33580adaf 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java
@@ -16,6 +16,8 @@
  */
 package org.apache.rocketmq.proxy.service.route;
 
+import com.google.common.collect.Lists;
+import com.google.common.net.HostAndPort;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -27,6 +29,58 @@
 import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
 
 public class ProxyTopicRouteData {
+    public ProxyTopicRouteData() {
+    }
+
+    public ProxyTopicRouteData(TopicRouteData topicRouteData) {
+        this.queueDatas = topicRouteData.getQueueDatas();
+        this.brokerDatas = new ArrayList<>();
+
+        for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {
+            ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData();
+            proxyBrokerData.setCluster(brokerData.getCluster());
+            proxyBrokerData.setBrokerName(brokerData.getBrokerName());
+            brokerData.getBrokerAddrs().forEach((brokerId, brokerAddr) -> {
+                HostAndPort brokerHostAndPort = HostAndPort.fromString(brokerAddr);
+
+                proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(brokerHostAndPort)));
+            });
+            this.brokerDatas.add(proxyBrokerData);
+        }
+    }
+
+    public ProxyTopicRouteData(TopicRouteData topicRouteData, int port) {
+        this.queueDatas = topicRouteData.getQueueDatas();
+        this.brokerDatas = new ArrayList<>();
+
+        for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {
+            ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData();
+            proxyBrokerData.setCluster(brokerData.getCluster());
+            proxyBrokerData.setBrokerName(brokerData.getBrokerName());
+            brokerData.getBrokerAddrs().forEach((brokerId, brokerAddr) -> {
+                HostAndPort brokerHostAndPort = HostAndPort.fromString(brokerAddr);
+                HostAndPort proxyHostAndPort = HostAndPort.fromParts(brokerHostAndPort.getHost(), port);
+
+                proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(proxyHostAndPort)));
+            });
+            this.brokerDatas.add(proxyBrokerData);
+        }
+    }
+
+    public ProxyTopicRouteData(TopicRouteData topicRouteData, List<Address> requestHostAndPortList) {
+        this.queueDatas = topicRouteData.getQueueDatas();
+        this.brokerDatas = new ArrayList<>();
+
+        for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {
+            ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData();
+            proxyBrokerData.setCluster(brokerData.getCluster());
+            proxyBrokerData.setBrokerName(brokerData.getBrokerName());
+            for (Long brokerId : brokerData.getBrokerAddrs().keySet()) {
+                proxyBrokerData.getBrokerAddrs().put(brokerId, requestHostAndPortList);
+            }
+            this.brokerDatas.add(proxyBrokerData);
+        }
+    }
 
     public static class ProxyBrokerData {
         private String cluster;
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
index ccf094c03aa..bcdf8140bc5 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java
@@ -19,9 +19,10 @@
 import com.github.benmanes.caffeine.cache.CacheLoader;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.benmanes.caffeine.cache.LoadingCache;
-import com.google.common.base.Optional;
+
 import java.time.Duration;
 import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -139,7 +140,7 @@ public String resolve(String name) {
     // pickup one topic in the topic cache
     private Optional<String> pickTopic() {
         if (topicCache.asMap().isEmpty()) {
-            return Optional.absent();
+            return Optional.empty();
         }
         return Optional.of(topicCache.asMap().keySet().iterator().next());
     }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java
index 734fbeba1b8..6c19edf2f86 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/AbstractSystemMessageSyncer.java
@@ -156,10 +156,6 @@ public void start() throws Exception {
     }
 
     protected void createSysTopic() {
-        if (this.adminService.topicExist(this.getBroadcastTopicName())) {
-            return;
-        }
-
         String clusterName = this.getBroadcastTopicClusterName();
         if (StringUtils.isEmpty(clusterName)) {
             throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "system topic cluster cannot be empty");
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionService.java
index f0e083adead..5c9820d336f 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionService.java
@@ -29,19 +29,20 @@ public abstract class AbstractTransactionService implements TransactionService,
     protected TransactionDataManager transactionDataManager = new TransactionDataManager();
 
     @Override
-    public TransactionData addTransactionDataByBrokerAddr(ProxyContext ctx, String brokerAddr, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId,
+    public TransactionData addTransactionDataByBrokerAddr(ProxyContext ctx, String brokerAddr, String topic, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId,
         Message message) {
-        return this.addTransactionDataByBrokerName(ctx, this.getBrokerNameByAddr(brokerAddr), producerGroup, tranStateTableOffset, commitLogOffset, transactionId, message);
+        return this.addTransactionDataByBrokerName(ctx, this.getBrokerNameByAddr(brokerAddr), topic, producerGroup, tranStateTableOffset, commitLogOffset, transactionId, message);
     }
 
     @Override
-    public TransactionData addTransactionDataByBrokerName(ProxyContext ctx, String brokerName, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId,
+    public TransactionData addTransactionDataByBrokerName(ProxyContext ctx, String brokerName, String topic, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId,
         Message message) {
         if (StringUtils.isBlank(brokerName)) {
             return null;
         }
         TransactionData transactionData = new TransactionData(
             brokerName,
+            topic,
             tranStateTableOffset, commitLogOffset, transactionId,
             System.currentTimeMillis(),
             ConfigurationManager.getProxyConfig().getTransactionDataExpireMillis());
@@ -55,13 +56,14 @@ public TransactionData addTransactionDataByBrokerName(ProxyContext ctx, String b
     }
 
     @Override
-    public EndTransactionRequestData genEndTransactionRequestHeader(ProxyContext ctx, String producerGroup, Integer commitOrRollback,
+    public EndTransactionRequestData genEndTransactionRequestHeader(ProxyContext ctx, String topic, String producerGroup, Integer commitOrRollback,
         boolean fromTransactionCheck, String msgId, String transactionId) {
         TransactionData transactionData = this.transactionDataManager.pollNoExpireTransactionData(producerGroup, transactionId);
         if (transactionData == null) {
             return null;
         }
         EndTransactionRequestHeader header = new EndTransactionRequestHeader();
+        header.setTopic(topic);
         header.setProducerGroup(producerGroup);
         header.setCommitOrRollback(commitOrRollback);
         header.setFromTransactionCheck(fromTransactionCheck);
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionData.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionData.java
index 88fbf44396e..b4bfe402e26 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionData.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionData.java
@@ -23,15 +23,17 @@
 
 public class TransactionData implements Comparable<TransactionData> {
     private final String brokerName;
+    private final String topic;
     private final long tranStateTableOffset;
     private final long commitLogOffset;
     private final String transactionId;
     private final long checkTimestamp;
     private final long expireMs;
 
-    public TransactionData(String brokerName, long tranStateTableOffset, long commitLogOffset, String transactionId,
+    public TransactionData(String brokerName, String topic, long tranStateTableOffset, long commitLogOffset, String transactionId,
         long checkTimestamp, long expireMs) {
         this.brokerName = brokerName;
+        this.topic = topic;
         this.tranStateTableOffset = tranStateTableOffset;
         this.commitLogOffset = commitLogOffset;
         this.transactionId = transactionId;
@@ -43,6 +45,10 @@ public String getBrokerName() {
         return brokerName;
     }
 
+    public String getTopic() {
+        return topic;
+    }
+
     public long getTranStateTableOffset() {
         return tranStateTableOffset;
     }
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionService.java
index a7ab3532424..89ebca4b6e0 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionService.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionService.java
@@ -30,13 +30,13 @@ public interface TransactionService {
 
     void unSubscribeAllTransactionTopic(ProxyContext ctx, String group);
 
-    TransactionData addTransactionDataByBrokerAddr(ProxyContext ctx, String brokerAddr, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId,
+    TransactionData addTransactionDataByBrokerAddr(ProxyContext ctx, String brokerAddr, String topic, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId,
         Message message);
 
-    TransactionData addTransactionDataByBrokerName(ProxyContext ctx, String brokerName, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId,
+    TransactionData addTransactionDataByBrokerName(ProxyContext ctx, String brokerName, String topic, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId,
         Message message);
 
-    EndTransactionRequestData genEndTransactionRequestHeader(ProxyContext ctx, String producerGroup, Integer commitOrRollback,
+    EndTransactionRequestData genEndTransactionRequestHeader(ProxyContext ctx, String topic, String producerGroup, Integer commitOrRollback,
         boolean fromTransactionCheck, String msgId, String transactionId);
 
     void onSendCheckTransactionStateFailed(ProxyContext context, String producerGroup, TransactionData transactionData);
diff --git a/proxy/src/main/resources/rmq.proxy.logback.xml b/proxy/src/main/resources/rmq.proxy.logback.xml
index f968a45e631..f485dcbe3ca 100644
--- a/proxy/src/main/resources/rmq.proxy.logback.xml
+++ b/proxy/src/main/resources/rmq.proxy.logback.xml
@@ -52,7 +52,7 @@
             <maxFileSize>128MB</maxFileSize>
         </triggeringPolicy>
         <encoder>
-            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8}%m%n</pattern>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %m%n</pattern>
             <charset class="java.nio.charset.Charset">UTF-8</charset>
         </encoder>
     </appender>
@@ -343,6 +343,31 @@
         </encoder>
     </appender>
 
+    <appender name="RocketmqAuthAuditAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>
+            ${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}${brokerLogDir:-${file.separator}}${file.separator}auth_audit.log
+        </file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>
+                ${user.home}${file.separator}logs${file.separator}rocketmqlogs${file.separator}otherdays${file.separator}${brokerLogDir:-${file.separator}}${file.separator}auth_audit.%i.log.gz
+            </fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>100MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="RocketmqAuthAuditAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqAuthAuditAppender_inner"/>
+    </appender>
+
     <appender name="RocketmqPopAppender" class="ch.qos.logback.classic.AsyncAppender">
         <appender-ref ref="RocketmqPopAppender_inner"/>
     </appender>
@@ -427,6 +452,10 @@
         <appender-ref ref="RocketmqProxyMetricsAppender" />
     </logger>
 
+    <logger name="RocketmqAuthAudit" additivity="false" level="INFO">
+        <appender-ref ref="RocketmqAuthAuditAppender"/>
+    </logger>
+
     <root level="INFO">
         <appender-ref ref="DefaultAppender"/>
     </root>
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/common/AddressTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/common/AddressTest.java
new file mode 100644
index 00000000000..b0df5bafc14
--- /dev/null
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/common/AddressTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.common;
+
+import com.google.common.net.HostAndPort;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+public class AddressTest {
+
+    @Test
+    public void testConstructorWithIPv4() {
+        HostAndPort hostAndPort = HostAndPort.fromString("192.168.1.1:8080");
+        Address address = new Address(hostAndPort);
+
+        assertEquals(Address.AddressScheme.IPv4, address.getAddressScheme());
+        assertEquals(hostAndPort, address.getHostAndPort());
+    }
+
+    @Test
+    public void testConstructorWithIPv6() {
+        HostAndPort hostAndPort = HostAndPort.fromString("[2001:db8::1]:8080");
+        Address address = new Address(hostAndPort);
+
+        assertEquals(Address.AddressScheme.IPv6, address.getAddressScheme());
+        assertEquals(hostAndPort, address.getHostAndPort());
+    }
+
+    @Test
+    public void testConstructorWithDomainName() {
+        HostAndPort hostAndPort = HostAndPort.fromString("example.com:8080");
+        Address address = new Address(hostAndPort);
+
+        assertEquals(Address.AddressScheme.DOMAIN_NAME, address.getAddressScheme());
+        assertEquals(hostAndPort, address.getHostAndPort());
+    }
+
+    @Test
+    public void testConstructorWithNullHostAndPort() {
+        Address address = new Address(null);
+
+        assertEquals(Address.AddressScheme.UNRECOGNIZED, address.getAddressScheme());
+        assertNull(address.getHostAndPort());
+    }
+}
\ No newline at end of file
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java
index 0a7e2f757d4..fdc1c5a3965 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java
@@ -26,7 +26,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.rocketmq.common.consumer.ReceiptHandle;
 import org.apache.rocketmq.common.message.MessageClientIDSetter;
-import org.apache.rocketmq.proxy.common.utils.FutureUtils;
+import org.apache.rocketmq.common.utils.FutureUtils;
 import org.apache.rocketmq.proxy.config.InitConfigTest;
 import org.junit.Before;
 import org.junit.Test;
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java
index 524945bd6fe..5ae16eb350d 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java
@@ -24,7 +24,7 @@
 import org.apache.rocketmq.proxy.common.ContextVariable;
 import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.proxy.config.InitConfigTest;
-import org.apache.rocketmq.proxy.grpc.interceptor.InterceptorConstants;
+import org.apache.rocketmq.common.constant.GrpcConstants;
 import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager;
 import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager;
 import org.apache.rocketmq.proxy.processor.MessagingProcessor;
@@ -65,10 +65,10 @@ public void before() throws Throwable {
         receiptHandleProcessor = mock(ReceiptHandleProcessor.class);
         metadataService = mock(MetadataService.class);
 
-        metadata.put(InterceptorConstants.CLIENT_ID, CLIENT_ID);
-        metadata.put(InterceptorConstants.LANGUAGE, JAVA);
-        metadata.put(InterceptorConstants.REMOTE_ADDRESS, REMOTE_ADDR);
-        metadata.put(InterceptorConstants.LOCAL_ADDRESS, LOCAL_ADDR);
+        metadata.put(GrpcConstants.CLIENT_ID, CLIENT_ID);
+        metadata.put(GrpcConstants.LANGUAGE, JAVA);
+        metadata.put(GrpcConstants.REMOTE_ADDRESS, REMOTE_ADDR);
+        metadata.put(GrpcConstants.LOCAL_ADDRESS, LOCAL_ADDR);
         when(messagingProcessor.getProxyRelayService()).thenReturn(proxyRelayService);
         when(messagingProcessor.getMetadataService()).thenReturn(metadataService);
         grpcChannelManager = new GrpcChannelManager(messagingProcessor.getProxyRelayService(), grpcClientSettingsManager);
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplicationTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplicationTest.java
index 7ba03fdbf1f..74d59a21131 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplicationTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplicationTest.java
@@ -30,9 +30,11 @@
 import io.grpc.stub.StreamObserver;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.common.constant.GrpcConstants;
 import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.proxy.config.InitConfigTest;
-import org.apache.rocketmq.proxy.grpc.interceptor.InterceptorConstants;
+import org.apache.rocketmq.proxy.grpc.pipeline.ContextInitPipeline;
+import org.apache.rocketmq.proxy.grpc.pipeline.RequestPipeline;
 import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;
 import org.junit.Assert;
 import org.junit.Before;
@@ -68,19 +70,22 @@ public class GrpcMessagingApplicationTest extends InitConfigTest {
     @Before
     public void setUp() throws Throwable {
         super.before();
-        grpcMessagingApplication = new GrpcMessagingApplication(grpcMessingActivity);
+        RequestPipeline pipeline = (context, headers, request) -> {
+        };
+        pipeline = pipeline.pipe(new ContextInitPipeline());
+        grpcMessagingApplication = new GrpcMessagingApplication(grpcMessingActivity, pipeline);
     }
 
     @Test
     public void testQueryRoute() {
         Metadata metadata = new Metadata();
-        metadata.put(InterceptorConstants.CLIENT_ID, CLIENT_ID);
-        metadata.put(InterceptorConstants.LANGUAGE, JAVA);
-        metadata.put(InterceptorConstants.REMOTE_ADDRESS, REMOTE_ADDR);
-        metadata.put(InterceptorConstants.LOCAL_ADDRESS, LOCAL_ADDR);
+        metadata.put(GrpcConstants.CLIENT_ID, CLIENT_ID);
+        metadata.put(GrpcConstants.LANGUAGE, JAVA);
+        metadata.put(GrpcConstants.REMOTE_ADDRESS, REMOTE_ADDR);
+        metadata.put(GrpcConstants.LOCAL_ADDRESS, LOCAL_ADDR);
         
         Assert.assertNotNull(Context.current()
-            .withValue(InterceptorConstants.METADATA, metadata)
+            .withValue(GrpcConstants.METADATA, metadata)
             .attach());
 
         CompletableFuture<QueryRouteResponse> future = new CompletableFuture<>();
@@ -104,12 +109,12 @@ public void testQueryRoute() {
     @Test
     public void testQueryRouteWithBadClientID() {
         Metadata metadata = new Metadata();
-        metadata.put(InterceptorConstants.LANGUAGE, JAVA);
-        metadata.put(InterceptorConstants.REMOTE_ADDRESS, REMOTE_ADDR);
-        metadata.put(InterceptorConstants.LOCAL_ADDRESS, LOCAL_ADDR);
+        metadata.put(GrpcConstants.LANGUAGE, JAVA);
+        metadata.put(GrpcConstants.REMOTE_ADDRESS, REMOTE_ADDR);
+        metadata.put(GrpcConstants.LOCAL_ADDRESS, LOCAL_ADDR);
 
         Assert.assertNotNull(Context.current()
-            .withValue(InterceptorConstants.METADATA, metadata)
+            .withValue(GrpcConstants.METADATA, metadata)
             .attach());
 
         QueryRouteRequest request = QueryRouteRequest.newBuilder()
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivityTest.java
new file mode 100644
index 00000000000..e42aeadbb6b
--- /dev/null
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivityTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.grpc.v2.producer;
+
+import apache.rocketmq.v2.Code;
+import apache.rocketmq.v2.RecallMessageRequest;
+import apache.rocketmq.v2.RecallMessageResponse;
+import apache.rocketmq.v2.Resource;
+import org.apache.rocketmq.proxy.common.ProxyException;
+import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
+import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.when;
+
+public class RecallMessageActivityTest extends BaseActivityTest {
+    private RecallMessageActivity recallMessageActivity;
+
+    @Before
+    public void before() throws Throwable {
+        super.before();
+        this.recallMessageActivity =
+            new RecallMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager);
+    }
+
+    @Test
+    public void testRecallMessage_success() {
+        when(this.messagingProcessor.recallMessage(any(), any(), any(), anyLong()))
+            .thenReturn(CompletableFuture.completedFuture("msgId"));
+
+        RecallMessageResponse response = this.recallMessageActivity.recallMessage(
+            createContext(),
+            RecallMessageRequest.newBuilder()
+                .setRecallHandle("handle")
+                .setTopic(Resource.newBuilder().setResourceNamespace("ns").setName("topic"))
+                .build()
+        ).join();
+
+        assertEquals(Code.OK, response.getStatus().getCode());
+        assertEquals("msgId", response.getMessageId());
+    }
+
+    @Test
+    public void testRecallMessage_fail() {
+        CompletableFuture<String> exceptionFuture = new CompletableFuture();
+        when(this.messagingProcessor.recallMessage(any(), any(), any(), anyLong())).thenReturn(exceptionFuture);
+        exceptionFuture.completeExceptionally(
+            new ProxyException(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, "info"));
+
+        CompletionException exception = Assert.assertThrows(CompletionException.class, () -> {
+            this.recallMessageActivity.recallMessage(
+                createContext(),
+                RecallMessageRequest.newBuilder()
+                    .setRecallHandle("handle")
+                    .setTopic(Resource.newBuilder().setResourceNamespace("ns").setName("topic"))
+                    .build()
+            ).join();
+        });
+        Assert.assertTrue(exception.getCause() instanceof ProxyException);
+        ProxyException cause = (ProxyException) exception.getCause();
+        Assert.assertEquals(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, cause.getCode());
+    }
+}
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivityTest.java
index a7ba69098bc..abbf82452ef 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivityTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivityTest.java
@@ -272,6 +272,26 @@ public void testGenPartitionFromQueueData() throws Exception {
         assertEquals(4, partitionWith4R8WPermRW.stream().filter(a -> a.getPermission() == Permission.WRITE).count());
         assertEquals(4, partitionWith4R8WPermRW.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count());
         assertEquals(0, partitionWith4R8WPermRW.stream().filter(a -> a.getPermission() == Permission.READ).count());
+
+        // test queueData with 2 read queues, 2 write queues, and no permission, expect 2 no permission queues.
+        QueueData queueDataWith2R2WNoPerm = createQueueData(2, 2, 0);
+        List<MessageQueue> partitionWith2R2WNoPerm = this.routeActivity.genMessageQueueFromQueueData(queueDataWith2R2WNoPerm, GRPC_TOPIC, TopicMessageType.UNSPECIFIED, GRPC_BROKER);
+        assertEquals(2, partitionWith2R2WNoPerm.size());
+        assertEquals(2, partitionWith2R2WNoPerm.stream().filter(a -> a.getAcceptMessageTypesValue(0) == MessageType.MESSAGE_TYPE_UNSPECIFIED.getNumber()).count());
+        assertEquals(2, partitionWith2R2WNoPerm.stream().filter(a -> a.getPermission() == Permission.NONE).count());
+        assertEquals(0, partitionWith2R2WNoPerm.stream().filter(a -> a.getPermission() == Permission.WRITE).count());
+        assertEquals(0, partitionWith2R2WNoPerm.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count());
+        assertEquals(0, partitionWith2R2WNoPerm.stream().filter(a -> a.getPermission() == Permission.READ).count());
+
+        // test queueData with 0 read queues, 0 write queues, and no permission, expect 1 no permission queue.
+        QueueData queueDataWith0R0WNoPerm = createQueueData(0, 0, 0);
+        List<MessageQueue> partitionWith0R0WNoPerm = this.routeActivity.genMessageQueueFromQueueData(queueDataWith0R0WNoPerm, GRPC_TOPIC, TopicMessageType.UNSPECIFIED, GRPC_BROKER);
+        assertEquals(1, partitionWith0R0WNoPerm.size());
+        assertEquals(1, partitionWith0R0WNoPerm.stream().filter(a -> a.getAcceptMessageTypesValue(0) == MessageType.MESSAGE_TYPE_UNSPECIFIED.getNumber()).count());
+        assertEquals(1, partitionWith0R0WNoPerm.stream().filter(a -> a.getPermission() == Permission.NONE).count());
+        assertEquals(0, partitionWith0R0WNoPerm.stream().filter(a -> a.getPermission() == Permission.WRITE).count());
+        assertEquals(0, partitionWith0R0WNoPerm.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count());
+        assertEquals(0, partitionWith0R0WNoPerm.stream().filter(a -> a.getPermission() == Permission.READ).count());
     }
 
     private static QueueData createQueueData(int r, int w, int perm) {
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivityTest.java
index 07a3abb9937..3f4d3a2d3a5 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivityTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivityTest.java
@@ -67,7 +67,7 @@ public void before() throws Throwable {
     public void testEndTransaction() throws Throwable {
         ArgumentCaptor<TransactionStatus> transactionStatusCaptor = ArgumentCaptor.forClass(TransactionStatus.class);
         ArgumentCaptor<Boolean> fromTransactionCheckCaptor = ArgumentCaptor.forClass(Boolean.class);
-        when(this.messagingProcessor.endTransaction(any(), any(), anyString(), anyString(),
+        when(this.messagingProcessor.endTransaction(any(), any(), anyString(), anyString(), anyString(),
             transactionStatusCaptor.capture(),
             fromTransactionCheckCaptor.capture())).thenReturn(CompletableFuture.completedFuture(null));
 
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java
index f154033e45f..9720938cf9e 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java
@@ -44,7 +44,7 @@
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
-import org.apache.rocketmq.proxy.common.utils.FutureUtils;
+import org.apache.rocketmq.common.utils.FutureUtils;
 import org.apache.rocketmq.proxy.common.utils.ProxyUtils;
 import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage;
 import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ProducerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ProducerProcessorTest.java
index af61c803153..6729ef0c4b3 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ProducerProcessorTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ProducerProcessorTest.java
@@ -21,6 +21,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
 import java.util.concurrent.Executors;
 import org.apache.rocketmq.client.producer.SendResult;
 import org.apache.rocketmq.client.producer.SendStatus;
@@ -34,20 +35,25 @@
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.common.utils.NetworkUtil;
+import org.apache.rocketmq.proxy.common.ProxyException;
+import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
 import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
 import org.apache.rocketmq.proxy.service.transaction.TransactionData;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
 import org.assertj.core.util.Lists;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -100,6 +106,7 @@ public void testSendMessage() throws Throwable {
             any(),
             brokerNameCaptor.capture(),
             anyString(),
+            anyString(),
             tranStateTableOffsetCaptor.capture(),
             commitLogOffsetCaptor.capture(),
             anyString(), any())).thenReturn(mock(TransactionData.class));
@@ -155,6 +162,7 @@ public void testSendRetryMessage() throws Throwable {
             any(),
             brokerNameCaptor.capture(),
             anyString(),
+            anyString(),
             tranStateTableOffsetCaptor.capture(),
             commitLogOffsetCaptor.capture(),
             anyString(), any())).thenReturn(mock(TransactionData.class));
@@ -203,6 +211,39 @@ public void testForwardMessageToDeadLetterQueue() throws Throwable {
         assertEquals(CONSUMER_GROUP, requestHeader.getGroup());
     }
 
+    @Test
+    public void testRecallMessage_notDelayMessage() {
+        when(metadataService.getTopicMessageType(any(), any())).thenReturn(TopicMessageType.NORMAL);
+        CompletionException exception = Assert.assertThrows(CompletionException.class, () -> {
+            producerProcessor.recallMessage(createContext(), TOPIC, "handle", 3000).join();
+        });
+        assertTrue(exception.getCause() instanceof ProxyException);
+        ProxyException cause = (ProxyException) exception.getCause();
+        assertEquals(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, cause.getCode());
+    }
+
+    @Test
+    public void testRecallMessage_invalidRecallHandle() {
+        when(metadataService.getTopicMessageType(any(), any())).thenReturn(TopicMessageType.DELAY);
+        CompletionException exception = Assert.assertThrows(CompletionException.class, () -> {
+            producerProcessor.recallMessage(createContext(), TOPIC, "handle", 3000).join();
+        });
+        assertTrue(exception.getCause() instanceof ProxyException);
+        ProxyException cause = (ProxyException) exception.getCause();
+        assertEquals("recall handle is invalid", cause.getMessage());
+    }
+
+    @Test
+    public void testRecallMessage_success() {
+        when(metadataService.getTopicMessageType(any(), any())).thenReturn(TopicMessageType.DELAY);
+        when(this.messageService.recallMessage(any(), any(), any(), anyLong()))
+            .thenReturn(CompletableFuture.completedFuture("msgId"));
+
+        String handle = RecallMessageHandle.HandleV1.buildHandle(TOPIC, "brokerName", "timestampStr", "whateverId");
+        String msgId = producerProcessor.recallMessage(createContext(), TOPIC, handle, 3000).join();
+        assertEquals("msgId", msgId);
+    }
+
     private static String createOffsetMsgId(long commitLogOffset) {
         int msgIDLength = 4 + 4 + 8;
         ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength);
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/TransactionProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/TransactionProcessorTest.java
index 6bffb15bd13..769b10ac8b7 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/TransactionProcessorTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/TransactionProcessorTest.java
@@ -54,11 +54,12 @@ public void testEndTransaction() throws Throwable {
     protected void testEndTransaction(int sysFlag, TransactionStatus transactionStatus) throws Throwable {
         when(this.messageService.endTransactionOneway(any(), any(), any(), anyLong())).thenReturn(CompletableFuture.completedFuture(null));
         ArgumentCaptor<Integer> commitOrRollbackCaptor = ArgumentCaptor.forClass(Integer.class);
-        when(transactionService.genEndTransactionRequestHeader(any(), anyString(), commitOrRollbackCaptor.capture(), anyBoolean(), anyString(), anyString()))
+        when(transactionService.genEndTransactionRequestHeader(any(), anyString(), anyString(), commitOrRollbackCaptor.capture(), anyBoolean(), anyString(), anyString()))
             .thenReturn(new EndTransactionRequestData("brokerName", new EndTransactionRequestHeader()));
 
         this.transactionProcessor.endTransaction(
             createContext(),
+            "topic",
             "transactionId",
             "msgId",
             PRODUCER_GROUP,
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java
index b2bd3a35f16..11dd6bc40c4 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/AbstractRemotingActivityTest.java
@@ -21,6 +21,7 @@
 import io.netty.channel.ChannelFuture;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelPromise;
+import java.util.concurrent.CompletableFuture;
 import org.apache.rocketmq.acl.common.AclException;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
@@ -30,7 +31,6 @@
 import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
 import org.apache.rocketmq.proxy.config.InitConfigTest;
 import org.apache.rocketmq.proxy.processor.MessagingProcessor;
-import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType;
 import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
 import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
@@ -39,7 +39,6 @@
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -48,8 +47,6 @@
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
-import java.util.concurrent.CompletableFuture;
-
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
@@ -89,19 +86,6 @@ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCom
         RemotingHelper.setPropertyToAttr(channel, AttributeKeys.VERSION_KEY, MQVersion.CURRENT_VERSION);
     }
 
-    @Test
-    public void testCreateContext() {
-        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
-        ProxyContext context = remotingActivity.createContext(ctx, request);
-
-        Assert.assertEquals(context.getAction(), RemotingHelper.getRequestCodeDesc(RequestCode.PULL_MESSAGE));
-        Assert.assertEquals(context.getProtocolType(), ChannelProtocolType.REMOTING.getName());
-        Assert.assertEquals(context.getLanguage(), LanguageCode.JAVA.name());
-        Assert.assertEquals(context.getClientID(), CLIENT_ID);
-        Assert.assertEquals(context.getClientVersion(), MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION));
-
-    }
-
     @Test
     public void testRequest() throws Exception {
         String brokerName = "broker";
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivityTest.java
new file mode 100644
index 00000000000..7d64923d774
--- /dev/null
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivityTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.remoting.activity;
+
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelPromise;
+import org.apache.rocketmq.common.attribute.TopicMessageType;
+import org.apache.rocketmq.proxy.common.ProxyException;
+import org.apache.rocketmq.proxy.common.ProxyExceptionCode;
+import org.apache.rocketmq.proxy.config.InitConfigTest;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.apache.rocketmq.proxy.service.channel.SimpleChannel;
+import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext;
+import org.apache.rocketmq.proxy.service.metadata.MetadataService;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.concurrent.CompletableFuture;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RecallMessageActivityTest extends InitConfigTest {
+    private static final String TOPIC = "topic";
+    private static final String GROUP = "group";
+    private static final String BROKER_NAME = "brokerName";
+
+    private RecallMessageActivity recallMessageActivity;
+    @Mock
+    private MessagingProcessor messagingProcessor;
+    @Mock
+    private MetadataService metadataService;
+
+    @Spy
+    private ChannelHandlerContext ctx = new SimpleChannelHandlerContext(new SimpleChannel(null, "1", "2")) {
+        @Override
+        public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) {
+            return null;
+        }
+    };
+
+    @Before
+    public void init() {
+        recallMessageActivity = new RecallMessageActivity(null, messagingProcessor);
+        when(messagingProcessor.getMetadataService()).thenReturn(metadataService);
+    }
+
+    @Test
+    public void testRecallMessage_notDelayMessage() {
+        when(metadataService.getTopicMessageType(any(), eq(TOPIC))).thenReturn(TopicMessageType.NORMAL);
+        ProxyException exception = Assert.assertThrows(ProxyException.class, () -> {
+            recallMessageActivity.processRequest0(ctx, mockRequest(), null);
+        });
+        Assert.assertEquals(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, exception.getCode());
+    }
+
+    @Test
+    public void testRecallMessage_success() throws Exception {
+        when(metadataService.getTopicMessageType(any(), eq(TOPIC))).thenReturn(TopicMessageType.DELAY);
+        RemotingCommand request = mockRequest();
+        RemotingCommand expectResponse = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "");
+        when(messagingProcessor.request(any(), eq(BROKER_NAME), eq(request), anyLong()))
+            .thenReturn(CompletableFuture.completedFuture(expectResponse));
+        RemotingCommand response = recallMessageActivity.processRequest0(ctx, request, null);
+        Assert.assertNull(response);
+        verify(ctx, times(1)).writeAndFlush(eq(expectResponse));
+    }
+
+    private RemotingCommand mockRequest() {
+        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();
+        requestHeader.setProducerGroup(GROUP);
+        requestHeader.setTopic(TOPIC);
+        requestHeader.setRecallHandle("handle");
+        requestHeader.setBrokerName(BROKER_NAME);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader);
+        request.makeCustomHeaderToNet();
+        return request;
+    }
+}
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandlerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandlerTest.java
index bf03786d348..4a417ea68a2 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandlerTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/Http2ProtocolProxyHandlerTest.java
@@ -20,7 +20,6 @@
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelPipeline;
 import io.netty.handler.codec.haproxy.HAProxyMessageEncoder;
-import org.apache.rocketmq.remoting.netty.AttributeKeys;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -28,7 +27,6 @@
 import org.mockito.junit.MockitoJUnitRunner;
 
 import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
@@ -51,7 +49,6 @@ public void setUp() throws Exception {
 
     @Test
     public void configPipeline() {
-        when(inboundChannel.hasAttr(eq(AttributeKeys.PROXY_PROTOCOL_ADDR))).thenReturn(true);
         when(inboundChannel.pipeline()).thenReturn(inboundPipeline);
         when(inboundPipeline.addLast(any(HAProxyMessageForwarder.class))).thenReturn(inboundPipeline);
         when(outboundChannel.pipeline()).thenReturn(outboundPipeline);
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java
index 3e3d37086b5..20ce2a16848 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.rocketmq.proxy.service.message;
 
+import io.netty.channel.Channel;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
@@ -25,12 +26,14 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.processor.AckMessageProcessor;
 import org.apache.rocketmq.broker.processor.ChangeInvisibleTimeProcessor;
 import org.apache.rocketmq.broker.processor.EndTransactionProcessor;
 import org.apache.rocketmq.broker.processor.PopMessageProcessor;
+import org.apache.rocketmq.broker.processor.RecallMessageProcessor;
 import org.apache.rocketmq.broker.processor.SendMessageProcessor;
 import org.apache.rocketmq.client.consumer.AckResult;
 import org.apache.rocketmq.client.consumer.AckStatus;
@@ -67,8 +70,11 @@
 import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
 import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -93,6 +99,8 @@ public class LocalMessageServiceTest extends InitConfigTest {
     @Mock
     private AckMessageProcessor ackMessageProcessorMock;
     @Mock
+    private RecallMessageProcessor recallMessageProcessorMock;
+    @Mock
     private BrokerController brokerControllerMock;
 
     private ProxyContext proxyContext;
@@ -121,6 +129,7 @@ public void setUp() throws Throwable {
         Mockito.when(brokerControllerMock.getChangeInvisibleTimeProcessor()).thenReturn(changeInvisibleTimeProcessorMock);
         Mockito.when(brokerControllerMock.getAckMessageProcessor()).thenReturn(ackMessageProcessorMock);
         Mockito.when(brokerControllerMock.getEndTransactionProcessor()).thenReturn(endTransactionProcessorMock);
+        Mockito.when(brokerControllerMock.getRecallMessageProcessor()).thenReturn(recallMessageProcessorMock);
         Mockito.when(brokerControllerMock.getBrokerConfig()).thenReturn(new BrokerConfig());
         localMessageService = new LocalMessageService(brokerControllerMock, channelManager, null);
         proxyContext = ProxyContext.create().withVal(ContextVariable.REMOTE_ADDRESS, "0.0.0.1")
@@ -370,11 +379,11 @@ public void testChangeInvisibleTime() throws Exception {
         responseHeader.setReviveQid(newReviveQueueId);
         responseHeader.setInvisibleTime(newInvisibleTime);
         responseHeader.setPopTime(newPopTime);
-        Mockito.when(changeInvisibleTimeProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> {
+        Mockito.when(changeInvisibleTimeProcessorMock.processRequestAsync(Mockito.any(Channel.class), Mockito.argThat(argument -> {
             boolean first = argument.getCode() == RequestCode.CHANGE_MESSAGE_INVISIBLETIME;
             boolean second = argument.readCustomHeader() instanceof ChangeInvisibleTimeRequestHeader;
             return first && second;
-        }))).thenReturn(remotingCommand);
+        }), Mockito.any(Boolean.class))).thenReturn(CompletableFuture.completedFuture(remotingCommand));
         ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();
         CompletableFuture<AckResult> future = localMessageService.changeInvisibleTime(proxyContext, handle, messageId,
             requestHeader, 1000L);
@@ -423,6 +432,31 @@ public void testAckMessage() throws Exception {
         assertThat(ackResult.getStatus()).isEqualTo(AckStatus.OK);
     }
 
+    @Test
+    public void testRecallMessage_success() throws Exception {
+        RecallMessageResponseHeader responseHeader = new RecallMessageResponseHeader();
+        responseHeader.setMsgId("msgId");
+        RemotingCommand response = RemotingCommand.createResponseCommandWithHeader(ResponseCode.SUCCESS, responseHeader);
+        Mockito.when(recallMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class),
+            Mockito.any())).thenReturn(response);
+        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();
+        String msgId = localMessageService.recallMessage(proxyContext, "brokerName", requestHeader, 1000L).join();
+        assertThat(msgId).isEqualTo("msgId");
+    }
+
+    @Test
+    public void testRecallMessage_fail() throws Exception {
+        RecallMessageResponseHeader responseHeader = new RecallMessageResponseHeader();
+        RemotingCommand response = RemotingCommand.createResponseCommandWithHeader(ResponseCode.SLAVE_NOT_AVAILABLE, responseHeader);
+        Mockito.when(recallMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class),
+            Mockito.any())).thenReturn(response);
+        RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader();
+        CompletionException exception = Assert.assertThrows(CompletionException.class, () -> {
+            localMessageService.recallMessage(proxyContext, "brokerName", requestHeader, 1000L).join();
+        });
+        Assert.assertTrue(exception.getCause() instanceof ProxyException);
+    }
+
     private MessageExt buildMessageExt(String topic, int queueId, long queueOffset) {
         MessageExt message1 = new MessageExt();
         message1.setTopic(topic);
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataServiceTest.java
index 98bf1104f8b..5894f871994 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataServiceTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataServiceTest.java
@@ -18,10 +18,16 @@
 package org.apache.rocketmq.proxy.service.metadata;
 
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.attribute.TopicMessageType;
 import org.apache.rocketmq.proxy.common.ProxyContext;
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
 import org.apache.rocketmq.proxy.service.BaseServiceTest;
+import org.apache.rocketmq.proxy.service.route.MessageQueueView;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
 import org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;
 import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
 import org.junit.Before;
@@ -29,6 +35,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
@@ -38,6 +45,8 @@ public class ClusterMetadataServiceTest extends BaseServiceTest {
 
     private ClusterMetadataService clusterMetadataService;
 
+    protected static final String BROKER2_ADDR = "127.0.0.2:10911";
+
     @Before
     public void before() throws Throwable {
         super.before();
@@ -51,6 +60,16 @@ public void before() throws Throwable {
         when(this.mqClientAPIExt.getSubscriptionGroupConfig(anyString(), eq(GROUP), anyLong())).thenReturn(new SubscriptionGroupConfig());
 
         this.clusterMetadataService = new ClusterMetadataService(this.topicRouteService, this.mqClientAPIFactory);
+
+        BrokerData brokerData2 = new BrokerData();
+        brokerData2.setBrokerName("brokerName2");
+        HashMap<Long, String> addrs = new HashMap<>();
+        addrs.put(MixAll.MASTER_ID, BROKER2_ADDR);
+        brokerData2.setBrokerAddrs(addrs);
+        brokerData2.setCluster(CLUSTER_NAME);
+        topicRouteData.getBrokerDatas().add(brokerData2);
+        when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData, null));
+
     }
 
     @Test
@@ -70,4 +89,22 @@ public void testGetSubscriptionGroupConfig() {
         assertNotNull(this.clusterMetadataService.getSubscriptionGroupConfig(ctx, GROUP));
         assertEquals(1, this.clusterMetadataService.subscriptionGroupConfigCache.asMap().size());
     }
+
+    @Test
+    public void findOneBroker() {
+
+        Set<String> resultBrokerNames = new HashSet<>();
+        // run 1000 times to test the random
+        for (int i = 0; i < 1000; i++) {
+            Optional<BrokerData> brokerData = null;
+            try {
+                brokerData = this.clusterMetadataService.findOneBroker(TOPIC);
+                resultBrokerNames.add(brokerData.get().getBrokerName());
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+        // we should choose two brokers
+        assertEquals(2, resultBrokerNames.size());
+    }
 }
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/ProxyClientRemotingProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/ProxyClientRemotingProcessorTest.java
index a6d807937e2..2cdd92ba5be 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/ProxyClientRemotingProcessorTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/ProxyClientRemotingProcessorTest.java
@@ -29,6 +29,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.rocketmq.broker.client.ProducerManager;
+import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor;
 import org.apache.rocketmq.common.message.MessageAccessor;
 import org.apache.rocketmq.common.message.MessageConst;
@@ -72,10 +73,14 @@ public class ProxyClientRemotingProcessorTest {
 
     @Test
     public void testTransactionCheck() throws Exception {
+        // Temporarily skip this test on the Mac system as it is flaky
+        if (MixAll.isMac()) {
+            return;
+        }
         CompletableFuture<ProxyRelayResult<Void>> proxyRelayResultFuture = new CompletableFuture<>();
         when(proxyRelayService.processCheckTransactionState(any(), any(), any(), any()))
             .thenReturn(new RelayData<>(
-                new TransactionData("brokerName", 0, 0, "id", System.currentTimeMillis(), 3000),
+                new TransactionData("brokerName", "topic", 0, 0, "id", System.currentTimeMillis(), 3000),
                 proxyRelayResultFuture));
 
         GrpcClientChannel grpcClientChannel = new GrpcClientChannel(proxyRelayService, grpcClientSettingsManager, null,
@@ -123,7 +128,7 @@ public void testTransactionCheck() throws Exception {
                 }
             });
         }
-        await().atMost(Duration.ofSeconds(1)).until(() -> count.get() == 100);
+        await().atMost(Duration.ofSeconds(3)).until(() -> count.get() == 100);
         verify(observer, times(2)).onNext(any());
     }
 
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManagerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManagerTest.java
index 25ae1509a95..a01c356f779 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManagerTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManagerTest.java
@@ -227,7 +227,7 @@ public void testRenewExceedMaxRenewTimes() {
                 Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(retryPolicy.nextDelayDuration(messageReceiptHandle.getRenewTimes()))))
             .thenReturn(ackResultFuture);
 
-        await().atMost(Duration.ofSeconds(1)).until(() -> {
+        await().atMost(Duration.ofSeconds(3)).until(() -> {
             receiptHandleManager.scheduleRenewTask();
             try {
                 ReceiptHandleGroup receiptHandleGroup = receiptHandleManager.receiptHandleGroupMap.values().stream().findFirst().get();
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionServiceTest.java
index 81de5ec843a..e4f655bee05 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionServiceTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionServiceTest.java
@@ -84,6 +84,7 @@ public void testAddAndGenEndHeader() {
         TransactionData transactionData = transactionService.addTransactionDataByBrokerName(
             ctx,
             BROKER_NAME,
+            "Topic",
             PRODUCER_GROUP,
             RANDOM.nextLong(),
             RANDOM.nextLong(),
@@ -94,6 +95,7 @@ public void testAddAndGenEndHeader() {
 
         EndTransactionRequestData requestData = transactionService.genEndTransactionRequestHeader(
             ctx,
+            "topic",
             PRODUCER_GROUP,
             MessageSysFlag.TRANSACTION_COMMIT_TYPE,
             true,
@@ -108,6 +110,7 @@ public void testAddAndGenEndHeader() {
 
         assertNull(transactionService.genEndTransactionRequestHeader(
             ctx,
+            "topic",
             "group",
             MessageSysFlag.TRANSACTION_COMMIT_TYPE,
             true,
@@ -125,6 +128,7 @@ public void testOnSendCheckTransactionStateFailedFailed() {
         TransactionData transactionData = transactionService.addTransactionDataByBrokerName(
             ctx,
             BROKER_NAME,
+            "Topic",
             PRODUCER_GROUP,
             RANDOM.nextLong(),
             RANDOM.nextLong(),
@@ -134,6 +138,7 @@ public void testOnSendCheckTransactionStateFailedFailed() {
         transactionService.onSendCheckTransactionStateFailed(ProxyContext.createForInner(this.getClass()), PRODUCER_GROUP, transactionData);
         assertNull(transactionService.genEndTransactionRequestHeader(
             ctx,
+            "topic",
             PRODUCER_GROUP,
             MessageSysFlag.TRANSACTION_COMMIT_TYPE,
             true,
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/TransactionDataManagerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/TransactionDataManagerTest.java
index f92a7846c5d..90a7f6b78cc 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/TransactionDataManagerTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/TransactionDataManagerTest.java
@@ -133,6 +133,7 @@ private static TransactionData createTransactionData(String txId, long checkTime
     private static TransactionData createTransactionData(String txId, long checkTimestamp, long checkImmunityTime) {
         return new TransactionData(
             "brokerName",
+            "topicName",
             RANDOM.nextLong(),
             RANDOM.nextLong(),
             txId,
diff --git a/remoting/BUILD.bazel b/remoting/BUILD.bazel
index 072148bc08c..9f806be7635 100644
--- a/remoting/BUILD.bazel
+++ b/remoting/BUILD.bazel
@@ -39,6 +39,7 @@ java_library(
         "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
         "@maven//:io_github_aliyunmq_rocketmq_logback_classic",
         "@maven//:commons_collections_commons_collections",
+        "@maven//:org_reflections_reflections",
     ],
 )
 
@@ -66,6 +67,7 @@ java_library(
         "@maven//:org_apache_tomcat_annotations_api",
         "@maven//:org_apache_commons_commons_lang3",
         "@maven//:org_jetbrains_annotations",
+        "@maven//:org_reflections_reflections",
     ],
     resources = glob(["src/test/resources/certs/*.pem"]) + glob(["src/test/resources/certs/*.key"])
 )
diff --git a/remoting/pom.xml b/remoting/pom.xml
index f7848068055..623f87f353e 100644
--- a/remoting/pom.xml
+++ b/remoting/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -40,6 +40,10 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.reflections</groupId>
+            <artifactId>reflections</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/PermissionChecker.java b/remoting/src/main/java/org/apache/rocketmq/remoting/CommandCallback.java
similarity index 84%
rename from acl/src/main/java/org/apache/rocketmq/acl/PermissionChecker.java
rename to remoting/src/main/java/org/apache/rocketmq/remoting/CommandCallback.java
index a38d3ec4787..884f3d9e5d1 100644
--- a/acl/src/main/java/org/apache/rocketmq/acl/PermissionChecker.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/CommandCallback.java
@@ -14,9 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+package org.apache.rocketmq.remoting;
 
-package org.apache.rocketmq.acl;
+public interface CommandCallback {
 
-public interface PermissionChecker {
-    void check(AccessResource checkedAccess, AccessResource ownedAccess);
+    void accept();
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingService.java b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingService.java
index c718f2e65c0..e7425ea1fb6 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingService.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingService.java
@@ -17,6 +17,8 @@
 
 package org.apache.rocketmq.remoting;
 
+import org.apache.rocketmq.remoting.pipeline.RequestPipeline;
+
 public interface RemotingService {
     void start();
 
@@ -24,6 +26,8 @@ public interface RemotingService {
 
     void registerRPCHook(RPCHook rpcHook);
 
+    void setRequestPipeline(RequestPipeline pipeline);
+
     /**
      * Remove all rpc hooks.
      */
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java
index 363b22eac71..d94efe71e49 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java
@@ -21,6 +21,15 @@
 import io.netty.channel.ChannelFutureListener;
 import io.netty.util.Attribute;
 import io.netty.util.AttributeKey;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.SocketChannel;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.utils.NetworkUtil;
@@ -36,15 +45,6 @@
 import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
 
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.nio.ByteBuffer;
-import java.nio.channels.SocketChannel;
-import java.util.HashMap;
-import java.util.Map;
-
 public class RemotingHelper {
     public static final String DEFAULT_CHARSET = "UTF-8";
     public static final String DEFAULT_CIDR_ALL = "0.0.0.0/0";
@@ -243,6 +243,22 @@ private static String parseChannelRemoteAddr0(final Channel channel) {
         return "";
     }
 
+    public static String parseChannelLocalAddr(final Channel channel) {
+        SocketAddress remote = channel.localAddress();
+        final String addr = remote != null ? remote.toString() : "";
+
+        if (addr.length() > 0) {
+            int index = addr.lastIndexOf("/");
+            if (index >= 0) {
+                return addr.substring(index + 1);
+            }
+
+            return addr;
+        }
+
+        return "";
+    }
+
     public static String parseHostFromAddress(String address) {
         if (address == null) {
             return "";
@@ -339,6 +355,18 @@ public void operationComplete(ChannelFuture future) throws Exception {
         }
     }
 
+    public static CompletableFuture<Void> convertChannelFutureToCompletableFuture(ChannelFuture channelFuture) {
+        CompletableFuture<Void> completableFuture = new CompletableFuture<>();
+        channelFuture.addListener((ChannelFutureListener) future -> {
+            if (future.isSuccess()) {
+                completableFuture.complete(null);
+            } else {
+                completableFuture.completeExceptionally(new RemotingConnectException(channelFuture.channel().remoteAddress().toString(), future.cause()));
+            }
+        });
+        return completableFuture;
+    }
+
     public static String getRequestCodeDesc(int code) {
         return REQUEST_CODE_MAP.getOrDefault(code, String.valueOf(code));
     }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/FileRegionEncoder.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/FileRegionEncoder.java
index 3522c7965c1..7373a560703 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/FileRegionEncoder.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/FileRegionEncoder.java
@@ -18,9 +18,6 @@
 package org.apache.rocketmq.remoting.netty;
 
 import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.buffer.CompositeByteBuf;
-import io.netty.buffer.Unpooled;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.FileRegion;
 import io.netty.handler.codec.MessageToByteEncoder;
@@ -54,12 +51,9 @@ protected void encode(ChannelHandlerContext ctx, FileRegion msg, final ByteBuf o
         WritableByteChannel writableByteChannel = new WritableByteChannel() {
             @Override
             public int write(ByteBuffer src) {
-                // To prevent mem_copy.
-                CompositeByteBuf b = (CompositeByteBuf) out;
-                // Have to increase writerIndex manually.
-                ByteBuf unpooled = Unpooled.wrappedBuffer(src);
-                b.addComponent(true, unpooled);
-                return unpooled.readableBytes();
+                int prev = out.writerIndex();
+                out.writeBytes(src);
+                return out.writerIndex() - prev;
             }
 
             @Override
@@ -82,10 +76,4 @@ public void close() throws IOException {
             msg.transferTo(writableByteChannel, transferred);
         }
     }
-
-    @Override
-    protected ByteBuf allocateBuffer(ChannelHandlerContext ctx, FileRegion msg, boolean preferDirect) throws Exception {
-        ByteBufAllocator allocator = ctx.alloc();
-        return preferDirect ? allocator.compositeDirectBuffer() : allocator.compositeHeapBuffer();
-    }
-}
\ No newline at end of file
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java
index c28288786a3..82601636403 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java
@@ -31,6 +31,8 @@ public class NettyClientConfig {
     private int connectTimeoutMillis = NettySystemConfig.connectTimeoutMillis;
     private long channelNotActiveInterval = 1000 * 60;
 
+    private boolean isScanAvailableNameSrv = true;
+
     /**
      * IdleStateEvent will be triggered when neither read nor write was performed for
      * the specified period of this time. Specify {@code 0} to disable
@@ -57,8 +59,6 @@ public class NettyClientConfig {
 
     private boolean enableReconnectForGoAway = true;
 
-    private boolean enableTransparentRetry = true;
-
     public boolean isClientCloseSocketIfTimeout() {
         return clientCloseSocketIfTimeout;
     }
@@ -203,14 +203,6 @@ public void setEnableReconnectForGoAway(boolean enableReconnectForGoAway) {
         this.enableReconnectForGoAway = enableReconnectForGoAway;
     }
 
-    public boolean isEnableTransparentRetry() {
-        return enableTransparentRetry;
-    }
-
-    public void setEnableTransparentRetry(boolean enableTransparentRetry) {
-        this.enableTransparentRetry = enableTransparentRetry;
-    }
-
     public String getSocksProxyConfig() {
         return socksProxyConfig;
     }
@@ -218,4 +210,12 @@ public String getSocksProxyConfig() {
     public void setSocksProxyConfig(String socksProxyConfig) {
         this.socksProxyConfig = socksProxyConfig;
     }
+
+    public boolean isScanAvailableNameSrv() {
+        return isScanAvailableNameSrv;
+    }
+
+    public void setScanAvailableNameSrv(boolean scanAvailableNameSrv) {
+        this.isScanAvailableNameSrv = scanAvailableNameSrv;
+    }
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java
index 955ffc1bc4f..0a2b2dac595 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java
@@ -74,7 +74,7 @@ public void log(InternalLogLevel internalLogLevel, String s) {
                 logger.debug(s);
             }
             if (internalLogLevel.equals(InternalLogLevel.TRACE)) {
-                logger.info(s);
+                logger.trace(s);
             }
             if (internalLogLevel.equals(InternalLogLevel.INFO)) {
                 logger.info(s);
@@ -93,7 +93,7 @@ public void log(InternalLogLevel internalLogLevel, String s, Object o) {
                 logger.debug(s, o);
             }
             if (internalLogLevel.equals(InternalLogLevel.TRACE)) {
-                logger.info(s, o);
+                logger.trace(s, o);
             }
             if (internalLogLevel.equals(InternalLogLevel.INFO)) {
                 logger.info(s, o);
@@ -112,7 +112,7 @@ public void log(InternalLogLevel internalLogLevel, String s, Object o, Object o1
                 logger.debug(s, o, o1);
             }
             if (internalLogLevel.equals(InternalLogLevel.TRACE)) {
-                logger.info(s, o, o1);
+                logger.trace(s, o, o1);
             }
             if (internalLogLevel.equals(InternalLogLevel.INFO)) {
                 logger.info(s, o, o1);
@@ -131,7 +131,7 @@ public void log(InternalLogLevel internalLogLevel, String s, Object... objects)
                 logger.debug(s, objects);
             }
             if (internalLogLevel.equals(InternalLogLevel.TRACE)) {
-                logger.info(s, objects);
+                logger.trace(s, objects);
             }
             if (internalLogLevel.equals(InternalLogLevel.INFO)) {
                 logger.info(s, objects);
@@ -150,7 +150,7 @@ public void log(InternalLogLevel internalLogLevel, String s, Throwable throwable
                 logger.debug(s, throwable);
             }
             if (internalLogLevel.equals(InternalLogLevel.TRACE)) {
-                logger.info(s, throwable);
+                logger.trace(s, throwable);
             }
             if (internalLogLevel.equals(InternalLogLevel.INFO)) {
                 logger.info(s, throwable);
@@ -169,7 +169,7 @@ public void log(InternalLogLevel internalLogLevel, Throwable throwable) {
                 logger.debug(EXCEPTION_MESSAGE, throwable);
             }
             if (internalLogLevel.equals(InternalLogLevel.TRACE)) {
-                logger.info(EXCEPTION_MESSAGE, throwable);
+                logger.trace(EXCEPTION_MESSAGE, throwable);
             }
             if (internalLogLevel.equals(InternalLogLevel.INFO)) {
                 logger.info(EXCEPTION_MESSAGE, throwable);
@@ -189,32 +189,32 @@ public boolean isTraceEnabled() {
 
         @Override
         public void trace(String var1) {
-            logger.info(var1);
+            logger.trace(var1);
         }
 
         @Override
         public void trace(String var1, Object var2) {
-            logger.info(var1, var2);
+            logger.trace(var1, var2);
         }
 
         @Override
         public void trace(String var1, Object var2, Object var3) {
-            logger.info(var1, var2, var3);
+            logger.trace(var1, var2, var3);
         }
 
         @Override
         public void trace(String var1, Object... var2) {
-            logger.info(var1, var2);
+            logger.trace(var1, var2);
         }
 
         @Override
         public void trace(String var1, Throwable var2) {
-            logger.info(var1, var2);
+            logger.trace(var1, var2);
         }
 
         @Override
         public void trace(Throwable var1) {
-            logger.info(EXCEPTION_MESSAGE, var1);
+            logger.trace(EXCEPTION_MESSAGE, var1);
         }
 
         @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
index 62a8a72901c..a4f23f181a3 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
@@ -39,8 +39,8 @@
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import javax.annotation.Nullable;
 import org.apache.rocketmq.common.AbortProcessException;
@@ -49,6 +49,7 @@
 import org.apache.rocketmq.common.ServiceThread;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.utils.ExceptionUtils;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.ChannelEventListener;
@@ -60,6 +61,7 @@
 import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
 import org.apache.rocketmq.remoting.metrics.RemotingMetricsManager;
+import org.apache.rocketmq.remoting.pipeline.RequestPipeline;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
@@ -123,6 +125,8 @@ public abstract class NettyRemotingAbstract {
      */
     protected List<RPCHook> rpcHooks = new ArrayList<>();
 
+    protected RequestPipeline requestPipeline;
+
     protected AtomicBoolean isShuttingDown = new AtomicBoolean(false);
 
     static {
@@ -270,11 +274,12 @@ public void processRequestCommand(final ChannelHandlerContext ctx, final Remotin
         Runnable run = buildProcessRequestHandler(ctx, cmd, pair, opaque);
 
         if (isShuttingDown.get()) {
-            if (cmd.getVersion() > MQVersion.Version.V5_1_4.ordinal()) {
+            if (cmd.getVersion() > MQVersion.Version.V5_3_1.ordinal()) {
                 final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.GO_AWAY,
                     "please go away");
                 response.setOpaque(opaque);
                 writeResponse(ctx.channel(), cmd, response);
+                log.info("proxy is shutting down, write response GO_AWAY. channel={}, requestCode={}, opaque={}", ctx.channel(), cmd.getCode(), opaque);
                 return;
             }
         }
@@ -316,9 +321,10 @@ private Runnable buildProcessRequestHandler(ChannelHandlerContext ctx, RemotingC
         return () -> {
             Exception exception = null;
             RemotingCommand response;
+            String remoteAddr = null;
 
             try {
-                String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
+                remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
                 try {
                     doBeforeRpcHooks(remoteAddr, cmd);
                 } catch (AbortProcessException e) {
@@ -327,6 +333,10 @@ private Runnable buildProcessRequestHandler(ChannelHandlerContext ctx, RemotingC
                     exception = e;
                 }
 
+                if (this.requestPipeline != null) {
+                    this.requestPipeline.execute(ctx, cmd);
+                }
+
                 if (exception == null) {
                     response = pair.getObject1().processRequest(ctx, cmd);
                 } else {
@@ -351,7 +361,7 @@ private Runnable buildProcessRequestHandler(ChannelHandlerContext ctx, RemotingC
                 response.setOpaque(opaque);
                 writeResponse(ctx.channel(), cmd, response);
             } catch (Throwable e) {
-                log.error("process request exception", e);
+                log.error("process request exception, remoteAddr: {}", remoteAddr, e);
                 log.error(cmd.toString());
 
                 if (!cmd.isOnewayRPC()) {
@@ -385,7 +395,7 @@ public void processResponseCommand(ChannelHandlerContext ctx, RemotingCommand cm
                 responseFuture.release();
             }
         } else {
-            log.warn("receive response, cmd={}, but not matched any request, address={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+            log.warn("receive response, cmd={}, but not matched any request, address={}, channelId={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), ctx.channel().id());
         }
     }
 
@@ -440,6 +450,10 @@ public void registerRPCHook(RPCHook rpcHook) {
         }
     }
 
+    public void setRequestPipeline(RequestPipeline pipeline) {
+        this.requestPipeline = pipeline;
+    }
+
     public void clearRPCHook() {
         rpcHooks.clear();
     }
@@ -496,13 +510,7 @@ public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingComma
 
     public CompletableFuture<ResponseFuture> invokeImpl(final Channel channel, final RemotingCommand request,
         final long timeoutMillis) {
-        String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);
-        doBeforeRpcHooks(channelRemoteAddr, request);
-        return invoke0(channel, request, timeoutMillis).whenComplete((v, t) -> {
-            if (t == null) {
-                doAfterRpcHooks(channelRemoteAddr, request, v.getResponseCommand());
-            }
-        });
+        return invoke0(channel, request, timeoutMillis);
     }
 
     protected CompletableFuture<ResponseFuture> invoke0(final Channel channel, final RemotingCommand request,
@@ -554,13 +562,13 @@ public void operationFail(Throwable throwable) {
                         return;
                     }
                     requestFail(opaque);
-                    log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel));
+                    log.warn("send a request command to channel <{}>, channelId={}, failed.", RemotingHelper.parseChannelRemoteAddr(channel), channel.id());
                 });
                 return future;
             } catch (Exception e) {
                 responseTable.remove(opaque);
                 responseFuture.release();
-                log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e);
+                log.warn("send a request command to channel <{}> channelId={} Exception", RemotingHelper.parseChannelRemoteAddr(channel), channel.id(), e);
                 future.completeExceptionally(new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e));
                 return future;
             }
@@ -595,7 +603,7 @@ public void invokeAsyncImpl(final Channel channel, final RemotingCommand request
             })
             .thenAccept(responseFuture -> invokeCallback.operationSucceed(responseFuture.getResponseCommand()))
             .exceptionally(t -> {
-                invokeCallback.operationFail(t);
+                invokeCallback.operationFail(ExceptionUtils.getRealException(t));
                 return null;
             });
     }
@@ -665,6 +673,10 @@ public void invokeOnewayImpl(final Channel channel, final RemotingCommand reques
         }
     }
 
+    public HashMap<Integer, Pair<NettyRequestProcessor, ExecutorService>> getProcessorTable() {
+        return processorTable;
+    }
+
     class NettyEventExecutor extends ServiceThread {
         private final LinkedBlockingQueue<NettyEvent> eventQueue = new LinkedBlockingQueue<>();
 
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
index f5157d03049..92ced6b01af 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
@@ -49,7 +49,6 @@
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 import java.security.cert.CertificateException;
-import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -74,6 +73,8 @@
 import org.apache.rocketmq.common.Pair;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.utils.FutureUtils;
+import org.apache.rocketmq.common.utils.NetworkUtil;
 import org.apache.rocketmq.common.utils.ThreadUtils;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
@@ -86,16 +87,19 @@
 import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
 import org.apache.rocketmq.remoting.proxy.SocksProxyConfig;
 
+import static org.apache.rocketmq.remoting.common.RemotingHelper.convertChannelFutureToCompletableFuture;
+
 public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient {
     private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);
 
     private static final long LOCK_TIMEOUT_MILLIS = 3000;
     private static final long MIN_CLOSE_TIMEOUT_MILLIS = 100;
 
-    private final NettyClientConfig nettyClientConfig;
+    protected final NettyClientConfig nettyClientConfig;
     private final Bootstrap bootstrap = new Bootstrap();
     private final EventLoopGroup eventLoopGroupWorker;
     private final Lock lockChannelTables = new ReentrantLock();
@@ -251,20 +255,24 @@ public void run(Timeout timeout) {
         };
         this.timer.newTimeout(timerTaskScanResponseTable, 1000 * 3, TimeUnit.MILLISECONDS);
 
-        int connectTimeoutMillis = this.nettyClientConfig.getConnectTimeoutMillis();
-        TimerTask timerTaskScanAvailableNameSrv = new TimerTask() {
-            @Override
-            public void run(Timeout timeout) {
-                try {
-                    NettyRemotingClient.this.scanAvailableNameSrv();
-                } catch (Exception e) {
-                    LOGGER.error("scanAvailableNameSrv exception", e);
-                } finally {
-                    timer.newTimeout(this, connectTimeoutMillis, TimeUnit.MILLISECONDS);
+        if (nettyClientConfig.isScanAvailableNameSrv()) {
+            int connectTimeoutMillis = this.nettyClientConfig.getConnectTimeoutMillis();
+            TimerTask timerTaskScanAvailableNameSrv = new TimerTask() {
+                @Override
+                public void run(Timeout timeout) {
+                    try {
+                        NettyRemotingClient.this.scanAvailableNameSrv();
+                    } catch (Exception e) {
+                        LOGGER.error("scanAvailableNameSrv exception", e);
+                    } finally {
+                        timer.newTimeout(this, connectTimeoutMillis, TimeUnit.MILLISECONDS);
+                    }
                 }
-            }
-        };
-        this.timer.newTimeout(timerTaskScanAvailableNameSrv, 0, TimeUnit.MILLISECONDS);
+            };
+            this.timer.newTimeout(timerTaskScanAvailableNameSrv, 0, TimeUnit.MILLISECONDS);
+        }
+        
+        
     }
 
     private Map.Entry<String, SocksProxyConfig> getProxy(String addr) {
@@ -281,6 +289,13 @@ private Map.Entry<String, SocksProxyConfig> getProxy(String addr) {
         return null;
     }
 
+    protected ChannelFuture doConnect(String addr) {
+        String[] hostAndPort = getHostAndPort(addr);
+        String host = hostAndPort[0];
+        int port = Integer.parseInt(hostAndPort[1]);
+        return fetchBootstrap(addr).connect(host, port);
+    }
+
     private Bootstrap fetchBootstrap(String addr) {
         Map.Entry<String, SocksProxyConfig> proxyEntry = getProxy(addr);
         if (proxyEntry == null) {
@@ -352,7 +367,7 @@ public void initChannel(SocketChannel ch) {
     }
 
     // Do not use RemotingHelper.string2SocketAddress(), it will directly resolve the domain
-    private String[] getHostAndPort(String address) {
+    protected String[] getHostAndPort(String address) {
         int split = address.lastIndexOf(":");
         return split < 0 ? new String[]{address} : new String[]{address.substring(0, split), address.substring(split + 1)};
     }
@@ -362,8 +377,8 @@ public void shutdown() {
         try {
             this.timer.stop();
 
-            for (String addr : this.channelTables.keySet()) {
-                this.channelTables.get(addr).close();
+            for (Map.Entry<String, ChannelWrapper> channel : this.channelTables.entrySet()) {
+                channel.getValue().close();
             }
 
             this.channelWrapperTables.clear();
@@ -412,14 +427,14 @@ public void closeChannel(final String addr, final Channel channel) {
                     boolean removeItemFromTable = true;
                     final ChannelWrapper prevCW = this.channelTables.get(addrRemote);
 
-                    LOGGER.info("closeChannel: begin close the channel[{}] Found: {}", addrRemote, prevCW != null);
+                    LOGGER.info("closeChannel: begin close the channel[addr={}, id={}] Found: {}", addrRemote, channel.id(), prevCW != null);
 
                     if (null == prevCW) {
-                        LOGGER.info("closeChannel: the channel[{}] has been removed from the channel table before", addrRemote);
+                        LOGGER.info("closeChannel: the channel[addr={}, id={}] has been removed from the channel table before", addrRemote, channel.id());
                         removeItemFromTable = false;
-                    } else if (prevCW.getChannel() != channel) {
-                        LOGGER.info("closeChannel: the channel[{}] has been closed before, and has been created again, nothing to do.",
-                            addrRemote);
+                    } else if (prevCW.isWrapperOf(channel)) {
+                        LOGGER.info("closeChannel: the channel[addr={}, id={}] has been closed before, and has been created again, nothing to do.",
+                            addrRemote, channel.id());
                         removeItemFromTable = false;
                     }
 
@@ -428,7 +443,7 @@ public void closeChannel(final String addr, final Channel channel) {
                         if (channelWrapper != null && channelWrapper.tryClose(channel)) {
                             this.channelTables.remove(addrRemote);
                         }
-                        LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote);
+                        LOGGER.info("closeChannel: the channel[addr={}, id={}] was removed from channel table", addrRemote, channel.id());
                     }
 
                     RemotingHelper.closeChannel(channel);
@@ -459,17 +474,15 @@ public void closeChannel(final Channel channel) {
                     for (Map.Entry<String, ChannelWrapper> entry : channelTables.entrySet()) {
                         String key = entry.getKey();
                         ChannelWrapper prev = entry.getValue();
-                        if (prev.getChannel() != null) {
-                            if (prev.getChannel() == channel) {
-                                prevCW = prev;
-                                addrRemote = key;
-                                break;
-                            }
+                        if (prev.isWrapperOf(channel)) {
+                            prevCW = prev;
+                            addrRemote = key;
+                            break;
                         }
                     }
 
                     if (null == prevCW) {
-                        LOGGER.info("eventCloseChannel: the channel[{}] has been removed from the channel table before", addrRemote);
+                        LOGGER.info("eventCloseChannel: the channel[addr={}, id={}] has been removed from the channel table before", RemotingHelper.parseChannelRemoteAddr(channel), channel.id());
                         removeItemFromTable = false;
                     }
 
@@ -478,11 +491,11 @@ public void closeChannel(final Channel channel) {
                         if (channelWrapper != null && channelWrapper.tryClose(channel)) {
                             this.channelTables.remove(addrRemote);
                         }
-                        LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote);
+                        LOGGER.info("closeChannel: the channel[addr={}, id={}] was removed from channel table", addrRemote, channel.id());
                         RemotingHelper.closeChannel(channel);
                     }
                 } catch (Exception e) {
-                    LOGGER.error("closeChannel: close the channel exception", e);
+                    LOGGER.error("closeChannel: close the channel[id={}] exception", channel.id(), e);
                 } finally {
                     this.lockChannelTables.unlock();
                 }
@@ -519,10 +532,11 @@ public void updateNameServerAddressList(List<String> addrs) {
                 this.namesrvAddrList.set(addrs);
 
                 // should close the channel if choosed addr is not exist.
-                if (this.namesrvAddrChoosed.get() != null && !addrs.contains(this.namesrvAddrChoosed.get())) {
-                    String namesrvAddr = this.namesrvAddrChoosed.get();
+                String chosenNameServerAddr = this.namesrvAddrChoosed.get();
+                if (chosenNameServerAddr != null && !addrs.contains(chosenNameServerAddr)) {
+                    namesrvAddrChoosed.compareAndSet(chosenNameServerAddr, null);
                     for (String addr : this.channelTables.keySet()) {
-                        if (addr.contains(namesrvAddr)) {
+                        if (addr.contains(chosenNameServerAddr)) {
                             ChannelWrapper channelWrapper = this.channelTables.get(addr);
                             if (channelWrapper != null) {
                                 channelWrapper.close();
@@ -552,7 +566,7 @@ public RemotingCommand invokeSync(String addr, final RemotingCommand request, lo
                 updateChannelLastResponseTime(addr);
                 return response;
             } catch (RemotingSendRequestException e) {
-                LOGGER.warn("invokeSync: send request exception, so close the channel[{}]", channelRemoteAddr);
+                LOGGER.warn("invokeSync: send request exception, so close the channel[addr={}, id={}]", channelRemoteAddr, channel.id());
                 this.closeChannel(addr, channel);
                 throw e;
             } catch (RemotingTimeoutException e) {
@@ -560,9 +574,9 @@ public RemotingCommand invokeSync(String addr, final RemotingCommand request, lo
                 boolean shouldClose = left > MIN_CLOSE_TIMEOUT_MILLIS || left > timeoutMillis / 4;
                 if (nettyClientConfig.isClientCloseSocketIfTimeout() && shouldClose) {
                     this.closeChannel(addr, channel);
-                    LOGGER.warn("invokeSync: close socket because of timeout, {}ms, {}", timeoutMillis, channelRemoteAddr);
+                    LOGGER.warn("invokeSync: close socket because of timeout, {}ms, channel[addr={}, id={}]", timeoutMillis, channelRemoteAddr, channel.id());
                 }
-                LOGGER.warn("invokeSync: wait response timeout exception, the channel[{}]", channelRemoteAddr);
+                LOGGER.warn("invokeSync: wait response timeout exception, the channel[addr={}, id={}]", channelRemoteAddr, channel.id());
                 throw e;
             }
         } else {
@@ -591,7 +605,7 @@ private void interruptPullRequests(Set<String> brokerAddrSet) {
             }
             String remoteAddr = RemotingHelper.parseChannelRemoteAddr(responseFuture.getChannel());
             // interrupt only pull message request
-            if (brokerAddrSet.contains(remoteAddr) && (cmd.getCode() == 11 || cmd.getCode() == 361)) {
+            if (brokerAddrSet.contains(remoteAddr) && (cmd.getCode() == RequestCode.PULL_MESSAGE || cmd.getCode() == RequestCode.LITE_PULL_MESSAGE)) {
                 LOGGER.info("interrupt {}", cmd);
                 responseFuture.interrupt();
             }
@@ -613,25 +627,33 @@ private void updateChannelLastResponseTime(final String addr) {
         }
     }
 
-    private Channel getAndCreateChannel(final String addr) throws InterruptedException {
+    private ChannelFuture getAndCreateChannelAsync(final String addr) throws InterruptedException {
         if (null == addr) {
-            return getAndCreateNameserverChannel();
+            return getAndCreateNameserverChannelAsync();
         }
 
         ChannelWrapper cw = this.channelTables.get(addr);
         if (cw != null && cw.isOK()) {
-            return cw.getChannel();
+            return cw.getChannelFuture();
         }
 
-        return this.createChannel(addr);
+        return this.createChannelAsync(addr);
+    }
+
+    private Channel getAndCreateChannel(final String addr) throws InterruptedException {
+        ChannelFuture channelFuture = getAndCreateChannelAsync(addr);
+        if (channelFuture == null) {
+            return null;
+        }
+        return channelFuture.awaitUninterruptibly().channel();
     }
 
-    private Channel getAndCreateNameserverChannel() throws InterruptedException {
+    private ChannelFuture getAndCreateNameserverChannelAsync() throws InterruptedException {
         String addr = this.namesrvAddrChoosed.get();
         if (addr != null) {
             ChannelWrapper cw = this.channelTables.get(addr);
             if (cw != null && cw.isOK()) {
-                return cw.getChannel();
+                return cw.getChannelFuture();
             }
         }
 
@@ -642,25 +664,19 @@ private Channel getAndCreateNameserverChannel() throws InterruptedException {
                 if (addr != null) {
                     ChannelWrapper cw = this.channelTables.get(addr);
                     if (cw != null && cw.isOK()) {
-                        return cw.getChannel();
+                        return cw.getChannelFuture();
                     }
                 }
 
                 if (addrList != null && !addrList.isEmpty()) {
-                    for (int i = 0; i < addrList.size(); i++) {
-                        int index = this.namesrvIndex.incrementAndGet();
-                        index = Math.abs(index);
-                        index = index % addrList.size();
-                        String newAddr = addrList.get(index);
-
-                        this.namesrvAddrChoosed.set(newAddr);
-                        LOGGER.info("new name server is chosen. OLD: {} , NEW: {}. namesrvIndex = {}", addr, newAddr, namesrvIndex);
-                        Channel channelNew = this.createChannel(newAddr);
-                        if (channelNew != null) {
-                            return channelNew;
-                        }
-                    }
-                    throw new RemotingConnectException(addrList.toString());
+                    int index = this.namesrvIndex.incrementAndGet();
+                    index = Math.abs(index);
+                    index = index % addrList.size();
+                    String newAddr = addrList.get(index);
+
+                    this.namesrvAddrChoosed.set(newAddr);
+                    LOGGER.info("new name server is chosen. OLD: {} , NEW: {}. namesrvIndex = {}", addr, newAddr, namesrvIndex);
+                    return this.createChannelAsync(newAddr);
                 }
             } catch (Exception e) {
                 LOGGER.error("getAndCreateNameserverChannel: create name server channel exception", e);
@@ -674,39 +690,23 @@ private Channel getAndCreateNameserverChannel() throws InterruptedException {
         return null;
     }
 
-    private Channel createChannel(final String addr) throws InterruptedException {
+    private ChannelFuture createChannelAsync(final String addr) throws InterruptedException {
         ChannelWrapper cw = this.channelTables.get(addr);
         if (cw != null && cw.isOK()) {
-            return cw.getChannel();
+            return cw.getChannelFuture();
         }
 
         if (this.lockChannelTables.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
             try {
-                boolean createNewConnection;
                 cw = this.channelTables.get(addr);
                 if (cw != null) {
-
-                    if (cw.isOK()) {
-                        return cw.getChannel();
-                    } else if (!cw.getChannelFuture().isDone()) {
-                        createNewConnection = false;
+                    if (cw.isOK() || !cw.getChannelFuture().isDone()) {
+                        return cw.getChannelFuture();
                     } else {
                         this.channelTables.remove(addr);
-                        createNewConnection = true;
                     }
-                } else {
-                    createNewConnection = true;
-                }
-
-                if (createNewConnection) {
-                    String[] hostAndPort = getHostAndPort(addr);
-                    ChannelFuture channelFuture = fetchBootstrap(addr)
-                        .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1]));
-                    LOGGER.info("createChannel: begin to connect remote host[{}] asynchronously", addr);
-                    cw = new ChannelWrapper(addr, channelFuture);
-                    this.channelTables.put(addr, cw);
-                    this.channelWrapperTables.put(channelFuture.channel(), cw);
                 }
+                return createChannel(addr).getChannelFuture();
             } catch (Exception e) {
                 LOGGER.error("createChannel: create channel exception", e);
             } finally {
@@ -716,27 +716,16 @@ private Channel createChannel(final String addr) throws InterruptedException {
             LOGGER.warn("createChannel: try to lock channel table, but timeout, {}ms", LOCK_TIMEOUT_MILLIS);
         }
 
-        if (cw != null) {
-            return waitChannelFuture(addr, cw);
-        }
-
         return null;
     }
 
-    private Channel waitChannelFuture(String addr, ChannelWrapper cw) {
-        ChannelFuture channelFuture = cw.getChannelFuture();
-        if (channelFuture.awaitUninterruptibly(this.nettyClientConfig.getConnectTimeoutMillis())) {
-            if (cw.isOK()) {
-                LOGGER.info("createChannel: connect remote host[{}] success, {}", addr, channelFuture.toString());
-                return cw.getChannel();
-            } else {
-                LOGGER.warn("createChannel: connect remote host[{}] failed, {}", addr, channelFuture.toString());
-            }
-        } else {
-            LOGGER.warn("createChannel: connect remote host[{}] timeout {}ms, {}", addr, this.nettyClientConfig.getConnectTimeoutMillis(),
-                channelFuture.toString());
-        }
-        return null;
+    private ChannelWrapper createChannel(String addr) {
+        ChannelFuture channelFuture = doConnect(addr);
+        LOGGER.info("createChannel: begin to connect remote host[{}] asynchronously", addr);
+        ChannelWrapper cw = new ChannelWrapper(addr, channelFuture);
+        this.channelTables.put(addr, cw);
+        this.channelWrapperTables.put(channelFuture.channel(), cw);
+        return cw;
     }
 
     @Override
@@ -744,38 +733,50 @@ public void invokeAsync(String addr, RemotingCommand request, long timeoutMillis
         throws InterruptedException, RemotingConnectException, RemotingTooMuchRequestException, RemotingTimeoutException,
         RemotingSendRequestException {
         long beginStartTime = System.currentTimeMillis();
-        final Channel channel = this.getAndCreateChannel(addr);
-        String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);
-        if (channel != null && channel.isActive()) {
-            long costTime = System.currentTimeMillis() - beginStartTime;
-            if (timeoutMillis < costTime) {
-                throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout");
-            }
-            this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr));
-        } else {
-            this.closeChannel(addr, channel);
-            throw new RemotingConnectException(addr);
+        final ChannelFuture channelFuture = this.getAndCreateChannelAsync(addr);
+        if (channelFuture == null) {
+            invokeCallback.operationFail(new RemotingConnectException(addr));
+            return;
         }
+        channelFuture.addListener(future -> {
+            if (future.isSuccess()) {
+                Channel channel = channelFuture.channel();
+                String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);
+                if (channel != null && channel.isActive()) {
+                    long costTime = System.currentTimeMillis() - beginStartTime;
+                    if (timeoutMillis < costTime) {
+                        invokeCallback.operationFail(new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout"));
+                    }
+                    this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr));
+                } else {
+                    this.closeChannel(addr, channel);
+                    invokeCallback.operationFail(new RemotingConnectException(addr));
+                }
+            } else {
+                invokeCallback.operationFail(new RemotingConnectException(addr));
+            }
+        });
     }
 
     @Override
     public void invokeOneway(String addr, RemotingCommand request, long timeoutMillis) throws InterruptedException,
         RemotingConnectException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {
-        final Channel channel = this.getAndCreateChannel(addr);
-        String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);
-        if (channel != null && channel.isActive()) {
-            try {
-                doBeforeRpcHooks(channelRemoteAddr, request);
-                this.invokeOnewayImpl(channel, request, timeoutMillis);
-            } catch (RemotingSendRequestException e) {
-                LOGGER.warn("invokeOneway: send request exception, so close the channel[{}]", channelRemoteAddr);
-                this.closeChannel(addr, channel);
-                throw e;
-            }
-        } else {
-            this.closeChannel(addr, channel);
+        final ChannelFuture channelFuture = this.getAndCreateChannelAsync(addr);
+        if (channelFuture == null) {
             throw new RemotingConnectException(addr);
         }
+        channelFuture.addListener(future -> {
+            if (future.isSuccess()) {
+                Channel channel = channelFuture.channel();
+                String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);
+                if (channel != null && channel.isActive()) {
+                    doBeforeRpcHooks(channelRemoteAddr, request);
+                    this.invokeOnewayImpl(channel, request, timeoutMillis);
+                } else {
+                    this.closeChannel(addr, channel);
+                }
+            }
+        });
     }
 
     @Override
@@ -783,17 +784,34 @@ public CompletableFuture<RemotingCommand> invoke(String addr, RemotingCommand re
         long timeoutMillis) {
         CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
         try {
-            final Channel channel = this.getAndCreateChannel(addr);
-            if (channel != null && channel.isActive()) {
-                return invokeImpl(channel, request, timeoutMillis).whenComplete((v, t) -> {
-                    if (t == null) {
-                        updateChannelLastResponseTime(addr);
-                    }
-                }).thenApply(ResponseFuture::getResponseCommand);
-            } else {
-                this.closeChannel(addr, channel);
+            final ChannelFuture channelFuture = this.getAndCreateChannelAsync(addr);
+            if (channelFuture == null) {
                 future.completeExceptionally(new RemotingConnectException(addr));
+                return future;
             }
+            channelFuture.addListener(f -> {
+                if (f.isSuccess()) {
+                    Channel channel = channelFuture.channel();
+                    if (channel != null && channel.isActive()) {
+                        invokeImpl(channel, request, timeoutMillis).whenComplete((v, t) -> {
+                            if (t == null) {
+                                updateChannelLastResponseTime(addr);
+                            }
+                        }).thenApply(ResponseFuture::getResponseCommand).whenComplete((v, t) -> {
+                            if (t != null) {
+                                future.completeExceptionally(t);
+                            } else {
+                                future.complete(v);
+                            }
+                        });
+                    } else {
+                        this.closeChannel(addr, channel);
+                        future.completeExceptionally(new RemotingConnectException(addr));
+                    }
+                } else {
+                    future.completeExceptionally(new RemotingConnectException(addr));
+                }
+            });
         } catch (Throwable t) {
             future.completeExceptionally(t);
         }
@@ -804,14 +822,18 @@ public CompletableFuture<RemotingCommand> invoke(String addr, RemotingCommand re
     public CompletableFuture<ResponseFuture> invokeImpl(final Channel channel, final RemotingCommand request,
         final long timeoutMillis) {
         Stopwatch stopwatch = Stopwatch.createStarted();
+        String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);
+        doBeforeRpcHooks(channelRemoteAddr, request);
+
         return super.invokeImpl(channel, request, timeoutMillis).thenCompose(responseFuture -> {
             RemotingCommand response = responseFuture.getResponseCommand();
             if (response.getCode() == ResponseCode.GO_AWAY) {
                 if (nettyClientConfig.isEnableReconnectForGoAway()) {
+                    LOGGER.info("Receive go away from channelId={}, channel={}", channel.id(), channel);
                     ChannelWrapper channelWrapper = channelWrapperTables.computeIfPresent(channel, (channel0, channelWrapper0) -> {
                         try {
-                            if (channelWrapper0.reconnect()) {
-                                LOGGER.info("Receive go away from channel {}, recreate the channel", channel0);
+                            if (channelWrapper0.reconnect(channel0)) {
+                                LOGGER.info("Receive go away from channelId={}, channel={}, recreate the channelId={}", channel0.id(), channel0, channelWrapper0.getChannel().id());
                                 channelWrapperTables.put(channelWrapper0.getChannel(), channelWrapper0);
                             }
                         } catch (Throwable t) {
@@ -819,26 +841,34 @@ public CompletableFuture<ResponseFuture> invokeImpl(final Channel channel, final
                         }
                         return channelWrapper0;
                     });
-                    if (channelWrapper != null) {
-                        if (nettyClientConfig.isEnableTransparentRetry()) {
+                    if (channelWrapper != null && !channelWrapper.isWrapperOf(channel)) {
+                        RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader());
+                        retryRequest.setBody(request.getBody());
+                        retryRequest.setExtFields(request.getExtFields());
+                        CompletableFuture<Void> future = convertChannelFutureToCompletableFuture(channelWrapper.getChannelFuture());
+                        return future.thenCompose(v -> {
                             long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS);
                             stopwatch.stop();
-                            RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader());
-                            retryRequest.setBody(request.getBody());
-                            Channel retryChannel;
-                            if (channelWrapper.isOK()) {
-                                retryChannel = channelWrapper.getChannel();
-                            } else {
-                                retryChannel = waitChannelFuture(channelWrapper.getChannelAddress(), channelWrapper);
-                            }
-                            if (retryChannel != null && channel != retryChannel) {
-                                return super.invokeImpl(retryChannel, retryRequest, timeoutMillis - duration);
-                            }
-                        }
+                            return super.invokeImpl(channelWrapper.getChannel(), retryRequest, timeoutMillis - duration)
+                                .thenCompose(r -> {
+                                    if (r.getResponseCommand().getCode() == ResponseCode.GO_AWAY) {
+                                        return FutureUtils.completeExceptionally(new RemotingSendRequestException(channelRemoteAddr,
+                                            new Throwable("Receive GO_AWAY twice in request from channelId=" + channel.id())));
+                                    }
+                                    return CompletableFuture.completedFuture(r);
+                                });
+                        });
+                    } else {
+                        LOGGER.warn("invokeImpl receive GO_AWAY, channelWrapper is null or channel is the same in wrapper, channelId={}", channel.id());
                     }
                 }
+                return FutureUtils.completeExceptionally(new RemotingSendRequestException(channelRemoteAddr, new Throwable("Receive GO_AWAY from channelId=" + channel.id())));
             }
             return CompletableFuture.completedFuture(responseFuture);
+        }).whenComplete((v, t) -> {
+            if (t == null) {
+                doAfterRpcHooks(channelRemoteAddr, request, v.getResponseCommand());
+            }
         });
     }
 
@@ -911,8 +941,9 @@ protected void scanChannelTablesOfNameServer() {
             return;
         }
 
-        for (String addr : this.channelTables.keySet()) {
-            ChannelWrapper channelWrapper = this.channelTables.get(addr);
+        for (Map.Entry<String, ChannelWrapper> entry : this.channelTables.entrySet()) {
+            String addr = entry.getKey();
+            ChannelWrapper channelWrapper = entry.getValue();
             if (channelWrapper == null) {
                 continue;
             }
@@ -967,7 +998,6 @@ class ChannelWrapper {
         // only affected by sync or async request, oneway is not included.
         private ChannelFuture channelToClose;
         private long lastResponseTime;
-        private volatile long lastReconnectTimestamp = 0L;
         private final String channelAddress;
 
         public ChannelWrapper(String address, ChannelFuture channelFuture) {
@@ -985,6 +1015,10 @@ public boolean isWritable() {
             return getChannel().isWritable();
         }
 
+        public boolean isWrapperOf(Channel channel) {
+            return this.channelFuture.channel() != null && this.channelFuture.channel() == channel;
+        }
+
         private Channel getChannel() {
             return getChannelFuture().channel();
         }
@@ -1010,20 +1044,25 @@ public String getChannelAddress() {
             return channelAddress;
         }
 
-        public boolean reconnect() {
+        public boolean reconnect(Channel channel) {
+            if (!isWrapperOf(channel)) {
+                LOGGER.warn("channelWrapper has reconnect, so do nothing, now channelId={}, input channelId={}",getChannel().id(), channel.id());
+                return false;
+            }
             if (lock.writeLock().tryLock()) {
                 try {
-                    if (lastReconnectTimestamp == 0L || System.currentTimeMillis() - lastReconnectTimestamp > Duration.ofSeconds(nettyClientConfig.getMaxReconnectIntervalTimeSeconds()).toMillis()) {
+                    if (isWrapperOf(channel)) {
                         channelToClose = channelFuture;
-                        String[] hostAndPort = getHostAndPort(channelAddress);
-                        channelFuture = fetchBootstrap(channelAddress)
-                            .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1]));
-                        lastReconnectTimestamp = System.currentTimeMillis();
+                        channelFuture = doConnect(channelAddress);
                         return true;
+                    } else {
+                        LOGGER.warn("channelWrapper has reconnect, so do nothing, now channelId={}, input channelId={}",getChannel().id(), channel.id());
                     }
                 } finally {
                     lock.writeLock().unlock();
                 }
+            } else {
+                LOGGER.warn("channelWrapper reconnect try lock fail, now channelId={}", getChannel().id());
             }
             return false;
         }
@@ -1084,19 +1123,18 @@ public void operationFail(final Throwable throwable) {
         }
     }
 
-    class NettyClientHandler extends SimpleChannelInboundHandler<RemotingCommand> {
-
+    public class NettyClientHandler extends SimpleChannelInboundHandler<RemotingCommand> {
         @Override
         protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception {
             processMessageReceived(ctx, msg);
         }
     }
 
-    class NettyConnectManageHandler extends ChannelDuplexHandler {
+    public class NettyConnectManageHandler extends ChannelDuplexHandler {
         @Override
         public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress,
             ChannelPromise promise) throws Exception {
-            final String local = localAddress == null ? "UNKNOWN" : RemotingHelper.parseSocketAddressAddr(localAddress);
+            final String local = localAddress == null ? NetworkUtil.getLocalAddress() : RemotingHelper.parseSocketAddressAddr(localAddress);
             final String remote = remoteAddress == null ? "UNKNOWN" : RemotingHelper.parseSocketAddressAddr(remoteAddress);
             LOGGER.info("NETTY CLIENT PIPELINE: CONNECT  {} => {}", local, remote);
 
@@ -1110,7 +1148,7 @@ public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, Sock
         @Override
         public void channelActive(ChannelHandlerContext ctx) throws Exception {
             final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
-            LOGGER.info("NETTY CLIENT PIPELINE: ACTIVE, {}", remoteAddress);
+            LOGGER.info("NETTY CLIENT PIPELINE: ACTIVE, {}, channelId={}", remoteAddress, ctx.channel().id());
             super.channelActive(ctx);
 
             if (NettyRemotingClient.this.channelEventListener != null) {
@@ -1133,7 +1171,7 @@ public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws
         @Override
         public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception {
             final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
-            LOGGER.info("NETTY CLIENT PIPELINE: CLOSE {}", remoteAddress);
+            LOGGER.info("NETTY CLIENT PIPELINE: CLOSE channel[addr={}, id={}]", remoteAddress, ctx.channel().id());
             closeChannel(ctx.channel());
             super.close(ctx, promise);
             NettyRemotingClient.this.failFast(ctx.channel());
@@ -1145,7 +1183,7 @@ public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exce
         @Override
         public void channelInactive(ChannelHandlerContext ctx) throws Exception {
             final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
-            LOGGER.info("NETTY CLIENT PIPELINE: channelInactive, the channel[{}]", remoteAddress);
+            LOGGER.info("NETTY CLIENT PIPELINE: channelInactive, the channel[addr={}, id={}]", remoteAddress, ctx.channel().id());
             closeChannel(ctx.channel());
             super.channelInactive(ctx);
         }
@@ -1156,7 +1194,7 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc
                 IdleStateEvent event = (IdleStateEvent) evt;
                 if (event.state().equals(IdleState.ALL_IDLE)) {
                     final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
-                    LOGGER.warn("NETTY CLIENT PIPELINE: IDLE exception [{}]", remoteAddress);
+                    LOGGER.warn("NETTY CLIENT PIPELINE: IDLE exception channel[addr={}, id={}]", remoteAddress, ctx.channel().id());
                     closeChannel(ctx.channel());
                     if (NettyRemotingClient.this.channelEventListener != null) {
                         NettyRemotingClient.this
@@ -1171,8 +1209,7 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc
         @Override
         public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
             final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
-            LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught {}", remoteAddress);
-            LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught exception.", cause);
+            LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught channel[addr={}, id={}]", remoteAddress, ctx.channel().id(), cause);
             closeChannel(ctx.channel());
             if (NettyRemotingClient.this.channelEventListener != null) {
                 NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress, ctx.channel()));
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
index 51f8b85009e..7ed804483be 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
@@ -54,19 +54,6 @@
 import io.netty.util.Timeout;
 import io.netty.util.TimerTask;
 import io.netty.util.concurrent.DefaultEventExecutorGroup;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.security.cert.CertificateException;
-import java.time.Duration;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.Pair;
@@ -88,15 +75,28 @@
 import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
-@SuppressWarnings("NullableProblems")
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.cert.CertificateException;
+import java.time.Duration;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
 public class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);
     private static final Logger TRAFFIC_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_TRAFFIC_NAME);
 
     private final ServerBootstrap serverBootstrap;
-    private final EventLoopGroup eventLoopGroupSelector;
-    private final EventLoopGroup eventLoopGroupBoss;
-    private final NettyServerConfig nettyServerConfig;
+    protected final EventLoopGroup eventLoopGroupSelector;
+    protected final EventLoopGroup eventLoopGroupBoss;
+    protected final NettyServerConfig nettyServerConfig;
 
     private final ExecutorService publicExecutor;
     private final ScheduledExecutorService scheduledExecutorService;
@@ -120,18 +120,18 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
     public static final String FILE_REGION_ENCODER_NAME = "fileRegionEncoder";
 
     // sharable handlers
-    private TlsModeHandler tlsModeHandler;
-    private NettyEncoder encoder;
-    private NettyConnectManageHandler connectionManageHandler;
-    private NettyServerHandler serverHandler;
-    private RemotingCodeDistributionHandler distributionHandler;
+    protected final TlsModeHandler tlsModeHandler = new TlsModeHandler(TlsSystemConfig.tlsMode);
+    protected final NettyEncoder encoder = new NettyEncoder();
+    protected final NettyConnectManageHandler connectionManageHandler = new NettyConnectManageHandler();
+    protected final NettyServerHandler serverHandler = new NettyServerHandler();
+    protected final RemotingCodeDistributionHandler distributionHandler = new RemotingCodeDistributionHandler();
 
     public NettyRemotingServer(final NettyServerConfig nettyServerConfig) {
         this(nettyServerConfig, null);
     }
 
     public NettyRemotingServer(final NettyServerConfig nettyServerConfig,
-        final ChannelEventListener channelEventListener) {
+                               final ChannelEventListener channelEventListener) {
         super(nettyServerConfig.getServerOnewaySemaphoreValue(), nettyServerConfig.getServerAsyncSemaphoreValue());
         this.serverBootstrap = new ServerBootstrap();
         this.nettyServerConfig = nettyServerConfig;
@@ -140,13 +140,13 @@ public NettyRemotingServer(final NettyServerConfig nettyServerConfig,
         this.publicExecutor = buildPublicExecutor(nettyServerConfig);
         this.scheduledExecutorService = buildScheduleExecutor();
 
-        this.eventLoopGroupBoss = buildBossEventLoopGroup();
+        this.eventLoopGroupBoss = buildEventLoopGroupBoss();
         this.eventLoopGroupSelector = buildEventLoopGroupSelector();
 
         loadSslContext();
     }
 
-    private EventLoopGroup buildEventLoopGroupSelector() {
+    protected EventLoopGroup buildEventLoopGroupSelector() {
         if (useEpoll()) {
             return new EpollEventLoopGroup(nettyServerConfig.getServerSelectorThreads(), new ThreadFactoryImpl("NettyServerEPOLLSelector_"));
         } else {
@@ -154,7 +154,7 @@ private EventLoopGroup buildEventLoopGroupSelector() {
         }
     }
 
-    private EventLoopGroup buildBossEventLoopGroup() {
+    protected EventLoopGroup buildEventLoopGroupBoss() {
         if (useEpoll()) {
             return new EpollEventLoopGroup(1, new ThreadFactoryImpl("NettyEPOLLBoss_"));
         } else {
@@ -197,13 +197,7 @@ private boolean useEpoll() {
             && Epoll.isAvailable();
     }
 
-    @Override
-    public void start() {
-        this.defaultEventExecutorGroup = new DefaultEventExecutorGroup(nettyServerConfig.getServerWorkerThreads(),
-            new ThreadFactoryImpl("NettyServerCodecThread_"));
-
-        prepareSharableHandlers();
-
+    protected void initServerBootstrap(ServerBootstrap serverBootstrap) {
         serverBootstrap.group(this.eventLoopGroupBoss, this.eventLoopGroupSelector)
             .channel(useEpoll() ? EpollServerSocketChannel.class : NioServerSocketChannel.class)
             .option(ChannelOption.SO_BACKLOG, 1024)
@@ -220,6 +214,14 @@ public void initChannel(SocketChannel ch) {
             });
 
         addCustomConfig(serverBootstrap);
+    }
+
+    @Override
+    public void start() {
+        this.defaultEventExecutorGroup = new DefaultEventExecutorGroup(nettyServerConfig.getServerWorkerThreads(),
+            new ThreadFactoryImpl("NettyServerCodecThread_"));
+
+        initServerBootstrap(serverBootstrap);
 
         try {
             ChannelFuture sync = serverBootstrap.bind().sync();
@@ -270,8 +272,9 @@ public void run(Timeout timeout) {
      */
     protected ChannelPipeline configChannel(SocketChannel ch) {
         return ch.pipeline()
-            .addLast(defaultEventExecutorGroup, HANDSHAKE_HANDLER_NAME, new HandshakeHandler())
-            .addLast(defaultEventExecutorGroup,
+            .addLast(nettyServerConfig.isServerNettyWorkerGroupEnable() ? defaultEventExecutorGroup : null,
+                HANDSHAKE_HANDLER_NAME, new HandshakeHandler())
+            .addLast(nettyServerConfig.isServerNettyWorkerGroupEnable() ? defaultEventExecutorGroup : null,
                 encoder,
                 new NettyDecoder(),
                 distributionHandler,
@@ -410,14 +413,6 @@ public ExecutorService getCallbackExecutor() {
         return this.publicExecutor;
     }
 
-    private void prepareSharableHandlers() {
-        tlsModeHandler = new TlsModeHandler(TlsSystemConfig.tlsMode);
-        encoder = new NettyEncoder();
-        connectionManageHandler = new NettyConnectManageHandler();
-        serverHandler = new NettyServerHandler();
-        distributionHandler = new RemotingCodeDistributionHandler();
-    }
-
     private void printRemotingCodeDistribution() {
         if (distributionHandler != null) {
             String inBoundSnapshotString = distributionHandler.getInBoundSnapshotString();
@@ -468,8 +463,8 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf byteBuf, List<Object> o
                 }
                 if (detectionResult.state() == ProtocolDetectionState.DETECTED) {
                     ctx.pipeline().addAfter(defaultEventExecutorGroup, ctx.name(), HA_PROXY_DECODER, new HAProxyMessageDecoder())
-                            .addAfter(defaultEventExecutorGroup, HA_PROXY_DECODER, HA_PROXY_HANDLER, new HAProxyMessageHandler())
-                            .addAfter(defaultEventExecutorGroup, HA_PROXY_HANDLER, TLS_MODE_HANDLER, tlsModeHandler);
+                        .addAfter(defaultEventExecutorGroup, HA_PROXY_DECODER, HA_PROXY_HANDLER, new HAProxyMessageHandler())
+                        .addAfter(defaultEventExecutorGroup, HA_PROXY_HANDLER, TLS_MODE_HANDLER, tlsModeHandler);
                 } else {
                     ctx.pipeline().addAfter(defaultEventExecutorGroup, ctx.name(), TLS_MODE_HANDLER, tlsModeHandler);
                 }
@@ -663,7 +658,7 @@ class SubRemotingServer extends NettyRemotingAbstract implements RemotingServer
 
         @Override
         public void registerProcessor(final int requestCode, final NettyRequestProcessor processor,
-            final ExecutorService executor) {
+                                      final ExecutorService executor) {
             ExecutorService executorThis = executor;
             if (null == executor) {
                 executorThis = NettyRemotingServer.this.publicExecutor;
@@ -707,19 +702,19 @@ public void removeRemotingServer(final int port) {
 
         @Override
         public RemotingCommand invokeSync(final Channel channel, final RemotingCommand request,
-            final long timeoutMillis) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException {
+                                          final long timeoutMillis) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException {
             return this.invokeSyncImpl(channel, request, timeoutMillis);
         }
 
         @Override
         public void invokeAsync(final Channel channel, final RemotingCommand request, final long timeoutMillis,
-            final InvokeCallback invokeCallback) throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {
+                                final InvokeCallback invokeCallback) throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {
             this.invokeAsyncImpl(channel, request, timeoutMillis, invokeCallback);
         }
 
         @Override
         public void invokeOneway(final Channel channel, final RemotingCommand request,
-            final long timeoutMillis) throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {
+                                 final long timeoutMillis) throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {
             this.invokeOnewayImpl(channel, request, timeoutMillis);
         }
 
@@ -782,16 +777,16 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
         private void handleWithMessage(HAProxyMessage msg, Channel channel) {
             try {
                 if (StringUtils.isNotBlank(msg.sourceAddress())) {
-                    channel.attr(AttributeKeys.PROXY_PROTOCOL_ADDR).set(msg.sourceAddress());
+                    RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_ADDR, msg.sourceAddress());
                 }
                 if (msg.sourcePort() > 0) {
-                    channel.attr(AttributeKeys.PROXY_PROTOCOL_PORT).set(String.valueOf(msg.sourcePort()));
+                    RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_PORT, String.valueOf(msg.sourcePort()));
                 }
                 if (StringUtils.isNotBlank(msg.destinationAddress())) {
-                    channel.attr(AttributeKeys.PROXY_PROTOCOL_SERVER_ADDR).set(msg.destinationAddress());
+                    RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_SERVER_ADDR, msg.destinationAddress());
                 }
                 if (msg.destinationPort() > 0) {
-                    channel.attr(AttributeKeys.PROXY_PROTOCOL_SERVER_PORT).set(String.valueOf(msg.destinationPort()));
+                    RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_SERVER_PORT, String.valueOf(msg.destinationPort()));
                 }
                 if (CollectionUtils.isNotEmpty(msg.tlvs())) {
                     msg.tlvs().forEach(tlv -> {
@@ -811,6 +806,6 @@ protected void handleHAProxyTLV(HAProxyTLV tlv, Channel channel) {
         }
         AttributeKey<String> key = AttributeKeys.valueOf(
             HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue()));
-        channel.attr(key).set(new String(valueBytes, CharsetUtil.UTF_8));
+        RemotingHelper.setPropertyToAttr(channel, key, new String(valueBytes, CharsetUtil.UTF_8));
     }
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java
index 756661f623f..664dee8371c 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java
@@ -36,6 +36,7 @@ public class NettyServerConfig implements Cloneable {
     private int writeBufferHighWaterMark = NettySystemConfig.writeBufferHighWaterMark;
     private int writeBufferLowWaterMark = NettySystemConfig.writeBufferLowWaterMark;
     private int serverSocketBacklog = NettySystemConfig.socketBacklog;
+    private boolean serverNettyWorkerGroupEnable = true;
     private boolean serverPooledByteBufAllocatorEnable = true;
 
     private boolean enableShutdownGracefully = false;
@@ -156,7 +157,7 @@ public void setUseEpollNativeSelector(boolean useEpollNativeSelector) {
 
     @Override
     public Object clone() throws CloneNotSupportedException {
-        return (NettyServerConfig) super.clone();
+        return super.clone();
     }
 
     public int getWriteBufferLowWaterMark() {
@@ -175,6 +176,14 @@ public void setWriteBufferHighWaterMark(int writeBufferHighWaterMark) {
         this.writeBufferHighWaterMark = writeBufferHighWaterMark;
     }
 
+    public boolean isServerNettyWorkerGroupEnable() {
+        return serverNettyWorkerGroupEnable;
+    }
+
+    public void setServerNettyWorkerGroupEnable(boolean serverNettyWorkerGroupEnable) {
+        this.serverNettyWorkerGroupEnable = serverNettyWorkerGroupEnable;
+    }
+
     public boolean isEnableShutdownGracefully() {
         return enableShutdownGracefully;
     }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java
index 9e73ad7ae0a..d7a8dfb22c3 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java
@@ -197,18 +197,16 @@ private static void extractTlsConfigFromFile(final File configFile) {
     private static void logTheFinalUsedTlsConfig() {
         LOGGER.info("Log the final used tls related configuration");
         LOGGER.info("{} = {}", TLS_TEST_MODE_ENABLE, tlsTestModeEnable);
-        LOGGER.info("{} = {}", TLS_SERVER_NEED_CLIENT_AUTH, tlsServerNeedClientAuth);
-        LOGGER.info("{} = {}", TLS_SERVER_KEYPATH, tlsServerKeyPath);
-        LOGGER.info("{} = {}", TLS_SERVER_KEYPASSWORD, tlsServerKeyPassword);
-        LOGGER.info("{} = {}", TLS_SERVER_CERTPATH, tlsServerCertPath);
-        LOGGER.info("{} = {}", TLS_SERVER_AUTHCLIENT, tlsServerAuthClient);
-        LOGGER.info("{} = {}", TLS_SERVER_TRUSTCERTPATH, tlsServerTrustCertPath);
-
-        LOGGER.info("{} = {}", TLS_CLIENT_KEYPATH, tlsClientKeyPath);
-        LOGGER.info("{} = {}", TLS_CLIENT_KEYPASSWORD, tlsClientKeyPassword);
-        LOGGER.info("{} = {}", TLS_CLIENT_CERTPATH, tlsClientCertPath);
-        LOGGER.info("{} = {}", TLS_CLIENT_AUTHSERVER, tlsClientAuthServer);
-        LOGGER.info("{} = {}", TLS_CLIENT_TRUSTCERTPATH, tlsClientTrustCertPath);
+        LOGGER.debug("{} = {}", TLS_SERVER_NEED_CLIENT_AUTH, tlsServerNeedClientAuth);
+        LOGGER.debug("{} = {}", TLS_SERVER_KEYPATH, tlsServerKeyPath);
+        LOGGER.debug("{} = {}", TLS_SERVER_CERTPATH, tlsServerCertPath);
+        LOGGER.debug("{} = {}", TLS_SERVER_AUTHCLIENT, tlsServerAuthClient);
+        LOGGER.debug("{} = {}", TLS_SERVER_TRUSTCERTPATH, tlsServerTrustCertPath);
+
+        LOGGER.debug("{} = {}", TLS_CLIENT_KEYPATH, tlsClientKeyPath);
+        LOGGER.debug("{} = {}", TLS_CLIENT_CERTPATH, tlsClientCertPath);
+        LOGGER.debug("{} = {}", TLS_CLIENT_AUTHSERVER, tlsClientAuthServer);
+        LOGGER.debug("{} = {}", TLS_CLIENT_TRUSTCERTPATH, tlsClientTrustCertPath);
     }
 
     private static ClientAuth parseClientAuthMode(String authMode) {
@@ -216,8 +214,9 @@ private static ClientAuth parseClientAuthMode(String authMode) {
             return ClientAuth.NONE;
         }
 
+        String authModeUpper = authMode.toUpperCase();
         for (ClientAuth clientAuth : ClientAuth.values()) {
-            if (clientAuth.name().equals(authMode.toUpperCase())) {
+            if (clientAuth.name().equals(authModeUpper)) {
                 return clientAuth;
             }
         }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/pipeline/RequestPipeline.java b/remoting/src/main/java/org/apache/rocketmq/remoting/pipeline/RequestPipeline.java
new file mode 100644
index 00000000000..575f0ef1ab9
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/pipeline/RequestPipeline.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.remoting.pipeline;
+
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public interface RequestPipeline {
+
+    void execute(ChannelHandlerContext ctx, RemotingCommand request) throws Exception;
+
+    default RequestPipeline pipe(RequestPipeline source) {
+        return (ctx, request) -> {
+            source.execute(ctx, request);
+            execute(ctx, request);
+        };
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ForbiddenType.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ForbiddenType.java
index 0701dc57fc5..7c561f5721a 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ForbiddenType.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ForbiddenType.java
@@ -37,11 +37,11 @@ public interface ForbiddenType {
      */
     int TOPIC_FORBIDDEN                = 3;
     /**
-     * 4=forbidden by brocasting mode
+     * 4=forbidden by broadcasting mode
      */
     int BROADCASTING_DISABLE_FORBIDDEN = 4;
     /**
-     * 5=forbidden for a substription(group with a topic)
+     * 5=forbidden for a subscription(group with a topic)
      */
     int SUBSCRIPTION_FORBIDDEN         = 5;
 
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java
index 2df9fbf0278..cf43cbab3dd 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java
@@ -35,7 +35,8 @@ public enum LanguageCode {
     GO((byte) 9),
     PHP((byte) 10),
     OMS((byte) 11),
-    RUST((byte) 12);
+    RUST((byte) 12),
+    NODE_JS((byte) 13);
 
     private byte code;
 
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/NamespaceUtil.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/NamespaceUtil.java
index 54016b392d4..074a089f014 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/NamespaceUtil.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/NamespaceUtil.java
@@ -136,7 +136,7 @@ public static String getNamespaceFromResource(String resource) {
         return index > 0 ? resourceWithoutRetryAndDLQ.substring(0, index) : STRING_BLANK;
     }
 
-    private static String withOutRetryAndDLQ(String originalResource) {
+    public static String withOutRetryAndDLQ(String originalResource) {
         if (StringUtils.isEmpty(originalResource)) {
             return STRING_BLANK;
         }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java
index 0fa275e822e..9b2b0f07b4f 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java
@@ -29,6 +29,7 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -38,6 +39,7 @@
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.CommandCallback;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -96,6 +98,7 @@ public class RemotingCommand {
     private transient byte[] body;
     private boolean suspended;
     private transient Stopwatch processTimer;
+    private transient List<CommandCallback> callbackList;
 
     protected RemotingCommand() {
     }
@@ -259,32 +262,29 @@ public void writeCustomHeader(CommandCustomHeader customHeader) {
         this.customHeader = customHeader;
     }
 
-    public CommandCustomHeader decodeCommandCustomHeader(
-        Class<? extends CommandCustomHeader> classHeader) throws RemotingCommandException {
+    public <T extends CommandCustomHeader> T decodeCommandCustomHeader(
+        Class<T> classHeader) throws RemotingCommandException {
         return decodeCommandCustomHeader(classHeader, false);
     }
 
-    public CommandCustomHeader decodeCommandCustomHeader(
-        Class<? extends CommandCustomHeader> classHeader, boolean isCached) throws RemotingCommandException {
+    public <T extends CommandCustomHeader> T decodeCommandCustomHeader(
+        Class<T> classHeader, boolean isCached) throws RemotingCommandException {
         if (isCached && cachedHeader != null) {
-            return cachedHeader;
+            return classHeader.cast(cachedHeader);
         }
         cachedHeader = decodeCommandCustomHeaderDirectly(classHeader, true);
-        return cachedHeader;
+        if (cachedHeader == null) {
+            return null;
+        }
+        return classHeader.cast(cachedHeader);
     }
 
-    public CommandCustomHeader decodeCommandCustomHeaderDirectly(Class<? extends CommandCustomHeader> classHeader,
+    public <T extends CommandCustomHeader> T decodeCommandCustomHeaderDirectly(Class<T> classHeader,
         boolean useFastEncode) throws RemotingCommandException {
-        CommandCustomHeader objectHeader;
+        T objectHeader;
         try {
             objectHeader = classHeader.getDeclaredConstructor().newInstance();
-        } catch (InstantiationException e) {
-            return null;
-        } catch (IllegalAccessException e) {
-            return null;
-        } catch (InvocationTargetException e) {
-            return null;
-        } catch (NoSuchMethodException e) {
+        } catch (Exception e) {
             return null;
         }
 
@@ -642,4 +642,12 @@ public Stopwatch getProcessTimer() {
     public void setProcessTimer(Stopwatch processTimer) {
         this.processTimer = processTimer;
     }
+
+    public List<CommandCallback> getCallbackList() {
+        return callbackList;
+    }
+
+    public void setCallbackList(List<CommandCallback> callbackList) {
+        this.callbackList = callbackList;
+    }
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSerializable.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSerializable.java
index 60cc4e3e227..139a7043d84 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSerializable.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSerializable.java
@@ -18,9 +18,9 @@
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.serializer.SerializerFeature;
-
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.util.List;
 
 public abstract class RemotingSerializable {
     private final static Charset CHARSET_UTF8 = StandardCharsets.UTF_8;
@@ -38,9 +38,20 @@ public static String toJson(final Object obj, boolean prettyFormat) {
     }
 
     public static <T> T decode(final byte[] data, Class<T> classOfT) {
+        if (data == null) {
+            return null;
+        }
         return fromJson(data, classOfT);
     }
 
+    public static <T> List<T> decodeList(final byte[] data, Class<T> classOfT) {
+        if (data == null) {
+            return null;
+        }
+        String json = new String(data, CHARSET_UTF8);
+        return JSON.parseArray(json, classOfT);
+    }
+
     public static <T> T fromJson(String json, Class<T> classOfT) {
         return JSON.parseObject(json, classOfT);
     }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java
index 00d14acd223..8b2749eaae2 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java
@@ -28,6 +28,7 @@ public class RequestCode {
     public static final int QUERY_CONSUMER_OFFSET = 14;
     public static final int UPDATE_CONSUMER_OFFSET = 15;
     public static final int UPDATE_AND_CREATE_TOPIC = 17;
+    public static final int UPDATE_AND_CREATE_TOPIC_LIST = 18;
     public static final int GET_ALL_TOPIC_CONFIG = 21;
     public static final int GET_TOPIC_CONFIG_LIST = 22;
 
@@ -72,17 +73,6 @@ public class RequestCode {
 
     public static final int GET_CLIENT_CONFIG = 47;
 
-    public static final int UPDATE_AND_CREATE_ACL_CONFIG = 50;
-
-    public static final int DELETE_ACL_CONFIG = 51;
-
-    public static final int GET_BROKER_CLUSTER_ACL_INFO = 52;
-
-    public static final int UPDATE_GLOBAL_WHITE_ADDRS_CONFIG = 53;
-
-    @Deprecated
-    public static final int GET_BROKER_CLUSTER_ACL_CONFIG = 54;
-
     public static final int GET_TIMER_CHECK_POINT = 60;
 
     public static final int GET_TIMER_METRICS = 61;
@@ -94,6 +84,7 @@ public class RequestCode {
     public static final int CHANGE_MESSAGE_INVISIBLETIME = 200053;
     public static final int NOTIFICATION = 200054;
     public static final int POLLING_INFO = 200055;
+    public static final int POP_ROLLBACK = 200056;
 
     public static final int PUT_KV_CONFIG = 100;
 
@@ -147,6 +138,8 @@ public class RequestCode {
 
     public static final int GET_TOPICS_BY_CLUSTER = 224;
 
+    public static final int UPDATE_AND_CREATE_SUBSCRIPTIONGROUP_LIST = 225;
+
     public static final int QUERY_TOPICS_BY_CONSUMER = 343;
     public static final int QUERY_SUBSCRIPTION_BY_CONSUMER = 345;
 
@@ -209,13 +202,20 @@ public class RequestCode {
     public static final int PUSH_REPLY_MESSAGE_TO_CLIENT = 326;
 
     public static final int ADD_WRITE_PERM_OF_BROKER = 327;
+    
+    public static final int GET_ALL_PRODUCER_INFO = 328;
+    
+    public static final int DELETE_EXPIRED_COMMITLOG = 329;
 
     public static final int GET_TOPIC_CONFIG = 351;
 
     public static final int GET_SUBSCRIPTIONGROUP_CONFIG = 352;
     public static final int UPDATE_AND_GET_GROUP_FORBIDDEN = 353;
+    public static final int CHECK_ROCKSDB_CQ_WRITE_PROGRESS = 354;
+    public static final int EXPORT_ROCKSDB_CONFIG_TO_JSON = 355;
 
     public static final int LITE_PULL_MESSAGE = 361;
+    public static final int RECALL_MESSAGE = 370;
 
     public static final int QUERY_ASSIGNMENT = 400;
     public static final int SET_MESSAGE_REQUEST_MODE = 401;
@@ -239,10 +239,6 @@ public class RequestCode {
 
     public static final int RESET_MASTER_FLUSH_OFFSET = 908;
 
-    public static final int GET_ALL_PRODUCER_INFO = 328;
-
-    public static final int DELETE_EXPIRED_COMMITLOG = 329;
-
     /**
      * Controller code
      */
@@ -289,4 +285,16 @@ public class RequestCode {
     public static final int REMOVE_COLD_DATA_FLOW_CTR_CONFIG = 2002;
     public static final int GET_COLD_DATA_FLOW_CTR_INFO = 2003;
     public static final int SET_COMMITLOG_READ_MODE = 2004;
+
+    public static final int AUTH_CREATE_USER = 3001;
+    public static final int AUTH_UPDATE_USER = 3002;
+    public static final int AUTH_DELETE_USER = 3003;
+    public static final int AUTH_GET_USER = 3004;
+    public static final int AUTH_LIST_USER = 3005;
+
+    public static final int AUTH_CREATE_ACL = 3006;
+    public static final int AUTH_UPDATE_ACL = 3007;
+    public static final int AUTH_DELETE_ACL = 3008;
+    public static final int AUTH_GET_ACL = 3009;
+    public static final int AUTH_LIST_ACL = 3010;
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestHeaderRegistry.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestHeaderRegistry.java
new file mode 100644
index 00000000000..082827b56a2
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestHeaderRegistry.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.reflections.Reflections;
+import org.reflections.scanners.SubTypesScanner;
+import org.reflections.util.ClasspathHelper;
+import org.reflections.util.ConfigurationBuilder;
+
+public class RequestHeaderRegistry {
+
+    private static final String PACKAGE_NAME = "org.apache.rocketmq.remoting.protocol.header";
+
+    private final Map<Integer, Class<? extends CommandCustomHeader>> requestHeaderMap = new HashMap<>();
+
+    public static RequestHeaderRegistry getInstance() {
+        return RequestHeaderRegistryHolder.INSTANCE;
+    }
+
+    public void initialize() {
+        Reflections reflections = new Reflections(new ConfigurationBuilder()
+            .setUrls(ClasspathHelper.forPackage(PACKAGE_NAME))
+            .setScanners(new SubTypesScanner(false)));
+
+        Set<Class<? extends CommandCustomHeader>> classes = reflections.getSubTypesOf(CommandCustomHeader.class);
+
+        classes.forEach(this::registerHeader);
+    }
+
+    public Class<? extends CommandCustomHeader> getRequestHeader(int requestCode) {
+        return this.requestHeaderMap.get(requestCode);
+    }
+
+    private void registerHeader(Class<? extends CommandCustomHeader> clazz) {
+        if (!clazz.isAnnotationPresent(RocketMQAction.class)) {
+            return;
+        }
+        RocketMQAction action = clazz.getAnnotation(RocketMQAction.class);
+        this.requestHeaderMap.putIfAbsent(action.value(), clazz);
+    }
+
+    private static class RequestHeaderRegistryHolder {
+        private static final RequestHeaderRegistry INSTANCE = new RequestHeaderRegistry();
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java
index 0e595fabc55..68f77ab31be 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java
@@ -55,6 +55,8 @@ public class ResponseCode extends RemotingSysResponseCode {
 
     public static final int FILTER_DATA_NOT_LATEST = 28;
 
+    public static final int INVALID_PARAMETER = 29;
+
     public static final int TRANSACTION_SHOULD_COMMIT = 200;
 
     public static final int TRANSACTION_SHOULD_ROLLBACK = 201;
@@ -72,12 +74,6 @@ public class ResponseCode extends RemotingSysResponseCode {
 
     public static final int NO_MESSAGE = 208;
 
-    public static final int UPDATE_AND_CREATE_ACL_CONFIG_FAILED = 209;
-
-    public static final int DELETE_ACL_CONFIG_FAILED = 210;
-
-    public static final int UPDATE_GLOBAL_WHITE_ADDRS_CONFIG_FAILED = 211;
-
     public static final int POLLING_FULL = 209;
 
     public static final int POLLING_TIMEOUT = 210;
@@ -130,4 +126,8 @@ public class ResponseCode extends RemotingSysResponseCode {
     public static final int CONTROLLER_JRAFT_INTERNAL_ERROR = 2015;
 
     public static final int CONTROLLER_BROKER_LIVE_INFO_NOT_EXISTS = 2016;
+
+    public static final int USER_NOT_EXIST = 3001;
+
+    public static final int POLICY_NOT_EXIST = 3002;
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/admin/TopicStatsTable.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/admin/TopicStatsTable.java
index 9f467e7449e..5cb2af6373f 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/admin/TopicStatsTable.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/admin/TopicStatsTable.java
@@ -22,6 +22,8 @@
 import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
 
 public class TopicStatsTable extends RemotingSerializable {
+    private double topicPutTps;
+
     private Map<MessageQueue, TopicOffset> offsetTable = new ConcurrentHashMap<>();
 
     public Map<MessageQueue, TopicOffset> getOffsetTable() {
@@ -31,4 +33,12 @@ public Map<MessageQueue, TopicOffset> getOffsetTable() {
     public void setOffsetTable(Map<MessageQueue, TopicOffset> offsetTable) {
         this.offsetTable = offsetTable;
     }
+
+    public double getTopicPutTps() {
+        return topicPutTps;
+    }
+
+    public void setTopicPutTps(double topicPutTps) {
+        this.topicPutTps = topicPutTps;
+    }
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/AclInfo.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/AclInfo.java
new file mode 100644
index 00000000000..4607a3028c7
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/AclInfo.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.body;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class AclInfo {
+
+    private String subject;
+
+    private List<PolicyInfo> policies;
+
+    public static AclInfo of(String subject, List<String> resources, List<String> actions,
+        List<String> sourceIps,
+        String decision) {
+        AclInfo aclInfo = new AclInfo();
+        aclInfo.setSubject(subject);
+        PolicyInfo policyInfo = PolicyInfo.of(resources, actions, sourceIps, decision);
+        aclInfo.setPolicies(Collections.singletonList(policyInfo));
+        return aclInfo;
+    }
+
+    public static class PolicyInfo {
+
+        private String policyType;
+
+        private List<PolicyEntryInfo> entries;
+
+        public static PolicyInfo of(List<String> resources, List<String> actions,
+            List<String> sourceIps, String decision) {
+            PolicyInfo policyInfo = new PolicyInfo();
+            List<PolicyEntryInfo> entries = resources.stream()
+                .map(resource -> PolicyEntryInfo.of(resource, actions, sourceIps, decision))
+                .collect(Collectors.toList());
+            policyInfo.setEntries(entries);
+            return policyInfo;
+        }
+
+        public String getPolicyType() {
+            return policyType;
+        }
+
+        public void setPolicyType(String policyType) {
+            this.policyType = policyType;
+        }
+
+        public List<PolicyEntryInfo> getEntries() {
+            return entries;
+        }
+
+        public void setEntries(List<PolicyEntryInfo> entries) {
+            this.entries = entries;
+        }
+    }
+
+    public static class PolicyEntryInfo {
+        private String resource;
+
+        private List<String> actions;
+
+        private List<String> sourceIps;
+
+        private String decision;
+
+        public static PolicyEntryInfo of(String resource, List<String> actions, List<String> sourceIps,
+            String decision) {
+            PolicyEntryInfo policyEntryInfo = new PolicyEntryInfo();
+            policyEntryInfo.setResource(resource);
+            policyEntryInfo.setActions(actions);
+            policyEntryInfo.setSourceIps(sourceIps);
+            policyEntryInfo.setDecision(decision);
+            return policyEntryInfo;
+        }
+
+        public String getResource() {
+            return resource;
+        }
+
+        public void setResource(String resource) {
+            this.resource = resource;
+        }
+
+        public List<String> getActions() {
+            return actions;
+        }
+
+        public void setActions(List<String> actions) {
+            this.actions = actions;
+        }
+
+        public List<String> getSourceIps() {
+            return sourceIps;
+        }
+
+        public void setSourceIps(List<String> sourceIps) {
+            this.sourceIps = sourceIps;
+        }
+
+        public String getDecision() {
+            return decision;
+        }
+
+        public void setDecision(String decision) {
+            this.decision = decision;
+        }
+    }
+
+    public String getSubject() {
+        return subject;
+    }
+
+    public void setSubject(String subject) {
+        this.subject = subject;
+    }
+
+    public List<PolicyInfo> getPolicies() {
+        return policies;
+    }
+
+    public void setPolicies(List<PolicyInfo> policies) {
+        this.policies = policies;
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ClusterAclVersionInfo.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ClusterAclVersionInfo.java
deleted file mode 100644
index 3ec6cf64f32..00000000000
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/ClusterAclVersionInfo.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.remoting.protocol.body;
-
-import java.util.Map;
-import org.apache.rocketmq.remoting.protocol.DataVersion;
-import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
-
-public class ClusterAclVersionInfo extends RemotingSerializable {
-
-    private String brokerName;
-
-    private String brokerAddr;
-
-    @Deprecated
-    private DataVersion aclConfigDataVersion;
-
-    private Map<String, DataVersion> allAclConfigDataVersion;
-
-    private String clusterName;
-
-    public String getBrokerName() {
-        return brokerName;
-    }
-
-    public void setBrokerName(String brokerName) {
-        this.brokerName = brokerName;
-    }
-
-    public String getBrokerAddr() {
-        return brokerAddr;
-    }
-
-    public void setBrokerAddr(String brokerAddr) {
-        this.brokerAddr = brokerAddr;
-    }
-
-    public String getClusterName() {
-        return clusterName;
-    }
-
-    public void setClusterName(String clusterName) {
-        this.clusterName = clusterName;
-    }
-
-    public DataVersion getAclConfigDataVersion() {
-        return aclConfigDataVersion;
-    }
-
-    public void setAclConfigDataVersion(DataVersion aclConfigDataVersion) {
-        this.aclConfigDataVersion = aclConfigDataVersion;
-    }
-
-    public Map<String, DataVersion> getAllAclConfigDataVersion() {
-        return allAclConfigDataVersion;
-    }
-
-    public void setAllAclConfigDataVersion(
-        Map<String, DataVersion> allAclConfigDataVersion) {
-        this.allAclConfigDataVersion = allAclConfigDataVersion;
-    }
-}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CreateTopicListRequestBody.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CreateTopicListRequestBody.java
new file mode 100644
index 00000000000..a72be31ac92
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CreateTopicListRequestBody.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.body;
+
+import java.util.List;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.remoting.annotation.CFNotNull;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+
+public class CreateTopicListRequestBody extends RemotingSerializable {
+    @CFNotNull
+    private List<TopicConfig> topicConfigList;
+
+    public CreateTopicListRequestBody() {}
+
+    public CreateTopicListRequestBody(List<TopicConfig> topicConfigList) {
+        this.topicConfigList = topicConfigList;
+    }
+
+    public List<TopicConfig> getTopicConfigList() {
+        return topicConfigList;
+    }
+
+    public void setTopicConfigList(List<TopicConfig> topicConfigList) {
+        this.topicConfigList = topicConfigList;
+    }
+
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/SubscriptionGroupList.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/SubscriptionGroupList.java
new file mode 100644
index 00000000000..c343ce21118
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/SubscriptionGroupList.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.body;
+
+import java.util.List;
+import org.apache.rocketmq.remoting.annotation.CFNotNull;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+
+public class SubscriptionGroupList extends RemotingSerializable {
+    @CFNotNull
+    private List<SubscriptionGroupConfig> groupConfigList;
+
+    public SubscriptionGroupList() {}
+
+    public SubscriptionGroupList(List<SubscriptionGroupConfig> groupConfigList) {
+        this.groupConfigList = groupConfigList;
+    }
+
+    public List<SubscriptionGroupConfig> getGroupConfigList() {
+        return groupConfigList;
+    }
+
+    public void setGroupConfigList(List<SubscriptionGroupConfig> groupConfigList) {
+        this.groupConfigList = groupConfigList;
+    }
+
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/UserInfo.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/UserInfo.java
new file mode 100644
index 00000000000..fdcabbf22e8
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/UserInfo.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.body;
+
+public class UserInfo {
+
+    private String username;
+
+    private String password;
+
+    private String userType;
+
+    private String userStatus;
+
+    public static UserInfo of(String username, String password, String userType) {
+        UserInfo userInfo = new UserInfo();
+        userInfo.setUsername(username);
+        userInfo.setPassword(password);
+        userInfo.setUserType(userType);
+        return userInfo;
+    }
+
+    public static UserInfo of(String username, String password, String userType, String userStatus) {
+        UserInfo userInfo = new UserInfo();
+        userInfo.setUsername(username);
+        userInfo.setPassword(password);
+        userInfo.setUserType(userType);
+        userInfo.setUserStatus(userStatus);
+        return userInfo;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public String getUserType() {
+        return userType;
+    }
+
+    public void setUserType(String userType) {
+        this.userType = userType;
+    }
+
+    public String getUserStatus() {
+        return userStatus;
+    }
+
+    public void setUserStatus(String userStatus) {
+        this.userStatus = userStatus;
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/AckMessageRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/AckMessageRequestHeader.java
index 9c5d4d8b1c7..28313fab9f0 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/AckMessageRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/AckMessageRequestHeader.java
@@ -17,14 +17,22 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
+@RocketMQAction(value = RequestCode.ACK_MESSAGE, action = Action.SUB)
 public class AckMessageRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private Integer queueId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/AddBrokerRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/AddBrokerRequestHeader.java
index 8ec19833323..6feae1d759b 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/AddBrokerRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/AddBrokerRequestHeader.java
@@ -17,10 +17,15 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.ADD_BROKER, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class AddBrokerRequestHeader implements CommandCustomHeader {
     @CFNullable
     private String configPath;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ChangeInvisibleTimeRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ChangeInvisibleTimeRequestHeader.java
index fd63de0fb7e..ebd32cc534c 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ChangeInvisibleTimeRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ChangeInvisibleTimeRequestHeader.java
@@ -17,14 +17,22 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
+@RocketMQAction(value = RequestCode.CHANGE_MESSAGE_INVISIBLETIME, action = Action.SUB)
 public class ChangeInvisibleTimeRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private Integer queueId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetBrokerAclConfigResponseHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java
similarity index 51%
rename from remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetBrokerAclConfigResponseHeader.java
rename to remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java
index 50f5713b8b5..f679077fdde 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetBrokerAclConfigResponseHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java
@@ -16,67 +16,43 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
-public class GetBrokerAclConfigResponseHeader implements CommandCustomHeader {
+@RocketMQAction(value = RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS, action = Action.GET)
+public class CheckRocksdbCqWriteProgressRequestHeader implements CommandCustomHeader {
 
     @CFNotNull
-    private String version;
+    @RocketMQResource(ResourceType.TOPIC)
+    private String topic;
 
-    private String allAclFileVersion;
-
-    @CFNotNull
-    private String brokerName;
-
-    @CFNotNull
-    private String brokerAddr;
-
-    @CFNotNull
-    private String clusterName;
+    private long checkStoreTime;
 
     @Override
     public void checkFields() throws RemotingCommandException {
-    }
-
-    public String getVersion() {
-        return version;
-    }
-
-    public void setVersion(String version) {
-        this.version = version;
-    }
-
-    public String getBrokerName() {
-        return brokerName;
-    }
-
-    public void setBrokerName(String brokerName) {
-        this.brokerName = brokerName;
-    }
 
-    public String getBrokerAddr() {
-        return brokerAddr;
     }
 
-    public void setBrokerAddr(String brokerAddr) {
-        this.brokerAddr = brokerAddr;
+    public String getTopic() {
+        return topic;
     }
 
-    public String getClusterName() {
-        return clusterName;
+    public void setTopic(String topic) {
+        this.topic = topic;
     }
 
-    public void setClusterName(String clusterName) {
-        this.clusterName = clusterName;
+    public long getCheckStoreTime() {
+        return checkStoreTime;
     }
 
-    public String getAllAclFileVersion() {
-        return allAclFileVersion;
+    public void setCheckStoreTime(long checkStoreTime) {
+        this.checkStoreTime = checkStoreTime;
     }
 
-    public void setAllAclFileVersion(String allAclFileVersion) {
-        this.allAclFileVersion = allAclFileVersion;
-    }
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckTransactionStateRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckTransactionStateRequestHeader.java
index 8c04eaa2e2c..7251488dd9a 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckTransactionStateRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckTransactionStateRequestHeader.java
@@ -21,11 +21,19 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
 
+@RocketMQAction(value = RequestCode.CHECK_TRANSACTION_STATE, action = Action.PUB)
 public class CheckTransactionStateRequestHeader extends RpcRequestHeader {
+    @RocketMQResource(ResourceType.TOPIC)
+    private String topic;
     @CFNotNull
     private Long tranStateTableOffset;
     @CFNotNull
@@ -38,6 +46,14 @@ public class CheckTransactionStateRequestHeader extends RpcRequestHeader {
     public void checkFields() throws RemotingCommandException {
     }
 
+    public String getTopic() {
+        return topic;
+    }
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
     public Long getTranStateTableOffset() {
         return tranStateTableOffset;
     }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CloneGroupOffsetRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CloneGroupOffsetRequestHeader.java
index 0589c0fa828..7475d260b20 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CloneGroupOffsetRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CloneGroupOffsetRequestHeader.java
@@ -21,15 +21,23 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
 
+@RocketMQAction(value = RequestCode.CLONE_GROUP_OFFSET, action = Action.UPDATE)
 public class CloneGroupOffsetRequestHeader extends RpcRequestHeader {
     @CFNotNull
     private String srcGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String destGroup;
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     private boolean offline;
 
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java
index f56ad5b59da..d160c1e8bfd 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java
@@ -18,13 +18,20 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
 
+@RocketMQAction(value = RequestCode.CONSUME_MESSAGE_DIRECTLY, action = Action.SUB)
 public class ConsumeMessageDirectlyResultRequestHeader extends TopicRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
     @CFNullable
     private String clientId;
@@ -33,6 +40,7 @@ public class ConsumeMessageDirectlyResultRequestHeader extends TopicRequestHeade
     @CFNullable
     private String brokerName;
     @CFNullable
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNullable
     private Integer topicSysFlag;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ConsumerSendMsgBackRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ConsumerSendMsgBackRequestHeader.java
index f69e016c2c9..078aba85e0d 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ConsumerSendMsgBackRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ConsumerSendMsgBackRequestHeader.java
@@ -18,19 +18,27 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
 
+@RocketMQAction(value = RequestCode.CONSUMER_SEND_MSG_BACK, action = Action.SUB)
 public class ConsumerSendMsgBackRequestHeader extends RpcRequestHeader {
     @CFNotNull
     private Long offset;
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String group;
     @CFNotNull
     private Integer delayLevel;
     private String originMsgId;
+    @RocketMQResource(ResourceType.TOPIC)
     private String originTopic;
     @CFNullable
     private boolean unitMode = false;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateAccessConfigRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateAccessConfigRequestHeader.java
deleted file mode 100644
index d02a23858d1..00000000000
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateAccessConfigRequestHeader.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.remoting.protocol.header;
-
-import com.google.common.base.MoreObjects;
-import org.apache.rocketmq.remoting.CommandCustomHeader;
-import org.apache.rocketmq.remoting.annotation.CFNotNull;
-import org.apache.rocketmq.remoting.exception.RemotingCommandException;
-
-public class CreateAccessConfigRequestHeader implements CommandCustomHeader {
-
-    @CFNotNull
-    private String accessKey;
-
-    private String secretKey;
-
-    private String whiteRemoteAddress;
-
-    private boolean admin;
-
-    private String defaultTopicPerm;
-
-    private String defaultGroupPerm;
-
-    // list string,eg: topicA=DENY,topicD=SUB
-    private String topicPerms;
-
-    // list string,eg: groupD=DENY,groupD=SUB
-    private String groupPerms;
-
-
-    @Override
-    public void checkFields() throws RemotingCommandException {
-
-    }
-
-    public String getAccessKey() {
-        return accessKey;
-    }
-
-    public void setAccessKey(String accessKey) {
-        this.accessKey = accessKey;
-    }
-
-    public String getSecretKey() {
-        return secretKey;
-    }
-
-    public void setSecretKey(String secretKey) {
-        this.secretKey = secretKey;
-    }
-
-    public String getWhiteRemoteAddress() {
-        return whiteRemoteAddress;
-    }
-
-    public void setWhiteRemoteAddress(String whiteRemoteAddress) {
-        this.whiteRemoteAddress = whiteRemoteAddress;
-    }
-
-    public boolean isAdmin() {
-        return admin;
-    }
-
-    public void setAdmin(boolean admin) {
-        this.admin = admin;
-    }
-
-    public String getDefaultTopicPerm() {
-        return defaultTopicPerm;
-    }
-
-    public void setDefaultTopicPerm(String defaultTopicPerm) {
-        this.defaultTopicPerm = defaultTopicPerm;
-    }
-
-    public String getDefaultGroupPerm() {
-        return defaultGroupPerm;
-    }
-
-    public void setDefaultGroupPerm(String defaultGroupPerm) {
-        this.defaultGroupPerm = defaultGroupPerm;
-    }
-
-    public String getTopicPerms() {
-        return topicPerms;
-    }
-
-    public void setTopicPerms(String topicPerms) {
-        this.topicPerms = topicPerms;
-    }
-
-    public String getGroupPerms() {
-        return groupPerms;
-    }
-
-    public void setGroupPerms(String groupPerms) {
-        this.groupPerms = groupPerms;
-    }
-
-    @Override
-    public String toString() {
-        return MoreObjects.toStringHelper(this)
-            .add("accessKey", accessKey)
-            .add("secretKey", secretKey)
-            .add("whiteRemoteAddress", whiteRemoteAddress)
-            .add("admin", admin)
-            .add("defaultTopicPerm", defaultTopicPerm)
-            .add("defaultGroupPerm", defaultGroupPerm)
-            .add("topicPerms", topicPerms)
-            .add("groupPerms", groupPerms)
-            .toString();
-    }
-}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateAclRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateAclRequestHeader.java
new file mode 100644
index 00000000000..5839d3eb011
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateAclRequestHeader.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+
+@RocketMQAction(value = RequestCode.AUTH_CREATE_ACL, resource = ResourceType.CLUSTER, action = Action.UPDATE)
+public class CreateAclRequestHeader implements CommandCustomHeader {
+
+    private String subject;
+
+    public CreateAclRequestHeader() {
+    }
+
+    public CreateAclRequestHeader(String subject) {
+        this.subject = subject;
+    }
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
+    }
+
+    public String getSubject() {
+        return subject;
+    }
+
+    public void setSubject(String subject) {
+        this.subject = subject;
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicListRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicListRequestHeader.java
new file mode 100644
index 00000000000..615de750c48
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicListRequestHeader.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
+
+@RocketMQAction(value = RequestCode.UPDATE_AND_CREATE_TOPIC_LIST, action = Action.CREATE)
+public class CreateTopicListRequestHeader extends RpcRequestHeader {
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicRequestHeader.java
index faddd9f461e..6a1f1cbd803 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicRequestHeader.java
@@ -22,13 +22,20 @@
 
 import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.common.TopicFilterType;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
 
+@RocketMQAction(value = RequestCode.UPDATE_AND_CREATE_TOPIC, action = Action.CREATE)
 public class CreateTopicRequestHeader extends TopicRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private String defaultTopic;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateUserRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateUserRequestHeader.java
new file mode 100644
index 00000000000..32e34ed229e
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateUserRequestHeader.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+
+@RocketMQAction(value = RequestCode.AUTH_CREATE_USER, resource = ResourceType.CLUSTER, action = Action.UPDATE)
+public class CreateUserRequestHeader implements CommandCustomHeader {
+
+    private String username;
+
+    public CreateUserRequestHeader() {
+    }
+
+    public CreateUserRequestHeader(String username) {
+        this.username = username;
+    }
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteAclRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteAclRequestHeader.java
new file mode 100644
index 00000000000..a1f06a2b8de
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteAclRequestHeader.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+
+@RocketMQAction(value = RequestCode.AUTH_DELETE_ACL, resource = ResourceType.CLUSTER, action = Action.UPDATE)
+public class DeleteAclRequestHeader implements CommandCustomHeader {
+
+    private String subject;
+
+    private String policyType;
+
+    private String resource;
+
+    public DeleteAclRequestHeader() {
+    }
+
+    public DeleteAclRequestHeader(String subject, String resource) {
+        this.subject = subject;
+        this.resource = resource;
+    }
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
+    }
+
+    public String getSubject() {
+        return subject;
+    }
+
+    public void setSubject(String subject) {
+        this.subject = subject;
+    }
+
+    public String getPolicyType() {
+        return policyType;
+    }
+
+    public void setPolicyType(String policyType) {
+        this.policyType = policyType;
+    }
+
+    public String getResource() {
+        return resource;
+    }
+
+    public void setResource(String resource) {
+        this.resource = resource;
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteSubscriptionGroupRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteSubscriptionGroupRequestHeader.java
index 4548454853f..e9369637373 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteSubscriptionGroupRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteSubscriptionGroupRequestHeader.java
@@ -17,12 +17,19 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
 
+@RocketMQAction(value = RequestCode.DELETE_SUBSCRIPTIONGROUP, action = Action.DELETE)
 public class DeleteSubscriptionGroupRequestHeader extends RpcRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String groupName;
 
     private boolean cleanOffset = false;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteTopicRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteTopicRequestHeader.java
index d24c1da0786..ea66ed94c7e 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteTopicRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteTopicRequestHeader.java
@@ -20,12 +20,19 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
 
+@RocketMQAction(value = RequestCode.DELETE_TOPIC_IN_BROKER, action = Action.DELETE)
 public class DeleteTopicRequestHeader extends TopicRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteUserRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteUserRequestHeader.java
new file mode 100644
index 00000000000..00eb1e8ac5d
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteUserRequestHeader.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+
+@RocketMQAction(value = RequestCode.AUTH_DELETE_USER, resource = ResourceType.CLUSTER, action = Action.UPDATE)
+public class DeleteUserRequestHeader implements CommandCustomHeader {
+
+    private String username;
+
+    public DeleteUserRequestHeader() {
+    }
+
+    public DeleteUserRequestHeader(String username) {
+        this.username = username;
+    }
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/EndTransactionRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/EndTransactionRequestHeader.java
index 3f5515e272e..cef464a71cf 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/EndTransactionRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/EndTransactionRequestHeader.java
@@ -18,13 +18,21 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
 
+@RocketMQAction(value = RequestCode.END_TRANSACTION, action = Action.PUB)
 public class EndTransactionRequestHeader extends RpcRequestHeader {
+    @RocketMQResource(ResourceType.TOPIC)
+    private String topic;
     @CFNotNull
     private String producerGroup;
     @CFNotNull
@@ -61,6 +69,14 @@ public void checkFields() throws RemotingCommandException {
         throw new RemotingCommandException("commitOrRollback field wrong");
     }
 
+    public String getTopic() {
+        return topic;
+    }
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
     public String getProducerGroup() {
         return producerGroup;
     }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExchangeHAInfoRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExchangeHAInfoRequestHeader.java
index b636e36ec95..103b2ace9bc 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExchangeHAInfoRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExchangeHAInfoRequestHeader.java
@@ -17,10 +17,15 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.EXCHANGE_BROKER_HA_INFO,resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class ExchangeHAInfoRequestHeader implements CommandCustomHeader {
     @CFNullable
     public String masterHaAddress;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExportRocksDBConfigToJsonRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExportRocksDBConfigToJsonRequestHeader.java
new file mode 100644
index 00000000000..8354f83053d
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExportRocksDBConfigToJsonRequestHeader.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.annotation.CFNotNull;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+
+@RocketMQAction(value = RequestCode.EXPORT_ROCKSDB_CONFIG_TO_JSON, resource = ResourceType.CLUSTER, action = Action.GET)
+public class ExportRocksDBConfigToJsonRequestHeader implements CommandCustomHeader {
+    private static final String CONFIG_TYPE_SEPARATOR = ";";
+
+    public enum ConfigType {
+        TOPICS("topics"),
+        SUBSCRIPTION_GROUPS("subscriptionGroups"),
+        CONSUMER_OFFSETS("consumerOffsets");
+
+        private final String typeName;
+
+        ConfigType(String typeName) {
+            this.typeName = typeName;
+        }
+
+        public static ConfigType getConfigTypeByName(String typeName) {
+            for (ConfigType configType : ConfigType.values()) {
+                if (configType.getTypeName().equalsIgnoreCase(typeName.trim())) {
+                    return configType;
+                }
+            }
+            throw new IllegalArgumentException("Unknown config type: " + typeName);
+        }
+
+        public static List<ConfigType> fromString(String ordinal) {
+            String[] configTypeNames = StringUtils.split(ordinal, CONFIG_TYPE_SEPARATOR);
+            List<ConfigType> configTypes = new ArrayList<>();
+            for (String configTypeName : configTypeNames) {
+                if (StringUtils.isNotEmpty(configTypeName)) {
+                    configTypes.add(getConfigTypeByName(configTypeName));
+                }
+            }
+            return configTypes;
+        }
+
+        public static String toString(List<ConfigType> configTypes) {
+            StringBuilder sb = new StringBuilder();
+            for (ConfigType configType : configTypes) {
+                sb.append(configType.getTypeName()).append(CONFIG_TYPE_SEPARATOR);
+            }
+            return sb.toString();
+        }
+
+        public String getTypeName() {
+            return typeName;
+        }
+    }
+
+    @CFNotNull
+    private String configType;
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
+    }
+
+    public List<ConfigType> fetchConfigType() {
+        return ConfigType.fromString(configType);
+    }
+
+    public void updateConfigType(List<ConfigType> configType) {
+        this.configType = ConfigType.toString(configType);
+    }
+
+    public String getConfigType() {
+        return configType;
+    }
+
+    public void setConfigType(String configType) {
+        this.configType = configType;
+    }
+}
\ No newline at end of file
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateGlobalWhiteAddrsConfigRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAclRequestHeader.java
similarity index 62%
rename from remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateGlobalWhiteAddrsConfigRequestHeader.java
rename to remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAclRequestHeader.java
index 59e93d32630..09b9df66151 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateGlobalWhiteAddrsConfigRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAclRequestHeader.java
@@ -16,35 +16,35 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
-import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
-public class UpdateGlobalWhiteAddrsConfigRequestHeader implements CommandCustomHeader {
+@RocketMQAction(value = RequestCode.AUTH_GET_ACL, resource = ResourceType.CLUSTER, action = Action.GET)
+public class GetAclRequestHeader implements CommandCustomHeader {
 
-    @CFNotNull
-    private String globalWhiteAddrs;
-    @CFNotNull
-    private String aclFileFullPath;
-
-    @Override
-    public void checkFields() throws RemotingCommandException {
+    private String subject;
 
+    public GetAclRequestHeader() {
     }
 
-    public String getGlobalWhiteAddrs() {
-        return globalWhiteAddrs;
+    public GetAclRequestHeader(String subject) {
+        this.subject = subject;
     }
 
-    public void setGlobalWhiteAddrs(String globalWhiteAddrs) {
-        this.globalWhiteAddrs = globalWhiteAddrs;
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
     }
 
-    public String getAclFileFullPath() {
-        return aclFileFullPath;
+    public String getSubject() {
+        return subject;
     }
 
-    public void setAclFileFullPath(String aclFileFullPath) {
-        this.aclFileFullPath = aclFileFullPath;
+    public void setSubject(String subject) {
+        this.subject = subject;
     }
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllProducerInfoRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllProducerInfoRequestHeader.java
index a24de24fd9d..e57a589062b 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllProducerInfoRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllProducerInfoRequestHeader.java
@@ -17,9 +17,14 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.GET_ALL_PRODUCER_INFO, resource = ResourceType.CLUSTER, action = Action.GET)
 public class GetAllProducerInfoRequestHeader implements CommandCustomHeader {
     @Override
     public void checkFields() throws RemotingCommandException {
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllTopicConfigResponseHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllTopicConfigResponseHeader.java
index cd8da68c60c..566ce16fa49 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllTopicConfigResponseHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetAllTopicConfigResponseHeader.java
@@ -20,9 +20,14 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.GET_ALL_TOPIC_CONFIG, resource = ResourceType.TOPIC, action = Action.LIST)
 public class GetAllTopicConfigResponseHeader implements CommandCustomHeader {
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetBrokerMemberGroupRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetBrokerMemberGroupRequestHeader.java
index 964025f5e6f..d2fd6d47035 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetBrokerMemberGroupRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetBrokerMemberGroupRequestHeader.java
@@ -17,12 +17,19 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.GET_BROKER_MEMBER_GROUP, action = Action.GET)
 public class GetBrokerMemberGroupRequestHeader implements CommandCustomHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.CLUSTER)
     private String clusterName;
 
     @CFNotNull
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsInBrokerHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsInBrokerHeader.java
index 156f6dbefd0..7d47d53abdc 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsInBrokerHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsInBrokerHeader.java
@@ -17,10 +17,15 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.GET_BROKER_CONSUME_STATS, resource = ResourceType.CLUSTER, action = Action.GET)
 public class GetConsumeStatsInBrokerHeader implements CommandCustomHeader {
     @CFNotNull
     private boolean isOrder;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsRequestHeader.java
index 474811b3bd5..2c51c3f529b 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsRequestHeader.java
@@ -17,19 +17,62 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
 
+@RocketMQAction(value = RequestCode.GET_CONSUME_STATS, action = Action.GET)
 public class GetConsumeStatsRequestHeader extends TopicRequestHeader {
+    private static final String TOPIC_NAME_SEPARATOR = ";";
+
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
+
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
 
+    // if topicList is provided, topic will be ignored
+    @RocketMQResource(value = ResourceType.TOPIC, splitter = TOPIC_NAME_SEPARATOR)
+    private String topicList;
+
     @Override
     public void checkFields() throws RemotingCommandException {
     }
 
+    public List<String> fetchTopicList() {
+        if (StringUtils.isBlank(topicList)) {
+            return Collections.emptyList();
+        }
+        return Arrays.asList(StringUtils.split(topicList, TOPIC_NAME_SEPARATOR));
+    }
+
+    public void updateTopicList(List<String> topicList) {
+        if (topicList == null || topicList.isEmpty()) {
+            return;
+        }
+        StringBuilder sb = new StringBuilder();
+        topicList.forEach(topic -> sb.append(topic).append(TOPIC_NAME_SEPARATOR));
+        this.setTopicList(sb.toString());
+    }
+
+    public String getTopicList() {
+        return topicList;
+    }
+
+    public void setTopicList(String topicList) {
+        this.topicList = topicList;
+    }
+
     public String getConsumerGroup() {
         return consumerGroup;
     }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerConnectionListRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerConnectionListRequestHeader.java
index b572c82f35b..64f0e2d6488 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerConnectionListRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerConnectionListRequestHeader.java
@@ -17,12 +17,19 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
 
+@RocketMQAction(value = RequestCode.GET_CONSUMER_CONNECTION_LIST, action = Action.GET)
 public class GetConsumerConnectionListRequestHeader extends RpcRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerListByGroupRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerListByGroupRequestHeader.java
index 43161ef3627..cd34cbfc54a 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerListByGroupRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerListByGroupRequestHeader.java
@@ -18,12 +18,19 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.GET_CONSUMER_LIST_BY_GROUP, action = Action.SUB)
 public class GetConsumerListByGroupRequestHeader extends RpcRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerRunningInfoRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerRunningInfoRequestHeader.java
index c67e9a15eb3..894e7266581 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerRunningInfoRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerRunningInfoRequestHeader.java
@@ -18,13 +18,20 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.GET_CONSUMER_RUNNING_INFO, action = Action.GET)
 public class GetConsumerRunningInfoRequestHeader extends RpcRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
     @CFNotNull
     private String clientId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerStatusRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerStatusRequestHeader.java
index de9115a0142..8e2b850d6c2 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerStatusRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetConsumerStatusRequestHeader.java
@@ -18,15 +18,23 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.INVOKE_BROKER_TO_GET_CONSUMER_STATUS, action = Action.GET)
 public class GetConsumerStatusRequestHeader extends TopicRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String group;
     @CFNullable
     private String clientAddr;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetEarliestMsgStoretimeRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetEarliestMsgStoretimeRequestHeader.java
index b6a3a2d47dd..b8715f40551 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetEarliestMsgStoretimeRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetEarliestMsgStoretimeRequestHeader.java
@@ -20,12 +20,19 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
+@RocketMQAction(value = RequestCode.GET_EARLIEST_MSG_STORETIME, action = Action.GET)
 public class GetEarliestMsgStoretimeRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private Integer queueId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetMaxOffsetRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetMaxOffsetRequestHeader.java
index ec4219874a6..68b36a24056 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetMaxOffsetRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetMaxOffsetRequestHeader.java
@@ -21,13 +21,20 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
+@RocketMQAction(value = RequestCode.GET_MAX_OFFSET, action = Action.GET)
 public class GetMaxOffsetRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private Integer queueId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetMinOffsetRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetMinOffsetRequestHeader.java
index 11087e6730f..8515dc33587 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetMinOffsetRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetMinOffsetRequestHeader.java
@@ -21,12 +21,19 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
+@RocketMQAction(value = RequestCode.GET_MIN_OFFSET, action = Action.GET)
 public class GetMinOffsetRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private Integer queueId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetProducerConnectionListRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetProducerConnectionListRequestHeader.java
index 701335191d8..f3f2d50fa76 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetProducerConnectionListRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetProducerConnectionListRequestHeader.java
@@ -17,10 +17,15 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
 
+@RocketMQAction(value = RequestCode.GET_PRODUCER_CONNECTION_LIST, resource = ResourceType.CLUSTER, action = Action.GET)
 public class GetProducerConnectionListRequestHeader extends RpcRequestHeader {
     @CFNotNull
     private String producerGroup;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetSubscriptionGroupConfigRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetSubscriptionGroupConfigRequestHeader.java
index 2ad06b4966e..5d8ac7c715e 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetSubscriptionGroupConfigRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetSubscriptionGroupConfigRequestHeader.java
@@ -20,10 +20,16 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
 
+@RocketMQAction(value = RequestCode.GET_SUBSCRIPTIONGROUP_CONFIG, action = Action.GET)
 public class GetSubscriptionGroupConfigRequestHeader extends RpcRequestHeader {
 
     @Override
@@ -31,6 +37,7 @@ public void checkFields() throws RemotingCommandException {
     }
 
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String group;
 
     /**
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicConfigRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicConfigRequestHeader.java
index 69b07f188fc..62f3de2b091 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicConfigRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicConfigRequestHeader.java
@@ -17,16 +17,23 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
 
+@RocketMQAction(value = RequestCode.GET_TOPIC_CONFIG, action = Action.GET)
 public class GetTopicConfigRequestHeader extends TopicRequestHeader {
     @Override
     public void checkFields() throws RemotingCommandException {
     }
 
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
 
 
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicStatsInfoRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicStatsInfoRequestHeader.java
index 7d1e8d6b7d9..7c909956ebd 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicStatsInfoRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicStatsInfoRequestHeader.java
@@ -17,12 +17,19 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
 
+@RocketMQAction(value = RequestCode.GET_TOPIC_STATS_INFO, action = Action.GET)
 public class GetTopicStatsInfoRequestHeader extends TopicRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicsByClusterRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicsByClusterRequestHeader.java
index 08af6f875c9..b2e78988b46 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicsByClusterRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetTopicsByClusterRequestHeader.java
@@ -17,10 +17,15 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.GET_TOPICS_BY_CLUSTER, resource = ResourceType.TOPIC, action = Action.LIST)
 public class GetTopicsByClusterRequestHeader implements CommandCustomHeader {
     @CFNotNull
     private String cluster;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetUserRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetUserRequestHeader.java
new file mode 100644
index 00000000000..cd95264d665
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/GetUserRequestHeader.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+
+@RocketMQAction(value = RequestCode.AUTH_GET_USER, resource = ResourceType.CLUSTER, action = Action.GET)
+public class GetUserRequestHeader implements CommandCustomHeader {
+
+    private String username;
+
+    public GetUserRequestHeader() {
+    }
+
+    public GetUserRequestHeader(String username) {
+        this.username = username;
+    }
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/HeartbeatRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/HeartbeatRequestHeader.java
index a37383f3a18..5a3c0fdf4ef 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/HeartbeatRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/HeartbeatRequestHeader.java
@@ -17,9 +17,14 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
 
+@RocketMQAction(value = RequestCode.HEART_BEAT, resource = ResourceType.GROUP, action = {Action.PUB, Action.SUB})
 public class HeartbeatRequestHeader extends RpcRequestHeader {
     // for namespace
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ListAclsRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ListAclsRequestHeader.java
new file mode 100644
index 00000000000..c929eef2658
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ListAclsRequestHeader.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+
+@RocketMQAction(value = RequestCode.AUTH_LIST_ACL, resource = ResourceType.CLUSTER, action = Action.GET)
+public class ListAclsRequestHeader implements CommandCustomHeader {
+
+    private String subjectFilter;
+
+    private String resourceFilter;
+
+    public ListAclsRequestHeader() {
+    }
+
+    public ListAclsRequestHeader(String subjectFilter, String resourceFilter) {
+        this.subjectFilter = subjectFilter;
+        this.resourceFilter = resourceFilter;
+    }
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
+    }
+
+    public String getSubjectFilter() {
+        return subjectFilter;
+    }
+
+    public void setSubjectFilter(String subjectFilter) {
+        this.subjectFilter = subjectFilter;
+    }
+
+    public String getResourceFilter() {
+        return resourceFilter;
+    }
+
+    public void setResourceFilter(String resourceFilter) {
+        this.resourceFilter = resourceFilter;
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ListUsersRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ListUsersRequestHeader.java
new file mode 100644
index 00000000000..3bf94245ae3
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ListUsersRequestHeader.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+
+@RocketMQAction(value = RequestCode.AUTH_LIST_USER, resource = ResourceType.CLUSTER, action = Action.GET)
+public class ListUsersRequestHeader implements CommandCustomHeader {
+
+    private String filter;
+
+    public ListUsersRequestHeader() {
+    }
+
+    public ListUsersRequestHeader(String filter) {
+        this.filter = filter;
+    }
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
+    }
+
+    public String getFilter() {
+        return filter;
+    }
+
+    public void setFilter(String filter) {
+        this.filter = filter;
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/LockBatchMqRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/LockBatchMqRequestHeader.java
index 3484fa7d3e7..970974cd3e7 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/LockBatchMqRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/LockBatchMqRequestHeader.java
@@ -17,9 +17,13 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
 
+@RocketMQAction(value = RequestCode.LOCK_BATCH_MQ, action = Action.SUB)
 public class LockBatchMqRequestHeader extends RpcRequestHeader {
     @Override
     public void checkFields() throws RemotingCommandException {
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotificationRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotificationRequestHeader.java
index 2ccf564df56..0e484f82c0d 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotificationRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotificationRequestHeader.java
@@ -17,15 +17,22 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
-
+@RocketMQAction(value = RequestCode.NOTIFICATION, action = Action.SUB)
 public class NotificationRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private int queueId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotificationResponseHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotificationResponseHeader.java
index cbab5974015..027717e006f 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotificationResponseHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotificationResponseHeader.java
@@ -26,10 +26,20 @@ public class NotificationResponseHeader implements CommandCustomHeader {
     @CFNotNull
     private boolean hasMsg = false;
 
+    private boolean pollingFull = false;
+
     public boolean isHasMsg() {
         return hasMsg;
     }
 
+    public boolean isPollingFull() {
+        return pollingFull;
+    }
+
+    public void setPollingFull(boolean pollingFull) {
+        this.pollingFull = pollingFull;
+    }
+
     public void setHasMsg(boolean hasMsg) {
         this.hasMsg = hasMsg;
     }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyBrokerRoleChangedRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyBrokerRoleChangedRequestHeader.java
index 3a112a57824..e9a348a8fcc 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyBrokerRoleChangedRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyBrokerRoleChangedRequestHeader.java
@@ -16,9 +16,14 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.NOTIFY_BROKER_ROLE_CHANGED, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class NotifyBrokerRoleChangedRequestHeader implements CommandCustomHeader {
     private String masterAddress;
     private Integer masterEpoch;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyConsumerIdsChangedRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyConsumerIdsChangedRequestHeader.java
index 38271f97549..eb109dc126a 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyConsumerIdsChangedRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyConsumerIdsChangedRequestHeader.java
@@ -17,12 +17,19 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.NOTIFY_CONSUMER_IDS_CHANGED, action = Action.SUB)
 public class NotifyConsumerIdsChangedRequestHeader extends RpcRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyMinBrokerIdChangeRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyMinBrokerIdChangeRequestHeader.java
index 2b02006abcb..8b451e2c429 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyMinBrokerIdChangeRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/NotifyMinBrokerIdChangeRequestHeader.java
@@ -17,10 +17,15 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.NOTIFY_MIN_BROKER_ID_CHANGE, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class NotifyMinBrokerIdChangeRequestHeader implements CommandCustomHeader {
     @CFNullable
     private Long minBrokerId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PeekMessageRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PeekMessageRequestHeader.java
index a6f23dc2ee0..61746a6bbb2 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PeekMessageRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PeekMessageRequestHeader.java
@@ -16,18 +16,26 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
+@RocketMQAction(value = RequestCode.PEEK_MESSAGE, action = Action.SUB)
 public class PeekMessageRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private int queueId;
     @CFNotNull
     private int maxMsgNums;
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PollingInfoRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PollingInfoRequestHeader.java
index 558fb3f5061..1959938aae2 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PollingInfoRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PollingInfoRequestHeader.java
@@ -17,15 +17,22 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
-
+@RocketMQAction(value = RequestCode.POLLING_INFO, action = Action.GET)
 public class PollingInfoRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private int queueId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PopMessageRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PopMessageRequestHeader.java
index 34b97987ddd..8a7ab4fec74 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PopMessageRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PopMessageRequestHeader.java
@@ -17,14 +17,22 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
+@RocketMQAction(value = RequestCode.POP_MESSAGE, action = Action.SUB)
 public class PopMessageRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private int queueId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PullMessageRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PullMessageRequestHeader.java
index a6d6d3b64c3..5785615b204 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PullMessageRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/PullMessageRequestHeader.java
@@ -23,17 +23,25 @@
 import com.google.common.base.MoreObjects;
 import io.netty.buffer.ByteBuf;
 import java.util.HashMap;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.protocol.FastCodesHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
+@RocketMQAction(value = RequestCode.PULL_MESSAGE, action = Action.SUB)
 public class PullMessageRequestHeader extends TopicQueueRequestHeader implements FastCodesHeader {
 
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private Integer queueId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumeQueueRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumeQueueRequestHeader.java
index a1b45e9b26c..c436f1b77cc 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumeQueueRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumeQueueRequestHeader.java
@@ -17,15 +17,23 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.QUERY_CONSUME_QUEUE, action = Action.GET)
 public class QueryConsumeQueueRequestHeader extends TopicQueueRequestHeader {
 
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     private int queueId;
     private long index;
     private int count;
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
 
     public String getTopic() {
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumeTimeSpanRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumeTimeSpanRequestHeader.java
index 9c0aa34b5fa..d9f134dba09 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumeTimeSpanRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumeTimeSpanRequestHeader.java
@@ -17,14 +17,22 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
 
+@RocketMQAction(value = RequestCode.QUERY_CONSUME_TIME_SPAN, action = Action.GET)
 public class QueryConsumeTimeSpanRequestHeader extends TopicRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String group;
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumerOffsetRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumerOffsetRequestHeader.java
index e16d38a7a3e..f56aa9639d3 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumerOffsetRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryConsumerOffsetRequestHeader.java
@@ -21,14 +21,22 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
+@RocketMQAction(value = RequestCode.QUERY_CONSUMER_OFFSET, action = Action.GET)
 public class QueryConsumerOffsetRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private Integer queueId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryCorrectionOffsetHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryCorrectionOffsetHeader.java
index 8e03ce5c6aa..27a3d35c5af 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryCorrectionOffsetHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryCorrectionOffsetHeader.java
@@ -20,15 +20,24 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.QUERY_CORRECTION_OFFSET, action = Action.GET)
 public class QueryCorrectionOffsetHeader extends TopicRequestHeader {
+    @RocketMQResource(value = ResourceType.GROUP, splitter = ",")
     private String filterGroups;
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String compareGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryMessageRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryMessageRequestHeader.java
index 1a78b9e4fab..1d2a53a6ce1 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryMessageRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryMessageRequestHeader.java
@@ -20,12 +20,19 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.QUERY_MESSAGE, action = {Action.SUB, Action.GET})
 public class QueryMessageRequestHeader extends TopicRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private String key;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QuerySubscriptionByConsumerRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QuerySubscriptionByConsumerRequestHeader.java
index 1e5a48c5887..981c7e2ef50 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QuerySubscriptionByConsumerRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QuerySubscriptionByConsumerRequestHeader.java
@@ -20,13 +20,21 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.QUERY_SUBSCRIPTION_BY_CONSUMER, action = Action.GET)
 public class QuerySubscriptionByConsumerRequestHeader extends TopicRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String group;
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryTopicConsumeByWhoRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryTopicConsumeByWhoRequestHeader.java
index 5cd9b931c2a..ee86323bc17 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryTopicConsumeByWhoRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryTopicConsumeByWhoRequestHeader.java
@@ -20,12 +20,19 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
 
+@RocketMQAction(value = RequestCode.QUERY_TOPIC_CONSUME_BY_WHO, action = Action.GET)
 public class QueryTopicConsumeByWhoRequestHeader extends TopicRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryTopicsByConsumerRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryTopicsByConsumerRequestHeader.java
index e90afdd8ba6..fe6723d3817 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryTopicsByConsumerRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/QueryTopicsByConsumerRequestHeader.java
@@ -20,12 +20,19 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
 
+@RocketMQAction(value = RequestCode.QUERY_TOPICS_BY_CONSUMER, action = Action.GET)
 public class QueryTopicsByConsumerRequestHeader extends RpcRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String group;
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageRequestHeader.java
new file mode 100644
index 00000000000..c29883682a0
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageRequestHeader.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
+import org.apache.rocketmq.remoting.annotation.CFNotNull;
+import org.apache.rocketmq.remoting.annotation.CFNullable;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
+
+@RocketMQAction(value = RequestCode.RECALL_MESSAGE, action = Action.PUB)
+public class RecallMessageRequestHeader extends TopicRequestHeader {
+    @CFNullable
+    private String producerGroup;
+
+    @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
+    private String topic;
+
+    @CFNotNull
+    private String recallHandle;
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+    }
+
+    public String getProducerGroup() {
+        return producerGroup;
+    }
+
+    public void setProducerGroup(String producerGroup) {
+        this.producerGroup = producerGroup;
+    }
+
+    public String getTopic() {
+        return topic;
+    }
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
+    public String getRecallHandle() {
+        return recallHandle;
+    }
+
+    public void setRecallHandle(String recallHandle) {
+        this.recallHandle = recallHandle;
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+                .add("producerGroup", producerGroup)
+                .add("topic", topic)
+                .add("recallHandle", recallHandle)
+                .toString();
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteAccessConfigRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageResponseHeader.java
similarity index 81%
rename from remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteAccessConfigRequestHeader.java
rename to remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageResponseHeader.java
index ef5cbdc9f1a..1833cfcd053 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/DeleteAccessConfigRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageResponseHeader.java
@@ -21,21 +21,18 @@
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 
-public class DeleteAccessConfigRequestHeader implements CommandCustomHeader {
-
+public class RecallMessageResponseHeader implements CommandCustomHeader {
     @CFNotNull
-    private String accessKey;
-
+    private String msgId;
     @Override
     public void checkFields() throws RemotingCommandException {
-
     }
 
-    public String getAccessKey() {
-        return accessKey;
+    public String getMsgId() {
+        return msgId;
     }
 
-    public void setAccessKey(String accessKey) {
-        this.accessKey = accessKey;
+    public void setMsgId(String msgId) {
+        this.msgId = msgId;
     }
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RemoveBrokerRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RemoveBrokerRequestHeader.java
index a641340eeb9..2786b72cbb6 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RemoveBrokerRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RemoveBrokerRequestHeader.java
@@ -17,14 +17,21 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.REMOVE_BROKER, resource = ResourceType.CLUSTER,action = Action.UPDATE)
 public class RemoveBrokerRequestHeader implements CommandCustomHeader {
     @CFNotNull
     private String brokerName;
     @CFNotNull
+    @RocketMQResource(ResourceType.CLUSTER)
     private String brokerClusterName;
     @CFNotNull
     private Long brokerId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ReplyMessageRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ReplyMessageRequestHeader.java
index eeb022d6d69..1a232bbaee2 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ReplyMessageRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ReplyMessageRequestHeader.java
@@ -17,15 +17,23 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT, action = Action.SUB)
 public class ReplyMessageRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String producerGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private String defaultTopic;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetMasterFlushOffsetHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetMasterFlushOffsetHeader.java
index ddee7224c7d..801ad08abca 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetMasterFlushOffsetHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetMasterFlushOffsetHeader.java
@@ -17,10 +17,15 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.RESET_MASTER_FLUSH_OFFSET, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class ResetMasterFlushOffsetHeader implements CommandCustomHeader {
     @CFNotNull
     private Long masterFlushOffset;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetOffsetRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetOffsetRequestHeader.java
index dd5c043ef62..f72fe57136c 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetOffsetRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetOffsetRequestHeader.java
@@ -17,17 +17,25 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, action = Action.UPDATE)
 public class ResetOffsetRequestHeader extends TopicQueueRequestHeader {
 
     @CFNotNull
-    private String topic;
+    @RocketMQResource(ResourceType.GROUP)
+    private String group;
 
     @CFNotNull
-    private String group;
+    @RocketMQResource(ResourceType.TOPIC)
+    private String topic;
 
     private int queueId = -1;
 
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResumeCheckHalfMessageRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResumeCheckHalfMessageRequestHeader.java
index 6265cdfd4be..923fd37ea6d 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResumeCheckHalfMessageRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResumeCheckHalfMessageRequestHeader.java
@@ -17,11 +17,20 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.RESUME_CHECK_HALF_MESSAGE, action = Action.UPDATE)
 public class ResumeCheckHalfMessageRequestHeader implements CommandCustomHeader {
+
+    @RocketMQResource(ResourceType.TOPIC)
+    private String topic;
     @CFNullable
     private String msgId;
 
@@ -30,6 +39,14 @@ public void checkFields() throws RemotingCommandException {
 
     }
 
+    public String getTopic() {
+        return topic;
+    }
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
     public String getMsgId() {
         return msgId;
     }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SearchOffsetRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SearchOffsetRequestHeader.java
index 79c3d337be0..bbefa8c1e5b 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SearchOffsetRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SearchOffsetRequestHeader.java
@@ -22,12 +22,19 @@
 
 import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.common.BoundaryType;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
+@RocketMQAction(value = RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, action = Action.GET)
 public class SearchOffsetRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private Integer queueId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeader.java
index 17ce5126313..2857fb516a6 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeader.java
@@ -21,6 +21,10 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -28,10 +32,12 @@
 import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
+@RocketMQAction(value = RequestCode.SEND_MESSAGE, action = Action.PUB)
 public class SendMessageRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
     private String producerGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private String defaultTopic;
@@ -50,9 +56,9 @@ public class SendMessageRequestHeader extends TopicQueueRequestHeader {
     @CFNullable
     private Integer reconsumeTimes;
     @CFNullable
-    private boolean unitMode = false;
+    private Boolean unitMode;
     @CFNullable
-    private boolean batch = false;
+    private Boolean batch;
     private Integer maxReconsumeTimes;
 
     @Override
@@ -136,6 +142,9 @@ public void setProperties(String properties) {
     }
 
     public Integer getReconsumeTimes() {
+        if (null == reconsumeTimes) {
+            return 0;
+        }
         return reconsumeTimes;
     }
 
@@ -144,10 +153,13 @@ public void setReconsumeTimes(Integer reconsumeTimes) {
     }
 
     public boolean isUnitMode() {
+        if (null == unitMode) {
+            return false;
+        }
         return unitMode;
     }
 
-    public void setUnitMode(boolean isUnitMode) {
+    public void setUnitMode(Boolean isUnitMode) {
         this.unitMode = isUnitMode;
     }
 
@@ -160,10 +172,13 @@ public void setMaxReconsumeTimes(final Integer maxReconsumeTimes) {
     }
 
     public boolean isBatch() {
+        if (null == batch) {
+            return false;
+        }
         return batch;
     }
 
-    public void setBatch(boolean batch) {
+    public void setBatch(Boolean batch) {
         this.batch = batch;
     }
 
@@ -173,14 +188,10 @@ public static SendMessageRequestHeader parseRequestHeader(RemotingCommand reques
         switch (request.getCode()) {
             case RequestCode.SEND_BATCH_MESSAGE:
             case RequestCode.SEND_MESSAGE_V2:
-                requestHeaderV2 =
-                    (SendMessageRequestHeaderV2) request
-                        .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class);
+                requestHeaderV2 = request.decodeCommandCustomHeader(SendMessageRequestHeaderV2.class);
             case RequestCode.SEND_MESSAGE:
                 if (null == requestHeaderV2) {
-                    requestHeader =
-                        (SendMessageRequestHeader) request
-                            .decodeCommandCustomHeader(SendMessageRequestHeader.class);
+                    requestHeader = request.decodeCommandCustomHeader(SendMessageRequestHeader.class);
                 } else {
                     requestHeader = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV1(requestHeaderV2);
                 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeaderV2.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeaderV2.java
index 4b0d795bcd5..4812e90f221 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeaderV2.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageRequestHeaderV2.java
@@ -20,20 +20,27 @@
 import com.google.common.base.MoreObjects;
 import io.netty.buffer.ByteBuf;
 import java.util.HashMap;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.protocol.FastCodesHeader;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
 /**
  * Use short variable name to speed up FastJson deserialization process.
  */
+@RocketMQAction(value = RequestCode.SEND_MESSAGE_V2, action = Action.PUB)
 public class SendMessageRequestHeaderV2 extends TopicQueueRequestHeader implements CommandCustomHeader, FastCodesHeader {
     @CFNotNull
     private String a; // producerGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String b; // topic;
     @CFNotNull
     private String c; // defaultTopic;
@@ -52,7 +59,7 @@ public class SendMessageRequestHeaderV2 extends TopicQueueRequestHeader implemen
     @CFNullable
     private Integer j; // reconsumeTimes;
     @CFNullable
-    private Boolean k; // unitMode = false;
+    private Boolean k; // unitMode;
 
     private Integer l; // consumeRetryTimes
 
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java
index fe1e8533e54..7563b910331 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java
@@ -36,6 +36,7 @@ public class SendMessageResponseHeader implements CommandCustomHeader, FastCodes
     private Long queueOffset;
     private String transactionId;
     private String batchUniqId;
+    private String recallHandle;
 
     @Override
     public void checkFields() throws RemotingCommandException {
@@ -48,6 +49,7 @@ public void encode(ByteBuf out) {
         writeIfNotNull(out, "queueOffset", queueOffset);
         writeIfNotNull(out, "transactionId", transactionId);
         writeIfNotNull(out, "batchUniqId", batchUniqId);
+        writeIfNotNull(out, "recallHandle", recallHandle);
     }
 
     @Override
@@ -76,6 +78,11 @@ public void decode(HashMap<String, String> fields) throws RemotingCommandExcepti
         if (str != null) {
             this.batchUniqId = str;
         }
+
+        str = fields.get("recallHandle");
+        if (str != null) {
+            this.recallHandle = str;
+        }
     }
 
     public String getMsgId() {
@@ -117,4 +124,12 @@ public String getBatchUniqId() {
     public void setBatchUniqId(String batchUniqId) {
         this.batchUniqId = batchUniqId;
     }
+
+    public String getRecallHandle() {
+        return recallHandle;
+    }
+
+    public void setRecallHandle(String recallHandle) {
+        this.recallHandle = recallHandle;
+    }
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UnlockBatchMqRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UnlockBatchMqRequestHeader.java
index e7a44f2f8b8..8a1bf4ba4c1 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UnlockBatchMqRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UnlockBatchMqRequestHeader.java
@@ -17,9 +17,13 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
 
+@RocketMQAction(value = RequestCode.UNLOCK_BATCH_MQ, action = Action.SUB)
 public class UnlockBatchMqRequestHeader extends RpcRequestHeader {
     @Override
     public void checkFields() throws RemotingCommandException {
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UnregisterClientRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UnregisterClientRequestHeader.java
index 79072195b5e..0fc4c145641 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UnregisterClientRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UnregisterClientRequestHeader.java
@@ -17,11 +17,17 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.RpcRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.UNREGISTER_CLIENT, action = {Action.PUB, Action.SUB})
 public class UnregisterClientRequestHeader extends RpcRequestHeader {
     @CFNotNull
     private String clientID;
@@ -29,6 +35,7 @@ public class UnregisterClientRequestHeader extends RpcRequestHeader {
     @CFNullable
     private String producerGroup;
     @CFNullable
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
 
     public String getClientID() {
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateAclRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateAclRequestHeader.java
new file mode 100644
index 00000000000..8e6a9a2fbfa
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateAclRequestHeader.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+
+@RocketMQAction(value = RequestCode.AUTH_UPDATE_ACL, resource = ResourceType.CLUSTER, action = Action.UPDATE)
+public class UpdateAclRequestHeader implements CommandCustomHeader {
+
+    private String subject;
+
+    public UpdateAclRequestHeader() {
+    }
+
+    public UpdateAclRequestHeader(String subject) {
+        this.subject = subject;
+    }
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
+    }
+
+    public String getSubject() {
+        return subject;
+    }
+
+    public void setSubject(String subject) {
+        this.subject = subject;
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateConsumerOffsetRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateConsumerOffsetRequestHeader.java
index f131c36f057..d22139f8dc4 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateConsumerOffsetRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateConsumerOffsetRequestHeader.java
@@ -21,14 +21,22 @@
 package org.apache.rocketmq.remoting.protocol.header;
 
 import com.google.common.base.MoreObjects;
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicQueueRequestHeader;
 
+@RocketMQAction(value = RequestCode.UPDATE_CONSUMER_OFFSET, action = Action.SUB)
 public class UpdateConsumerOffsetRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String consumerGroup;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String topic;
     @CFNotNull
     private Integer queueId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateGroupForbiddenRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateGroupForbiddenRequestHeader.java
index cfb28df218d..d46c79a19eb 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateGroupForbiddenRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateGroupForbiddenRequestHeader.java
@@ -20,14 +20,22 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.UPDATE_AND_GET_GROUP_FORBIDDEN, action = Action.UPDATE)
 public class UpdateGroupForbiddenRequestHeader extends TopicRequestHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.GROUP)
     private String  group;
     @CFNotNull
+    @RocketMQResource(ResourceType.TOPIC)
     private String  topic;
 
     private Boolean readable;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateUserRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateUserRequestHeader.java
new file mode 100644
index 00000000000..68d03418497
--- /dev/null
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/UpdateUserRequestHeader.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+
+@RocketMQAction(value = RequestCode.AUTH_UPDATE_USER, resource = ResourceType.CLUSTER, action = Action.UPDATE)
+public class UpdateUserRequestHeader implements CommandCustomHeader {
+
+    private String username;
+
+    public UpdateUserRequestHeader() {
+    }
+
+    public UpdateUserRequestHeader(String username) {
+        this.username = username;
+    }
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+}
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ViewBrokerStatsDataRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ViewBrokerStatsDataRequestHeader.java
index b879bfbac24..5bd8b0e32bc 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ViewBrokerStatsDataRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ViewBrokerStatsDataRequestHeader.java
@@ -17,10 +17,15 @@
 
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.VIEW_BROKER_STATS_DATA, resource = ResourceType.CLUSTER, action = Action.GET)
 public class ViewBrokerStatsDataRequestHeader implements CommandCustomHeader {
     @CFNotNull
     private String statsName;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ViewMessageRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ViewMessageRequestHeader.java
index 79421fee4c0..2dff436215c 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ViewMessageRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ViewMessageRequestHeader.java
@@ -20,11 +20,19 @@
  */
 package org.apache.rocketmq.remoting.protocol.header;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.VIEW_MESSAGE_BY_ID, action = Action.GET)
 public class ViewMessageRequestHeader implements CommandCustomHeader {
+    @RocketMQResource(ResourceType.TOPIC)
+    private String topic;
     @CFNotNull
     private Long offset;
 
@@ -32,6 +40,14 @@ public class ViewMessageRequestHeader implements CommandCustomHeader {
     public void checkFields() throws RemotingCommandException {
     }
 
+    public String getTopic() {
+        return topic;
+    }
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
     public Long getOffset() {
         return offset;
     }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/AlterSyncStateSetRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/AlterSyncStateSetRequestHeader.java
index d1c605c19a9..89a12b70240 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/AlterSyncStateSetRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/AlterSyncStateSetRequestHeader.java
@@ -16,9 +16,14 @@
  */
 package org.apache.rocketmq.remoting.protocol.header.controller;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.CONTROLLER_ALTER_SYNC_STATE_SET, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class AlterSyncStateSetRequestHeader implements CommandCustomHeader {
     private String brokerName;
     private Long masterBrokerId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/ElectMasterRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/ElectMasterRequestHeader.java
index 79000f184fb..e8b596a8af1 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/ElectMasterRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/ElectMasterRequestHeader.java
@@ -16,13 +16,20 @@
  */
 package org.apache.rocketmq.remoting.protocol.header.controller;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.CONTROLLER_ELECT_MASTER, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class ElectMasterRequestHeader implements CommandCustomHeader {
 
     @CFNotNull
+    @RocketMQResource(ResourceType.CLUSTER)
     private String clusterName = "";
 
     @CFNotNull
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/GetReplicaInfoRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/GetReplicaInfoRequestHeader.java
index efc7afc8498..8097f37903d 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/GetReplicaInfoRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/GetReplicaInfoRequestHeader.java
@@ -16,9 +16,14 @@
  */
 package org.apache.rocketmq.remoting.protocol.header.controller;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.CONTROLLER_GET_REPLICA_INFO, resource = ResourceType.CLUSTER, action = Action.GET)
 public class GetReplicaInfoRequestHeader implements CommandCustomHeader {
     private String brokerName;
 
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/admin/CleanControllerBrokerDataRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/admin/CleanControllerBrokerDataRequestHeader.java
index 9482f4af5f4..2ab87f82a6a 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/admin/CleanControllerBrokerDataRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/admin/CleanControllerBrokerDataRequestHeader.java
@@ -17,14 +17,21 @@
 
 package org.apache.rocketmq.remoting.protocol.header.controller.admin;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.CLEAN_BROKER_DATA, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class CleanControllerBrokerDataRequestHeader implements CommandCustomHeader {
 
     @CFNullable
+    @RocketMQResource(ResourceType.CLUSTER)
     private String clusterName;
 
     @CFNotNull
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/ApplyBrokerIdRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/ApplyBrokerIdRequestHeader.java
index c577d6a736a..55222b54f9b 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/ApplyBrokerIdRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/ApplyBrokerIdRequestHeader.java
@@ -17,11 +17,18 @@
 
 package org.apache.rocketmq.remoting.protocol.header.controller.register;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.CONTROLLER_APPLY_BROKER_ID, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class ApplyBrokerIdRequestHeader implements CommandCustomHeader {
 
+    @RocketMQResource(ResourceType.CLUSTER)
     private String clusterName;
 
     private String brokerName;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/GetNextBrokerIdRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/GetNextBrokerIdRequestHeader.java
index aeb222955e0..1ba0d30a871 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/GetNextBrokerIdRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/GetNextBrokerIdRequestHeader.java
@@ -17,11 +17,18 @@
 
 package org.apache.rocketmq.remoting.protocol.header.controller.register;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, resource = ResourceType.CLUSTER, action = Action.GET)
 public class GetNextBrokerIdRequestHeader implements CommandCustomHeader {
 
+    @RocketMQResource(ResourceType.CLUSTER)
     private String clusterName;
 
     private String brokerName;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/RegisterBrokerToControllerRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/RegisterBrokerToControllerRequestHeader.java
index 6efab166666..38247c3a78e 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/RegisterBrokerToControllerRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/controller/register/RegisterBrokerToControllerRequestHeader.java
@@ -17,11 +17,18 @@
 
 package org.apache.rocketmq.remoting.protocol.header.controller.register;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.CONTROLLER_REGISTER_BROKER, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class RegisterBrokerToControllerRequestHeader implements CommandCustomHeader {
 
+    @RocketMQResource(ResourceType.CLUSTER)
     private String clusterName;
 
     private String brokerName;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/AddWritePermOfBrokerRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/AddWritePermOfBrokerRequestHeader.java
index 6c50916282e..a250def7427 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/AddWritePermOfBrokerRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/AddWritePermOfBrokerRequestHeader.java
@@ -16,10 +16,15 @@
  */
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.ADD_WRITE_PERM_OF_BROKER, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class AddWritePermOfBrokerRequestHeader implements CommandCustomHeader {
     @CFNotNull
     private String brokerName;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/BrokerHeartbeatRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/BrokerHeartbeatRequestHeader.java
index eb7332fdf40..7566afa902e 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/BrokerHeartbeatRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/BrokerHeartbeatRequestHeader.java
@@ -17,13 +17,20 @@
 
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.BROKER_HEARTBEAT, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class BrokerHeartbeatRequestHeader implements CommandCustomHeader {
     @CFNotNull
+    @RocketMQResource(ResourceType.CLUSTER)
     private String clusterName;
     @CFNotNull
     private String brokerAddr;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/DeleteKVConfigRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/DeleteKVConfigRequestHeader.java
index 7fcbe97e8db..ab747a57748 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/DeleteKVConfigRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/DeleteKVConfigRequestHeader.java
@@ -17,10 +17,15 @@
 
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.DELETE_KV_CONFIG, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class DeleteKVConfigRequestHeader implements CommandCustomHeader {
     @CFNotNull
     private String namespace;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/DeleteTopicFromNamesrvRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/DeleteTopicFromNamesrvRequestHeader.java
index 5e5bd8b172e..81d80f1fed2 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/DeleteTopicFromNamesrvRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/DeleteTopicFromNamesrvRequestHeader.java
@@ -16,14 +16,21 @@
  */
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
 
+@RocketMQAction(value = RequestCode.DELETE_TOPIC_IN_NAMESRV, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class DeleteTopicFromNamesrvRequestHeader extends TopicRequestHeader {
     @CFNotNull
     private String topic;
 
+    @RocketMQResource(ResourceType.CLUSTER)
     private String clusterName;
 
     @Override
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetKVConfigRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetKVConfigRequestHeader.java
index 3a069afeb8a..21bfe983d23 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetKVConfigRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetKVConfigRequestHeader.java
@@ -17,10 +17,15 @@
 
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.GET_KV_CONFIG, resource = ResourceType.CLUSTER, action = Action.GET)
 public class GetKVConfigRequestHeader implements CommandCustomHeader {
     @CFNotNull
     private String namespace;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetKVListByNamespaceRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetKVListByNamespaceRequestHeader.java
index 9876161e173..85bc514aa11 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetKVListByNamespaceRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetKVListByNamespaceRequestHeader.java
@@ -17,10 +17,15 @@
 
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.GET_KVLIST_BY_NAMESPACE, resource = ResourceType.CLUSTER, action = Action.GET)
 public class GetKVListByNamespaceRequestHeader implements CommandCustomHeader {
     @CFNotNull
     private String namespace;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetRouteInfoRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetRouteInfoRequestHeader.java
index 8de15f3fd6d..760c3931e86 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetRouteInfoRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/GetRouteInfoRequestHeader.java
@@ -20,11 +20,16 @@
  */
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
 
+@RocketMQAction(value = RequestCode.GET_ROUTEINFO_BY_TOPIC, resource = ResourceType.CLUSTER, action = Action.GET)
 public class GetRouteInfoRequestHeader extends TopicRequestHeader {
 
     @CFNotNull
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/PutKVConfigRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/PutKVConfigRequestHeader.java
index 60f16cbea35..a253996bde0 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/PutKVConfigRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/PutKVConfigRequestHeader.java
@@ -17,10 +17,15 @@
 
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.PUT_KV_CONFIG, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class PutKVConfigRequestHeader implements CommandCustomHeader {
     @CFNotNull
     private String namespace;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/QueryDataVersionRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/QueryDataVersionRequestHeader.java
index 2a1e95b2ce7..26fc5dfec56 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/QueryDataVersionRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/QueryDataVersionRequestHeader.java
@@ -17,16 +17,23 @@
 
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.QUERY_DATA_VERSION, resource = ResourceType.CLUSTER, action = Action.GET)
 public class QueryDataVersionRequestHeader implements CommandCustomHeader {
     @CFNotNull
     private String brokerName;
     @CFNotNull
     private String brokerAddr;
     @CFNotNull
+    @RocketMQResource(ResourceType.CLUSTER)
     private String clusterName;
     @CFNotNull
     private Long brokerId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/RegisterBrokerRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/RegisterBrokerRequestHeader.java
index f97d40daa9d..eb2889a894f 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/RegisterBrokerRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/RegisterBrokerRequestHeader.java
@@ -20,17 +20,24 @@
  */
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.REGISTER_BROKER, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class RegisterBrokerRequestHeader implements CommandCustomHeader {
     @CFNotNull
     private String brokerName;
     @CFNotNull
     private String brokerAddr;
     @CFNotNull
+    @RocketMQResource(ResourceType.CLUSTER)
     private String clusterName;
     @CFNotNull
     private String haServerAddr;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/RegisterTopicRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/RegisterTopicRequestHeader.java
index d93a05824e5..73893f6b52d 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/RegisterTopicRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/RegisterTopicRequestHeader.java
@@ -16,10 +16,15 @@
  */
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.rpc.TopicRequestHeader;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.REGISTER_TOPIC_IN_NAMESRV, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class RegisterTopicRequestHeader extends TopicRequestHeader {
     @CFNotNull
     private String topic;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/UnRegisterBrokerRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/UnRegisterBrokerRequestHeader.java
index e0609791f77..800d608c0b4 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/UnRegisterBrokerRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/UnRegisterBrokerRequestHeader.java
@@ -20,16 +20,23 @@
  */
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
+import org.apache.rocketmq.common.resource.RocketMQResource;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.UNREGISTER_BROKER, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class UnRegisterBrokerRequestHeader implements CommandCustomHeader {
     @CFNotNull
     private String brokerName;
     @CFNotNull
     private String brokerAddr;
     @CFNotNull
+    @RocketMQResource(ResourceType.CLUSTER)
     private String clusterName;
     @CFNotNull
     private Long brokerId;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/WipeWritePermOfBrokerRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/WipeWritePermOfBrokerRequestHeader.java
index edd87d1111d..e712d586f63 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/WipeWritePermOfBrokerRequestHeader.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/namesrv/WipeWritePermOfBrokerRequestHeader.java
@@ -16,10 +16,15 @@
  */
 package org.apache.rocketmq.remoting.protocol.header.namesrv;
 
+import org.apache.rocketmq.common.action.Action;
+import org.apache.rocketmq.common.action.RocketMQAction;
+import org.apache.rocketmq.common.resource.ResourceType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
 
+@RocketMQAction(value = RequestCode.WIPE_WRITE_PERM_OF_BROKER, resource = ResourceType.CLUSTER, action = Action.UPDATE)
 public class WipeWritePermOfBrokerRequestHeader implements CommandCustomHeader {
     @CFNotNull
     private String brokerName;
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingUtils.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingUtils.java
index 45cbed75727..647669fde24 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingUtils.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingUtils.java
@@ -172,21 +172,21 @@ public static Map.Entry<Long, Integer> checkNameEpochNumConsistence(String topic
 
             if (scope != null
                 && !scope.equals(mappingDetail.getScope())) {
-                throw new RuntimeException(String.format("scope dose not match %s != %s in %s", mappingDetail.getScope(), scope, broker));
+                throw new RuntimeException(String.format("scope does not match %s != %s in %s", mappingDetail.getScope(), scope, broker));
             } else {
                 scope = mappingDetail.getScope();
             }
 
             if (maxEpoch != -1
                 && maxEpoch != mappingDetail.getEpoch()) {
-                throw new RuntimeException(String.format("epoch dose not match %d != %d in %s", maxEpoch, mappingDetail.getEpoch(), mappingDetail.getBname()));
+                throw new RuntimeException(String.format("epoch does not match %d != %d in %s", maxEpoch, mappingDetail.getEpoch(), mappingDetail.getBname()));
             } else {
                 maxEpoch = mappingDetail.getEpoch();
             }
 
             if (maxNum != -1
                 && maxNum != mappingDetail.getTotalQueues()) {
-                throw new RuntimeException(String.format("total queue number dose not match %d != %d in %s", maxNum, mappingDetail.getTotalQueues(), mappingDetail.getBname()));
+                throw new RuntimeException(String.format("total queue number does not match %d != %d in %s", maxNum, mappingDetail.getTotalQueues(), mappingDetail.getBname()));
             } else {
                 maxNum = mappingDetail.getTotalQueues();
             }
@@ -266,7 +266,7 @@ public static void checkLogicQueueMappingItemOffset(List<LogicQueueMappingItem>
                 throw new RuntimeException("The non-latest item has negative logic offset");
             }
             if (lastGen != -1 && item.getGen() >= lastGen) {
-                throw new RuntimeException("The gen dose not increase monotonically");
+                throw new RuntimeException("The gen does not increase monotonically");
             }
 
             if (item.getEndOffset() != -1
@@ -276,10 +276,10 @@ public static void checkLogicQueueMappingItemOffset(List<LogicQueueMappingItem>
 
             if (lastOffset != -1 && item.getLogicOffset() != -1) {
                 if (item.getLogicOffset() >= lastOffset) {
-                    throw new RuntimeException("The base logic offset dose not increase monotonically");
+                    throw new RuntimeException("The base logic offset does not increase monotonically");
                 }
                 if (item.computeMaxStaticQueueOffset() >= lastOffset) {
-                    throw new RuntimeException("The max logic offset dose not increase monotonically");
+                    throw new RuntimeException("The max logic offset does not increase monotonically");
                 }
             }
             lastGen = item.getGen();
@@ -321,11 +321,11 @@ public static void  checkPhysicalQueueConsistence(Map<String, TopicConfigAndQueu
             for (List<LogicQueueMappingItem> items: configMapping.getMappingDetail().getHostedQueues().values()) {
                 for (LogicQueueMappingItem item: items) {
                     if (item.getStartOffset() != 0) {
-                        throw new RuntimeException("The start offset dose not begin from 0");
+                        throw new RuntimeException("The start offset does not begin from 0");
                     }
                     TopicConfig topicConfig = brokerConfigMap.get(item.getBname());
                     if (topicConfig == null) {
-                        throw new RuntimeException("The broker of item dose not exist");
+                        throw new RuntimeException("The broker of item does not exist");
                     }
                     if (item.getQueueId() >= topicConfig.getWriteQueueNums()) {
                         throw new RuntimeException("The physical queue id is overflow the write queues");
@@ -365,7 +365,7 @@ public static Map<Integer, TopicQueueMappingOne> checkAndBuildMappingItems(List<
         }
         if (checkConsistence) {
             if (maxNum != globalIdMap.size()) {
-                throw new RuntimeException(String.format("The total queue number in config dose not match the real hosted queues %d != %d", maxNum, globalIdMap.size()));
+                throw new RuntimeException(String.format("The total queue number in config does not match the real hosted queues %d != %d", maxNum, globalIdMap.size()));
             }
             for (int i = 0; i < maxNum; i++) {
                 if (!globalIdMap.containsKey(i)) {
@@ -417,7 +417,7 @@ public static void checkTargetBrokersComplete(Set<String> targetBrokers, Map<Str
                 continue;
             }
             if (!targetBrokers.contains(broker)) {
-                throw new RuntimeException("The existed broker " + broker + " dose not in target brokers ");
+                throw new RuntimeException("The existed broker " + broker + " does not in target brokers ");
             }
         }
     }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/SimpleSubscriptionData.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/SimpleSubscriptionData.java
index ec2b51e0b96..bb5c3074b44 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/SimpleSubscriptionData.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/SimpleSubscriptionData.java
@@ -65,7 +65,8 @@ public void setVersion(long version) {
         this.version = version;
     }
 
-    @Override public boolean equals(Object o) {
+    @Override
+    public boolean equals(Object o) {
         if (this == o) {
             return true;
         }
@@ -73,11 +74,12 @@ public void setVersion(long version) {
             return false;
         }
         SimpleSubscriptionData that = (SimpleSubscriptionData) o;
-        return version == that.version && Objects.equals(topic, that.topic);
+        return Objects.equals(topic, that.topic) && Objects.equals(expressionType, that.expressionType) && Objects.equals(expression, that.expression);
     }
 
-    @Override public int hashCode() {
-        return Objects.hash(topic, version);
+    @Override
+    public int hashCode() {
+        return Objects.hash(topic, expressionType, expression);
     }
 
     @Override public String toString() {
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/SubscriptionGroupConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/SubscriptionGroupConfig.java
index 5522059aaf5..85cbce9b6a9 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/SubscriptionGroupConfig.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/subscription/SubscriptionGroupConfig.java
@@ -182,6 +182,7 @@ public int hashCode() {
         result = prime * result + (consumeEnable ? 1231 : 1237);
         result = prime * result + (consumeFromMinEnable ? 1231 : 1237);
         result = prime * result + (notifyConsumerIdsChangedEnable ? 1231 : 1237);
+        result = prime * result + (consumeMessageOrderly ? 1231 : 1237);
         result = prime * result + ((groupName == null) ? 0 : groupName.hashCode());
         result = prime * result + retryMaxTimes;
         result = prime * result + retryQueueNums;
@@ -208,6 +209,7 @@ public boolean equals(Object obj) {
             .append(consumeEnable, other.consumeEnable)
             .append(consumeFromMinEnable, other.consumeFromMinEnable)
             .append(consumeBroadcastEnable, other.consumeBroadcastEnable)
+            .append(consumeMessageOrderly, other.consumeMessageOrderly)
             .append(retryQueueNums, other.retryQueueNums)
             .append(retryMaxTimes, other.retryMaxTimes)
             .append(whichBrokerWhenConsumeSlowly, other.whichBrokerWhenConsumeSlowly)
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java
index bca2d79d995..c8b404dd696 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java
@@ -174,6 +174,7 @@ public void operationSucceed(RemotingCommand response) {
                             PullMessageResponseHeader responseHeader =
                                 (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class);
                             rpcResponsePromise.setSuccess(new RpcResponse(response.getCode(), responseHeader, response.getBody()));
+                            break;
                         default:
                             RpcResponse rpcResponse = new RpcResponse(new RpcException(response.getCode(), "unexpected remote response code"));
                             rpcResponsePromise.setSuccess(rpcResponse);
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java
index 0cbe627d801..0e35acc01f9 100644
--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java
@@ -21,15 +21,14 @@
 import io.netty.channel.DefaultFileRegion;
 import io.netty.channel.FileRegion;
 import io.netty.channel.embedded.EmbeddedChannel;
-import org.junit.Assert;
-import org.junit.Test;
-
 import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.Random;
 import java.util.UUID;
+import org.junit.Assert;
+import org.junit.Test;
 
 public class FileRegionEncoderTest {
 
@@ -50,7 +49,7 @@ public void testEncode() throws IOException {
         random.nextBytes(data);
         write(file, data);
         FileRegion fileRegion = new DefaultFileRegion(file, 0, dataLength);
-        Assert.assertEquals(0, fileRegion.transfered());
+        Assert.assertEquals(0, fileRegion.transferred());
         Assert.assertEquals(dataLength, fileRegion.count());
         Assert.assertTrue(channel.writeOutbound(fileRegion));
         ByteBuf out = (ByteBuf) channel.readOutbound();
@@ -78,4 +77,4 @@ private static void write(File file, byte[] data) throws IOException {
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/ExportRocksDBConfigToJsonRequestHeaderTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/ExportRocksDBConfigToJsonRequestHeaderTest.java
new file mode 100644
index 00000000000..bbe625a42af
--- /dev/null
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/ExportRocksDBConfigToJsonRequestHeaderTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ExportRocksDBConfigToJsonRequestHeaderTest {
+    @Test
+    public void configTypeTest() {
+        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configTypes = new ArrayList<>();
+        configTypes.add(ExportRocksDBConfigToJsonRequestHeader.ConfigType.TOPICS);
+        configTypes.add(ExportRocksDBConfigToJsonRequestHeader.ConfigType.SUBSCRIPTION_GROUPS);
+
+        String string = ExportRocksDBConfigToJsonRequestHeader.ConfigType.toString(configTypes);
+
+        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> newConfigTypes = ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString(string);
+        assert newConfigTypes.size() == 2;
+        assert configTypes.equals(newConfigTypes);
+
+        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> topics = ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString("topics");
+        assert topics.size() == 1;
+        assert topics.get(0).equals(ExportRocksDBConfigToJsonRequestHeader.ConfigType.TOPICS);
+
+        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> mix = ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString("toPics; subScriptiongroups");
+        assert mix.size() == 2;
+        assert mix.get(0).equals(ExportRocksDBConfigToJsonRequestHeader.ConfigType.TOPICS);
+        assert mix.get(1).equals(ExportRocksDBConfigToJsonRequestHeader.ConfigType.SUBSCRIPTION_GROUPS);
+
+        Assert.assertThrows(IllegalArgumentException.class, () -> {
+            ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString("topics; subscription");
+        });
+
+    }
+}
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsRequestHeaderTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsRequestHeaderTest.java
new file mode 100644
index 00000000000..8004305e17a
--- /dev/null
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/GetConsumeStatsRequestHeaderTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.protocol.header;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class GetConsumeStatsRequestHeaderTest {
+
+    private GetConsumeStatsRequestHeader header;
+
+    @Before
+    public void setUp() {
+        header = new GetConsumeStatsRequestHeader();
+    }
+
+    @Test
+    public void updateTopicList_NullTopicList_DoesNotUpdate() {
+        header.updateTopicList(null);
+        assertNull(header.getTopicList());
+    }
+
+    @Test
+    public void updateTopicList_EmptyTopicList_SetsEmptyString() {
+        header.updateTopicList(Collections.emptyList());
+        assertNull(header.getTopicList());
+    }
+
+    @Test
+    public void updateTopicList_SingleTopic_SetsSingleTopicString() {
+        List<String> topicList = Collections.singletonList("TopicA");
+        header.updateTopicList(topicList);
+        assertEquals("TopicA;", header.getTopicList());
+    }
+
+    @Test
+    public void updateTopicList_MultipleTopics_SetsMultipleTopicsString() {
+        List<String> topicList = Arrays.asList("TopicA", "TopicB", "TopicC");
+        header.updateTopicList(topicList);
+        assertEquals("TopicA;TopicB;TopicC;", header.getTopicList());
+    }
+
+    @Test
+    public void updateTopicList_RepeatedTopics_SetsRepeatedTopicsString() {
+        List<String> topicList = Arrays.asList("TopicA", "TopicA", "TopicB");
+        header.updateTopicList(topicList);
+        assertEquals("TopicA;TopicA;TopicB;", header.getTopicList());
+    }
+
+    @Test
+    public void fetchTopicList_NullTopicList_ReturnsEmptyList() {
+        header.setTopicList(null);
+        List<String> topicList = header.fetchTopicList();
+        assertEquals(Collections.emptyList(), topicList);
+
+        header.updateTopicList(new ArrayList<>());
+        topicList = header.fetchTopicList();
+        assertEquals(Collections.emptyList(), topicList);
+    }
+
+    @Test
+    public void fetchTopicList_EmptyTopicList_ReturnsEmptyList() {
+        header.setTopicList("");
+        List<String> topicList = header.fetchTopicList();
+        assertEquals(Collections.emptyList(), topicList);
+    }
+
+    @Test
+    public void fetchTopicList_BlankTopicList_ReturnsEmptyList() {
+        header.setTopicList("   ");
+        List<String> topicList = header.fetchTopicList();
+        assertEquals(Collections.emptyList(), topicList);
+    }
+
+    @Test
+    public void fetchTopicList_SingleTopic_ReturnsSingleTopicList() {
+        header.setTopicList("TopicA");
+        List<String> topicList = header.fetchTopicList();
+        assertEquals(Collections.singletonList("TopicA"), topicList);
+    }
+
+    @Test
+    public void fetchTopicList_MultipleTopics_ReturnsTopicList() {
+        header.setTopicList("TopicA;TopicB;TopicC");
+        List<String> topicList = header.fetchTopicList();
+        assertEquals(Arrays.asList("TopicA", "TopicB", "TopicC"), topicList);
+    }
+
+    @Test
+    public void fetchTopicList_TopicListEndsWithSeparator_ReturnsTopicList() {
+        header.setTopicList("TopicA;TopicB;");
+        List<String> topicList = header.fetchTopicList();
+        assertEquals(Arrays.asList("TopicA", "TopicB"), topicList);
+    }
+
+    @Test
+    public void fetchTopicList_TopicListStartsWithSeparator_ReturnsTopicList() {
+        header.setTopicList(";TopicA;TopicB");
+        List<String> topicList = header.fetchTopicList();
+        assertEquals(Arrays.asList("TopicA", "TopicB"), topicList);
+    }
+}
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/subscription/SimpleSubscriptionDataTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/subscription/SimpleSubscriptionDataTest.java
new file mode 100644
index 00000000000..fa8e4ba6ae4
--- /dev/null
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/subscription/SimpleSubscriptionDataTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.remoting.protocol.subscription;
+
+import com.google.common.collect.Sets;
+import java.util.Set;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SimpleSubscriptionDataTest {
+    @Test
+    public void testNotEqual() {
+        String topic = "test-topic";
+        String expressionType = "TAG";
+        String expression1 = "test-expression-1";
+        String expression2 = "test-expression-2";
+        SimpleSubscriptionData simpleSubscriptionData1 = new SimpleSubscriptionData(topic, expressionType, expression1, 1);
+        SimpleSubscriptionData simpleSubscriptionData2 = new SimpleSubscriptionData(topic, expressionType, expression2, 1);
+        assertThat(simpleSubscriptionData1.equals(simpleSubscriptionData2)).isFalse();
+    }
+
+    @Test
+    public void testEqual() {
+        String topic = "test-topic";
+        String expressionType = "TAG";
+        String expression1 = "test-expression-1";
+        String expression2 = "test-expression-1";
+        SimpleSubscriptionData simpleSubscriptionData1 = new SimpleSubscriptionData(topic, expressionType, expression1, 1);
+        SimpleSubscriptionData simpleSubscriptionData2 = new SimpleSubscriptionData(topic, expressionType, expression2, 1);
+        assertThat(simpleSubscriptionData1.equals(simpleSubscriptionData2)).isTrue();
+    }
+
+    @Test
+    public void testSetNotEqual() {
+        String topic = "test-topic";
+        String expressionType = "TAG";
+        String expression1 = "test-expression-1";
+        String expression2 = "test-expression-2";
+        Set<SimpleSubscriptionData> set1 = Sets.newHashSet(new SimpleSubscriptionData(topic, expressionType, expression1, 1));
+        Set<SimpleSubscriptionData> set2 = Sets.newHashSet(new SimpleSubscriptionData(topic, expressionType, expression2, 1));
+        assertThat(set1.equals(set2)).isFalse();
+    }
+
+    @Test
+    public void testSetEqual() {
+        String topic = "test-topic";
+        String expressionType = "TAG";
+        String expression1 = "test-expression-1";
+        String expression2 = "test-expression-1";
+        Set<SimpleSubscriptionData> set1 = Sets.newHashSet(new SimpleSubscriptionData(topic, expressionType, expression1, 1));
+        Set<SimpleSubscriptionData> set2 = Sets.newHashSet(new SimpleSubscriptionData(topic, expressionType, expression2, 1));
+        assertThat(set1.equals(set2)).isTrue();
+    }
+}
\ No newline at end of file
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/rpc/ClientMetadataTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/rpc/ClientMetadataTest.java
new file mode 100644
index 00000000000..a9f38854586
--- /dev/null
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/rpc/ClientMetadataTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.rpc;
+
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ClientMetadataTest {
+
+    private ClientMetadata clientMetadata;
+
+    private final ConcurrentMap<String, TopicRouteData> topicRouteTable = new ConcurrentHashMap<>();
+
+    private final ConcurrentMap<String, ConcurrentMap<MessageQueue, String>> topicEndPointsTable = new ConcurrentHashMap<>();
+
+    private final ConcurrentMap<String, HashMap<Long, String>> brokerAddrTable = new ConcurrentHashMap<>();
+
+    private final String defaultTopic = "defaultTopic";
+
+    private final String defaultBroker = "defaultBroker";
+
+    @Before
+    public void init() throws IllegalAccessException {
+        clientMetadata = new ClientMetadata();
+
+        FieldUtils.writeDeclaredField(clientMetadata, "topicRouteTable", topicRouteTable, true);
+        FieldUtils.writeDeclaredField(clientMetadata, "topicEndPointsTable", topicEndPointsTable, true);
+        FieldUtils.writeDeclaredField(clientMetadata, "brokerAddrTable", brokerAddrTable, true);
+    }
+
+    @Test
+    public void testGetBrokerNameFromMessageQueue() {
+        MessageQueue mq1 = new MessageQueue(defaultTopic, "broker0", 0);
+        MessageQueue mq2 = new MessageQueue(defaultTopic, "broker1", 0);
+        ConcurrentMap<MessageQueue, String> messageQueueMap = new ConcurrentHashMap<>();
+        messageQueueMap.put(mq1, "broker0");
+        messageQueueMap.put(mq2, "broker1");
+        topicEndPointsTable.put(defaultTopic, messageQueueMap);
+
+        String actual = clientMetadata.getBrokerNameFromMessageQueue(mq1);
+        assertEquals("broker0", actual);
+    }
+
+    @Test
+    public void testGetBrokerNameFromMessageQueueNotFound() {
+        MessageQueue mq = new MessageQueue("topic1", "broker0", 0);
+        topicEndPointsTable.put(defaultTopic, new ConcurrentHashMap<>());
+
+        String actual = clientMetadata.getBrokerNameFromMessageQueue(mq);
+        assertEquals("broker0", actual);
+    }
+
+    @Test
+    public void testFindMasterBrokerAddrNotFound() {
+        assertNull(clientMetadata.findMasterBrokerAddr(defaultBroker));
+    }
+
+    @Test
+    public void testFindMasterBrokerAddr() {
+        String defaultBrokerAddr = "127.0.0.1:10911";
+        brokerAddrTable.put(defaultBroker, new HashMap<>());
+        brokerAddrTable.get(defaultBroker).put(0L, defaultBrokerAddr);
+
+        String actual = clientMetadata.findMasterBrokerAddr(defaultBroker);
+        assertEquals(defaultBrokerAddr, actual);
+    }
+
+    @Test
+    public void testTopicRouteData2EndpointsForStaticTopicNotFound() {
+        TopicRouteData topicRouteData = new TopicRouteData();
+        topicRouteData.setTopicQueueMappingByBroker(null);
+
+        ConcurrentMap<MessageQueue, String> actual = ClientMetadata.topicRouteData2EndpointsForStaticTopic(defaultTopic, topicRouteData);
+        assertTrue(actual.isEmpty());
+    }
+
+    @Test
+    public void testTopicRouteData2EndpointsForStaticTopic() {
+        TopicRouteData topicRouteData = new TopicRouteData();
+        Map<String, TopicQueueMappingInfo> mappingInfos = new HashMap<>();
+        TopicQueueMappingInfo info = new TopicQueueMappingInfo();
+        info.setScope("scope");
+        info.setCurrIdMap(new ConcurrentHashMap<>());
+        info.getCurrIdMap().put(0, 0);
+        info.setTotalQueues(1);
+        info.setBname("bname");
+        mappingInfos.put(defaultBroker, info);
+        topicRouteData.setTopicQueueMappingByBroker(mappingInfos);
+
+        ConcurrentMap<MessageQueue, String> actual = ClientMetadata.topicRouteData2EndpointsForStaticTopic(defaultTopic, topicRouteData);
+        assertEquals(1, actual.size());
+    }
+}
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/rpc/RpcClientImplTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/rpc/RpcClientImplTest.java
new file mode 100644
index 00000000000..c33511a9764
--- /dev/null
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/rpc/RpcClientImplTest.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.remoting.rpc;
+
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.InvokeCallback;
+import org.apache.rocketmq.remoting.RemotingClient;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.apache.rocketmq.remoting.protocol.RequestCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;
+import org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.GetMinOffsetResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader;
+import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetResponseHeader;
+import org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.concurrent.Future;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RpcClientImplTest {
+
+    @Mock
+    private RemotingClient remotingClient;
+
+    @Mock
+    private ClientMetadata clientMetadata;
+
+    private RpcClientImpl rpcClient;
+
+    private MessageQueue mq;
+
+    @Mock
+    private RpcRequest request;
+
+    private final long defaultTimeout = 3000L;
+
+    @Before
+    public void init() throws IllegalAccessException {
+        rpcClient = new RpcClientImpl(clientMetadata, remotingClient);
+
+        String defaultBroker = "brokerName";
+        mq = new MessageQueue("defaultTopic", defaultBroker, 0);
+        RpcRequestHeader header = mock(RpcRequestHeader.class);
+        when(request.getHeader()).thenReturn(header);
+        when(clientMetadata.getBrokerNameFromMessageQueue(mq)).thenReturn(defaultBroker);
+        when(clientMetadata.findMasterBrokerAddr(any())).thenReturn("127.0.0.1:10911");
+    }
+
+    @Test
+    public void testInvoke_PULL_MESSAGE() throws Exception {
+        when(request.getCode()).thenReturn(RequestCode.PULL_MESSAGE);
+
+        doAnswer(invocation -> {
+            InvokeCallback callback = invocation.getArgument(3);
+            RemotingCommand response = mock(RemotingCommand.class);
+            when(response.getBody()).thenReturn("success".getBytes());
+            PullMessageResponseHeader responseHeader = mock(PullMessageResponseHeader.class);
+            when(response.decodeCommandCustomHeader(PullMessageResponseHeader.class)).thenReturn(responseHeader);
+            callback.operationSucceed(response);
+            return null;
+        }).when(remotingClient).invokeAsync(
+                any(),
+                any(RemotingCommand.class),
+                anyLong(),
+                any(InvokeCallback.class));
+
+        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);
+        RpcResponse actual = future.get();
+
+        assertEquals(ResponseCode.SUCCESS, actual.getCode());
+        assertEquals("success", new String((byte[]) actual.getBody()));
+    }
+
+    @Test
+    public void testInvoke_GET_MIN_OFFSET() throws Exception {
+        when(request.getCode()).thenReturn(RequestCode.GET_MIN_OFFSET);
+
+        RemotingCommand responseCommand = mock(RemotingCommand.class);
+        when(responseCommand.getBody()).thenReturn("1".getBytes());
+        GetMinOffsetResponseHeader responseHeader = mock(GetMinOffsetResponseHeader.class);
+        when(responseCommand.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class)).thenReturn(responseHeader);
+        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);
+
+        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);
+        RpcResponse actual = future.get();
+
+        assertEquals(ResponseCode.SUCCESS, actual.getCode());
+        assertEquals("1", new String((byte[]) actual.getBody()));
+    }
+
+    @Test
+    public void testInvoke_GET_MAX_OFFSET() throws Exception {
+        when(request.getCode()).thenReturn(RequestCode.GET_MAX_OFFSET);
+
+        RemotingCommand responseCommand = mock(RemotingCommand.class);
+        when(responseCommand.getBody()).thenReturn("1000".getBytes());
+        GetMaxOffsetResponseHeader responseHeader = mock(GetMaxOffsetResponseHeader.class);
+        when(responseCommand.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class)).thenReturn(responseHeader);
+        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);
+
+        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);
+        RpcResponse actual = future.get();
+
+        assertEquals(ResponseCode.SUCCESS, actual.getCode());
+        assertEquals("1000", new String((byte[]) actual.getBody()));
+    }
+
+    @Test
+    public void testInvoke_SEARCH_OFFSET_BY_TIMESTAMP() throws Exception {
+        when(request.getCode()).thenReturn(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP);
+
+        RemotingCommand responseCommand = mock(RemotingCommand.class);
+        when(responseCommand.getBody()).thenReturn("1000".getBytes());
+        SearchOffsetResponseHeader responseHeader = mock(SearchOffsetResponseHeader.class);
+        when(responseCommand.decodeCommandCustomHeader(SearchOffsetResponseHeader.class)).thenReturn(responseHeader);
+        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);
+
+        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);
+        RpcResponse actual = future.get();
+
+        assertEquals(ResponseCode.SUCCESS, actual.getCode());
+        assertEquals("1000", new String((byte[]) actual.getBody()));
+    }
+
+    @Test
+    public void testInvoke_GET_EARLIEST_MSG_STORETIME() throws Exception {
+        when(request.getCode()).thenReturn(RequestCode.GET_EARLIEST_MSG_STORETIME);
+
+        RemotingCommand responseCommand = mock(RemotingCommand.class);
+        when(responseCommand.getBody()).thenReturn("10000".getBytes());
+        GetEarliestMsgStoretimeResponseHeader responseHeader = mock(GetEarliestMsgStoretimeResponseHeader.class);
+        when(responseCommand.decodeCommandCustomHeader(GetEarliestMsgStoretimeResponseHeader.class)).thenReturn(responseHeader);
+        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);
+
+        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);
+        RpcResponse actual = future.get();
+
+        assertEquals(ResponseCode.SUCCESS, actual.getCode());
+        assertEquals("10000", new String((byte[]) actual.getBody()));
+    }
+
+    @Test
+    public void testInvoke_QUERY_CONSUMER_OFFSET() throws Exception {
+        when(request.getCode()).thenReturn(RequestCode.QUERY_CONSUMER_OFFSET);
+
+        RemotingCommand responseCommand = mock(RemotingCommand.class);
+        when(responseCommand.getBody()).thenReturn("1000".getBytes());
+        QueryConsumerOffsetResponseHeader responseHeader = mock(QueryConsumerOffsetResponseHeader.class);
+        when(responseCommand.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class)).thenReturn(responseHeader);
+        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);
+
+        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);
+        RpcResponse actual = future.get();
+
+        assertEquals(ResponseCode.SUCCESS, actual.getCode());
+        assertEquals("1000", new String((byte[]) actual.getBody()));
+    }
+
+    @Test
+    public void testInvoke_UPDATE_CONSUMER_OFFSET() throws Exception {
+        when(request.getCode()).thenReturn(RequestCode.UPDATE_CONSUMER_OFFSET);
+
+        RemotingCommand responseCommand = mock(RemotingCommand.class);
+        when(responseCommand.getBody()).thenReturn("success".getBytes());
+        UpdateConsumerOffsetResponseHeader responseHeader = mock(UpdateConsumerOffsetResponseHeader.class);
+        when(responseCommand.decodeCommandCustomHeader(UpdateConsumerOffsetResponseHeader.class)).thenReturn(responseHeader);
+        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);
+
+        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);
+        RpcResponse actual = future.get();
+
+        assertEquals(ResponseCode.SUCCESS, actual.getCode());
+        assertEquals("success", new String((byte[]) actual.getBody()));
+    }
+
+    @Test
+    public void testInvoke_GET_TOPIC_STATS_INFO() throws Exception {
+        when(request.getCode()).thenReturn(RequestCode.GET_TOPIC_STATS_INFO);
+
+        RemotingCommand responseCommand = mock(RemotingCommand.class);
+        TopicStatsTable topicStatsTable = new TopicStatsTable();
+        when(responseCommand.getBody()).thenReturn(topicStatsTable.encode());
+        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);
+
+        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);
+        RpcResponse actual = future.get();
+
+        assertEquals(ResponseCode.SUCCESS, actual.getCode());
+        assertTrue(actual.getBody() instanceof TopicStatsTable);
+    }
+
+    @Test
+    public void testInvoke_GET_TOPIC_CONFIG() throws Exception {
+        when(request.getCode()).thenReturn(RequestCode.GET_TOPIC_CONFIG);
+
+        RemotingCommand responseCommand = mock(RemotingCommand.class);
+        TopicConfigAndQueueMapping topicConfigAndQueueMapping = new TopicConfigAndQueueMapping();
+        when(responseCommand.getBody()).thenReturn(RemotingSerializable.encode(topicConfigAndQueueMapping));
+        when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand);
+
+        Future<RpcResponse> future = rpcClient.invoke(mq, request, defaultTimeout);
+        RpcResponse actual = future.get();
+
+        assertEquals(ResponseCode.SUCCESS, actual.getCode());
+        assertTrue(actual.getBody() instanceof TopicConfigAndQueueMapping);
+    }
+}
diff --git a/srvutil/pom.xml b/srvutil/pom.xml
index 894e9cc6fd6..c117e8e3c2f 100644
--- a/srvutil/pom.xml
+++ b/srvutil/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/AclFileWatchService.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/AclFileWatchService.java
deleted file mode 100644
index eff9b422857..00000000000
--- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/AclFileWatchService.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.srvutil;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.rocketmq.common.ServiceThread;
-import org.apache.rocketmq.common.UtilAll;
-import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.MessageDigest;
-import java.util.HashMap;
-import java.util.Map;
-
-public class AclFileWatchService extends ServiceThread {
-    private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
-
-    private final String aclPath;
-    private int aclFilesNum;
-    @Deprecated
-    private final Map<String, String> fileCurrentHash;
-    private Map<String, Long> fileLastModifiedTime;
-    private List<String/**absolute pathname **/> fileList = new ArrayList<>();
-    private final AclFileWatchService.Listener listener;
-    private static final int WATCH_INTERVAL = 5000;
-    private MessageDigest md = MessageDigest.getInstance("MD5");
-    private String defaultAclFile;
-
-    public AclFileWatchService(String path, String defaultAclFile, final AclFileWatchService.Listener listener) throws Exception {
-        this.aclPath = path;
-        this.defaultAclFile = defaultAclFile;
-        this.fileCurrentHash = new HashMap<>();
-        this.fileLastModifiedTime = new HashMap<>();
-        this.listener = listener;
-
-        getAllAclFiles(path);
-        if (new File(this.defaultAclFile).exists() && !fileList.contains(this.defaultAclFile)) {
-            fileList.add(this.defaultAclFile);
-        }
-        this.aclFilesNum = fileList.size();
-        for (int i = 0; i < aclFilesNum; i++) {
-            String fileAbsolutePath = fileList.get(i);
-            this.fileLastModifiedTime.put(fileAbsolutePath, new File(fileAbsolutePath).lastModified());
-        }
-
-    }
-
-    public void getAllAclFiles(String path) {
-        File file = new File(path);
-        if (!file.exists()) {
-            log.info("The default acl dir {} is not exist", path);
-            return;
-        }
-        File[] files = file.listFiles();
-        for (int i = 0; i < files.length; i++) {
-            String fileName = files[i].getAbsolutePath();
-            File f = new File(fileName);
-            if (fileName.equals(aclPath + File.separator + "tools.yml")) {
-                continue;
-            } else if (fileName.endsWith(".yml") || fileName.endsWith(".yaml")) {
-                fileList.add(fileName);
-            } else if (f.isDirectory()) {
-                getAllAclFiles(fileName);
-            }
-        }
-    }
-
-    @Override
-    public String getServiceName() {
-        return "AclFileWatchService";
-    }
-
-    @Override
-    public void run() {
-        log.info(this.getServiceName() + " service started");
-
-        while (!this.isStopped()) {
-            try {
-                this.waitForRunning(WATCH_INTERVAL);
-
-                if (fileList.size() > 0) {
-                    fileList.clear();
-                }
-                getAllAclFiles(aclPath);
-                if (new File(defaultAclFile).exists() && !fileList.contains(defaultAclFile)) {
-                    fileList.add(defaultAclFile);
-                }
-                int realAclFilesNum = fileList.size();
-
-                if (aclFilesNum != realAclFilesNum) {
-                    log.info("aclFilesNum: " + aclFilesNum + "  realAclFilesNum: " + realAclFilesNum);
-                    aclFilesNum = realAclFilesNum;
-                    log.info("aclFilesNum: " + aclFilesNum + "  realAclFilesNum: " + realAclFilesNum);
-                    Map<String, Long> fileLastModifiedTime = new HashMap<>(realAclFilesNum);
-                    for (int i = 0; i < realAclFilesNum; i++) {
-                        String fileAbsolutePath = fileList.get(i);
-                        fileLastModifiedTime.put(fileAbsolutePath, new File(fileAbsolutePath).lastModified());
-                    }
-                    this.fileLastModifiedTime = fileLastModifiedTime;
-                    listener.onFileNumChanged(aclPath);
-                } else {
-                    for (int i = 0; i < aclFilesNum; i++) {
-                        String fileName = fileList.get(i);
-                        Long newLastModifiedTime = new File(fileName).lastModified();
-                        if (!newLastModifiedTime.equals(fileLastModifiedTime.get(fileName))) {
-                            fileLastModifiedTime.put(fileName, newLastModifiedTime);
-                            listener.onFileChanged(fileName);
-                        }
-                    }
-                }
-            } catch (Exception e) {
-                log.warn(this.getServiceName() + " service has exception. ", e);
-            }
-        }
-        log.info(this.getServiceName() + " service end");
-    }
-
-    @Deprecated
-    private String hash(String filePath) throws IOException {
-        Path path = Paths.get(filePath);
-        md.update(Files.readAllBytes(path));
-        byte[] hash = md.digest();
-        return UtilAll.bytes2string(hash);
-    }
-
-    public interface Listener {
-        /**
-         * Will be called when the target file is changed
-         *
-         * @param aclFileName the changed file absolute path
-         */
-        void onFileChanged(String aclFileName);
-
-        /**
-         * Will be called when the number of the acl file is changed
-         *
-         * @param path the path of the acl dir
-         */
-        void onFileNumChanged(String path);
-    }
-}
diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java
index 06c301bec9c..0c0690da5ce 100644
--- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java
+++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java
@@ -64,7 +64,7 @@ public void run0() {
                 this.waitForRunning(WATCH_INTERVAL);
                 for (Map.Entry<String, String> entry : currentHash.entrySet()) {
                     String newHash = md5Digest(entry.getKey());
-                    if (!newHash.equals(currentHash.get(entry.getKey()))) {
+                    if (!newHash.equals(entry.getValue())) {
                         entry.setValue(newHash);
                         listener.onChanged(entry.getKey());
                     }
diff --git a/store/BUILD.bazel b/store/BUILD.bazel
index 8364a239c9a..269ff2d9b34 100644
--- a/store/BUILD.bazel
+++ b/store/BUILD.bazel
@@ -24,6 +24,7 @@ java_library(
         "//common",
         "//remoting",
         "@maven//:com_alibaba_fastjson",
+        "@maven//:com_alibaba_fastjson2_fastjson2",
         "@maven//:com_conversantmedia_disruptor",
         "@maven//:com_google_guava_guava",
         "@maven//:commons_collections_commons_collections",
@@ -42,6 +43,8 @@ java_library(
         "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
         "@maven//:io_github_aliyunmq_rocketmq_logback_classic",
         "@maven//:org_apache_rocketmq_rocketmq_rocksdb",
+        "@maven//:com_google_code_findbugs_jsr305",
+        "@maven//:commons_validator_commons_validator",
     ],
 )
 
@@ -63,6 +66,7 @@ java_library(
         "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
         "@maven//:io_github_aliyunmq_rocketmq_logback_classic",
         "@maven//:org_apache_rocketmq_rocketmq_rocksdb",
+        "@maven//:org_junit_jupiter_junit_jupiter_api",
     ],
 )
 
diff --git a/store/pom.xml b/store/pom.xml
index e1e61612354..8b5552b443c 100644
--- a/store/pom.xml
+++ b/store/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java
index c8420fea11f..d9cd602a65c 100644
--- a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java
+++ b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java
@@ -132,12 +132,13 @@ public void shutdown() {
         super.shutdown(true);
         for (AllocateRequest req : this.requestTable.values()) {
             if (req.mappedFile != null) {
-                log.info("delete pre allocated maped file, {}", req.mappedFile.getFileName());
+                log.info("delete pre allocated mapped file, {}", req.mappedFile.getFileName());
                 req.mappedFile.destroy(1000);
             }
         }
     }
 
+    @Override
     public void run() {
         log.info(this.getServiceName() + " service started");
 
diff --git a/store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java b/store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java
index 4d53f3b7bcd..c3534d06113 100644
--- a/store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java
+++ b/store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java
@@ -25,4 +25,5 @@ public enum AppendMessageStatus {
     MESSAGE_SIZE_EXCEEDED,
     PROPERTIES_SIZE_EXCEEDED,
     UNKNOWN_ERROR,
+    ROCKSDB_ERROR,
 }
diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
index cc29cca5d94..75000b25d25 100644
--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java
@@ -16,6 +16,7 @@
  */
 package org.apache.rocketmq.store;
 
+import com.google.common.base.Strings;
 import java.net.Inet6Address;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
@@ -35,7 +36,6 @@
 import java.util.stream.Collectors;
 import com.sun.jna.NativeLong;
 import com.sun.jna.Pointer;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.ServiceThread;
 import org.apache.rocketmq.common.SystemClock;
@@ -58,9 +58,15 @@
 import org.apache.rocketmq.store.config.BrokerRole;
 import org.apache.rocketmq.store.config.FlushDiskType;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
+import org.apache.rocketmq.store.exception.StoreException;
 import org.apache.rocketmq.store.ha.HAService;
 import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService;
+import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLockImpl;
 import org.apache.rocketmq.store.logfile.MappedFile;
+import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
+import org.apache.rocketmq.store.queue.CqUnit;
+import org.apache.rocketmq.store.queue.ReferredIterator;
 import org.apache.rocketmq.store.util.LibC;
 import org.rocksdb.RocksDBException;
 
@@ -103,7 +109,6 @@ public class CommitLog implements Swappable {
     protected int commitLogSize;
 
     private final boolean enabledAppendPropCRC;
-    protected final MultiDispatch multiDispatch;
 
     public CommitLog(final DefaultMessageStore messageStore) {
         String storePath = messageStore.getMessageStoreConfig().getStorePathCommitLog();
@@ -129,7 +134,11 @@ protected PutMessageThreadLocal initialValue() {
                 return new PutMessageThreadLocal(defaultMessageStore.getMessageStoreConfig());
             }
         };
-        this.putMessageLock = messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock();
+
+        PutMessageLock adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl();
+
+        this.putMessageLock = messageStore.getMessageStoreConfig().getUseABSLock() ? adaptiveBackOffSpinLock :
+            messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock();
 
         this.flushDiskWatcher = new FlushDiskWatcher();
 
@@ -138,8 +147,6 @@ protected PutMessageThreadLocal initialValue() {
         this.commitLogSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();
 
         this.enabledAppendPropCRC = messageStore.getMessageStoreConfig().isEnabledAppendPropCRC();
-
-        this.multiDispatch = new MultiDispatch(defaultMessageStore);
     }
 
     public void setFullStorePaths(Set<String> fullStorePaths) {
@@ -322,22 +329,26 @@ public void recoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBExcep
         boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable();
         final List<MappedFile> mappedFiles = this.mappedFileQueue.getMappedFiles();
         if (!mappedFiles.isEmpty()) {
-            // Began to recover from the last third file
-            int index = mappedFiles.size() - 3;
-            if (index < 0) {
-                index = 0;
+            int index = mappedFiles.size() - 1;
+            while (index > 0) {
+                MappedFile mappedFile = mappedFiles.get(index);
+                if (mappedFile.getFileFromOffset() <= maxPhyOffsetOfConsumeQueue) {
+                    // It's safe to recover from this mapped file
+                    break;
+                }
+                index--;
             }
+            // TODO: Discuss if we need to load more commit-log mapped files into memory.
 
             MappedFile mappedFile = mappedFiles.get(index);
             ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();
             long processOffset = mappedFile.getFileFromOffset();
             long mappedFileOffset = 0;
             long lastValidMsgPhyOffset = this.getConfirmOffset();
-            // normal recover doesn't require dispatching
-            boolean doDispatch = false;
             while (true) {
                 DispatchRequest dispatchRequest = this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover, checkDupInfo);
                 int size = dispatchRequest.getMsgSize();
+                boolean doDispatch = dispatchRequest.getCommitLogOffset() > maxPhyOffsetOfConsumeQueue;
                 // Normal data
                 if (dispatchRequest.isSuccess() && size > 0) {
                     lastValidMsgPhyOffset = processOffset + mappedFileOffset;
@@ -424,8 +435,14 @@ private void doNothingForDeadCode(final Object obj) {
     public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer, final boolean checkCRC,
         final boolean checkDupInfo, final boolean readBody) {
         try {
+            if (byteBuffer.remaining() <= 4) {
+                return new DispatchRequest(-1, false /* fail */);
+            }
             // 1 TOTAL SIZE
             int totalSize = byteBuffer.getInt();
+            if (byteBuffer.remaining() < totalSize - 4) {
+                return new DispatchRequest(-1, false /* fail */);
+            }
 
             // 2 MAGIC CODE
             int magicCode = byteBuffer.getInt();
@@ -529,7 +546,7 @@ public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer,
                 }
 
                 String tags = propertiesMap.get(MessageConst.PROPERTY_TAGS);
-                if (tags != null && tags.length() > 0) {
+                if (!Strings.isNullOrEmpty(tags)) {
                     tagsCode = MessageExtBrokerInner.tagsString2tagsCode(MessageExt.parseTopicFilterType(sysFlag), tags);
                 }
 
@@ -569,7 +586,7 @@ public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer,
                             }
                         }
                     }
-                    if (expectedCRC > 0) {
+                    if (expectedCRC >= 0) {
                         ByteBuffer tmpBuffer = byteBuffer.duplicate();
                         tmpBuffer.position(tmpBuffer.position() - totalSize);
                         tmpBuffer.limit(tmpBuffer.position() + totalSize - CommitLog.CRC32_RESERVED_LEN);
@@ -620,6 +637,7 @@ public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer,
 
             return dispatchRequest;
         } catch (Exception e) {
+            log.error("checkMessageAndReturnSize failed, may can not dispatch", e);
         }
 
         return new DispatchRequest(-1, false /* success */);
@@ -651,7 +669,7 @@ public long getConfirmOffset() {
         } else if (this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {
             return this.confirmOffset;
         } else {
-            return getMaxOffset();
+            return this.defaultMessageStore.isSyncDiskFlush() ? getFlushedWhere() : getMaxOffset();
         }
     }
 
@@ -769,8 +787,11 @@ else if (size == 0) {
                 }
             }
 
-            // only for rocksdb mode
-            this.getMessageStore().finishCommitLogDispatch();
+            try {
+                this.getMessageStore().getQueueStore().flush();
+            } catch (StoreException e) {
+                log.error("Failed to flush ConsumeQueueStore", e);
+            }
 
             processOffset += mappedFileOffset;
             if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) {
@@ -987,7 +1008,7 @@ public CompletableFuture<PutMessageResult> asyncPutMessage(final MessageExtBroke
             msg.setEncodedBuff(putMessageThreadLocal.getEncoder().getEncoderBuffer());
             PutMessageContext putMessageContext = new PutMessageContext(topicQueueKey);
 
-            putMessageLock.lock(); //spin or ReentrantLock ,depending on store config
+            putMessageLock.lock(); //spin or ReentrantLock, depending on store config
             try {
                 long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
                 this.beginTimeInLock = beginLockTimestamp;
@@ -1834,12 +1855,13 @@ class DefaultAppendMessageCallback implements AppendMessageCallback {
         private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4;
         // Store the message content
         private final ByteBuffer msgStoreItemMemory;
-        private final int crc32ReservedLength = CommitLog.CRC32_RESERVED_LEN;
+        private final int crc32ReservedLength;
         private final MessageStoreConfig messageStoreConfig;
 
         DefaultAppendMessageCallback(MessageStoreConfig messageStoreConfig) {
             this.msgStoreItemMemory = ByteBuffer.allocate(END_FILE_MIN_BLANK_LENGTH);
             this.messageStoreConfig = messageStoreConfig;
+            this.crc32ReservedLength = messageStoreConfig.isEnabledAppendPropCRC() ? CommitLog.CRC32_RESERVED_LEN : 0;
         }
 
         public AppendMessageResult handlePropertiesForLmqMsg(ByteBuffer preEncodeBuffer,
@@ -1848,7 +1870,16 @@ public AppendMessageResult handlePropertiesForLmqMsg(ByteBuffer preEncodeBuffer,
                 return null;
             }
 
-            multiDispatch.wrapMultiDispatch(msgInner);
+            try {
+                LmqDispatch.wrapLmqDispatch(defaultMessageStore, msgInner);
+            } catch (ConsumeQueueException e) {
+                if (e.getCause() instanceof RocksDBException) {
+                    log.error("Failed to wrap multi-dispatch", e);
+                    return new AppendMessageResult(AppendMessageStatus.ROCKSDB_ERROR);
+                }
+                log.error("Failed to wrap multi-dispatch", e);
+                return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);
+            }
 
             msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
 
@@ -1902,7 +1933,7 @@ public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer
             // STORETIMESTAMP + STOREHOSTADDRESS + OFFSET <br>
 
             ByteBuffer preEncodeBuffer = msgInner.getEncodedBuff();
-            boolean isMultiDispatchMsg = messageStoreConfig.isEnableMultiDispatch() && CommitLog.isMultiDispatchMsg(msgInner);
+            boolean isMultiDispatchMsg = messageStoreConfig.isEnableLmq() && msgInner.needDispatchLMQ();
             if (isMultiDispatchMsg) {
                 AppendMessageResult appendMessageResult = handlePropertiesForLmqMsg(preEncodeBuffer, msgInner);
                 if (appendMessageResult != null) {
@@ -1964,23 +1995,28 @@ public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer
                     queueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills);
             }
 
-            int pos = 4 + 4 + 4 + 4 + 4;
+            int pos = 4     // 1 TOTALSIZE
+                + 4     // 2 MAGICCODE
+                + 4     // 3 BODYCRC
+                + 4     // 4 QUEUEID
+                + 4;    // 5 FLAG
             // 6 QUEUEOFFSET
             preEncodeBuffer.putLong(pos, queueOffset);
             pos += 8;
             // 7 PHYSICALOFFSET
             preEncodeBuffer.putLong(pos, fileFromOffset + byteBuffer.position());
+            pos += 8;
             int ipLen = (msgInner.getSysFlag() & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 + 4 : 16 + 4;
-            // 8 SYSFLAG, 9 BORNTIMESTAMP, 10 BORNHOST, 11 STORETIMESTAMP
-            pos += 8 + 4 + 8 + ipLen;
-            // refresh store time stamp in lock
+            // 8 SYSFLAG, 9 BORNTIMESTAMP, 10 BORNHOST
+            pos += 4 + 8 + ipLen;
+            // 11 STORETIMESTAMP refresh store time stamp in lock
             preEncodeBuffer.putLong(pos, msgInner.getStoreTimestamp());
             if (enabledAppendPropCRC) {
                 // 18 CRC32
                 int checkSize = msgLen - crc32ReservedLength;
                 ByteBuffer tmpBuffer = preEncodeBuffer.duplicate();
                 tmpBuffer.limit(tmpBuffer.position() + checkSize);
-                int crc32 = UtilAll.crc32(tmpBuffer);
+                int crc32 = UtilAll.crc32(tmpBuffer);   // UtilAll.crc32 function will change the position to limit of the buffer
                 tmpBuffer.limit(tmpBuffer.position() + crc32ReservedLength);
                 MessageDecoder.createCrc32(tmpBuffer, crc32);
             }
@@ -1993,7 +2029,12 @@ public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer
             msgInner.setEncodedBuff(null);
 
             if (isMultiDispatchMsg) {
-                CommitLog.this.multiDispatch.updateMultiQueueOffset(msgInner);
+                try {
+                    LmqDispatch.updateLmqOffsets(defaultMessageStore, msgInner);
+                } catch (ConsumeQueueException e) {
+                    // Increase in-memory max offset of the queue should not fail.
+                    return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);
+                }
             }
 
             return new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, msgLen, msgIdSupplier,
@@ -2116,7 +2157,8 @@ public DefaultFlushManager() {
             this.commitRealTimeService = new CommitLog.CommitRealTimeService();
         }
 
-        @Override public void start() {
+        @Override
+        public void start() {
             this.flushCommitLogService.start();
 
             if (defaultMessageStore.isTransientStorePoolEnable()) {
@@ -2124,6 +2166,7 @@ public DefaultFlushManager() {
             }
         }
 
+        @Override
         public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult,
             MessageExt messageExt) {
             // Synchronization flush
@@ -2150,9 +2193,13 @@ public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMess
             // Asynchronous flush
             else {
                 if (!CommitLog.this.defaultMessageStore.isTransientStorePoolEnable()) {
-                    flushCommitLogService.wakeup();
+                    if (defaultMessageStore.getMessageStoreConfig().isWakeFlushWhenPutMessage()) {
+                        flushCommitLogService.wakeup();
+                    }
                 } else {
-                    commitRealTimeService.wakeup();
+                    if (defaultMessageStore.getMessageStoreConfig().isWakeCommitWhenPutMessage()) {
+                        commitRealTimeService.wakeup();
+                    }
                 }
             }
         }
@@ -2175,9 +2222,13 @@ public CompletableFuture<PutMessageStatus> handleDiskFlush(AppendMessageResult r
             // Asynchronous flush
             else {
                 if (!CommitLog.this.defaultMessageStore.isTransientStorePoolEnable()) {
-                    flushCommitLogService.wakeup();
+                    if (defaultMessageStore.getMessageStoreConfig().isWakeFlushWhenPutMessage()) {
+                        flushCommitLogService.wakeup();
+                    }
                 } else {
-                    commitRealTimeService.wakeup();
+                    if (defaultMessageStore.getMessageStoreConfig().isWakeCommitWhenPutMessage()) {
+                        commitRealTimeService.wakeup();
+                    }
                 }
                 return CompletableFuture.completedFuture(PutMessageStatus.PUT_OK);
             }
@@ -2236,10 +2287,6 @@ public FlushManager getFlushManager() {
         return flushManager;
     }
 
-    public static boolean isMultiDispatchMsg(MessageExtBrokerInner msg) {
-        return StringUtils.isNoneBlank(msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)) && !msg.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX);
-    }
-
     private boolean isCloseReadAhead() {
         return !MixAll.isWindows() && !defaultMessageStore.getMessageStoreConfig().isDataReadAheadEnable();
     }
@@ -2398,16 +2445,15 @@ public boolean isMsgInColdArea(String group, String topic, int queueId, long off
                 return false;
             }
             try {
-                ConsumeQueue consumeQueue = (ConsumeQueue) defaultMessageStore.findConsumeQueue(topic, queueId);
+                ConsumeQueueInterface consumeQueue = defaultMessageStore.findConsumeQueue(topic, queueId);
                 if (null == consumeQueue) {
                     return false;
                 }
-                SelectMappedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(offset);
-                if (null == bufferConsumeQueue || null == bufferConsumeQueue.getByteBuffer()) {
+                ReferredIterator<CqUnit> bufferConsumeQueue = consumeQueue.iterateFrom(offset, 1);
+                if (null == bufferConsumeQueue || !bufferConsumeQueue.hasNext()) {
                     return false;
                 }
-                long offsetPy = bufferConsumeQueue.getByteBuffer().getLong();
-                return defaultMessageStore.checkInColdAreaByCommitOffset(offsetPy, getMaxOffset());
+                return defaultMessageStore.checkInColdAreaByCommitOffset(bufferConsumeQueue.next().getPos(), getMaxOffset());
             } catch (Exception e) {
                 log.error("isMsgInColdArea group: {}, topic: {}, queueId: {}, offset: {}",
                     group, topic, queueId, offset, e);
diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
index 453c9d1dc72..b6b9cff538d 100644
--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
+++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
@@ -727,8 +727,8 @@ private void multiDispatchLmqQueue(DispatchRequest request, int maxRetries) {
         Map<String, String> prop = request.getPropertiesMap();
         String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
         String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);
-        String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
-        String[] queueOffsets = multiQueueOffset.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
+        String[] queues = multiDispatchQueue.split(MixAll.LMQ_DISPATCH_SEPARATOR);
+        String[] queueOffsets = multiQueueOffset.split(MixAll.LMQ_DISPATCH_SEPARATOR);
         if (queues.length != queueOffsets.length) {
             log.error("[bug] queues.length!=queueOffsets.length ", request.getTopic());
             return;
@@ -816,7 +816,7 @@ private boolean putMessagePositionInfo(final long offset, final int size, final
                 long currentLogicOffset = mappedFile.getWrotePosition() + mappedFile.getFileFromOffset();
 
                 if (expectLogicOffset < currentLogicOffset) {
-                    log.warn("Build  consume queue repeatedly, expectLogicOffset: {} currentLogicOffset: {} Topic: {} QID: {} Diff: {}",
+                    log.warn("Build consume queue repeatedly, expectLogicOffset: {} currentLogicOffset: {} Topic: {} QID: {} Diff: {}",
                         expectLogicOffset, currentLogicOffset, this.topic, this.queueId, expectLogicOffset - currentLogicOffset);
                     return true;
                 }
@@ -833,7 +833,13 @@ private boolean putMessagePositionInfo(final long offset, final int size, final
                 }
             }
             this.setMaxPhysicOffset(offset + size);
-            return mappedFile.appendMessage(this.byteBufferIndex.array());
+            boolean appendResult;
+            if (messageStore.getMessageStoreConfig().isPutConsumeQueueDataByFileChannel()) {
+                appendResult = mappedFile.appendMessageUsingFileChannel(this.byteBufferIndex.array());
+            } else {
+                appendResult = mappedFile.appendMessage(this.byteBufferIndex.array());
+            }
+            return appendResult;
         }
         return false;
     }
@@ -846,7 +852,12 @@ private void fillPreBlank(final MappedFile mappedFile, final long untilWhere) {
 
         int until = (int) (untilWhere % this.mappedFileQueue.getMappedFileSize());
         for (int i = 0; i < until; i += CQ_STORE_UNIT_SIZE) {
-            mappedFile.appendMessage(byteBuffer.array());
+            if (messageStore.getMessageStoreConfig().isPutConsumeQueueDataByFileChannel()) {
+                mappedFile.appendMessageUsingFileChannel(byteBuffer.array());
+            } else {
+                mappedFile.appendMessage(byteBuffer.array());
+            }
+
         }
     }
 
diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java
index 780505c53d1..3f266378df3 100644
--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java
+++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueueExt.java
@@ -33,7 +33,7 @@
  * such as message store time, filter bit map and etc.
  * <p/>
  * <li>1. This class is used only by {@link ConsumeQueue}</li>
- * <li>2. And is week reliable.</li>
+ * <li>2. And is weakly reliable.</li>
  * <li>3. Be careful, address returned is always less than 0.</li>
  * <li>4. Pls keep this file small.</li>
  */
diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
index 48a3adcc086..360523be852 100644
--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
@@ -58,6 +58,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.validator.routines.InetAddressValidator;
 import org.apache.rocketmq.common.AbstractBrokerRunnable;
 import org.apache.rocketmq.common.BoundaryType;
 import org.apache.rocketmq.common.BrokerConfig;
@@ -92,6 +93,7 @@
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
 import org.apache.rocketmq.store.dledger.DLedgerCommitLog;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.ha.DefaultHAService;
 import org.apache.rocketmq.store.ha.HAService;
 import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService;
@@ -162,12 +164,14 @@ public class DefaultMessageStore implements MessageStore {
     private volatile boolean shutdown = true;
     protected boolean notifyMessageArriveInBatch = false;
 
-    private StoreCheckpoint storeCheckpoint;
+    protected StoreCheckpoint storeCheckpoint;
     private TimerMessageStore timerMessageStore;
 
     private final LinkedList<CommitLogDispatcher> dispatcherList;
 
-    private RandomAccessFile lockFile;
+    private RocksDBMessageStore rocksDBMessageStore;
+
+    private final RandomAccessFile lockFile;
 
     private FileLock lock;
 
@@ -187,7 +191,7 @@ public class DefaultMessageStore implements MessageStore {
 
     private volatile long brokerInitMaxOffset = -1L;
 
-    private List<PutMessageHook> putMessageHookList = new ArrayList<>();
+    private final List<PutMessageHook> putMessageHookList = new ArrayList<>();
 
     private SendMessageBackHook sendMessageBackHook;
 
@@ -200,20 +204,21 @@ public class DefaultMessageStore implements MessageStore {
 
     private final ConcurrentLinkedQueue<BatchDispatchRequest> batchDispatchRequestQueue = new ConcurrentLinkedQueue<>();
 
-    private int dispatchRequestOrderlyQueueSize = 16;
+    private final int dispatchRequestOrderlyQueueSize = 16;
 
     private final DispatchRequestOrderlyQueue dispatchRequestOrderlyQueue = new DispatchRequestOrderlyQueue(dispatchRequestOrderlyQueueSize);
 
     private long stateMachineVersion = 0L;
 
     // this is a unmodifiableMap
-    private ConcurrentMap<String, TopicConfig> topicConfigTable;
+    private final ConcurrentMap<String, TopicConfig> topicConfigTable;
 
     private final ScheduledExecutorService scheduledCleanQueueExecutorService =
         ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread"));
 
     public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
-        final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap<String, TopicConfig> topicConfigTable) throws IOException {
+        final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig,
+        final ConcurrentMap<String, TopicConfig> topicConfigTable) throws IOException {
         this.messageArrivingListener = messageArrivingListener;
         this.brokerConfig = brokerConfig;
         this.messageStoreConfig = messageStoreConfig;
@@ -353,12 +358,7 @@ public boolean load() {
             }
 
             if (result) {
-                this.storeCheckpoint =
-                    new StoreCheckpoint(
-                        StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig.getStorePathRootDir()));
-                this.masterFlushedOffset = this.storeCheckpoint.getMasterFlushedOffset();
-                setConfirmOffset(this.storeCheckpoint.getConfirmPhyOffset());
-
+                loadCheckPoint();
                 result = this.indexService.load(lastExitOK);
                 this.recover(lastExitOK);
                 LOGGER.info("message store recover end, and the max phy offset = {}", this.getMaxPhyOffset());
@@ -380,6 +380,14 @@ public boolean load() {
         return result;
     }
 
+    public void loadCheckPoint() throws IOException {
+        this.storeCheckpoint =
+            new StoreCheckpoint(
+                StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig.getStorePathRootDir()));
+        this.masterFlushedOffset = this.storeCheckpoint.getMasterFlushedOffset();
+        setConfirmOffset(this.storeCheckpoint.getConfirmPhyOffset());
+    }
+
     /**
      * @throws Exception
      */
@@ -432,7 +440,7 @@ private void doRecheckReputOffsetFromCq() throws InterruptedException {
             return;
         }
 
-        /**
+        /*
          * 1. Make sure the fast-forward messages to be truncated during the recovering according to the max physical offset of the commitlog;
          * 2. DLedger committedPos may be missing, so the maxPhysicalPosInLogicQueue maybe bigger that maxOffset returned by DLedgerCommitLog, just let it go;
          * 3. Calculate the reput offset according to the consume queue;
@@ -452,7 +460,7 @@ private void doRecheckReputOffsetFromCq() throws InterruptedException {
         }
         if (maxPhysicalPosInLogicQueue < this.commitLog.getMinOffset()) {
             maxPhysicalPosInLogicQueue = this.commitLog.getMinOffset();
-            /**
+            /*
              * This happens in following conditions:
              * 1. If someone removes all the consumequeue files or the disk get damaged.
              * 2. Launch a new broker, and copy the commitlog from other brokers.
@@ -509,10 +517,11 @@ public void shutdown() {
             if (this.compactionService != null) {
                 this.compactionService.shutdown();
             }
-
+            if (this.rocksDBMessageStore != null && this.rocksDBMessageStore.consumeQueueStore != null) {
+                this.rocksDBMessageStore.consumeQueueStore.shutdown();
+            }
             this.flushConsumeQueueService.shutdown();
             this.allocateMappedFileService.shutdown();
-            this.storeCheckpoint.flush();
             this.storeCheckpoint.shutdown();
 
             this.perfs.shutdown();
@@ -807,6 +816,7 @@ public GetMessageResult getMessage(final String group, final String topic, final
         long maxOffset = 0;
 
         GetMessageResult getResult = new GetMessageResult();
+        int filterMessageCount = 0;
 
         final long maxOffsetPy = this.commitLog.getMaxOffset();
 
@@ -865,7 +875,7 @@ public GetMessageResult getMessage(final String group, final String topic, final
 
                             boolean isInMem = estimateInMemByCommitOffset(offsetPy, maxOffsetPy);
 
-                            if ((cqUnit.getQueueOffset() - offset) * consumeQueue.getUnitSize() > maxFilterMessageSize) {
+                            if ((cqUnit.getQueueOffset() - offset) * consumeQueue.getUnitSize() >= maxFilterMessageSize) {
                                 break;
                             }
 
@@ -907,7 +917,7 @@ public GetMessageResult getMessage(final String group, final String topic, final
                                 continue;
                             }
 
-                            if (messageStoreConfig.isColdDataFlowControlEnable() && !MixAll.isSysConsumerGroupForNoColdReadLimit(group) && !selectResult.isInCache()) {
+                            if (messageStoreConfig.isColdDataFlowControlEnable() && !MixAll.isSysConsumerGroupPullMessage(group) && !selectResult.isInCache()) {
                                 getResult.setColdDataSum(getResult.getColdDataSum() + sizePy);
                             }
 
@@ -918,6 +928,7 @@ public GetMessageResult getMessage(final String group, final String topic, final
                                 }
                                 // release...
                                 selectResult.release();
+                                filterMessageCount++;
                                 continue;
                             }
                             this.storeStatsService.getGetMessageTransferredMsgCount().add(cqUnit.getBatchNum());
@@ -955,6 +966,12 @@ public GetMessageResult getMessage(final String group, final String topic, final
         } else {
             this.storeStatsService.getGetMessageTimesTotalMiss().add(1);
         }
+
+        if (this.messageStoreConfig.isDiskFallRecorded() && GetMessageStatus.OFFSET_OVERFLOW_ONE == status) {
+            brokerStatsManager.recordDiskFallBehindSize(group, topic, queueId, 0);
+            brokerStatsManager.recordDiskFallBehindTime(group, topic, queueId, 0);
+        }
+
         long elapsedTime = this.getSystemClock().now() - beginTime;
         this.storeStatsService.setGetMessageEntireTimeMax(elapsedTime);
 
@@ -967,6 +984,7 @@ public GetMessageResult getMessage(final String group, final String topic, final
         getResult.setNextBeginOffset(nextBeginOffset);
         getResult.setMaxOffset(maxOffset);
         getResult.setMinOffset(minOffset);
+        getResult.setFilterMessageCount(filterMessageCount);
         return getResult;
     }
 
@@ -977,14 +995,14 @@ public CompletableFuture<GetMessageResult> getMessageAsync(String group, String
     }
 
     @Override
-    public long getMaxOffsetInQueue(String topic, int queueId) {
+    public long getMaxOffsetInQueue(String topic, int queueId) throws ConsumeQueueException {
         return getMaxOffsetInQueue(topic, queueId, true);
     }
 
     @Override
-    public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) {
+    public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) throws ConsumeQueueException {
         if (committed) {
-            ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId);
+            ConsumeQueueInterface logic = this.getConsumeQueue(topic, queueId);
             if (logic != null) {
                 return logic.getMaxOffsetInQueue();
             }
@@ -1020,7 +1038,7 @@ public void setTimerMessageStore(TimerMessageStore timerMessageStore) {
 
     @Override
     public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) {
-        ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId);
+        ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId);
         if (consumeQueue != null) {
             CqUnit cqUnit = consumeQueue.get(consumeQueueOffset);
             if (cqUnit != null) {
@@ -1156,7 +1174,7 @@ public boolean getLastMappedFile(long startOffset) {
 
     @Override
     public long getEarliestMessageTime(String topic, int queueId) {
-        ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId);
+        ConsumeQueueInterface logicQueue = this.getConsumeQueue(topic, queueId);
         if (logicQueue != null) {
             Pair<CqUnit, Long> pair = logicQueue.getEarliestUnitAndStoreTime();
             if (pair != null && pair.getObject2() != null) {
@@ -1178,13 +1196,17 @@ public long getEarliestMessageTime() {
         if (this.getCommitLog() instanceof DLedgerCommitLog) {
             minPhyOffset += DLedgerEntry.BODY_OFFSET;
         }
-        final int size = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION + 8;
+        int size = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION + 8;
+        InetAddressValidator validator = InetAddressValidator.getInstance();
+        if (validator.isValidInet6Address(this.brokerConfig.getBrokerIP1())) {
+            size = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION + 20;
+        }
         return this.getCommitLog().pickupStoreTimestamp(minPhyOffset, size);
     }
 
     @Override
     public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) {
-        ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId);
+        ConsumeQueueInterface logicQueue = this.getConsumeQueue(topic, queueId);
         if (logicQueue != null) {
             Pair<CqUnit, Long> pair = logicQueue.getCqUnitAndStoreTime(consumeQueueOffset);
             if (pair != null && pair.getObject2() != null) {
@@ -1202,12 +1224,12 @@ public CompletableFuture<Long> getMessageStoreTimeStampAsync(String topic, int q
 
     @Override
     public long getMessageTotalInQueue(String topic, int queueId) {
-        ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId);
+        ConsumeQueueInterface logicQueue = this.getConsumeQueue(topic, queueId);
         if (logicQueue != null) {
             return logicQueue.getMessageTotalInQueue();
         }
 
-        return -1;
+        return 0;
     }
 
     @Override
@@ -1364,7 +1386,6 @@ public long now() {
      * If offset table is cleaned, and old messages are dispatching after the old consume queue is cleaned,
      * consume queue will be created with old offset, then later message with new offset table can not be
      * dispatched to consume queue.
-     * @throws RocksDBException only in rocksdb mode
      */
     @Override
     public int deleteTopics(final Set<String> deleteTopics) {
@@ -1491,7 +1512,7 @@ public boolean checkInDiskByConsumeOffset(final String topic, final int queueId,
 
         final long maxOffsetPy = this.commitLog.getMaxOffset();
 
-        ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId);
+        ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId);
         if (consumeQueue != null) {
             CqUnit cqUnit = consumeQueue.get(consumeOffset);
 
@@ -1507,7 +1528,7 @@ public boolean checkInDiskByConsumeOffset(final String topic, final int queueId,
 
     @Override
     public boolean checkInMemByConsumeOffset(final String topic, final int queueId, long consumeOffset, int batchSize) {
-        ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId);
+        ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId);
         if (consumeQueue != null) {
             CqUnit firstCQItem = consumeQueue.get(consumeOffset);
             if (firstCQItem == null) {
@@ -1541,9 +1562,17 @@ public boolean checkInStoreByConsumeOffset(String topic, int queueId, long consu
     public long dispatchBehindBytes() {
         return this.reputMessageService.behind();
     }
+    @Override
+    public long dispatchBehindMilliseconds() {
+        return this.reputMessageService.behindMs();
+    }
 
     public long flushBehindBytes() {
-        return this.commitLog.remainHowManyDataToCommit() + this.commitLog.remainHowManyDataToFlush();
+        if (this.messageStoreConfig.isTransientStorePoolEnable()) {
+            return this.commitLog.remainHowManyDataToCommit() + this.commitLog.remainHowManyDataToFlush();
+        } else {
+            return this.commitLog.remainHowManyDataToFlush();
+        }
     }
 
     @Override
@@ -1730,10 +1759,11 @@ public boolean checkInDiskByCommitOffset(long offsetPy) {
     /**
      * The ratio val is estimated by the experiment and experience
      * so that the result is not high accurate for different business
+     *
      * @return
      */
     public boolean checkInColdAreaByCommitOffset(long offsetPy, long maxOffsetPy) {
-        long memory = (long)(StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE * (this.messageStoreConfig.getAccessMessageInMemoryHotRatio() / 100.0));
+        long memory = (long) (StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE * (this.messageStoreConfig.getAccessMessageInMemoryHotRatio() / 100.0));
         return (maxOffsetPy - offsetPy) > memory;
     }
 
@@ -1911,11 +1941,6 @@ public MessageStoreConfig getMessageStoreConfig() {
         return messageStoreConfig;
     }
 
-    @Override
-    public void finishCommitLogDispatch() {
-        // ignore
-    }
-
     @Override
     public TransientStorePool getTransientStorePool() {
         return transientStorePool;
@@ -2183,7 +2208,7 @@ class CleanCommitLogService {
             System.getProperty("rocketmq.broker.diskSpaceCleanForciblyRatio", "");
         private long lastRedeleteTimestamp = 0;
 
-        private volatile int manualDeleteFileSeveralTimes = 0;
+        private final AtomicInteger manualDeleteFileSeveralTimes = new AtomicInteger();
 
         private volatile boolean cleanImmediately = false;
 
@@ -2226,7 +2251,7 @@ class CleanCommitLogService {
         }
 
         public void executeDeleteFilesManually() {
-            this.manualDeleteFileSeveralTimes = MAX_MANUAL_DELETE_FILE_TIMES;
+            this.manualDeleteFileSeveralTimes.set(MAX_MANUAL_DELETE_FILE_TIMES);
             DefaultMessageStore.LOGGER.info("executeDeleteFilesManually was invoked");
         }
 
@@ -2248,12 +2273,12 @@ private void deleteExpiredFiles() {
 
             boolean isTimeUp = this.isTimeToDelete();
             boolean isUsageExceedsThreshold = this.isSpaceToDelete();
-            boolean isManualDelete = this.manualDeleteFileSeveralTimes > 0;
+            boolean isManualDelete = this.manualDeleteFileSeveralTimes.get() > 0;
 
             if (isTimeUp || isUsageExceedsThreshold || isManualDelete) {
 
                 if (isManualDelete) {
-                    this.manualDeleteFileSeveralTimes--;
+                    this.manualDeleteFileSeveralTimes.decrementAndGet();
                 }
 
                 boolean cleanAtOnce = DefaultMessageStore.this.getMessageStoreConfig().isCleanFileForciblyEnable() && this.cleanImmediately;
@@ -2262,7 +2287,7 @@ private void deleteExpiredFiles() {
                     fileReservedTime,
                     isTimeUp,
                     isUsageExceedsThreshold,
-                    manualDeleteFileSeveralTimes,
+                    manualDeleteFileSeveralTimes.get(),
                     cleanAtOnce,
                     deleteFileBatchMax);
 
@@ -2407,11 +2432,11 @@ private boolean isSpaceToDelete() {
         }
 
         public int getManualDeleteFileSeveralTimes() {
-            return manualDeleteFileSeveralTimes;
+            return manualDeleteFileSeveralTimes.get();
         }
 
         public void setManualDeleteFileSeveralTimes(int manualDeleteFileSeveralTimes) {
-            this.manualDeleteFileSeveralTimes = manualDeleteFileSeveralTimes;
+            this.manualDeleteFileSeveralTimes.set(manualDeleteFileSeveralTimes);
         }
 
         public double calcStorePathPhysicRatio() {
@@ -2695,15 +2720,15 @@ public long getJoinTime() {
         }
     }
 
-    class BatchDispatchRequest {
+    static class BatchDispatchRequest {
 
-        private ByteBuffer byteBuffer;
+        private final ByteBuffer byteBuffer;
 
-        private int position;
+        private final int position;
 
-        private int size;
+        private final int size;
 
-        private long id;
+        private final long id;
 
         public BatchDispatchRequest(ByteBuffer byteBuffer, int position, int size, long id) {
             this.byteBuffer = byteBuffer;
@@ -2713,7 +2738,7 @@ public BatchDispatchRequest(ByteBuffer byteBuffer, int position, int size, long
         }
     }
 
-    class DispatchRequestOrderlyQueue {
+    static class DispatchRequestOrderlyQueue {
 
         DispatchRequest[][] buffer;
 
@@ -2778,6 +2803,7 @@ public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) {
     class ReputMessageService extends ServiceThread {
 
         protected volatile long reputFromOffset = 0;
+        protected volatile long currentReputTimestamp = System.currentTimeMillis();
 
         public long getReputFromOffset() {
             return reputFromOffset;
@@ -2787,6 +2813,10 @@ public void setReputFromOffset(long reputFromOffset) {
             this.reputFromOffset = reputFromOffset;
         }
 
+        public long getCurrentReputTimestamp() {
+            return currentReputTimestamp;
+        }
+
         @Override
         public void shutdown() {
             for (int i = 0; i < 50 && this.isCommitLogAvailable(); i++) {
@@ -2809,8 +2839,21 @@ public long behind() {
             return DefaultMessageStore.this.getConfirmOffset() - this.reputFromOffset;
         }
 
+        public long behindMs() {
+            long lastCommitLogFileTimeStamp = System.currentTimeMillis();
+            MappedFile lastMappedFile = DefaultMessageStore.this.commitLog.getMappedFileQueue().getLastMappedFile();
+            if (lastMappedFile != null) {
+                lastCommitLogFileTimeStamp = lastMappedFile.getStoreTimestamp();
+            }
+            return Math.max(0, lastCommitLogFileTimeStamp - this.currentReputTimestamp);
+        }
+
         public boolean isCommitLogAvailable() {
-            return this.reputFromOffset < DefaultMessageStore.this.getConfirmOffset();
+            return this.reputFromOffset < getReputEndOffset();
+        }
+
+        protected long getReputEndOffset() {
+            return DefaultMessageStore.this.getMessageStoreConfig().isReadUnCommitted() ? DefaultMessageStore.this.commitLog.getMaxOffset() : DefaultMessageStore.this.commitLog.getConfirmOffset();
         }
 
         public void doReput() {
@@ -2819,7 +2862,11 @@ public void doReput() {
                     this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset());
                 this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset();
             }
-            for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) {
+            boolean isCommitLogAvailable = isCommitLogAvailable();
+            if (!isCommitLogAvailable) {
+                currentReputTimestamp = System.currentTimeMillis();
+            }
+            for (boolean doNext = true; isCommitLogAvailable && doNext; ) {
 
                 SelectMappedBufferResult result = DefaultMessageStore.this.commitLog.getData(reputFromOffset);
 
@@ -2830,18 +2877,19 @@ public void doReput() {
                 try {
                     this.reputFromOffset = result.getStartOffset();
 
-                    for (int readSize = 0; readSize < result.getSize() && reputFromOffset < DefaultMessageStore.this.getConfirmOffset() && doNext; ) {
+                    for (int readSize = 0; readSize < result.getSize() && reputFromOffset < getReputEndOffset() && doNext; ) {
                         DispatchRequest dispatchRequest =
                             DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(result.getByteBuffer(), false, false, false);
                         int size = dispatchRequest.getBufferSize() == -1 ? dispatchRequest.getMsgSize() : dispatchRequest.getBufferSize();
 
-                        if (reputFromOffset + size > DefaultMessageStore.this.getConfirmOffset()) {
+                        if (reputFromOffset + size > getReputEndOffset()) {
                             doNext = false;
                             break;
                         }
 
                         if (dispatchRequest.isSuccess()) {
                             if (size > 0) {
+                                currentReputTimestamp = dispatchRequest.getStoreTimestamp();
                                 DefaultMessageStore.this.doDispatch(dispatchRequest);
 
                                 if (!notifyMessageArriveInBatch) {
@@ -2885,8 +2933,6 @@ public void doReput() {
                 } finally {
                     result.release();
                 }
-
-                finishCommitLogDispatch();
             }
         }
 
@@ -2900,8 +2946,8 @@ private void notifyMessageArrive4MultiQueue(DispatchRequest dispatchRequest) {
             if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) {
                 return;
             }
-            String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
-            String[] queueOffsets = multiQueueOffset.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
+            String[] queues = multiDispatchQueue.split(MixAll.LMQ_DISPATCH_SEPARATOR);
+            String[] queueOffsets = multiQueueOffset.split(MixAll.LMQ_DISPATCH_SEPARATOR);
             if (queues.length != queueOffsets.length) {
                 return;
             }
@@ -2910,7 +2956,7 @@ private void notifyMessageArrive4MultiQueue(DispatchRequest dispatchRequest) {
                 long queueOffset = Long.parseLong(queueOffsets[i]);
                 int queueId = dispatchRequest.getQueueId();
                 if (DefaultMessageStore.this.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {
-                    queueId = 0;
+                    queueId = MixAll.LMQ_QUEUE_ID;
                 }
                 DefaultMessageStore.this.messageArrivingListener.arriving(
                     queueName, queueId, queueOffset + 1, dispatchRequest.getTagsCode(),
@@ -2926,7 +2972,7 @@ public void run() {
                 try {
                     TimeUnit.MILLISECONDS.sleep(1);
                     this.doReput();
-                } catch (Exception e) {
+                } catch (Throwable e) {
                     DefaultMessageStore.LOGGER.warn(this.getServiceName() + " service has exception. ", e);
                 }
             }
@@ -2950,13 +2996,13 @@ class MainBatchDispatchRequestService extends ServiceThread {
 
         public MainBatchDispatchRequestService() {
             batchDispatchRequestExecutor = ThreadUtils.newThreadPoolExecutor(
-                    DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(),
-                    DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(),
-                    1000 * 60,
-                    TimeUnit.MICROSECONDS,
-                    new LinkedBlockingQueue<>(4096),
-                    new ThreadFactoryImpl("BatchDispatchRequestServiceThread_"),
-                    new ThreadPoolExecutor.AbortPolicy());
+                DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(),
+                DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(),
+                1000 * 60,
+                TimeUnit.MICROSECONDS,
+                new LinkedBlockingQueue<>(4096),
+                new ThreadFactoryImpl("BatchDispatchRequestServiceThread_"),
+                new ThreadPoolExecutor.AbortPolicy());
         }
 
         private void pollBatchDispatchRequest() {
@@ -3123,7 +3169,7 @@ public void doReput() {
                 try {
                     this.reputFromOffset = result.getStartOffset();
 
-                    for (int readSize = 0; readSize < result.getSize() && reputFromOffset < DefaultMessageStore.this.getConfirmOffset() && doNext; ) {
+                    for (int readSize = 0; readSize < result.getSize() && reputFromOffset < getReputEndOffset() && doNext; ) {
                         ByteBuffer byteBuffer = result.getByteBuffer();
 
                         int totalSize = preCheckMessageAndReturnSize(byteBuffer);
@@ -3166,9 +3212,6 @@ public void doReput() {
                     result.release();
                 }
             }
-
-            // only for rocksdb mode
-            finishCommitLogDispatch();
         }
 
         /**
@@ -3238,6 +3281,17 @@ public HARuntimeInfo getHARuntimeInfo() {
         }
     }
 
+    public void enableRocksdbCQWrite() {
+        try {
+            RocksDBMessageStore store = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, this.topicConfigTable);
+            this.rocksDBMessageStore = store;
+            store.loadAndStartConsumerServiceOnly();
+            addDispatcher(store.getDispatcherBuildRocksdbConsumeQueue());
+        } catch (Exception e) {
+            LOGGER.error("enableRocksdbCqWrite error", e);
+        }
+    }
+
     public int getMaxDelayLevel() {
         return maxDelayLevel;
     }
@@ -3325,4 +3379,12 @@ public boolean isTransientStorePoolEnable() {
     public long getReputFromOffset() {
         return this.reputMessageService.getReputFromOffset();
     }
+
+    public RocksDBMessageStore getRocksDBMessageStore() {
+        return this.rocksDBMessageStore;
+    }
+
+    public ConsumeQueueStoreInterface getConsumeQueueStore() {
+        return consumeQueueStore;
+    }
 }
diff --git a/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java b/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java
index 79d006bafc3..654760b88c8 100644
--- a/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java
+++ b/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java
@@ -17,6 +17,9 @@
 package org.apache.rocketmq.store;
 
 import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.message.MessageConst;
 
 public class DispatchRequest {
     private final String topic;
@@ -228,6 +231,18 @@ public void setOffsetId(String offsetId) {
         this.offsetId = offsetId;
     }
 
+    public boolean containsLMQ() {
+        if (!MixAll.topicAllowsLMQ(topic)) {
+            return false;
+        }
+        if (null == propertiesMap || propertiesMap.isEmpty()) {
+            return false;
+        }
+        String lmqNames = propertiesMap.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
+        String lmqOffsets = propertiesMap.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);
+        return !StringUtils.isBlank(lmqNames) && !StringUtils.isBlank(lmqOffsets);
+    }
+
     @Override
     public String toString() {
         return "DispatchRequest{" +
diff --git a/store/src/main/java/org/apache/rocketmq/store/GetMessageResult.java b/store/src/main/java/org/apache/rocketmq/store/GetMessageResult.java
index a7556dfb855..6f322a19e19 100644
--- a/store/src/main/java/org/apache/rocketmq/store/GetMessageResult.java
+++ b/store/src/main/java/org/apache/rocketmq/store/GetMessageResult.java
@@ -43,6 +43,8 @@ public class GetMessageResult {
 
     private long coldDataSum = 0L;
 
+    private int filterMessageCount;
+
     public static final GetMessageResult NO_MATCH_LOGIC_QUEUE =
         new GetMessageResult(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE, 0, 0, 0, Collections.emptyList(),
             Collections.emptyList(), Collections.emptyList());
@@ -177,10 +179,18 @@ public void setColdDataSum(long coldDataSum) {
         this.coldDataSum = coldDataSum;
     }
 
+    public int getFilterMessageCount() {
+        return filterMessageCount;
+    }
+
+    public void setFilterMessageCount(int filterMessageCount) {
+        this.filterMessageCount = filterMessageCount;
+    }
+
     @Override
     public String toString() {
         return "GetMessageResult [status=" + status + ", nextBeginOffset=" + nextBeginOffset + ", minOffset="
             + minOffset + ", maxOffset=" + maxOffset + ", bufferTotalSize=" + bufferTotalSize + ", messageCount=" + messageCount
-            + ", suggestPullingFromSlave=" + suggestPullingFromSlave + "]";
+            + ", filterMessageCount=" + filterMessageCount + ", suggestPullingFromSlave=" + suggestPullingFromSlave + "]";
     }
 }
diff --git a/store/src/main/java/org/apache/rocketmq/store/LmqDispatch.java b/store/src/main/java/org/apache/rocketmq/store/LmqDispatch.java
new file mode 100644
index 00000000000..2805f510140
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/LmqDispatch.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
+
+public class LmqDispatch {
+    private static final short VALUE_OF_EACH_INCREMENT = 1;
+
+    public static void wrapLmqDispatch(MessageStore messageStore, final MessageExtBrokerInner msg)
+        throws ConsumeQueueException {
+        String lmqNames = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
+        String[] queueNames = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR);
+        Long[] queueOffsets = new Long[queueNames.length];
+        if (messageStore.getMessageStoreConfig().isEnableLmq()) {
+            for (int i = 0; i < queueNames.length; i++) {
+                if (MixAll.isLmq(queueNames[i])) {
+                    queueOffsets[i] = messageStore.getQueueStore().getLmqQueueOffset(queueNames[i], MixAll.LMQ_QUEUE_ID);
+                }
+            }
+        }
+        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET,
+            StringUtils.join(queueOffsets, MixAll.LMQ_DISPATCH_SEPARATOR));
+        msg.removeWaitStorePropertyString();
+    }
+
+    public static void updateLmqOffsets(MessageStore messageStore, final MessageExtBrokerInner msgInner)
+        throws ConsumeQueueException {
+        String lmqNames = msgInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
+        String[] queueNames = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR);
+        for (String queueName : queueNames) {
+            if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {
+                messageStore.getQueueStore().increaseLmqOffset(queueName, MixAll.LMQ_QUEUE_ID, VALUE_OF_EACH_INCREMENT);
+            }
+        }
+    }
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java b/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java
index 20e9a652b7e..7531c96d119 100644
--- a/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java
+++ b/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java
@@ -175,11 +175,11 @@ public PutMessageResult encodeWithoutProperties(MessageExtBrokerInner msgInner)
     public PutMessageResult encode(MessageExtBrokerInner msgInner) {
         this.byteBuf.clear();
 
-        if (messageStoreConfig.isEnableMultiDispatch() && CommitLog.isMultiDispatchMsg(msgInner)) {
+        if (messageStoreConfig.isEnableLmq() && msgInner.needDispatchLMQ()) {
             return encodeWithoutProperties(msgInner);
         }
 
-        /**
+        /*
          * Serialize message
          */
         final byte[] propertiesData =
diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
index 814c6d1bfef..4bbee142a17 100644
--- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java
@@ -23,6 +23,7 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.Supplier;
+import javax.annotation.Nonnull;
 import org.apache.rocketmq.common.BoundaryType;
 import org.apache.rocketmq.common.Pair;
 import org.apache.rocketmq.common.SystemClock;
@@ -31,6 +32,7 @@
 import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.ha.HAService;
 import org.apache.rocketmq.store.hook.PutMessageHook;
 import org.apache.rocketmq.store.hook.SendMessageBackHook;
@@ -181,7 +183,7 @@ CompletableFuture<GetMessageResult> getMessageAsync(final String group, final St
      * @param queueId Queue ID.
      * @return Maximum offset at present.
      */
-    long getMaxOffsetInQueue(final String topic, final int queueId);
+    long getMaxOffsetInQueue(final String topic, final int queueId) throws ConsumeQueueException;
 
     /**
      * Get maximum offset of the topic queue.
@@ -191,7 +193,7 @@ CompletableFuture<GetMessageResult> getMessageAsync(final String group, final St
      * @param committed return the max offset in ConsumeQueue if true, or the max offset in CommitLog if false
      * @return Maximum offset at present.
      */
-    long getMaxOffsetInQueue(final String topic, final int queueId, final boolean committed);
+    long getMaxOffsetInQueue(final String topic, final int queueId, final boolean committed) throws ConsumeQueueException;
 
     /**
      * Get the minimum offset of the topic queue.
@@ -509,6 +511,13 @@ CompletableFuture<QueryMessageResult> queryMessageAsync(final String topic, fina
      */
     long dispatchBehindBytes();
 
+    /**
+     * Get number of the milliseconds that have been stored in commit log and not yet dispatched to consume queue.
+     *
+     * @return number of the milliseconds to dispatch.
+     */
+    long dispatchBehindMilliseconds();
+
     /**
      * Flush the message store to persist all data.
      *
@@ -626,14 +635,6 @@ CompletableFuture<QueryMessageResult> queryMessageAsync(final String topic, fina
     void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile,
         boolean isRecover, boolean isFileEnd) throws RocksDBException;
 
-    /**
-     * Only used in rocksdb mode, because we build consumeQueue in batch(default 16 dispatchRequests)
-     * It will be triggered in two cases:
-     * @see org.apache.rocketmq.store.DefaultMessageStore.ReputMessageService#doReput
-     * @see CommitLog#recoverAbnormally
-     */
-    void finishCommitLogDispatch();
-
     /**
      * Get the message store config
      *
@@ -724,6 +725,7 @@ void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, Ma
      *
      * @return the queue store
      */
+    @Nonnull
     ConsumeQueueStoreInterface getQueueStore();
 
     /**
diff --git a/store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java b/store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java
deleted file mode 100644
index 5bc587a8e03..00000000000
--- a/store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.store;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.common.message.MessageAccessor;
-import org.apache.rocketmq.common.message.MessageConst;
-import org.apache.rocketmq.common.message.MessageExtBrokerInner;
-
-/**
- * MultiDispatch for lmq, not-thread-safe
- */
-public class MultiDispatch {
-    private final StringBuilder keyBuilder = new StringBuilder();
-    private final DefaultMessageStore messageStore;
-    private static final short VALUE_OF_EACH_INCREMENT = 1;
-
-    public MultiDispatch(DefaultMessageStore messageStore) {
-        this.messageStore = messageStore;
-    }
-
-    public String queueKey(String queueName, MessageExtBrokerInner msgInner) {
-        keyBuilder.delete(0, keyBuilder.length());
-        keyBuilder.append(queueName);
-        keyBuilder.append('-');
-        int queueId = msgInner.getQueueId();
-        if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {
-            queueId = 0;
-        }
-        keyBuilder.append(queueId);
-        return keyBuilder.toString();
-    }
-
-    public void wrapMultiDispatch(final MessageExtBrokerInner msg) {
-
-        String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
-        String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
-        Long[] queueOffsets = new Long[queues.length];
-        if (messageStore.getMessageStoreConfig().isEnableLmq()) {
-            for (int i = 0; i < queues.length; i++) {
-                String key = queueKey(queues[i], msg);
-                if (MixAll.isLmq(key)) {
-                    queueOffsets[i] = messageStore.getQueueStore().getLmqQueueOffset(key);
-                }
-            }
-        }
-        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET,
-                StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER));
-        msg.removeWaitStorePropertyString();
-    }
-
-    public void updateMultiQueueOffset(final MessageExtBrokerInner msgInner) {
-        String multiDispatchQueue = msgInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
-        String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
-        for (String queue : queues) {
-            String key = queueKey(queue, msgInner);
-            if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) {
-                messageStore.getQueueStore().increaseLmqOffset(key, VALUE_OF_EACH_INCREMENT);
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java
index 6141b778bf7..321689ac8f5 100644
--- a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java
@@ -16,16 +16,16 @@
  */
 package org.apache.rocketmq.store;
 
+import io.opentelemetry.api.common.AttributesBuilder;
+import io.opentelemetry.api.metrics.Meter;
 import java.io.IOException;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.function.Supplier;
-
-import io.opentelemetry.api.common.AttributesBuilder;
-import io.opentelemetry.api.metrics.Meter;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
 import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager;
@@ -39,6 +39,8 @@
 
 public class RocksDBMessageStore extends DefaultMessageStore {
 
+    private CommitLogDispatcherBuildRocksdbConsumeQueue dispatcherBuildRocksdbConsumeQueue;
+
     public RocksDBMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
         final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap<String, TopicConfig> topicConfigTable) throws
         IOException {
@@ -77,15 +79,6 @@ public void recoverTopicQueueTable() {
         this.consumeQueueStore.setTopicQueueTable(new ConcurrentHashMap<>());
     }
 
-    @Override
-    public void finishCommitLogDispatch() {
-        try {
-            putMessagePositionInfo(null);
-        } catch (RocksDBException e) {
-            ERROR_LOG.info("try to finish commitlog dispatch error.", e);
-        }
-    }
-
     @Override
     public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) {
         return findConsumeQueue(topic, queueId);
@@ -166,16 +159,50 @@ public void run() {
         }
     }
 
-    @Override
-    public long estimateMessageCount(String topic, int queueId, long from, long to, MessageFilter filter) {
-        // todo
-        return 0;
-    }
-
     @Override
     public void initMetrics(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier) {
         DefaultStoreMetricsManager.init(meter, attributesBuilderSupplier, this);
         // Also add some metrics for rocksdb's monitoring.
         RocksDBStoreMetricsManager.init(meter, attributesBuilderSupplier, this);
     }
+
+    public CommitLogDispatcherBuildRocksdbConsumeQueue getDispatcherBuildRocksdbConsumeQueue() {
+        return dispatcherBuildRocksdbConsumeQueue;
+    }
+
+    class CommitLogDispatcherBuildRocksdbConsumeQueue implements CommitLogDispatcher {
+        @Override
+        public void dispatch(DispatchRequest request) throws RocksDBException {
+            boolean enable = getMessageStoreConfig().isRocksdbCQDoubleWriteEnable();
+            if (!enable) {
+                return;
+            }
+            final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag());
+            switch (tranType) {
+                case MessageSysFlag.TRANSACTION_NOT_TYPE:
+                case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
+                    putMessagePositionInfo(request);
+                    break;
+                case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
+                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
+                    break;
+            }
+        }
+    }
+
+    public void loadAndStartConsumerServiceOnly() {
+        try {
+            this.dispatcherBuildRocksdbConsumeQueue = new CommitLogDispatcherBuildRocksdbConsumeQueue();
+            boolean loadResult = this.consumeQueueStore.load();
+            if (!loadResult) {
+                throw new RuntimeException("load consume queue failed");
+            }
+            super.loadCheckPoint();
+            this.consumeQueueStore.start();
+        } catch (Exception e) {
+            ERROR_LOG.error("loadAndStartConsumerServiceOnly error", e);
+            throw new RuntimeException(e);
+        }
+    }
+
 }
diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
index 0ba02e4cb9d..0ea58415487 100644
--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
+++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
@@ -22,6 +22,8 @@
 import org.apache.rocketmq.store.ConsumeQueue;
 import org.apache.rocketmq.store.StoreType;
 import org.apache.rocketmq.store.queue.BatchConsumeQueue;
+import org.rocksdb.CompressionType;
+import org.rocksdb.util.SizeUnit;
 
 public class MessageStoreConfig {
 
@@ -50,7 +52,7 @@ public class MessageStoreConfig {
     // CommitLog file size,default is 1G
     private int mappedFileSizeCommitLog = 1024 * 1024 * 1024;
 
-    // CompactinLog file size, default is 100M
+    // CompactionLog file size, default is 100M
     private int compactionMappedFileSize = 100 * 1024 * 1024;
 
     // CompactionLog consumeQueue file size, default is 10M
@@ -97,6 +99,7 @@ public class MessageStoreConfig {
     private boolean timerSkipUnknownError = false;
     private boolean timerWarmEnable = false;
     private boolean timerStopDequeue = false;
+    private boolean timerEnableRetryUntilSuccess = false;
     private int timerCongestNumEachSlot = Integer.MAX_VALUE;
 
     private int timerMetricSmallThreshold = 1000000;
@@ -105,6 +108,7 @@ public class MessageStoreConfig {
     // default, defaultRocksDB
     @ImportantField
     private String storeType = StoreType.DEFAULT.getStoreType();
+
     // ConsumeQueue file size,default is 30W
     private int mappedFileSizeConsumeQueue = 300000 * ConsumeQueue.CQ_STORE_UNIT_SIZE;
     // enable consume queue ext
@@ -413,6 +417,70 @@ public class MessageStoreConfig {
 
     private int topicQueueLockNum = 32;
 
+    /**
+     * If readUnCommitted is true, the dispatch of the consume queue will exceed the confirmOffset, which may cause the client to read uncommitted messages.
+     * For example, reput offset exceeding the flush offset during synchronous disk flushing.
+     */
+    private boolean readUnCommitted = false;
+
+    private boolean putConsumeQueueDataByFileChannel = true;
+
+    private boolean rocksdbCQDoubleWriteEnable = false;
+
+    /**
+     * If ConsumeQueueStore is RocksDB based, this option is to configure bottom-most tier compression type.
+     * The following values are valid:
+     * <ul>
+     *     <li>snappy</li>
+     *     <li>z</li>
+     *     <li>bzip2</li>
+     *     <li>lz4</li>
+     *     <li>lz4hc</li>
+     *     <li>xpress</li>
+     *     <li>zstd</li>
+     * </ul>
+     *
+     * LZ4 is the recommended one.
+     */
+    private String bottomMostCompressionTypeForConsumeQueueStore = CompressionType.ZSTD_COMPRESSION.getLibraryName();
+
+    private String rocksdbCompressionType = CompressionType.LZ4_COMPRESSION.getLibraryName();
+
+    /**
+     * Flush RocksDB WAL frequency, aka, flush WAL every N write ops.
+     */
+    private int rocksdbFlushWalFrequency = 1024;
+
+    private long rocksdbWalFileRollingThreshold = SizeUnit.GB;
+
+    public String getRocksdbCompressionType() {
+        return rocksdbCompressionType;
+    }
+
+    public void setRocksdbCompressionType(String compressionType) {
+        this.rocksdbCompressionType = compressionType;
+    }
+
+    /**
+     * Spin number in the retreat strategy of spin lock
+     * Default is 1000
+     */
+    private int spinLockCollisionRetreatOptimalDegree = 1000;
+
+    /**
+     * Use AdaptiveBackOffLock
+     **/
+    private boolean useABSLock = false;
+
+    public boolean isRocksdbCQDoubleWriteEnable() {
+        return rocksdbCQDoubleWriteEnable;
+    }
+
+    public void setRocksdbCQDoubleWriteEnable(boolean rocksdbWriteEnable) {
+        this.rocksdbCQDoubleWriteEnable = rocksdbWriteEnable;
+    }
+
+
     public boolean isEnabledAppendPropCRC() {
         return enabledAppendPropCRC;
     }
@@ -672,7 +740,6 @@ public void setForceVerifyPropCRC(boolean forceVerifyPropCRC) {
         this.forceVerifyPropCRC = forceVerifyPropCRC;
     }
 
-
     public String getStorePathCommitLog() {
         if (storePathCommitLog == null) {
             return storePathRootDir + File.separator + "commitlog";
@@ -1623,6 +1690,14 @@ public void setTimerSkipUnknownError(boolean timerSkipUnknownError) {
         this.timerSkipUnknownError = timerSkipUnknownError;
     }
 
+    public boolean isTimerEnableRetryUntilSuccess() {
+        return timerEnableRetryUntilSuccess;
+    }
+
+    public void setTimerEnableRetryUntilSuccess(boolean timerEnableRetryUntilSuccess) {
+        this.timerEnableRetryUntilSuccess = timerEnableRetryUntilSuccess;
+    }
+
     public boolean isTimerWarmEnable() {
         return timerWarmEnable;
     }
@@ -1819,4 +1894,60 @@ public int getTopicQueueLockNum() {
     public void setTopicQueueLockNum(int topicQueueLockNum) {
         this.topicQueueLockNum = topicQueueLockNum;
     }
+
+    public boolean isReadUnCommitted() {
+        return readUnCommitted;
+    }
+
+    public void setReadUnCommitted(boolean readUnCommitted) {
+        this.readUnCommitted = readUnCommitted;
+    }
+
+    public boolean isPutConsumeQueueDataByFileChannel() {
+        return putConsumeQueueDataByFileChannel;
+    }
+
+    public void setPutConsumeQueueDataByFileChannel(boolean putConsumeQueueDataByFileChannel) {
+        this.putConsumeQueueDataByFileChannel = putConsumeQueueDataByFileChannel;
+    }
+
+    public String getBottomMostCompressionTypeForConsumeQueueStore() {
+        return bottomMostCompressionTypeForConsumeQueueStore;
+    }
+
+    public void setBottomMostCompressionTypeForConsumeQueueStore(String bottomMostCompressionTypeForConsumeQueueStore) {
+        this.bottomMostCompressionTypeForConsumeQueueStore = bottomMostCompressionTypeForConsumeQueueStore;
+    }
+
+    public int getRocksdbFlushWalFrequency() {
+        return rocksdbFlushWalFrequency;
+    }
+
+    public void setRocksdbFlushWalFrequency(int rocksdbFlushWalFrequency) {
+        this.rocksdbFlushWalFrequency = rocksdbFlushWalFrequency;
+    }
+
+    public long getRocksdbWalFileRollingThreshold() {
+        return rocksdbWalFileRollingThreshold;
+    }
+
+    public void setRocksdbWalFileRollingThreshold(long rocksdbWalFileRollingThreshold) {
+        this.rocksdbWalFileRollingThreshold = rocksdbWalFileRollingThreshold;
+    }
+
+    public int getSpinLockCollisionRetreatOptimalDegree() {
+        return spinLockCollisionRetreatOptimalDegree;
+    }
+
+    public void setSpinLockCollisionRetreatOptimalDegree(int spinLockCollisionRetreatOptimalDegree) {
+        this.spinLockCollisionRetreatOptimalDegree = spinLockCollisionRetreatOptimalDegree;
+    }
+
+    public void setUseABSLock(boolean useABSLock) {
+        this.useABSLock = useABSLock;
+    }
+
+    public boolean getUseABSLock() {
+        return useABSLock;
+    }
 }
diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
index 27a18abc9d9..29be9e7c614 100644
--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
@@ -16,13 +16,26 @@
  */
 package org.apache.rocketmq.store.dledger;
 
+import io.openmessaging.storage.dledger.AppendFuture;
+import io.openmessaging.storage.dledger.BatchAppendFuture;
+import io.openmessaging.storage.dledger.DLedgerConfig;
+import io.openmessaging.storage.dledger.DLedgerServer;
+import io.openmessaging.storage.dledger.entry.DLedgerEntry;
+import io.openmessaging.storage.dledger.protocol.AppendEntryRequest;
+import io.openmessaging.storage.dledger.protocol.AppendEntryResponse;
+import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest;
+import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode;
+import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore;
+import io.openmessaging.storage.dledger.store.file.MmapFile;
+import io.openmessaging.storage.dledger.store.file.MmapFileList;
+import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult;
+import io.openmessaging.storage.dledger.utils.DLedgerUtils;
 import java.net.Inet6Address;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
-
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExtBatch;
@@ -43,21 +56,6 @@
 import org.apache.rocketmq.store.logfile.MappedFile;
 import org.rocksdb.RocksDBException;
 
-import io.openmessaging.storage.dledger.AppendFuture;
-import io.openmessaging.storage.dledger.BatchAppendFuture;
-import io.openmessaging.storage.dledger.DLedgerConfig;
-import io.openmessaging.storage.dledger.DLedgerServer;
-import io.openmessaging.storage.dledger.entry.DLedgerEntry;
-import io.openmessaging.storage.dledger.protocol.AppendEntryRequest;
-import io.openmessaging.storage.dledger.protocol.AppendEntryResponse;
-import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest;
-import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode;
-import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore;
-import io.openmessaging.storage.dledger.store.file.MmapFile;
-import io.openmessaging.storage.dledger.store.file.MmapFileList;
-import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult;
-import io.openmessaging.storage.dledger.utils.DLedgerUtils;
-
 /**
  * Store all metadata downtime for recovery, data protection reliability
  */
@@ -318,6 +316,7 @@ private void dledgerRecoverNormally(long maxPhyOffsetOfConsumeQueue) throws Rock
 
     private void dledgerRecoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException {
         boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover();
+        boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable();
         dLedgerFileStore.load();
         if (!dLedgerFileList.getMappedFiles().isEmpty()) {
             dLedgerFileStore.recover();
@@ -346,7 +345,7 @@ private void dledgerRecoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws Ro
             long processOffset = mmapFile.getFileFromOffset();
             long mmapFileOffset = 0;
             while (true) {
-                DispatchRequest dispatchRequest = this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover, true);
+                DispatchRequest dispatchRequest = this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover, checkDupInfo);
                 int size = dispatchRequest.getMsgSize();
 
                 if (dispatchRequest.isSuccess()) {
@@ -427,7 +426,7 @@ private void setRecoverPosition() {
         log.info("Will set the initial commitlog offset={} for dledger", dividedCommitlogOffset);
     }
 
-    private boolean isMmapFileMatchedRecover(final MmapFile mmapFile) {
+    private boolean isMmapFileMatchedRecover(final MmapFile mmapFile) throws RocksDBException {
         ByteBuffer byteBuffer = mmapFile.sliceByteBuffer();
 
         int magicCode = byteBuffer.getInt(DLedgerEntry.BODY_OFFSET + MessageDecoder.MESSAGE_MAGIC_CODE_POSITION);
@@ -435,39 +434,46 @@ private boolean isMmapFileMatchedRecover(final MmapFile mmapFile) {
             return false;
         }
 
-        int storeTimestampPosition;
-        int sysFlag = byteBuffer.getInt(DLedgerEntry.BODY_OFFSET + MessageDecoder.SYSFLAG_POSITION);
-        if ((sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0) {
-            storeTimestampPosition = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION;
+        if (this.defaultMessageStore.getMessageStoreConfig().isEnableRocksDBStore()) {
+            final long maxPhyOffsetInConsumeQueue = this.defaultMessageStore.getQueueStore().getMaxPhyOffsetInConsumeQueue();
+            long phyOffset = byteBuffer.getLong(DLedgerEntry.BODY_OFFSET + MessageDecoder.MESSAGE_PHYSIC_OFFSET_POSITION);
+            if (phyOffset <= maxPhyOffsetInConsumeQueue) {
+                log.info("find check. beginPhyOffset: {}, maxPhyOffsetInConsumeQueue: {}", phyOffset, maxPhyOffsetInConsumeQueue);
+                return true;
+            }
         } else {
-            // v6 address is 12 byte larger than v4
-            storeTimestampPosition = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION + 12;
-        }
+            int storeTimestampPosition;
+            int sysFlag = byteBuffer.getInt(DLedgerEntry.BODY_OFFSET + MessageDecoder.SYSFLAG_POSITION);
+            if ((sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0) {
+                storeTimestampPosition = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION;
+            } else {
+                // v6 address is 12 byte larger than v4
+                storeTimestampPosition = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION + 12;
+            }
 
-        long storeTimestamp = byteBuffer.getLong(DLedgerEntry.BODY_OFFSET + storeTimestampPosition);
-        if (storeTimestamp == 0) {
-            return false;
-        }
+            long storeTimestamp = byteBuffer.getLong(DLedgerEntry.BODY_OFFSET + storeTimestampPosition);
+            if (storeTimestamp == 0) {
+                return false;
+            }
 
-        if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable()
+            if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable()
                 && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) {
-            if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) {
-                log.info("dledger find check timestamp, {} {}",
-                    storeTimestamp,
-                    UtilAll.timeMillisToHumanString(storeTimestamp));
-                return true;
-            }
-        } else {
-            if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) {
-                log.info("dledger find check timestamp, {} {}",
-                    storeTimestamp,
-                    UtilAll.timeMillisToHumanString(storeTimestamp));
-                return true;
+                if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) {
+                    log.info("dledger find check timestamp, {} {}",
+                        storeTimestamp,
+                        UtilAll.timeMillisToHumanString(storeTimestamp));
+                    return true;
+                }
+            } else {
+                if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) {
+                    log.info("dledger find check timestamp, {} {}",
+                        storeTimestamp,
+                        UtilAll.timeMillisToHumanString(storeTimestamp));
+                    return true;
+                }
             }
         }
-
         return false;
-
     }
 
     @Override
@@ -567,15 +573,16 @@ public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner
         AppendFuture<AppendEntryResponse> dledgerFuture;
         EncodeResult encodeResult;
 
+        encodeResult = this.messageSerializer.serialize(msg);
+        if (encodeResult.status != AppendMessageStatus.PUT_OK) {
+            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status)));
+        }
+
         String topicQueueKey = msg.getTopic() + "-" + msg.getQueueId();
         topicQueueLock.lock(topicQueueKey);
         try {
             defaultMessageStore.assignOffset(msg);
 
-            encodeResult = this.messageSerializer.serialize(msg);
-            if (encodeResult.status != AppendMessageStatus.PUT_OK) {
-                return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status)));
-            }
             putMessageLock.lock(); //spin or ReentrantLock ,depending on store config
             long elapsedTimeInLock;
             long queueOffset;
diff --git a/store/src/main/java/org/apache/rocketmq/store/exception/ConsumeQueueException.java b/store/src/main/java/org/apache/rocketmq/store/exception/ConsumeQueueException.java
new file mode 100644
index 00000000000..880e6347eb4
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/exception/ConsumeQueueException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.exception;
+
+public class ConsumeQueueException extends StoreException {
+    public ConsumeQueueException() {
+    }
+
+    public ConsumeQueueException(String message) {
+        super(message);
+    }
+
+    public ConsumeQueueException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public ConsumeQueueException(Throwable cause) {
+        super(cause);
+    }
+
+    public ConsumeQueueException(String message, Throwable cause, boolean enableSuppression,
+        boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/exception/StoreException.java b/store/src/main/java/org/apache/rocketmq/store/exception/StoreException.java
new file mode 100644
index 00000000000..8c99e8a05bb
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/exception/StoreException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.exception;
+
+public class StoreException extends Exception {
+    public StoreException() {
+    }
+
+    public StoreException(String message) {
+        super(message);
+    }
+
+    public StoreException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public StoreException(Throwable cause) {
+        super(cause);
+    }
+
+    public StoreException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAService.java
index 75e0afa4e89..c0e203862ca 100644
--- a/store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAService.java
+++ b/store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAService.java
@@ -350,8 +350,8 @@ public void run() {
                                         + sc.socket().getRemoteSocketAddress());
                                     try {
                                         HAConnection conn = createConnection(sc);
-                                        conn.start();
                                         DefaultHAService.this.addConnection(conn);
+                                        conn.start();
                                     } catch (Exception e) {
                                         log.error("new HAConnection exception", e);
                                         sc.close();
diff --git a/store/src/main/java/org/apache/rocketmq/store/index/IndexService.java b/store/src/main/java/org/apache/rocketmq/store/index/IndexService.java
index ef5d21ac7ce..2d325ee13a4 100644
--- a/store/src/main/java/org/apache/rocketmq/store/index/IndexService.java
+++ b/store/src/main/java/org/apache/rocketmq/store/index/IndexService.java
@@ -164,11 +164,10 @@ public void destroy() {
     }
 
     public QueryOffsetResult queryOffset(String topic, String key, int maxNum, long begin, long end) {
-        List<Long> phyOffsets = new ArrayList<>(maxNum);
-
         long indexLastUpdateTimestamp = 0;
         long indexLastUpdatePhyoffset = 0;
         maxNum = Math.min(maxNum, this.defaultMessageStore.getMessageStoreConfig().getMaxMsgsNumBatch());
+        List<Long> phyOffsets = new ArrayList<>(maxNum);
         try {
             this.readWriteLock.readLock().lock();
             if (!this.indexFileList.isEmpty()) {
diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java
new file mode 100644
index 00000000000..96200bcc15b
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.lock;
+
+import org.apache.rocketmq.store.PutMessageLock;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+
+public interface AdaptiveBackOffSpinLock extends PutMessageLock {
+    /**
+     * Configuration update
+     * @param messageStoreConfig
+     */
+    default void update(MessageStoreConfig messageStoreConfig) {
+    }
+
+    /**
+     * Locking mechanism switching
+     */
+    default void swap() {
+    }
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java
new file mode 100644
index 00000000000..b4abb082718
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.lock;
+
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+
+import java.time.LocalTime;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock {
+    private AdaptiveBackOffSpinLock adaptiveLock;
+    //state
+    private AtomicBoolean state = new AtomicBoolean(true);
+
+    // Used to determine the switchover between a mutex lock and a spin lock
+    private final static float SWAP_SPIN_LOCK_RATIO = 0.8f;
+
+    // It is used to adjust the spin number K of the escape spin lock
+    // When (retreat number / TPS) <= (1 / BASE_SWAP_ADAPTIVE_RATIO * SPIN_LOCK_ADAPTIVE_RATIO), K is decreased
+    private final static int SPIN_LOCK_ADAPTIVE_RATIO = 4;
+
+    // It is used to adjust the spin number K of the escape spin lock
+    // When (retreat number / TPS) >= (1 / BASE_SWAP_ADAPTIVE_RATIO), K is increased
+    private final static int BASE_SWAP_LOCK_RATIO = 320;
+
+    private final static String BACK_OFF_SPIN_LOCK = "SpinLock";
+
+    private final static String REENTRANT_LOCK = "ReentrantLock";
+
+    private Map<String, AdaptiveBackOffSpinLock> locks;
+
+    private final List<AtomicInteger> tpsTable;
+
+    private final List<Map<Thread, Byte>> threadTable;
+
+    private int swapCriticalPoint;
+
+    private AtomicInteger currentThreadNum = new AtomicInteger(0);
+
+    private AtomicBoolean isOpen = new AtomicBoolean(true);
+
+    public AdaptiveBackOffSpinLockImpl() {
+        this.locks = new HashMap<>();
+        this.locks.put(REENTRANT_LOCK, new BackOffReentrantLock());
+        this.locks.put(BACK_OFF_SPIN_LOCK, new BackOffSpinLock());
+
+        this.threadTable = new ArrayList<>(2);
+        this.threadTable.add(new ConcurrentHashMap<>());
+        this.threadTable.add(new ConcurrentHashMap<>());
+
+        this.tpsTable = new ArrayList<>(2);
+        this.tpsTable.add(new AtomicInteger(0));
+        this.tpsTable.add(new AtomicInteger(0));
+
+        adaptiveLock = this.locks.get(BACK_OFF_SPIN_LOCK);
+    }
+
+    @Override
+    public void lock() {
+        int slot = LocalTime.now().getSecond() % 2;
+        this.threadTable.get(slot).putIfAbsent(Thread.currentThread(), Byte.MAX_VALUE);
+        this.tpsTable.get(slot).getAndIncrement();
+        boolean state;
+        do {
+            state = this.state.get();
+        } while (!state);
+
+        currentThreadNum.incrementAndGet();
+        this.adaptiveLock.lock();
+    }
+
+    @Override
+    public void unlock() {
+        this.adaptiveLock.unlock();
+        currentThreadNum.decrementAndGet();
+        if (isOpen.get()) {
+            swap();
+        }
+    }
+
+    @Override
+    public void update(MessageStoreConfig messageStoreConfig) {
+        this.adaptiveLock.update(messageStoreConfig);
+    }
+
+    @Override
+    public void swap() {
+        if (!this.state.get()) {
+            return;
+        }
+        boolean needSwap = false;
+        int slot = 1 - LocalTime.now().getSecond() % 2;
+        int tps = this.tpsTable.get(slot).get() + 1;
+        int threadNum = this.threadTable.get(slot).size();
+        this.tpsTable.get(slot).set(-1);
+        this.threadTable.get(slot).clear();
+        if (tps == 0) {
+            return;
+        }
+
+        if (this.adaptiveLock instanceof BackOffSpinLock) {
+            BackOffSpinLock lock = (BackOffSpinLock) this.adaptiveLock;
+            // Avoid frequent adjustment of K, and make a reasonable range through experiments
+            // reasonable range : (retreat number / TPS) > (1 / BASE_SWAP_ADAPTIVE_RATIO * SPIN_LOCK_ADAPTIVE_RATIO) &&
+            // (retreat number / TPS) < (1 / BASE_SWAP_ADAPTIVE_RATIO)
+            if (lock.getNumberOfRetreat(slot) * BASE_SWAP_LOCK_RATIO >= tps) {
+                if (lock.isAdapt()) {
+                    lock.adapt(true);
+                } else {
+                    // It is used to switch between mutex lock and spin lock
+                    this.swapCriticalPoint = tps * threadNum;
+                    needSwap = true;
+                }
+            } else if (lock.getNumberOfRetreat(slot) * BASE_SWAP_LOCK_RATIO * SPIN_LOCK_ADAPTIVE_RATIO <= tps) {
+                lock.adapt(false);
+            }
+            lock.setNumberOfRetreat(slot, 0);
+        } else {
+            if (tps * threadNum <= this.swapCriticalPoint * SWAP_SPIN_LOCK_RATIO) {
+                needSwap = true;
+            }
+        }
+
+        if (needSwap) {
+            if (this.state.compareAndSet(true, false)) {
+                // Ensures that no threads are in contention locks as well as in critical zones
+                int currentThreadNum;
+                do {
+                    currentThreadNum = this.currentThreadNum.get();
+                } while (currentThreadNum != 0);
+
+                try {
+                    if (this.adaptiveLock instanceof BackOffSpinLock) {
+                        this.adaptiveLock = this.locks.get(REENTRANT_LOCK);
+                    } else {
+                        this.adaptiveLock = this.locks.get(BACK_OFF_SPIN_LOCK);
+                        ((BackOffSpinLock) this.adaptiveLock).adapt(false);
+                    }
+                } catch (Exception e) {
+                    //ignore
+                } finally {
+                    this.state.compareAndSet(false, true);
+                }
+            }
+        }
+    }
+
+    public List<AdaptiveBackOffSpinLock> getLocks() {
+        return (List<AdaptiveBackOffSpinLock>) this.locks.values();
+    }
+
+    public void setLocks(Map<String, AdaptiveBackOffSpinLock> locks) {
+        this.locks = locks;
+    }
+
+    public boolean getState() {
+        return this.state.get();
+    }
+
+    public void setState(boolean state) {
+        this.state.set(state);
+    }
+
+    public AdaptiveBackOffSpinLock getAdaptiveLock() {
+        return adaptiveLock;
+    }
+
+    public List<AtomicInteger> getTpsTable() {
+        return tpsTable;
+    }
+
+    public void setSwapCriticalPoint(int swapCriticalPoint) {
+        this.swapCriticalPoint = swapCriticalPoint;
+    }
+
+    public int getSwapCriticalPoint() {
+        return swapCriticalPoint;
+    }
+
+    public boolean isOpen() {
+        return this.isOpen.get();
+    }
+
+    public void setOpen(boolean open) {
+        this.isOpen.set(open);
+    }
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java
new file mode 100644
index 00000000000..90e416419bc
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.lock;
+
+import java.util.concurrent.locks.ReentrantLock;
+
+public class BackOffReentrantLock implements AdaptiveBackOffSpinLock {
+    private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync
+
+    @Override
+    public void lock() {
+        putMessageNormalLock.lock();
+    }
+
+    @Override
+    public void unlock() {
+        putMessageNormalLock.unlock();
+    }
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java
new file mode 100644
index 00000000000..f754970a054
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.lock;
+
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+
+import java.time.LocalTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class BackOffSpinLock implements AdaptiveBackOffSpinLock {
+
+    private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true);
+
+    private int optimalDegree;
+
+    private final static int INITIAL_DEGREE = 1000;
+
+    private final static int MAX_OPTIMAL_DEGREE = 10000;
+
+    private final List<AtomicInteger> numberOfRetreat;
+
+    public BackOffSpinLock() {
+        this.optimalDegree = INITIAL_DEGREE;
+
+        numberOfRetreat = new ArrayList<>(2);
+        numberOfRetreat.add(new AtomicInteger(0));
+        numberOfRetreat.add(new AtomicInteger(0));
+    }
+
+    @Override
+    public void lock() {
+        int spinDegree = this.optimalDegree;
+        while (true) {
+            for (int i = 0; i < spinDegree; i++) {
+                if (this.putMessageSpinLock.compareAndSet(true, false)) {
+                    return;
+                }
+            }
+            numberOfRetreat.get(LocalTime.now().getSecond() % 2).getAndIncrement();
+            try {
+                Thread.sleep(0);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
+    @Override
+    public void unlock() {
+        this.putMessageSpinLock.compareAndSet(false, true);
+    }
+
+    @Override
+    public void update(MessageStoreConfig messageStoreConfig) {
+        this.optimalDegree = messageStoreConfig.getSpinLockCollisionRetreatOptimalDegree();
+    }
+
+    public int getOptimalDegree() {
+        return this.optimalDegree;
+    }
+
+    public void setOptimalDegree(int optimalDegree) {
+        this.optimalDegree = optimalDegree;
+    }
+
+    public boolean isAdapt() {
+        return optimalDegree < MAX_OPTIMAL_DEGREE;
+    }
+
+    public synchronized void adapt(boolean isRise) {
+        if (isRise) {
+            if (optimalDegree * 2 <= MAX_OPTIMAL_DEGREE) {
+                optimalDegree *= 2;
+            } else {
+                if (optimalDegree + INITIAL_DEGREE <= MAX_OPTIMAL_DEGREE) {
+                    optimalDegree += INITIAL_DEGREE;
+                }
+            }
+        } else {
+            if (optimalDegree >= 2 * INITIAL_DEGREE) {
+                optimalDegree -= INITIAL_DEGREE;
+            }
+        }
+    }
+
+    public int getNumberOfRetreat(int pos) {
+        return numberOfRetreat.get(pos).get();
+    }
+
+    public void setNumberOfRetreat(int pos, int size) {
+        this.numberOfRetreat.get(pos).set(size);
+    }
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java b/store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java
index 03477c33249..c490d093a16 100644
--- a/store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java
+++ b/store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java
@@ -97,14 +97,14 @@ public class DefaultMappedFile extends AbstractMappedFile {
     protected long mappedByteBufferAccessCountSinceLastSwap = 0L;
 
     /**
-     * If this mapped file belongs to consume queue, this field stores store-timestamp of first message referenced
-     * by this logical queue.
+     * If this mapped file belongs to consume queue, this field stores store-timestamp of first message referenced by
+     * this logical queue.
      */
     private long startTimestamp = -1;
 
     /**
-     * If this mapped file belongs to consume queue, this field stores store-timestamp of last message referenced
-     * by this logical queue.
+     * If this mapped file belongs to consume queue, this field stores store-timestamp of last message referenced by
+     * this logical queue.
      */
     private long stopTimestamp = -1;
 
@@ -357,6 +357,24 @@ public boolean appendMessage(final byte[] data, final int offset, final int leng
         return false;
     }
 
+    @Override
+    public boolean appendMessageUsingFileChannel(byte[] data) {
+        int currentPos = WROTE_POSITION_UPDATER.get(this);
+
+        if ((currentPos + data.length) <= this.fileSize) {
+            try {
+                this.fileChannel.position(currentPos);
+                this.fileChannel.write(ByteBuffer.wrap(data, 0, data.length));
+            } catch (Throwable e) {
+                log.error("Error occurred when append message to mappedFile.", e);
+            }
+            WROTE_POSITION_UPDATER.addAndGet(this, data.length);
+            return true;
+        }
+
+        return false;
+    }
+
     /**
      * @return The current flushed position
      */
@@ -840,7 +858,6 @@ public void setStopTimestamp(long stopTimestamp) {
         this.stopTimestamp = stopTimestamp;
     }
 
-
     public Iterator<SelectMappedBufferResult> iterator(int startPos) {
         return new Itr(startPos);
     }
diff --git a/store/src/main/java/org/apache/rocketmq/store/logfile/MappedFile.java b/store/src/main/java/org/apache/rocketmq/store/logfile/MappedFile.java
index dfcf66f0882..fd70d6c5634 100644
--- a/store/src/main/java/org/apache/rocketmq/store/logfile/MappedFile.java
+++ b/store/src/main/java/org/apache/rocketmq/store/logfile/MappedFile.java
@@ -101,12 +101,23 @@ public interface MappedFile {
 
     /**
      * Appends a raw message data represents by a byte array to the current {@code MappedFile}.
+     * Using mappedByteBuffer
      *
      * @param data the byte array to append
      * @return true if success; false otherwise.
      */
     boolean appendMessage(byte[] data);
 
+
+    /**
+     * Appends a raw message data represents by a byte array to the current {@code MappedFile}.
+     * Using fileChannel
+     *
+     * @param data the byte array to append
+     * @return true if success; false otherwise.
+     */
+    boolean appendMessageUsingFileChannel(byte[] data);
+
     /**
      * Appends a raw message data represents by a byte array to the current {@code MappedFile}.
      *
diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java
index 2f2ce981257..d5d6236458e 100644
--- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java
@@ -46,6 +46,7 @@
 import org.apache.rocketmq.store.StoreStatsService;
 import org.apache.rocketmq.store.TransientStorePool;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.ha.HAService;
 import org.apache.rocketmq.store.hook.PutMessageHook;
 import org.apache.rocketmq.store.hook.SendMessageBackHook;
@@ -63,7 +64,7 @@
 import io.opentelemetry.sdk.metrics.ViewBuilder;
 
 public abstract class AbstractPluginMessageStore implements MessageStore {
-    protected MessageStore next = null;
+    protected MessageStore next;
     protected MessageStorePluginContext context;
 
     public AbstractPluginMessageStore(MessageStorePluginContext context, MessageStore next) {
@@ -139,12 +140,12 @@ public CompletableFuture<GetMessageResult> getMessageAsync(String group, String
     }
 
     @Override
-    public long getMaxOffsetInQueue(String topic, int queueId) {
+    public long getMaxOffsetInQueue(String topic, int queueId) throws ConsumeQueueException {
         return next.getMaxOffsetInQueue(topic, queueId);
     }
 
     @Override
-    public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) {
+    public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) throws ConsumeQueueException {
         return next.getMaxOffsetInQueue(topic, queueId, committed);
     }
 
@@ -292,6 +293,11 @@ public long dispatchBehindBytes() {
         return next.dispatchBehindBytes();
     }
 
+    @Override
+    public long dispatchBehindMilliseconds() {
+        return next.dispatchBehindMilliseconds();
+    }
+
     @Override
     public long flush() {
         return next.flush();
@@ -647,11 +653,6 @@ public void initMetrics(Meter meter, Supplier<AttributesBuilder> attributesBuild
         next.initMetrics(meter, attributesBuilderSupplier);
     }
 
-    @Override
-    public void finishCommitLogDispatch() {
-        next.finishCommitLogDispatch();
-    }
-
     @Override
     public void recoverTopicQueueTable() {
         next.recoverTopicQueueTable();
@@ -661,4 +662,8 @@ public void recoverTopicQueueTable() {
     public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) {
         next.notifyMessageArriveIfNecessary(dispatchRequest);
     }
+
+    public MessageStore getNext() {
+        return next;
+    }
 }
diff --git a/store/src/main/java/org/apache/rocketmq/store/pop/PopCheckPoint.java b/store/src/main/java/org/apache/rocketmq/store/pop/PopCheckPoint.java
index e041b66d9c5..38e0a207528 100644
--- a/store/src/main/java/org/apache/rocketmq/store/pop/PopCheckPoint.java
+++ b/store/src/main/java/org/apache/rocketmq/store/pop/PopCheckPoint.java
@@ -43,6 +43,8 @@ public class PopCheckPoint implements Comparable<PopCheckPoint> {
     private List<Integer> queueOffsetDiff;
     @JSONField(name = "bn")
     String brokerName;
+    @JSONField(name = "rp")
+    String rePutTimes; // ck rePut times
 
     public long getReviveOffset() {
         return reviveOffset;
@@ -136,6 +138,14 @@ public void setBrokerName(String brokerName) {
         this.brokerName = brokerName;
     }
 
+    public String getRePutTimes() {
+        return rePutTimes;
+    }
+
+    public void setRePutTimes(String rePutTimes) {
+        this.rePutTimes = rePutTimes;
+    }
+
     public void addDiff(int diff) {
         if (this.queueOffsetDiff == null) {
             this.queueOffsetDiff = new ArrayList<>(8);
@@ -171,10 +181,21 @@ public long ackOffsetByIndex(byte index) {
         return startOffset + queueOffsetDiff.get(index);
     }
 
+    public int parseRePutTimes() {
+        if (null == rePutTimes) {
+            return 0;
+        }
+        try {
+            return Integer.parseInt(rePutTimes);
+        } catch (Exception e) {
+        }
+        return Byte.MAX_VALUE;
+    }
+
     @Override
     public String toString() {
         return "PopCheckPoint [topic=" + topic + ", cid=" + cid + ", queueId=" + queueId + ", startOffset=" + startOffset + ", bitMap=" + bitMap + ", num=" + num + ", reviveTime=" + getReviveTime()
-            + ", reviveOffset=" + reviveOffset + ", diff=" + queueOffsetDiff + ", brokerName=" + brokerName + "]";
+            + ", reviveOffset=" + reviveOffset + ", diff=" + queueOffsetDiff + ", brokerName=" + brokerName + ", rePutTimes=" + rePutTimes + "]";
     }
 
     @Override
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java
index d76b0557737..ef693dc1e65 100644
--- a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java
@@ -25,6 +25,7 @@
 import org.apache.rocketmq.store.DefaultMessageStore;
 import org.apache.rocketmq.store.DispatchRequest;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.rocksdb.RocksDBException;
 
 public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInterface {
@@ -38,7 +39,11 @@ public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInte
     public AbstractConsumeQueueStore(DefaultMessageStore messageStore) {
         this.messageStore = messageStore;
         this.messageStoreConfig = messageStore.getMessageStoreConfig();
-        this.consumeQueueTable = new ConcurrentHashMap<>(32);
+        if (messageStoreConfig.isEnableLmq()) {
+            this.consumeQueueTable = new ConcurrentHashMap<>(32_768);
+        } else {
+            this.consumeQueueTable = new ConcurrentHashMap<>(32);
+        }
     }
 
     @Override
@@ -47,7 +52,7 @@ public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, Di
     }
 
     @Override
-    public Long getMaxOffset(String topic, int queueId) {
+    public Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException {
         return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId);
     }
 
@@ -58,7 +63,7 @@ public void setTopicQueueTable(ConcurrentMap<String, Long> topicQueueTable) {
     }
 
     @Override
-    public ConcurrentMap getTopicQueueTable() {
+    public ConcurrentMap<String, Long> getTopicQueueTable() {
         return this.queueOffsetOperator.getTopicQueueTable();
     }
 
@@ -75,13 +80,13 @@ public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) {
     }
 
     @Override
-    public void increaseLmqOffset(String queueKey, short messageNum) {
-        queueOffsetOperator.increaseLmqOffset(queueKey, messageNum);
+    public void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException {
+        queueOffsetOperator.increaseLmqOffset(topic, queueId, delta);
     }
 
     @Override
-    public long getLmqQueueOffset(String queueKey) {
-        return queueOffsetOperator.getLmqOffset(queueKey);
+    public long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException {
+        return queueOffsetOperator.getLmqOffset(topic, queueId, (t, q) -> 0L);
     }
 
     @Override
@@ -105,9 +110,9 @@ public long getStoreTime(CqUnit cqUnit) {
             try {
                 final long phyOffset = cqUnit.getPos();
                 final int size = cqUnit.getSize();
-                long storeTime = this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size);
-                return storeTime;
+                return this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size);
             } catch (Exception e) {
+                log.error("Failed to getStoreTime", e);
             }
         }
         return -1;
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java
index 7108c835c8e..16171827245 100644
--- a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java
@@ -587,7 +587,12 @@ public boolean putBatchMessagePositionInfo(final long offset, final int size, fi
         MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(this.mappedFileQueue.getMaxOffset());
         if (mappedFile != null) {
             boolean isNewFile = isNewFile(mappedFile);
-            boolean appendRes = mappedFile.appendMessage(this.byteBufferItem.array());
+            boolean appendRes;
+            if (messageStore.getMessageStoreConfig().isPutConsumeQueueDataByFileChannel()) {
+                appendRes = mappedFile.appendMessageUsingFileChannel(this.byteBufferItem.array());
+            } else {
+                appendRes = mappedFile.appendMessage(this.byteBufferItem.array());
+            }
             if (appendRes) {
                 maxMsgPhyOffsetInCommitLog = offset;
                 maxOffsetInQueue = msgBaseOffset + batchSize;
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java
index cbe9b4f5acd..5f9c2f90be3 100644
--- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java
@@ -47,6 +47,7 @@
 import org.apache.rocketmq.store.DefaultMessageStore;
 import org.apache.rocketmq.store.DispatchRequest;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.exception.StoreException;
 
 import static java.lang.String.format;
 import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathBatchConsumeQueue;
@@ -134,6 +135,12 @@ public boolean recoverConcurrently() {
 
     @Override
     public boolean shutdown() {
+        try {
+            flush();
+        } catch (StoreException e) {
+            log.error("Failed to flush all consume queues", e);
+            return false;
+        }
         return true;
     }
 
@@ -326,6 +333,15 @@ public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) {
         return fileQueueLifeCycle.flush(flushLeastPages);
     }
 
+    @Override
+    public void flush() throws StoreException {
+        for (Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>> topicEntry : this.consumeQueueTable.entrySet()) {
+            for (Map.Entry<Integer, ConsumeQueueInterface> cqEntry : topicEntry.getValue().entrySet()) {
+                flush(cqEntry.getValue(), 0);
+            }
+        }
+    }
+
     @Override
     public void destroy(ConsumeQueueInterface consumeQueue) {
         FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java
index e68880a828c..72a481bd57b 100644
--- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java
@@ -22,6 +22,8 @@
 import org.apache.rocketmq.common.BoundaryType;
 import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.DispatchRequest;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
+import org.apache.rocketmq.store.exception.StoreException;
 import org.rocksdb.RocksDBException;
 
 public interface ConsumeQueueStoreInterface {
@@ -79,10 +81,17 @@ public interface ConsumeQueueStoreInterface {
     boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages);
 
     /**
-     * clean expired data from minPhyOffset
-     * @param minPhyOffset
+     * Flush all nested consume queues to disk
+     *
+     * @throws StoreException if there is an error during flush
+     */
+    void flush() throws StoreException;
+
+    /**
+     * clean expired data from minCommitLogOffset
+     * @param minCommitLogOffset Minimum commit log offset
      */
-    void cleanExpired(long minPhyOffset);
+    void cleanExpired(long minCommitLogOffset);
 
     /**
      * Check files.
@@ -92,10 +101,10 @@ public interface ConsumeQueueStoreInterface {
     /**
      * Delete expired files ending at min commit log position.
      * @param consumeQueue
-     * @param minCommitLogPos min commit log position
+     * @param minCommitLogOffset min commit log position
      * @return deleted file numbers.
      */
-    int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos);
+    int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogOffset);
 
     /**
      * Is the first file available?
@@ -185,17 +194,19 @@ public interface ConsumeQueueStoreInterface {
 
     /**
      * Increase lmq offset
-     * @param queueKey
-     * @param messageNum
+     * @param topic Topic/Queue name
+     * @param queueId Queue ID
+     * @param delta amount to increase
      */
-    void increaseLmqOffset(String queueKey, short messageNum);
+    void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException;
 
     /**
      * get lmq queue offset
-     * @param queueKey
+     * @param topic
+     * @param queueId
      * @return
      */
-    long getLmqQueueOffset(String queueKey);
+    long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException;
 
     /**
      * recover topicQueue table by minPhyOffset
@@ -232,11 +243,13 @@ public interface ConsumeQueueStoreInterface {
 
     /**
      * get maxOffset of specific topic-queueId in topicQueue table
-     * @param topic
-     * @param queueId
+     *
+     * @param topic Topic name
+     * @param queueId Queue identifier
      * @return the max offset in QueueOffsetOperator
+     * @throws ConsumeQueueException if there is an error while retrieving max consume queue offset
      */
-    Long getMaxOffset(String topic, int queueId);
+    Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException;
 
     /**
      * get max physic offset in consumeQueue
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java b/store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java
index b8865fd9195..34f5cb142b6 100644
--- a/store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java
@@ -109,6 +109,7 @@ public String toString() {
                 ", size=" + size +
                 ", pos=" + pos +
                 ", batchNum=" + batchNum +
+                ", tagsCode=" + tagsCode +
                 ", compactedOffset=" + compactedOffset +
                 '}';
     }
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/DispatchEntry.java b/store/src/main/java/org/apache/rocketmq/store/queue/DispatchEntry.java
new file mode 100644
index 00000000000..a93ec0c50e1
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/DispatchEntry.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store.queue;
+
+import java.nio.charset.StandardCharsets;
+import javax.annotation.Nonnull;
+import org.apache.rocketmq.store.DispatchRequest;
+
+/**
+ * Use Record when Java 16 is available
+ */
+public class DispatchEntry {
+    public byte[] topic;
+    public int queueId;
+    public long queueOffset;
+    public long commitLogOffset;
+    public int messageSize;
+    public long tagCode;
+    public long storeTimestamp;
+
+    public static DispatchEntry from(@Nonnull DispatchRequest request) {
+        DispatchEntry entry = new DispatchEntry();
+        entry.topic = request.getTopic().getBytes(StandardCharsets.UTF_8);
+        entry.queueId = request.getQueueId();
+        entry.queueOffset = request.getConsumeQueueOffset();
+        entry.commitLogOffset = request.getCommitLogOffset();
+        entry.messageSize = request.getMsgSize();
+        entry.tagCode = request.getTagsCode();
+        entry.storeTimestamp = request.getStoreTimestamp();
+        return entry;
+    }
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializer.java b/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializer.java
new file mode 100644
index 00000000000..8d3a8cd3a49
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializer.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.queue;
+
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
+
+public interface OffsetInitializer {
+    long maxConsumeQueueOffset(String topic, int queueId) throws ConsumeQueueException;
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializerRocksDBImpl.java b/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializerRocksDBImpl.java
new file mode 100644
index 00000000000..4b889e1e448
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializerRocksDBImpl.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.queue;
+
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
+import org.rocksdb.RocksDBException;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+
+public class OffsetInitializerRocksDBImpl implements OffsetInitializer {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(OffsetInitializerRocksDBImpl.class);
+
+    private final RocksDBConsumeQueueStore consumeQueueStore;
+
+    public OffsetInitializerRocksDBImpl(RocksDBConsumeQueueStore consumeQueueStore) {
+        this.consumeQueueStore = consumeQueueStore;
+    }
+
+    @Override
+    public long maxConsumeQueueOffset(String topic, int queueId) throws ConsumeQueueException {
+        try {
+            long offset = consumeQueueStore.getMaxOffsetInQueue(topic, queueId);
+            LOGGER.info("Look up RocksDB for max-offset of LMQ[{}:{}]: {}", topic, queueId, offset);
+            return offset;
+        } catch (RocksDBException e) {
+            throw new ConsumeQueueException(e);
+        }
+    }
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java
index 5b4bf994e0e..7d388171618 100644
--- a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java
@@ -17,6 +17,7 @@
 
 package org.apache.rocketmq.store.queue;
 
+import com.google.common.base.Preconditions;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -26,6 +27,7 @@
 import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 
 /**
  * QueueOffsetOperator is a component for operating offsets for queues.
@@ -35,7 +37,11 @@ public class QueueOffsetOperator {
 
     private ConcurrentMap<String, Long> topicQueueTable = new ConcurrentHashMap<>(1024);
     private ConcurrentMap<String, Long> batchTopicQueueTable = new ConcurrentHashMap<>(1024);
-    private ConcurrentMap<String/* topic-queueid */, Long/* offset */> lmqTopicQueueTable = new ConcurrentHashMap<>(1024);
+
+    /**
+     * {TOPIC}-{QUEUE_ID} --> NEXT Consume Queue Offset
+     */
+    private ConcurrentMap<String/* topic-queue-id */, Long/* offset */> lmqTopicQueueTable = new ConcurrentHashMap<>(1024);
 
     public long getQueueOffset(String topicQueueKey) {
         return ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L);
@@ -63,17 +69,28 @@ public void increaseBatchQueueOffset(String topicQueueKey, short messageNum) {
         this.batchTopicQueueTable.put(topicQueueKey, batchQueueOffset + messageNum);
     }
 
-    public long getLmqOffset(String topicQueueKey) {
-        return ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L);
-    }
-
-    public Long getLmqTopicQueueNextOffset(String topicQueueKey) {
-        return this.lmqTopicQueueTable.get(topicQueueKey);
+    public long getLmqOffset(String topic, int queueId, OffsetInitializer callback) throws ConsumeQueueException {
+        Preconditions.checkNotNull(callback, "ConsumeQueueOffsetCallback cannot be null");
+        String topicQueue = topic + "-" + queueId;
+        if (!lmqTopicQueueTable.containsKey(topicQueue)) {
+            // Load from RocksDB on cache miss.
+            Long prev = lmqTopicQueueTable.putIfAbsent(topicQueue, callback.maxConsumeQueueOffset(topic, queueId));
+            if (null != prev) {
+                log.error("[BUG] Data racing, lmqTopicQueueTable should NOT contain key={}", topicQueue);
+            }
+        }
+        return lmqTopicQueueTable.get(topicQueue);
     }
 
-    public void increaseLmqOffset(String queueKey, short messageNum) {
-        Long lmqOffset = ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, queueKey, k -> 0L);
-        this.lmqTopicQueueTable.put(queueKey, lmqOffset + messageNum);
+    public void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException {
+        String topicQueue = topic + "-" + queueId;
+        if (!this.lmqTopicQueueTable.containsKey(topicQueue)) {
+            throw new ConsumeQueueException(String.format("Max offset of Queue[name=%s, id=%d] should have existed", topic, queueId));
+        }
+        long prev = lmqTopicQueueTable.get(topicQueue);
+        this.lmqTopicQueueTable.compute(topicQueue, (k, offset) -> offset + delta);
+        long current = lmqTopicQueueTable.get(topicQueue);
+        log.debug("Max offset of LMQ[{}:{}] increased: {} --> {}", topic, queueId, prev, current);
     }
 
     public long currentQueueOffset(String topicQueueKey) {
@@ -112,4 +129,4 @@ public ConcurrentMap<String, Long> getTopicQueueTable() {
     public void setBatchTopicQueueTable(ConcurrentMap<String, Long> batchTopicQueueTable) {
         this.batchTopicQueueTable = batchTopicQueueTable;
     }
-}
\ No newline at end of file
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java
index 5a981bb4df1..7bd3c2e3057 100644
--- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java
@@ -18,7 +18,6 @@
 
 import java.nio.ByteBuffer;
 import java.util.List;
-
 import org.apache.rocketmq.common.BoundaryType;
 import org.apache.rocketmq.common.Pair;
 import org.apache.rocketmq.common.attribute.CQType;
@@ -223,10 +222,47 @@ public void increaseQueueOffset(QueueOffsetOperator queueOffsetOperator, Message
 
     @Override
     public long estimateMessageCount(long from, long to, MessageFilter filter) {
-        // todo
-        return 0;
+        // Check from and to offset validity
+        Pair<CqUnit, Long> fromUnit = getCqUnitAndStoreTime(from);
+        if (fromUnit == null) {
+            return -1;
+        }
+
+        if (from >= to) {
+            return -1;
+        }
+
+        if (to > getMaxOffsetInQueue()) {
+            to = getMaxOffsetInQueue();
+        }
+
+        int maxSampleSize = messageStore.getMessageStoreConfig().getMaxConsumeQueueScan();
+        int sampleSize = to - from > maxSampleSize ? maxSampleSize : (int) (to - from);
+
+        int matchThreshold = messageStore.getMessageStoreConfig().getSampleCountThreshold();
+        int matchSize = 0;
+
+        for (int i = 0; i < sampleSize; i++) {
+            long index = from + i;
+            Pair<CqUnit, Long> pair = getCqUnitAndStoreTime(index);
+            if (pair == null) {
+                continue;
+            }
+            CqUnit cqUnit = pair.getObject1();
+            if (filter.isMatchedByConsumeQueue(cqUnit.getTagsCode(), cqUnit.getCqExtUnit())) {
+                matchSize++;
+                // if matchSize is plenty, early exit estimate
+                if (matchSize > matchThreshold) {
+                    sampleSize = i;
+                    break;
+                }
+            }
+        }
+        // Make sure the second half is a floating point number, otherwise it will be truncated to 0
+        return sampleSize == 0 ? 0 : (long) ((to - from) * (matchSize / (sampleSize * 1.0)));
     }
 
+
     @Override
     public long getMinOffsetInQueue() {
         return this.messageStore.getMinOffsetInQueue(this.topic, this.queueId);
@@ -235,22 +271,17 @@ public long getMinOffsetInQueue() {
     private int pullNum(long cqOffset, long maxCqOffset) {
         long diffLong = maxCqOffset - cqOffset;
         if (diffLong < Integer.MAX_VALUE) {
-            int diffInt = (int) diffLong;
-            return diffInt > 16 ? 16 : diffInt;
+            return (int) diffLong;
         }
-        return 16;
+        return Integer.MAX_VALUE;
     }
 
     @Override
     public ReferredIterator<CqUnit> iterateFrom(final long startIndex) {
-        try {
-            long maxCqOffset = getMaxOffsetInQueue();
-            if (startIndex < maxCqOffset) {
-                int num = pullNum(startIndex, maxCqOffset);
-                return iterateFrom0(startIndex, num);
-            }
-        } catch (RocksDBException e) {
-            log.error("[RocksDBConsumeQueue] iterateFrom error!", e);
+        long maxCqOffset = getMaxOffsetInQueue();
+        if (startIndex < maxCqOffset) {
+            int num = pullNum(startIndex, maxCqOffset);
+            return new LargeRocksDBConsumeQueueIterator(startIndex, num);
         }
         return null;
     }
@@ -311,7 +342,7 @@ public CqUnit getEarliestUnit() {
     public CqUnit getLatestUnit() {
         try {
             long maxOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId);
-            return get(maxOffset);
+            return get(maxOffset > 0 ? maxOffset - 1 : maxOffset);
         } catch (RocksDBException e) {
             ERROR_LOG.error("getLatestUnit Failed. topic: {}, queueId: {}, {}", topic, queueId, e.getMessage());
         }
@@ -392,4 +423,61 @@ public CqUnit nextAndRelease() {
             }
         }
     }
+
+    private class LargeRocksDBConsumeQueueIterator implements ReferredIterator<CqUnit> {
+        private final long startIndex;
+        private final int totalCount;
+        private int currentIndex;
+
+        public LargeRocksDBConsumeQueueIterator(final long startIndex, final int num) {
+            this.startIndex = startIndex;
+            this.totalCount = num;
+            this.currentIndex = 0;
+        }
+
+        @Override
+        public boolean hasNext() {
+            return this.currentIndex < this.totalCount;
+        }
+
+
+        @Override
+        public CqUnit next() {
+            if (!hasNext()) {
+                return null;
+            }
+
+            final ByteBuffer byteBuffer;
+            try {
+                byteBuffer = messageStore.getQueueStore().get(topic, queueId, startIndex + currentIndex);
+            } catch (RocksDBException e) {
+                ERROR_LOG.error("get cq from rocksdb failed. topic: {}, queueId: {}", topic, queueId, e);
+                return null;
+            }
+            if (byteBuffer == null || byteBuffer.remaining() < RocksDBConsumeQueueTable.CQ_UNIT_SIZE) {
+                return null;
+            }
+            CqUnit cqUnit = new CqUnit(this.startIndex + currentIndex, byteBuffer.getLong(), byteBuffer.getInt(), byteBuffer.getLong());
+            this.currentIndex++;
+            return cqUnit;
+        }
+
+        @Override
+        public void remove() {
+            throw new UnsupportedOperationException("remove");
+        }
+
+        @Override
+        public void release() {
+        }
+
+        @Override
+        public CqUnit nextAndRelease() {
+            try {
+                return next();
+            } finally {
+                release();
+            }
+        }
+    }
 }
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java
index 6fa66282e9d..cb989852fb9 100644
--- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java
@@ -16,7 +16,10 @@
  */
 package org.apache.rocketmq.store.queue;
 
+import io.netty.util.internal.PlatformDependent;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -24,31 +27,33 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
+import java.util.function.Consumer;
+import java.util.function.Function;
+import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.Pair;
-import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.store.ConsumeQueue;
 import org.apache.rocketmq.store.DefaultMessageStore;
-import org.apache.rocketmq.store.DispatchRequest;
+import org.apache.rocketmq.store.queue.offset.OffsetEntry;
+import org.apache.rocketmq.store.queue.offset.OffsetEntryType;
 import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.RocksDBException;
 import org.rocksdb.RocksIterator;
 import org.rocksdb.WriteBatch;
 
-import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8;
-import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1;
+import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_1;
 
 public class RocksDBConsumeQueueOffsetTable {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
     private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);
     private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);
 
-    private static final byte[] MAX_BYTES = "max".getBytes(CHARSET_UTF8);
-    private static final byte[] MIN_BYTES = "min".getBytes(CHARSET_UTF8);
+    private static final byte[] MAX_BYTES = "max".getBytes(StandardCharsets.UTF_8);
+    private static final byte[] MIN_BYTES = "min".getBytes(StandardCharsets.UTF_8);
 
     /**
      * Rocksdb ConsumeQueue's Offset unit. Format:
@@ -72,30 +77,31 @@ public class RocksDBConsumeQueueOffsetTable {
      * </pre>
      * ConsumeQueue's Offset unit. Size: CommitLog Physical Offset(8) + ConsumeQueue Offset(8) =  16 Bytes
      */
-    private static final int OFFSET_PHY_OFFSET = 0;
-    private static final int OFFSET_CQ_OFFSET = 8;
+    static final int OFFSET_PHY_OFFSET = 0;
+    static final int OFFSET_CQ_OFFSET = 8;
     /**
-     *
      * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬─────────────┐
      * │ Topic Bytes Array Size  │  CTRL_1   │  CTRL_1   │  Max(Min) │  CTRL_1   │   QueueId   │
      * │        (4 Bytes)        │ (1 Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │  (4 Bytes)  │
      * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴─────────────┤
      */
-    private static final int OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 3 + 1 + 4;
+    public static final int OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 3 + 1 + 4;
     private static final int OFFSET_VALUE_LENGTH = 8 + 8;
 
     /**
      * We use a new system topic='CHECKPOINT_TOPIC' to record the maxPhyOffset built by CQ dispatch thread.
+     *
      * @see ConsumeQueueStore#getMaxPhyOffsetInConsumeQueue(), we use it to find the maxPhyOffset built by CQ dispatch thread.
      * If we do not record the maxPhyOffset, it may take us a long time to start traversing from the head of
      * RocksDBConsumeQueueOffsetTable to find it.
      */
     private static final String MAX_PHYSICAL_OFFSET_CHECKPOINT = TopicValidator.RMQ_SYS_ROCKSDB_OFFSET_TOPIC;
-    private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(CHARSET_UTF8);
+    private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(StandardCharsets.UTF_8);
     private static final int INNER_CHECKPOINT_TOPIC_LEN = OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES.length;
     private static final ByteBuffer INNER_CHECKPOINT_TOPIC = ByteBuffer.allocateDirect(INNER_CHECKPOINT_TOPIC_LEN);
     private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY = new byte[INNER_CHECKPOINT_TOPIC_LEN];
     private final ByteBuffer maxPhyOffsetBB;
+
     static {
         buildOffsetKeyByteBuffer0(INNER_CHECKPOINT_TOPIC, MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES, 0, true);
         INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN);
@@ -111,69 +117,147 @@ public class RocksDBConsumeQueueOffsetTable {
     /**
      * Although we have already put max(min) consumeQueueOffset and physicalOffset in rocksdb, we still hope to get them
      * from heap to avoid accessing rocksdb.
+     *
      * @see ConsumeQueue#getMaxPhysicOffset(), maxPhysicOffset  --> topicQueueMaxCqOffset
      * @see ConsumeQueue#getMinLogicOffset(),   minLogicOffset  --> topicQueueMinOffset
      */
-    private final Map<String/* topic-queueId */, PhyAndCQOffset> topicQueueMinOffset;
-    private final Map<String/* topic-queueId */, Long> topicQueueMaxCqOffset;
+    private final ConcurrentMap<String/* topic-queueId */, PhyAndCQOffset> topicQueueMinOffset;
+    private final ConcurrentMap<String/* topic-queueId */, Long> topicQueueMaxCqOffset;
 
     public RocksDBConsumeQueueOffsetTable(RocksDBConsumeQueueTable rocksDBConsumeQueueTable,
         ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) {
         this.rocksDBConsumeQueueTable = rocksDBConsumeQueueTable;
         this.rocksDBStorage = rocksDBStorage;
         this.messageStore = messageStore;
-        this.topicQueueMinOffset = new ConcurrentHashMap(1024);
-        this.topicQueueMaxCqOffset = new ConcurrentHashMap(1024);
+        this.topicQueueMinOffset = new ConcurrentHashMap<>(1024);
+        this.topicQueueMaxCqOffset = new ConcurrentHashMap<>(1024);
 
         this.maxPhyOffsetBB = ByteBuffer.allocateDirect(8);
     }
 
     public void load() {
         this.offsetCFH = this.rocksDBStorage.getOffsetCFHandle();
+        loadMaxConsumeQueueOffsets();
     }
 
-    public void updateTempTopicQueueMaxOffset(final Pair<ByteBuffer, ByteBuffer> offsetBBPair,
-        final byte[] topicBytes, final DispatchRequest request,
-        final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap) {
-        buildOffsetKeyAndValueByteBuffer(offsetBBPair, topicBytes, request);
-        ByteBuffer topicQueueId = offsetBBPair.getObject1();
-        ByteBuffer maxOffsetBB = offsetBBPair.getObject2();
-        Pair<ByteBuffer, DispatchRequest> old = tempTopicQueueMaxOffsetMap.get(topicQueueId);
-        if (old == null) {
-            tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair(maxOffsetBB, request));
-        } else {
-            long oldMaxOffset = old.getObject1().getLong(OFFSET_CQ_OFFSET);
-            long maxOffset = maxOffsetBB.getLong(OFFSET_CQ_OFFSET);
-            if (maxOffset >= oldMaxOffset) {
-                ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset);
+    private void loadMaxConsumeQueueOffsets() {
+        Function<OffsetEntry, Boolean> predicate = entry -> entry.type == OffsetEntryType.MAXIMUM;
+        Consumer<OffsetEntry> fn = entry -> {
+            topicQueueMaxCqOffset.putIfAbsent(entry.topic + "-" + entry.queueId, entry.offset);
+            log.info("LoadMaxConsumeQueueOffsets Max {}:{} --> {}|{}", entry.topic, entry.queueId, entry.offset, entry.commitLogOffset);
+        };
+        try {
+            forEach(predicate, fn);
+        } catch (RocksDBException e) {
+            log.error("Failed to maximum consume queue offset", e);
+        }
+    }
+
+    public void forEach(Function<OffsetEntry, Boolean> predicate, Consumer<OffsetEntry> fn) throws RocksDBException {
+        try (RocksIterator iterator = this.rocksDBStorage.seekOffsetCF()) {
+            if (null == iterator) {
+                return;
+            }
+
+            int keyBufferCapacity = 256;
+            iterator.seekToFirst();
+            ByteBuffer keyBuffer = ByteBuffer.allocateDirect(keyBufferCapacity);
+            ByteBuffer valueBuffer = ByteBuffer.allocateDirect(16);
+            while (iterator.isValid()) {
+                // parse key buffer according to key layout
+                keyBuffer.clear(); // clear position and limit before reuse
+                int total = iterator.key(keyBuffer);
+                if (total > keyBufferCapacity) {
+                    keyBufferCapacity = total;
+                    PlatformDependent.freeDirectBuffer(keyBuffer);
+                    keyBuffer = ByteBuffer.allocateDirect(keyBufferCapacity);
+                    continue;
+                }
+
+                if (keyBuffer.remaining() <= OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES) {
+                    iterator.next();
+                    ROCKSDB_LOG.warn("Malformed Key/Value pair");
+                    continue;
+                }
+
+                int topicLength = keyBuffer.getInt();
+                byte ctrl1 = keyBuffer.get();
+                assert ctrl1 == CTRL_1;
+
+                byte[] topicBytes = new byte[topicLength];
+                keyBuffer.get(topicBytes);
+                ctrl1 = keyBuffer.get();
+                assert ctrl1 == CTRL_1;
+                String topic = new String(topicBytes, StandardCharsets.UTF_8);
+
+                byte[] minMax = new byte[3];
+                keyBuffer.get(minMax);
+                OffsetEntryType entryType;
+                if (Arrays.equals(minMax, MAX_BYTES)) {
+                    entryType = OffsetEntryType.MAXIMUM;
+                } else {
+                    entryType = OffsetEntryType.MINIMUM;
+                }
+                ctrl1 = keyBuffer.get();
+                assert ctrl1 == CTRL_1;
+
+                assert keyBuffer.remaining() == Integer.BYTES;
+                int queueId = keyBuffer.getInt();
+
+                // Read and parse value buffer according to value layout
+                valueBuffer.clear(); // clear position and limit before reuse
+                total = iterator.value(valueBuffer);
+                if (total != Long.BYTES + Long.BYTES) {
+                    // Skip system checkpoint topic as its value is only 8 bytes
+                    iterator.next();
+                    continue;
+                }
+                long commitLogOffset = valueBuffer.getLong();
+                long consumeOffset = valueBuffer.getLong();
+
+                OffsetEntry entry = new OffsetEntry();
+                entry.topic = topic;
+                entry.queueId = queueId;
+                entry.type = entryType;
+                entry.offset = consumeOffset;
+                entry.commitLogOffset = commitLogOffset;
+                if (predicate.apply(entry)) {
+                    fn.accept(entry);
+                }
+                iterator.next();
             }
+            // clean up direct buffers
+            PlatformDependent.freeDirectBuffer(keyBuffer);
+            PlatformDependent.freeDirectBuffer(valueBuffer);
         }
     }
 
-    public void putMaxPhyAndCqOffset(final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap,
+    public void putMaxPhyAndCqOffset(final Map<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> tempTopicQueueMaxOffsetMap,
         final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException {
-        for (Map.Entry<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> entry : tempTopicQueueMaxOffsetMap.entrySet()) {
+        for (Map.Entry<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> entry : tempTopicQueueMaxOffsetMap.entrySet()) {
             writeBatch.put(this.offsetCFH, entry.getKey(), entry.getValue().getObject1());
         }
 
         appendMaxPhyOffset(writeBatch, maxPhyOffset);
     }
 
-    public void putHeapMaxCqOffset(final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap) {
-        for (Map.Entry<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> entry : tempTopicQueueMaxOffsetMap.entrySet()) {
-            DispatchRequest request = entry.getValue().getObject2();
-            putHeapMaxCqOffset(request.getTopic(), request.getQueueId(), request.getConsumeQueueOffset());
+    public void putHeapMaxCqOffset(final Map<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> tempTopicQueueMaxOffsetMap) {
+        for (Map.Entry<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> entry : tempTopicQueueMaxOffsetMap.entrySet()) {
+            DispatchEntry dispatchEntry = entry.getValue().getObject2();
+            String topic = new String(dispatchEntry.topic, StandardCharsets.UTF_8);
+            putHeapMaxCqOffset(topic, dispatchEntry.queueId, dispatchEntry.queueOffset);
         }
     }
 
     /**
      * When topic is deleted, we clean up its offset info in rocksdb.
+     *
      * @param topic
      * @param queueId
      * @throws RocksDBException
      */
     public void destroyOffset(String topic, int queueId, WriteBatch writeBatch) throws RocksDBException {
-        final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
+        final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
         final ByteBuffer minOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, false);
         byte[] minOffsetBytes = this.rocksDBStorage.getOffset(minOffsetKey.array());
         Long startCQOffset = (minOffsetBytes != null) ? ByteBuffer.wrap(minOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null;
@@ -214,15 +298,14 @@ public long getMaxPhyOffset() throws RocksDBException {
 
     /**
      * Traverse the offset table to find dirty topic
+     *
      * @param existTopicSet
      * @return
      */
     public Map<String, Set<Integer>> iterateOffsetTable2FindDirty(final Set<String> existTopicSet) {
         Map<String/* topic */, Set<Integer/* queueId */>> topicQueueIdToBeDeletedMap = new HashMap<>();
 
-        RocksIterator iterator = null;
-        try {
-            iterator = rocksDBStorage.seekOffsetCF();
+        try (RocksIterator iterator = rocksDBStorage.seekOffsetCF()) {
             if (iterator == null) {
                 return topicQueueIdToBeDeletedMap;
             }
@@ -236,17 +319,22 @@ public Map<String, Set<Integer>> iterateOffsetTable2FindDirty(final Set<String>
                 ByteBuffer keyBB = ByteBuffer.wrap(key);
                 int topicLen = keyBB.getInt(0);
                 byte[] topicBytes = new byte[topicLen];
-                /**
+                /*
                  * "Topic Bytes Array Size" + "CTRL_1" = 4 + 1
                  */
                 keyBB.position(4 + 1);
                 keyBB.get(topicBytes);
-                String topic = new String(topicBytes, CHARSET_UTF8);
+                String topic = new String(topicBytes, StandardCharsets.UTF_8);
                 if (TopicValidator.isSystemTopic(topic)) {
                     continue;
                 }
 
-                /**
+                // LMQ topic offsets should NOT be removed
+                if (MixAll.isLmq(topic)) {
+                    continue;
+                }
+
+                /*
                  * "Topic Bytes Array Size" + "CTRL_1" + "Topic Bytes Array" + "CTRL_1"  + "Max(min)" + "CTRL_1"
                  *  = 4 + 1 + topicLen + 1 + 3 + 1
                  */
@@ -270,10 +358,6 @@ public Map<String, Set<Integer>> iterateOffsetTable2FindDirty(final Set<String>
             }
         } catch (Exception e) {
             ERROR_LOG.error("iterateOffsetTable2MarkDirtyCQ Failed.", e);
-        } finally {
-            if (iterator != null) {
-                iterator.close();
-            }
         }
         return topicQueueIdToBeDeletedMap;
     }
@@ -285,9 +369,13 @@ public Long getMaxCqOffset(String topic, int queueId) throws RocksDBException {
             final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId);
             maxCqOffset = (byteBuffer != null) ? byteBuffer.getLong(OFFSET_CQ_OFFSET) : null;
             String topicQueueId = buildTopicQueueId(topic, queueId);
-            this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, maxCqOffset != null ? maxCqOffset : -1L);
+            long offset = maxCqOffset != null ? maxCqOffset : -1L;
+            Long prev = this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, offset);
+            if (null == prev) {
+                ROCKSDB_LOG.info("Max offset of {} is initialized to {} according to RocksDB", topicQueueId, offset);
+            }
             if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
-                ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, maxCqOffset);
+                ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, offset);
             }
         }
 
@@ -296,34 +384,50 @@ public Long getMaxCqOffset(String topic, int queueId) throws RocksDBException {
 
     /**
      * truncate dirty offset in rocksdb
+     *
      * @param offsetToTruncate
      * @throws RocksDBException
      */
     public void truncateDirty(long offsetToTruncate) throws RocksDBException {
         correctMaxPyhOffset(offsetToTruncate);
 
-        ConcurrentMap<String, TopicConfig> allTopicConfigMap = this.messageStore.getTopicConfigs();
-        if (allTopicConfigMap == null) {
-            return;
-        }
-        for (TopicConfig topicConfig : allTopicConfigMap.values()) {
-            for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) {
-                truncateDirtyOffset(topicConfig.getTopicName(), i);
+        Function<OffsetEntry, Boolean> predicate = entry -> {
+            if (entry.type == OffsetEntryType.MINIMUM) {
+                return false;
             }
-        }
+            // Normal entry offset MUST have the following inequality
+            // entry commit-log offset + message-size-in-bytes <= offsetToTruncate;
+            // otherwise, the consume queue contains dirty records to truncate;
+            //
+            // If the broker node is configured to use async-flush, it's possible consume queues contain
+            //  pointers to message records that is not flushed and lost during restart.
+            return entry.commitLogOffset >= offsetToTruncate;
+        };
+
+        Consumer<OffsetEntry> fn = entry -> {
+            try {
+                truncateDirtyOffset(entry.topic, entry.queueId);
+            } catch (RocksDBException e) {
+                log.error("Failed to truncate maximum offset of consume queue[topic={}, queue-id={}]",
+                    entry.topic, entry.queueId, e);
+            }
+        };
+
+        forEach(predicate, fn);
     }
 
-    private Pair<Boolean, Long> isMinOffsetOk(final String topic, final int queueId, final long minPhyOffset) throws RocksDBException {
+    private Pair<Boolean, Long> isMinOffsetOk(final String topic, final int queueId,
+        final long minPhyOffset) throws RocksDBException {
         PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId);
         if (phyAndCQOffset != null) {
             final long phyOffset = phyAndCQOffset.getPhyOffset();
             final long cqOffset = phyAndCQOffset.getCqOffset();
 
-            return (phyOffset >= minPhyOffset) ? new Pair(true, cqOffset) : new Pair(false, cqOffset);
+            return (phyOffset >= minPhyOffset) ? new Pair<>(true, cqOffset) : new Pair<>(false, cqOffset);
         }
         ByteBuffer byteBuffer = getMinPhyAndCqOffsetInKV(topic, queueId);
         if (byteBuffer == null) {
-            return new Pair(false, 0L);
+            return new Pair<>(false, 0L);
         }
         final long phyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET);
         final long cqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET);
@@ -334,9 +438,9 @@ private Pair<Boolean, Long> isMinOffsetOk(final String topic, final int queueId,
             if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
                 ROCKSDB_LOG.warn("updateMinOffsetInQueue. {}, {}", topicQueueId, newPhyAndCQOffset);
             }
-            return new Pair(true, cqOffset);
+            return new Pair<>(true, cqOffset);
         }
-        return new Pair(false, cqOffset);
+        return new Pair<>(false, cqOffset);
     }
 
     private void truncateDirtyOffset(String topic, int queueId) throws RocksDBException {
@@ -361,8 +465,7 @@ private void correctMaxPyhOffset(long maxPhyOffset) throws RocksDBException {
         if (!this.rocksDBStorage.hold()) {
             return;
         }
-        try {
-            WriteBatch writeBatch = new WriteBatch();
+        try (WriteBatch writeBatch = new WriteBatch()) {
             long oldMaxPhyOffset = getMaxPhyOffset();
             if (oldMaxPhyOffset <= maxPhyOffset) {
                 return;
@@ -416,10 +519,10 @@ private ByteBuffer getMaxPhyAndCqOffsetInKV(String topic, int queueId) throws Ro
     }
 
     private ByteBuffer getPhyAndCqOffsetInKV(String topic, int queueId, boolean max) throws RocksDBException {
-        final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
+        final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
         final ByteBuffer keyBB = buildOffsetKeyByteBuffer(topicBytes, queueId, max);
 
-        byte[] value =  this.rocksDBStorage.getOffset(keyBB.array());
+        byte[] value = this.rocksDBStorage.getOffset(keyBB.array());
         return (value != null) ? ByteBuffer.wrap(value) : null;
     }
 
@@ -427,17 +530,19 @@ private String buildTopicQueueId(final String topic, final int queueId) {
         return topic + "-" + queueId;
     }
 
-    private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset, final long minCQOffset) {
+    private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset,
+        final long minCQOffset) {
         String topicQueueId = buildTopicQueueId(topic, queueId);
         PhyAndCQOffset phyAndCQOffset = new PhyAndCQOffset(minPhyOffset, minCQOffset);
         this.topicQueueMinOffset.put(topicQueueId, phyAndCQOffset);
     }
 
-    private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxCQOffset) {
+    private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxOffset) {
         String topicQueueId = buildTopicQueueId(topic, queueId);
-        Long oldMaxCqOffset = this.topicQueueMaxCqOffset.put(topicQueueId, maxCQOffset);
-        if (oldMaxCqOffset != null && oldMaxCqOffset > maxCQOffset) {
-            ERROR_LOG.error("cqOffset invalid0. old: {}, now: {}", oldMaxCqOffset, maxCQOffset);
+        Long prev = this.topicQueueMaxCqOffset.put(topicQueueId, maxOffset);
+        if (prev != null && prev > maxOffset) {
+            ERROR_LOG.error("Max offset of consume-queue[topic={}, queue-id={}] regressed. prev-max={}, current-max={}",
+                topic, queueId, prev, maxOffset);
         }
     }
 
@@ -463,9 +568,8 @@ private void updateCqOffset(final String topic, final int queueId, final long ph
         if (!this.rocksDBStorage.hold()) {
             return;
         }
-        WriteBatch writeBatch = new WriteBatch();
-        try {
-            final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
+        try (WriteBatch writeBatch = new WriteBatch()) {
+            final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
             final ByteBuffer offsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, max);
 
             final ByteBuffer offsetValue = buildOffsetValueByteBuffer(phyOffset, cqOffset);
@@ -481,7 +585,6 @@ private void updateCqOffset(final String topic, final int queueId, final long ph
             ERROR_LOG.error("updateCqOffset({}) failed.", max ? "max" : "min", e);
             throw e;
         } finally {
-            writeBatch.close();
             this.rocksDBStorage.release();
             if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
                 ROCKSDB_LOG.warn("updateCqOffset({}). topic: {}, queueId: {}, phyOffset: {}, cqOffset: {}",
@@ -504,10 +607,8 @@ private boolean correctMaxCqOffset(final String topic, final int queueId, final
             throw new RocksDBException("correctMaxCqOffset error");
         }
 
-        long high = maxCQOffset;
-        long low = minCQOffset;
-        PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high,
-            low, maxPhyOffsetInCQ, false);
+        PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, maxCQOffset,
+            minCQOffset, maxPhyOffsetInCQ, false);
 
         long targetCQOffset = targetPhyAndCQOffset.getCqOffset();
         long targetPhyOffset = targetPhyAndCQOffset.getPhyOffset();
@@ -541,10 +642,8 @@ private boolean correctMinCqOffset(final String topic, final int queueId,
             return true;
         }
 
-        long high = maxCQOffset;
-        long low = minCQOffset;
-        PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, low,
-            minPhyOffset, true);
+        PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, maxCQOffset,
+            minCQOffset, minPhyOffset, true);
         long targetCQOffset = phyAndCQOffset.getCqOffset();
         long targetPhyOffset = phyAndCQOffset.getPhyOffset();
 
@@ -568,28 +667,29 @@ public static Pair<ByteBuffer, ByteBuffer> getOffsetByteBufferPair() {
         return new Pair<>(offsetKey, offsetValue);
     }
 
-    private void buildOffsetKeyAndValueByteBuffer(final Pair<ByteBuffer, ByteBuffer> offsetBBPair,
-        final byte[] topicBytes, final DispatchRequest request) {
+    static void buildOffsetKeyAndValueByteBuffer(final Pair<ByteBuffer, ByteBuffer> offsetBBPair,
+        final DispatchEntry entry) {
         final ByteBuffer offsetKey = offsetBBPair.getObject1();
-        buildOffsetKeyByteBuffer(offsetKey, topicBytes, request.getQueueId(), true);
+        buildOffsetKeyByteBuffer(offsetKey, entry.topic, entry.queueId, true);
 
         final ByteBuffer offsetValue = offsetBBPair.getObject2();
-        buildOffsetValueByteBuffer(offsetValue, request.getCommitLogOffset(), request.getConsumeQueueOffset());
+        buildOffsetValueByteBuffer(offsetValue, entry.commitLogOffset, entry.queueOffset);
     }
 
-    private ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) {
+    private static ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) {
         ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);
         buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max);
         return byteBuffer;
     }
 
-    private void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final boolean max) {
+    public static void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes,
+        final int queueId, final boolean max) {
         byteBuffer.position(0).limit(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);
         buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max);
     }
 
-    private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId,
-        final boolean max) {
+    private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes,
+        final int queueId, final boolean max) {
         byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1);
         if (max) {
             byteBuffer.put(MAX_BYTES);
@@ -600,18 +700,20 @@ private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final
         byteBuffer.flip();
     }
 
-    private void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) {
+    private static void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset,
+        final long cqOffset) {
         byteBuffer.position(0).limit(OFFSET_VALUE_LENGTH);
         buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset);
     }
 
-    private ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) {
+    private static ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) {
         final ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_VALUE_LENGTH);
         buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset);
         return byteBuffer;
     }
 
-    private void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) {
+    private static void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset,
+        final long cqOffset) {
         byteBuffer.putLong(phyOffset).putLong(cqOffset);
         byteBuffer.flip();
     }
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java
index 4c66696e3c8..94ed0c926a7 100644
--- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java
@@ -18,6 +18,7 @@
 
 import java.io.File;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -28,22 +29,27 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
-
+import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.Nonnull;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.BoundaryType;
+import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.Pair;
+import org.apache.rocketmq.common.ServiceState;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageExtBrokerInner;
-import org.apache.rocketmq.common.utils.DataConverter;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.store.DefaultMessageStore;
 import org.apache.rocketmq.store.DispatchRequest;
-import org.apache.rocketmq.store.config.BrokerRole;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
+import org.apache.rocketmq.store.exception.StoreException;
 import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;
+import org.rocksdb.FlushOptions;
 import org.rocksdb.RocksDBException;
 import org.rocksdb.Statistics;
 import org.rocksdb.WriteBatch;
@@ -52,11 +58,8 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
     private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);
     private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);
 
-    public static final byte CTRL_0 = '\u0000';
-    public static final byte CTRL_1 = '\u0001';
-    public static final byte CTRL_2 = '\u0002';
+    private static final int DEFAULT_BYTE_BUFFER_CAPACITY = 16;
 
-    private static final int BATCH_SIZE = 16;
     public static final int MAX_KEY_LEN = 300;
 
     private final ScheduledExecutorService scheduledExecutorService;
@@ -71,26 +74,33 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
     private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable;
     private final RocksDBConsumeQueueOffsetTable rocksDBConsumeQueueOffsetTable;
 
-    private final WriteBatch writeBatch;
-    private final List<DispatchRequest> bufferDRList;
     private final List<Pair<ByteBuffer, ByteBuffer>> cqBBPairList;
     private final List<Pair<ByteBuffer, ByteBuffer>> offsetBBPairList;
-    private final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap;
+    private final Map<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> tempTopicQueueMaxOffsetMap;
     private volatile boolean isCQError = false;
 
+    private int consumeQueueByteBufferCacheIndex;
+    private int offsetBufferCacheIndex;
+
+    private final OffsetInitializer offsetInitializer;
+
+    private final RocksGroupCommitService groupCommitService;
+
+    private final AtomicReference<ServiceState> serviceState = new AtomicReference<>(ServiceState.CREATE_JUST);
+
     public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) {
         super(messageStore);
 
         this.storePath = StorePathConfigHelper.getStorePathConsumeQueue(messageStoreConfig.getStorePathRootDir());
-        this.rocksDBStorage = new ConsumeQueueRocksDBStorage(messageStore, storePath, 4);
+        this.rocksDBStorage = new ConsumeQueueRocksDBStorage(messageStore, storePath);
         this.rocksDBConsumeQueueTable = new RocksDBConsumeQueueTable(rocksDBStorage, messageStore);
         this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore);
 
-        this.writeBatch = new WriteBatch();
-        this.bufferDRList = new ArrayList(BATCH_SIZE);
-        this.cqBBPairList = new ArrayList(BATCH_SIZE);
-        this.offsetBBPairList = new ArrayList(BATCH_SIZE);
-        for (int i = 0; i < BATCH_SIZE; i++) {
+        this.offsetInitializer = new OffsetInitializerRocksDBImpl(this);
+        this.groupCommitService = new RocksGroupCommitService(this);
+        this.cqBBPairList = new ArrayList<>(16);
+        this.offsetBBPairList = new ArrayList<>(DEFAULT_BYTE_BUFFER_CAPACITY);
+        for (int i = 0; i < DEFAULT_BYTE_BUFFER_CAPACITY; i++) {
             this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair());
             this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair());
         }
@@ -100,16 +110,35 @@ public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) {
             new ThreadFactoryImpl("RocksDBConsumeQueueStoreScheduledThread", messageStore.getBrokerIdentity()));
     }
 
+    private Pair<ByteBuffer, ByteBuffer> getCQByteBufferPair() {
+        int idx = consumeQueueByteBufferCacheIndex++;
+        if (idx >= cqBBPairList.size()) {
+            this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair());
+        }
+        return cqBBPairList.get(idx);
+    }
+
+    private Pair<ByteBuffer, ByteBuffer> getOffsetByteBufferPair() {
+        int idx = offsetBufferCacheIndex++;
+        if (idx >= offsetBBPairList.size()) {
+            this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair());
+        }
+        return offsetBBPairList.get(idx);
+    }
+
     @Override
     public void start() {
-        log.info("RocksDB ConsumeQueueStore start!");
-        this.scheduledExecutorService.scheduleAtFixedRate(() -> {
-            this.rocksDBStorage.statRocksdb(ROCKSDB_LOG);
-        }, 10, this.messageStoreConfig.getStatRocksDBCQIntervalSec(), TimeUnit.SECONDS);
-
-        this.scheduledExecutorService.scheduleWithFixedDelay(() -> {
-            cleanDirty(messageStore.getTopicConfigs().keySet());
-        }, 10, this.messageStoreConfig.getCleanRocksDBDirtyCQIntervalMin(), TimeUnit.MINUTES);
+        if (serviceState.compareAndSet(ServiceState.CREATE_JUST, ServiceState.RUNNING)) {
+            log.info("RocksDB ConsumeQueueStore start!");
+            this.groupCommitService.start();
+            this.scheduledExecutorService.scheduleAtFixedRate(() -> {
+                this.rocksDBStorage.statRocksdb(ROCKSDB_LOG);
+            }, 10, this.messageStoreConfig.getStatRocksDBCQIntervalSec(), TimeUnit.SECONDS);
+
+            this.scheduledExecutorService.scheduleWithFixedDelay(() -> {
+                cleanDirty(messageStore.getTopicConfigs().keySet());
+            }, 10, this.messageStoreConfig.getCleanRocksDBDirtyCQIntervalMin(), TimeUnit.MINUTES);
+        }
     }
 
     private void cleanDirty(final Set<String> existTopicSet) {
@@ -144,18 +173,23 @@ public boolean loadAfterDestroy() {
 
     @Override
     public void recover() {
-        // ignored
+        start();
     }
 
     @Override
     public boolean recoverConcurrently() {
+        start();
         return true;
     }
 
     @Override
     public boolean shutdown() {
-        this.scheduledExecutorService.shutdown();
-        return shutdownInner();
+        if (serviceState.compareAndSet(ServiceState.RUNNING, ServiceState.SHUTDOWN_ALREADY)) {
+            this.groupCommitService.shutdown();
+            this.scheduledExecutorService.shutdown();
+            return shutdownInner();
+        }
+        return true;
     }
 
     private boolean shutdownInner() {
@@ -164,25 +198,28 @@ private boolean shutdownInner() {
 
     @Override
     public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException {
-        if (request == null || this.bufferDRList.size() >= BATCH_SIZE) {
-            putMessagePosition();
+        if (null == request) {
+            return;
         }
-        if (request != null) {
-            this.bufferDRList.add(request);
+
+        try {
+            groupCommitService.putRequest(request);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
         }
     }
 
-    public void putMessagePosition() throws RocksDBException {
+    public void putMessagePosition(List<DispatchRequest> requests) throws RocksDBException {
         final int maxRetries = 30;
         for (int i = 0; i < maxRetries; i++) {
-            if (putMessagePosition0()) {
+            if (putMessagePosition0(requests)) {
                 if (this.isCQError) {
                     this.messageStore.getRunningFlags().clearLogicsQueueError();
                     this.isCQError = false;
                 }
                 return;
             } else {
-                ERROR_LOG.warn("{} put cq Failed. retryTime: {}", i);
+                ERROR_LOG.warn("Put cq Failed. retryTime: {}", i);
                 try {
                     Thread.sleep(100);
                 } catch (InterruptedException ignored) {
@@ -197,30 +234,22 @@ public void putMessagePosition() throws RocksDBException {
         throw new RocksDBException("put CQ Failed");
     }
 
-    private boolean putMessagePosition0() {
+    private boolean putMessagePosition0(List<DispatchRequest> requests) {
         if (!this.rocksDBStorage.hold()) {
             return false;
         }
 
-        final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap = this.tempTopicQueueMaxOffsetMap;
-        try {
-            final List<DispatchRequest> bufferDRList = this.bufferDRList;
-            final int size = bufferDRList.size();
+        try (WriteBatch writeBatch = new WriteBatch()) {
+            final int size = requests.size();
             if (size == 0) {
                 return true;
             }
-            final List<Pair<ByteBuffer, ByteBuffer>> cqBBPairList = this.cqBBPairList;
-            final List<Pair<ByteBuffer, ByteBuffer>> offsetBBPairList = this.offsetBBPairList;
-            final WriteBatch writeBatch = this.writeBatch;
-
             long maxPhyOffset = 0;
             for (int i = size - 1; i >= 0; i--) {
-                final DispatchRequest request = bufferDRList.get(i);
-                final byte[] topicBytes = request.getTopic().getBytes(DataConverter.CHARSET_UTF8);
-
-                this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(cqBBPairList.get(i), topicBytes, request, writeBatch);
-                this.rocksDBConsumeQueueOffsetTable.updateTempTopicQueueMaxOffset(offsetBBPairList.get(i),
-                    topicBytes, request, tempTopicQueueMaxOffsetMap);
+                final DispatchRequest request = requests.get(i);
+                DispatchEntry entry = DispatchEntry.from(request);
+                dispatch(entry, writeBatch);
+                dispatchLMQ(request, writeBatch);
 
                 final int msgSize = request.getMsgSize();
                 final long phyOffset = request.getCommitLogOffset();
@@ -231,47 +260,92 @@ private boolean putMessagePosition0() {
 
             this.rocksDBConsumeQueueOffsetTable.putMaxPhyAndCqOffset(tempTopicQueueMaxOffsetMap, writeBatch, maxPhyOffset);
 
-            // clear writeBatch in batchPut
             this.rocksDBStorage.batchPut(writeBatch);
 
             this.rocksDBConsumeQueueOffsetTable.putHeapMaxCqOffset(tempTopicQueueMaxOffsetMap);
-
-            long storeTimeStamp = bufferDRList.get(size - 1).getStoreTimestamp();
-            if (this.messageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE
-                || this.messageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) {
-                this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimeStamp);
-            }
-            this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimeStamp);
-
-            notifyMessageArriveAndClear();
+            notifyMessageArriveAndClear(requests);
             return true;
         } catch (Exception e) {
-            ERROR_LOG.error("putMessagePosition0 Failed.", e);
+            ERROR_LOG.error("putMessagePosition0 failed.", e);
             return false;
         } finally {
             tempTopicQueueMaxOffsetMap.clear();
+            consumeQueueByteBufferCacheIndex = 0;
+            offsetBufferCacheIndex = 0;
             this.rocksDBStorage.release();
         }
     }
 
-    private void notifyMessageArriveAndClear() {
-        final List<DispatchRequest> bufferDRList = this.bufferDRList;
+    private void dispatch(@Nonnull DispatchEntry entry, @Nonnull final WriteBatch writeBatch) throws RocksDBException {
+        this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(getCQByteBufferPair(), entry, writeBatch);
+        updateTempTopicQueueMaxOffset(getOffsetByteBufferPair(), entry);
+    }
+
+    private void updateTempTopicQueueMaxOffset(final Pair<ByteBuffer, ByteBuffer> offsetBBPair,
+        final DispatchEntry entry) {
+        RocksDBConsumeQueueOffsetTable.buildOffsetKeyAndValueByteBuffer(offsetBBPair, entry);
+        ByteBuffer topicQueueId = offsetBBPair.getObject1();
+        ByteBuffer maxOffsetBB = offsetBBPair.getObject2();
+        Pair<ByteBuffer, DispatchEntry> old = tempTopicQueueMaxOffsetMap.get(topicQueueId);
+        if (old == null) {
+            tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair<>(maxOffsetBB, entry));
+        } else {
+            long oldMaxOffset = old.getObject1().getLong(RocksDBConsumeQueueOffsetTable.OFFSET_CQ_OFFSET);
+            long maxOffset = maxOffsetBB.getLong(RocksDBConsumeQueueOffsetTable.OFFSET_CQ_OFFSET);
+            if (maxOffset >= oldMaxOffset) {
+                ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset);
+            }
+        }
+    }
+
+    private void dispatchLMQ(@Nonnull DispatchRequest request, @Nonnull final WriteBatch writeBatch)
+        throws RocksDBException {
+        if (!messageStoreConfig.isEnableLmq() || !request.containsLMQ()) {
+            return;
+        }
+        Map<String, String> map = request.getPropertiesMap();
+        String lmqNames = map.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
+        String lmqOffsets = map.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);
+        String[] queues = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR);
+        String[] queueOffsets = lmqOffsets.split(MixAll.LMQ_DISPATCH_SEPARATOR);
+        if (queues.length != queueOffsets.length) {
+            ERROR_LOG.error("[bug] queues.length!=queueOffsets.length ", request.getTopic());
+            return;
+        }
+        for (int i = 0; i < queues.length; i++) {
+            String queueName = queues[i];
+            DispatchEntry entry = DispatchEntry.from(request);
+            long queueOffset = Long.parseLong(queueOffsets[i]);
+            int queueId = request.getQueueId();
+            if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {
+                queueId = MixAll.LMQ_QUEUE_ID;
+            }
+            entry.queueId = queueId;
+            entry.queueOffset = queueOffset;
+            entry.topic = queueName.getBytes(StandardCharsets.UTF_8);
+            log.debug("Dispatch LMQ[{}:{}]:{} --> {}", queueName, queueId, queueOffset, entry.commitLogOffset);
+            dispatch(entry, writeBatch);
+        }
+    }
+
+    private void notifyMessageArriveAndClear(List<DispatchRequest> requests) {
         try {
-            for (DispatchRequest dp : bufferDRList) {
+            for (DispatchRequest dp : requests) {
                 this.messageStore.notifyMessageArriveIfNecessary(dp);
             }
+            requests.clear();
         } catch (Exception e) {
             ERROR_LOG.error("notifyMessageArriveAndClear Failed.", e);
-        } finally {
-            bufferDRList.clear();
         }
     }
 
     public Statistics getStatistics() {
         return rocksDBStorage.getStatistics();
     }
+
     @Override
-    public List<ByteBuffer> rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException {
+    public List<ByteBuffer> rangeQuery(final String topic, final int queueId, final long startIndex,
+        final int num) throws RocksDBException {
         return this.rocksDBConsumeQueueTable.rangeQuery(topic, queueId, startIndex, num);
     }
 
@@ -283,6 +357,7 @@ public ByteBuffer get(final String topic, final int queueId, final long cqOffset
     /**
      * Ignored, we do not need to recover topicQueueTable and correct minLogicOffset. Because we will correct them
      * when we use them, we call it lazy correction.
+     *
      * @see RocksDBConsumeQueue#increaseQueueOffset(QueueOffsetOperator, MessageExtBrokerInner, short)
      * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int)
      */
@@ -309,8 +384,7 @@ public void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException
             return;
         }
 
-        WriteBatch writeBatch = new WriteBatch();
-        try {
+        try (WriteBatch writeBatch = new WriteBatch()) {
             this.rocksDBConsumeQueueTable.destroyCQ(topic, queueId, writeBatch);
             this.rocksDBConsumeQueueOffsetTable.destroyOffset(topic, queueId, writeBatch);
 
@@ -319,7 +393,6 @@ public void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException
             ERROR_LOG.error("kv deleteTopic {} Failed.", topic, e);
             throw e;
         } finally {
-            writeBatch.close();
             this.rocksDBStorage.release();
         }
     }
@@ -329,10 +402,22 @@ public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) {
         try {
             this.rocksDBStorage.flushWAL();
         } catch (Exception e) {
+            log.error("Failed to flush WAL", e);
         }
         return true;
     }
 
+    @Override
+    public void flush() throws StoreException {
+        try (FlushOptions flushOptions = new FlushOptions()) {
+            flushOptions.setWaitForFlush(true);
+            flushOptions.setAllowWriteStall(true);
+            this.rocksDBStorage.flush(flushOptions);
+        } catch (RocksDBException e) {
+            throw new StoreException(e);
+        }
+    }
+
     @Override
     public void checkSelf() {
         // ignored
@@ -349,8 +434,9 @@ public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitL
      * will be rewritten by new KV when new messages are appended or will be cleaned up when topics are deleted.
      * But dirty offset info in RocksDBConsumeQueueOffsetTable must be truncated, because we use offset info in
      * RocksDBConsumeQueueOffsetTable to rebuild topicQueueTable(@see RocksDBConsumeQueue#increaseQueueOffset).
-     * @param offsetToTruncate
-     * @throws RocksDBException
+     *
+     * @param offsetToTruncate CommitLog offset to truncate to
+     * @throws RocksDBException If there is any error.
      */
     @Override
     public void truncateDirty(long offsetToTruncate) throws RocksDBException {
@@ -368,16 +454,27 @@ public void cleanExpired(final long minPhyOffset) {
     }
 
     @Override
-    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException {
+    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp,
+        BoundaryType boundaryType) throws RocksDBException {
         final long minPhysicOffset = this.messageStore.getMinPhyOffset();
         long low = this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId);
         Long high = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId);
         if (high == null || high == -1) {
             return 0;
         }
-        return this.rocksDBConsumeQueueTable.binarySearchInCQByTime(topic, queueId, high, low, timestamp, minPhysicOffset);
+        return this.rocksDBConsumeQueueTable.binarySearchInCQByTime(topic, queueId, high, low, timestamp,
+            minPhysicOffset, boundaryType);
     }
 
+    /**
+     * This method actually returns NEXT slot index to use, starting from 0. For example, if the queue is empty,
+     * it returns 0, pointing to the first slot of the 0-based queue;
+     *
+     * @param topic   Topic name
+     * @param queueId Queue ID
+     * @return Index of the next slot to push into
+     * @throws RocksDBException if RocksDB fails to fulfill the request.
+     */
     @Override
     public long getMaxOffsetInQueue(String topic, int queueId) throws RocksDBException {
         Long maxOffset = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId);
@@ -403,7 +500,13 @@ public long getMaxPhyOffsetInConsumeQueue() throws RocksDBException {
     public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) {
         ConcurrentMap<Integer, ConsumeQueueInterface> map = this.consumeQueueTable.get(topic);
         if (null == map) {
-            ConcurrentMap<Integer, ConsumeQueueInterface> newMap = new ConcurrentHashMap<>(128);
+            ConcurrentMap<Integer, ConsumeQueueInterface> newMap;
+            if (MixAll.isLmq(topic)) {
+                // For LMQ, no need to over allocate internal hashtable
+                newMap = new ConcurrentHashMap<>(1, 1.0F);
+            } else {
+                newMap = new ConcurrentHashMap<>(8);
+            }
             ConcurrentMap<Integer, ConsumeQueueInterface> oldMap = this.consumeQueueTable.putIfAbsent(topic, newMap);
             if (oldMap != null) {
                 map = oldMap;
@@ -442,4 +545,21 @@ public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) {
     public long getTotalSize() {
         return 0;
     }
+
+    @Override
+    public long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException {
+        return queueOffsetOperator.getLmqOffset(topic, queueId, offsetInitializer);
+    }
+
+    @Override
+    public Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException {
+        if (MixAll.isLmq(topic)) {
+            return getLmqQueueOffset(topic, queueId);
+        }
+        return super.getMaxOffset(topic, queueId);
+    }
+
+    public boolean isStopped() {
+        return ServiceState.SHUTDOWN_ALREADY == serviceState.get();
+    }
 }
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java
index 0a735ea27c1..deee0295706 100644
--- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java
@@ -17,25 +17,25 @@
 package org.apache.rocketmq.store.queue;
 
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.rocketmq.common.BoundaryType;
 import org.apache.rocketmq.common.Pair;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.store.DefaultMessageStore;
-import org.apache.rocketmq.store.DispatchRequest;
 import org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable.PhyAndCQOffset;
 import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.RocksDBException;
 import org.rocksdb.WriteBatch;
 
-import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8;
-import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_0;
-import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1;
-import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_2;
+import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_0;
+import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_1;
+import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_2;
 
 /**
  * We use RocksDBConsumeQueueTable to store cqUnit.
@@ -104,30 +104,30 @@ public void load() {
         this.defaultCFH = this.rocksDBStorage.getDefaultCFHandle();
     }
 
-    public void buildAndPutCQByteBuffer(final Pair<ByteBuffer, ByteBuffer> cqBBPair,
-        final byte[] topicBytes, final DispatchRequest request, final WriteBatch writeBatch) throws RocksDBException {
+    public void buildAndPutCQByteBuffer(final Pair<ByteBuffer, ByteBuffer> cqBBPair, final DispatchEntry request,
+        final WriteBatch writeBatch) throws RocksDBException {
         final ByteBuffer cqKey = cqBBPair.getObject1();
-        buildCQKeyByteBuffer(cqKey, topicBytes, request.getQueueId(), request.getConsumeQueueOffset());
+        buildCQKeyByteBuffer(cqKey, request.topic, request.queueId, request.queueOffset);
 
         final ByteBuffer cqValue = cqBBPair.getObject2();
-        buildCQValueByteBuffer(cqValue, request.getCommitLogOffset(), request.getMsgSize(), request.getTagsCode(), request.getStoreTimestamp());
+        buildCQValueByteBuffer(cqValue, request.commitLogOffset, request.messageSize, request.tagCode, request.storeTimestamp);
 
         writeBatch.put(this.defaultCFH, cqKey, cqValue);
     }
 
     public ByteBuffer getCQInKV(final String topic, final int queueId, final long cqOffset) throws RocksDBException {
-        final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
+        final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
         final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, cqOffset);
         byte[] value = this.rocksDBStorage.getCQ(keyBB.array());
         return (value != null) ? ByteBuffer.wrap(value) : null;
     }
 
     public List<ByteBuffer> rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException {
-        final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
-        final List<ColumnFamilyHandle> defaultCFHList = new ArrayList(num);
+        final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
+        final List<ColumnFamilyHandle> defaultCFHList = new ArrayList<>(num);
         final ByteBuffer[] resultList = new ByteBuffer[num];
-        final List<Integer> kvIndexList = new ArrayList(num);
-        final List<byte[]> kvKeyList = new ArrayList(num);
+        final List<Integer> kvIndexList = new ArrayList<>(num);
+        final List<byte[]> kvKeyList = new ArrayList<>(num);
         for (int i = 0; i < num; i++) {
             final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, startIndex + i);
             kvIndexList.add(i);
@@ -152,7 +152,7 @@ public List<ByteBuffer> rangeQuery(final String topic, final int queueId, final
         }
 
         final int resultSize = resultList.length;
-        List<ByteBuffer> bbValueList = new ArrayList(resultSize);
+        List<ByteBuffer> bbValueList = new ArrayList<>(resultSize);
         for (int i = 0; i < resultSize; i++) {
             ByteBuffer byteBuffer = resultList[i];
             if (byteBuffer == null) {
@@ -170,7 +170,7 @@ public List<ByteBuffer> rangeQuery(final String topic, final int queueId, final
      * @throws RocksDBException
      */
     public void destroyCQ(final String topic, final int queueId, WriteBatch writeBatch) throws RocksDBException {
-        final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
+        final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
         final ByteBuffer cqStartKey = buildDeleteCQKey(true, topicBytes, queueId);
         final ByteBuffer cqEndKey = buildDeleteCQKey(false, topicBytes, queueId);
 
@@ -180,10 +180,43 @@ public void destroyCQ(final String topic, final int queueId, WriteBatch writeBat
     }
 
     public long binarySearchInCQByTime(String topic, int queueId, long high, long low, long timestamp,
-        long minPhysicOffset) throws RocksDBException {
-        long result = 0;
+        long minPhysicOffset, BoundaryType boundaryType) throws RocksDBException {
+        long result = -1L;
         long targetOffset = -1L, leftOffset = -1L, rightOffset = -1L;
-        long leftValue = -1L, rightValue = -1L;
+        long ceiling = high, floor = low;
+        // Handle the following corner cases first:
+        // 1. store time of (high) < timestamp
+        ByteBuffer buffer = getCQInKV(topic, queueId, ceiling);
+        if (buffer != null) {
+            long storeTime = buffer.getLong(MSG_STORE_TIME_SIZE_OFFSET);
+            if (storeTime < timestamp) {
+                switch (boundaryType) {
+                    case LOWER:
+                        return ceiling + 1;
+                    case UPPER:
+                        return ceiling;
+                    default:
+                        log.warn("Unknown boundary type");
+                        break;
+                }
+            }
+        }
+        // 2. store time of (low) > timestamp
+        buffer = getCQInKV(topic, queueId, floor);
+        if (buffer != null) {
+            long storeTime = buffer.getLong(MSG_STORE_TIME_SIZE_OFFSET);
+            if (storeTime > timestamp) {
+                switch (boundaryType) {
+                    case LOWER:
+                        return floor;
+                    case UPPER:
+                        return 0;
+                    default:
+                        log.warn("Unknown boundary type");
+                        break;
+                }
+            }
+        }
         while (high >= low) {
             long midOffset = low + ((high - low) >>> 1);
             ByteBuffer byteBuffer = getCQInKV(topic, queueId, midOffset);
@@ -209,22 +242,64 @@ public long binarySearchInCQByTime(String topic, int queueId, long high, long lo
             } else if (storeTime > timestamp) {
                 high = midOffset - 1;
                 rightOffset = midOffset;
-                rightValue = storeTime;
             } else {
                 low = midOffset + 1;
                 leftOffset = midOffset;
-                leftValue = storeTime;
             }
         }
         if (targetOffset != -1) {
+            // offset next to it might also share the same store-timestamp.
+            switch (boundaryType) {
+                case LOWER: {
+                    while (true) {
+                        long nextOffset = targetOffset - 1;
+                        if (nextOffset < floor) {
+                            break;
+                        }
+                        ByteBuffer byteBuffer = getCQInKV(topic, queueId, nextOffset);
+                        long storeTime = byteBuffer.getLong(MSG_STORE_TIME_SIZE_OFFSET);
+                        if (storeTime != timestamp) {
+                            break;
+                        }
+                        targetOffset = nextOffset;
+                    }
+                    break;
+                }
+                case UPPER: {
+                    while (true) {
+                        long nextOffset = targetOffset + 1;
+                        if (nextOffset > ceiling) {
+                            break;
+                        }
+                        ByteBuffer byteBuffer = getCQInKV(topic, queueId, nextOffset);
+                        long storeTime = byteBuffer.getLong(MSG_STORE_TIME_SIZE_OFFSET);
+                        if (storeTime != timestamp) {
+                            break;
+                        }
+                        targetOffset = nextOffset;
+                    }
+                    break;
+                }
+                default: {
+                    log.warn("Unknown boundary type");
+                    break;
+                }
+            }
             result = targetOffset;
         } else {
-            if (leftValue == -1) {
-                result = rightOffset;
-            } else if (rightValue == -1) {
-                result = leftOffset;
-            } else {
-                result = Math.abs(timestamp - leftValue) > Math.abs(timestamp - rightValue) ? rightOffset : leftOffset;
+            switch (boundaryType) {
+                case LOWER: {
+                    result = rightOffset;
+                    break;
+                }
+                case UPPER: {
+                    result = leftOffset;
+                    break;
+                }
+                default: {
+                    log.warn("Unknown boundary type");
+                    break;
+                }
             }
         }
         return result;
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksGroupCommitService.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksGroupCommitService.java
new file mode 100644
index 00000000000..e2f2c9ee2c1
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksGroupCommitService.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store.queue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.store.DispatchRequest;
+import org.rocksdb.RocksDBException;
+
+public class RocksGroupCommitService extends ServiceThread {
+
+    private static final int MAX_BUFFER_SIZE = 100_000;
+
+    private static final int PREFERRED_DISPATCH_REQUEST_COUNT = 256;
+
+    private final LinkedBlockingQueue<DispatchRequest> buffer;
+
+    private final RocksDBConsumeQueueStore store;
+
+    private final List<DispatchRequest> requests = new ArrayList<>(PREFERRED_DISPATCH_REQUEST_COUNT);
+
+    public RocksGroupCommitService(RocksDBConsumeQueueStore store) {
+        this.store = store;
+        this.buffer = new LinkedBlockingQueue<>(MAX_BUFFER_SIZE);
+    }
+
+    @Override
+    public String getServiceName() {
+        return "RocksGroupCommit";
+    }
+
+    @Override
+    public void run() {
+        log.info("{} service started", this.getServiceName());
+        while (!this.isStopped()) {
+            try {
+                this.waitForRunning(10);
+                this.doCommit();
+            } catch (Exception e) {
+                log.warn("{} service has exception. ", this.getServiceName(), e);
+            }
+        }
+        log.info("{} service end", this.getServiceName());
+    }
+
+    public void putRequest(final DispatchRequest request) throws InterruptedException {
+        while (!buffer.offer(request, 3, TimeUnit.SECONDS)) {
+            log.warn("RocksGroupCommitService#buffer is full, 3s elapsed before space becomes available");
+        }
+        this.wakeup();
+    }
+
+    private void doCommit() {
+        while (!buffer.isEmpty()) {
+            while (true) {
+                DispatchRequest dispatchRequest = buffer.poll();
+                if (null != dispatchRequest) {
+                    requests.add(dispatchRequest);
+                }
+
+                if (requests.isEmpty()) {
+                    // buffer has been drained
+                    break;
+                }
+
+                if (null == dispatchRequest || requests.size() >= PREFERRED_DISPATCH_REQUEST_COUNT) {
+                    groupCommit();
+                }
+            }
+        }
+    }
+
+    private void groupCommit() {
+        while (!store.isStopped()) {
+            try {
+                // putMessagePosition will clear requests after consume queue building completion
+                store.putMessagePosition(requests);
+                break;
+            } catch (RocksDBException e) {
+                log.error("Failed to build consume queue in RocksDB", e);
+            }
+        }
+    }
+
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/SparseConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/SparseConsumeQueue.java
index 4a5f3a93b1d..7e14de30abc 100644
--- a/store/src/main/java/org/apache/rocketmq/store/queue/SparseConsumeQueue.java
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/SparseConsumeQueue.java
@@ -262,7 +262,15 @@ public void putEndPositionInfo(MappedFile mappedFile) {
             this.byteBufferItem.putShort((short)0);
             this.byteBufferItem.putInt(INVALID_POS);
             this.byteBufferItem.putInt(0); // 4 bytes reserved
-            boolean appendRes = mappedFile.appendMessage(this.byteBufferItem.array());
+
+            boolean appendRes;
+
+            if (messageStore.getMessageStoreConfig().isPutConsumeQueueDataByFileChannel()) {
+                appendRes = mappedFile.appendMessageUsingFileChannel(this.byteBufferItem.array());
+            } else {
+                appendRes = mappedFile.appendMessage(this.byteBufferItem.array());
+            }
+
             if (!appendRes) {
                 log.error("append end position info into {} failed", mappedFile.getFileName());
             }
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntry.java b/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntry.java
new file mode 100644
index 00000000000..bac3aa430b4
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntry.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store.queue.offset;
+
+public class OffsetEntry {
+    /**
+     * Topic identifier. For now, it's topic name directly. In the future, we should use fixed length topic identifier.
+     */
+    public String topic;
+
+    /**
+     * Queue ID
+     */
+    public int queueId;
+
+    /**
+     * Flag if the entry is for maximum or minimum
+     */
+    public OffsetEntryType type;
+
+    /**
+     * Maximum or minimum consume-queue offset.
+     */
+    public long offset;
+
+    /**
+     * Maximum or minimum commit-log offset.
+     */
+    public long commitLogOffset;
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntryType.java b/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntryType.java
new file mode 100644
index 00000000000..78df4e15fa5
--- /dev/null
+++ b/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntryType.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store.queue.offset;
+
+public enum OffsetEntryType {
+    MAXIMUM,
+    MINIMUM
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java
index 362684560c8..b343a5b4b50 100644
--- a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java
+++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java
@@ -16,53 +16,45 @@
  */
 package org.apache.rocketmq.store.rocksdb;
 
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.config.AbstractRocksDBStorage;
-import org.apache.rocketmq.common.utils.DataConverter;
 import org.apache.rocketmq.store.MessageStore;
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.ColumnFamilyOptions;
-import org.rocksdb.CompactRangeOptions;
 import org.rocksdb.ReadOptions;
 import org.rocksdb.RocksDB;
 import org.rocksdb.RocksDBException;
 import org.rocksdb.RocksIterator;
 import org.rocksdb.WriteBatch;
-import org.rocksdb.WriteOptions;
 
 public class ConsumeQueueRocksDBStorage extends AbstractRocksDBStorage {
+
+    public static final byte[] OFFSET_COLUMN_FAMILY = "offset".getBytes(StandardCharsets.UTF_8);
+
     private final MessageStore messageStore;
     private volatile ColumnFamilyHandle offsetCFHandle;
 
-    public ConsumeQueueRocksDBStorage(final MessageStore messageStore, final String dbPath, final int prefixLen) {
+    public ConsumeQueueRocksDBStorage(final MessageStore messageStore, final String dbPath) {
+        super(dbPath);
         this.messageStore = messageStore;
-        this.dbPath = dbPath;
         this.readOnly = false;
     }
 
-    private void initOptions() {
+    protected void initOptions() {
         this.options = RocksDBOptionsFactory.createDBOptions();
+        super.initOptions();
+    }
 
-        this.writeOptions = new WriteOptions();
-        this.writeOptions.setSync(false);
-        this.writeOptions.setDisableWAL(true);
-        this.writeOptions.setNoSlowdown(true);
-
+    @Override
+    protected void initTotalOrderReadOptions() {
         this.totalOrderReadOptions = new ReadOptions();
         this.totalOrderReadOptions.setPrefixSameAsStart(false);
         this.totalOrderReadOptions.setTotalOrderSeek(false);
-
-        this.compactRangeOptions = new CompactRangeOptions();
-        this.compactRangeOptions.setBottommostLevelCompaction(CompactRangeOptions.BottommostLevelCompaction.kForce);
-        this.compactRangeOptions.setAllowWriteStall(true);
-        this.compactRangeOptions.setExclusiveManualCompaction(false);
-        this.compactRangeOptions.setChangeLevel(true);
-        this.compactRangeOptions.setTargetLevel(-1);
-        this.compactRangeOptions.setMaxSubcompactions(4);
     }
 
     @Override
@@ -72,7 +64,7 @@ protected boolean postLoad() {
 
             initOptions();
 
-            final List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList();
+            final List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList<>();
 
             ColumnFamilyOptions cqCfOptions = RocksDBOptionsFactory.createCQCFOptions(this.messageStore);
             this.cfOptions.add(cqCfOptions);
@@ -80,11 +72,8 @@ protected boolean postLoad() {
 
             ColumnFamilyOptions offsetCfOptions = RocksDBOptionsFactory.createOffsetCFOptions();
             this.cfOptions.add(offsetCfOptions);
-            cfDescriptors.add(new ColumnFamilyDescriptor("offset".getBytes(DataConverter.CHARSET_UTF8), offsetCfOptions));
-
-            final List<ColumnFamilyHandle> cfHandles = new ArrayList();
-            open(cfDescriptors, cfHandles);
-
+            cfDescriptors.add(new ColumnFamilyDescriptor(OFFSET_COLUMN_FAMILY, offsetCfOptions));
+            open(cfDescriptors);
             this.defaultCFHandle = cfHandles.get(0);
             this.offsetCFHandle = cfHandles.get(1);
         } catch (final Exception e) {
@@ -130,4 +119,4 @@ public RocksIterator seekOffsetCF() {
     public ColumnFamilyHandle getOffsetCFHandle() {
         return this.offsetCFHandle;
     }
-}
\ No newline at end of file
+}
diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java
index a3a99d3346c..5687d6a222d 100644
--- a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java
+++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java
@@ -16,12 +16,13 @@
  */
 package org.apache.rocketmq.store.rocksdb;
 
-import org.apache.rocketmq.common.config.ConfigRocksDBStorage;
+import org.apache.rocketmq.common.config.ConfigHelper;
 import org.apache.rocketmq.store.MessageStore;
 import org.rocksdb.BlockBasedTableConfig;
 import org.rocksdb.BloomFilter;
 import org.rocksdb.ColumnFamilyOptions;
 import org.rocksdb.CompactionOptionsUniversal;
+import org.rocksdb.CompactionPriority;
 import org.rocksdb.CompactionStopStyle;
 import org.rocksdb.CompactionStyle;
 import org.rocksdb.CompressionType;
@@ -65,14 +66,21 @@ public static ColumnFamilyOptions createCQCFOptions(final MessageStore messageSt
                 setMaxMergeWidth(Integer.MAX_VALUE).
                 setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize).
                 setCompressionSizePercent(-1);
+        String bottomMostCompressionTypeOpt = messageStore.getMessageStoreConfig()
+            .getBottomMostCompressionTypeForConsumeQueueStore();
+        String compressionTypeOpt = messageStore.getMessageStoreConfig()
+            .getRocksdbCompressionType();
+        CompressionType bottomMostCompressionType = CompressionType.getCompressionType(bottomMostCompressionTypeOpt);
+        CompressionType compressionType = CompressionType.getCompressionType(compressionTypeOpt);
         return columnFamilyOptions.setMaxWriteBufferNumber(4).
                 setWriteBufferSize(128 * SizeUnit.MB).
                 setMinWriteBufferNumberToMerge(1).
                 setTableFormatConfig(blockBasedTableConfig).
                 setMemTableConfig(new SkipListMemTableConfig()).
-                setCompressionType(CompressionType.LZ4_COMPRESSION).
-                setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION).
+                setCompressionType(compressionType).
+                setBottommostCompressionType(bottomMostCompressionType).
                 setNumLevels(7).
+                setCompactionPriority(CompactionPriority.MinOverlappingRatio).
                 setCompactionStyle(CompactionStyle.UNIVERSAL).
                 setCompactionOptionsUniversal(compactionOption).
                 setMaxCompactionBytes(100 * SizeUnit.GB).
@@ -123,6 +131,57 @@ public static ColumnFamilyOptions createOffsetCFOptions() {
                 setInplaceUpdateSupport(true);
     }
 
+    public static ColumnFamilyOptions createPopCFOptions() {
+        BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig()
+            .setFormatVersion(5)
+            .setIndexType(IndexType.kBinarySearch)
+            .setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash)
+            .setDataBlockHashTableUtilRatio(0.75)
+            .setBlockSize(32 * SizeUnit.KB)
+            .setMetadataBlockSize(4 * SizeUnit.KB)
+            .setFilterPolicy(new BloomFilter(16, false))
+            .setCacheIndexAndFilterBlocks(false)
+            .setCacheIndexAndFilterBlocksWithHighPriority(true)
+            .setPinL0FilterAndIndexBlocksInCache(false)
+            .setPinTopLevelIndexAndFilter(true)
+            .setBlockCache(new LRUCache(1024 * SizeUnit.MB, 8, false))
+            .setWholeKeyFiltering(true);
+
+        CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal()
+            .setSizeRatio(100)
+            .setMaxSizeAmplificationPercent(25)
+            .setAllowTrivialMove(true)
+            .setMinMergeWidth(2)
+            .setMaxMergeWidth(Integer.MAX_VALUE)
+            .setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize)
+            .setCompressionSizePercent(-1);
+
+        //noinspection resource
+        return new ColumnFamilyOptions()
+            .setMaxWriteBufferNumber(4)
+            .setWriteBufferSize(128 * SizeUnit.MB)
+            .setMinWriteBufferNumberToMerge(1)
+            .setTableFormatConfig(blockBasedTableConfig)
+            .setMemTableConfig(new SkipListMemTableConfig())
+            .setCompressionType(CompressionType.NO_COMPRESSION)
+            .setBottommostCompressionType(CompressionType.NO_COMPRESSION)
+            .setNumLevels(7)
+            .setCompactionPriority(CompactionPriority.MinOverlappingRatio)
+            .setCompactionStyle(CompactionStyle.UNIVERSAL)
+            .setCompactionOptionsUniversal(compactionOption)
+            .setMaxCompactionBytes(100 * SizeUnit.GB)
+            .setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB)
+            .setHardPendingCompactionBytesLimit(256 * SizeUnit.GB)
+            .setLevel0FileNumCompactionTrigger(2)
+            .setLevel0SlowdownWritesTrigger(8)
+            .setLevel0StopWritesTrigger(10)
+            .setTargetFileSizeBase(256 * SizeUnit.MB)
+            .setTargetFileSizeMultiplier(2)
+            .setMergeOperator(new StringAppendOperator())
+            .setReportBgIoStats(true)
+            .setOptimizeFiltersForHits(true);
+    }
+
     /**
      * Create a rocksdb db options, the user must take care to close it after closing db.
      * @return
@@ -134,22 +193,21 @@ public static DBOptions createDBOptions() {
         Statistics statistics = new Statistics();
         statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS);
         return options.
-                setDbLogDir(ConfigRocksDBStorage.getDBLogDir()).
+                setDbLogDir(ConfigHelper.getDBLogDir()).
                 setInfoLogLevel(InfoLogLevel.INFO_LEVEL).
                 setWalRecoveryMode(WALRecoveryMode.PointInTimeRecovery).
                 setManualWalFlush(true).
-                setMaxTotalWalSize(0).
-                setWalSizeLimitMB(0).
-                setWalTtlSeconds(0).
                 setCreateIfMissing(true).
+                setBytesPerSync(SizeUnit.MB).
                 setCreateMissingColumnFamilies(true).
                 setMaxOpenFiles(-1).
-                setMaxLogFileSize(1 * SizeUnit.GB).
+                setMaxLogFileSize(SizeUnit.GB).
                 setKeepLogFileNum(5).
-                setMaxManifestFileSize(1 * SizeUnit.GB).
+                setMaxManifestFileSize(SizeUnit.GB).
                 setAllowConcurrentMemtableWrite(false).
                 setStatistics(statistics).
                 setAtomicFlush(true).
+                setCompactionReadaheadSize(4 * SizeUnit.MB).
                 setMaxBackgroundJobs(32).
                 setMaxSubcompactions(8).
                 setParanoidChecks(true).
diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
index 489d7b4fbce..c272a302344 100644
--- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
+++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
@@ -18,6 +18,8 @@
 
 import java.util.HashMap;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
@@ -69,9 +71,9 @@ public class BrokerStatsManager {
     @Deprecated public static final String COMMERCIAL_PERM_FAILURES = Stats.COMMERCIAL_PERM_FAILURES;
 
     // Send message latency
-    public static final String TOPIC_PUT_LATENCY = "TOPIC_PUT_LATENCY";
-    public static final String GROUP_ACK_NUMS = "GROUP_ACK_NUMS";
-    public static final String GROUP_CK_NUMS = "GROUP_CK_NUMS";
+    @Deprecated public static final String TOPIC_PUT_LATENCY = "TOPIC_PUT_LATENCY";
+    @Deprecated public static final String GROUP_ACK_NUMS = "GROUP_ACK_NUMS";
+    @Deprecated public static final String GROUP_CK_NUMS = "GROUP_CK_NUMS";
     public static final String DLQ_PUT_NUMS = "DLQ_PUT_NUMS";
     public static final String BROKER_ACK_NUMS = "BROKER_ACK_NUMS";
     public static final String BROKER_CK_NUMS = "BROKER_CK_NUMS";
@@ -121,6 +123,8 @@ public class BrokerStatsManager {
     public static final String CHANNEL_ACTIVITY_IDLE = "IDLE";
     public static final String CHANNEL_ACTIVITY_EXCEPTION = "EXCEPTION";
     public static final String CHANNEL_ACTIVITY_CLOSE = "CLOSE";
+    private static final String[] NEED_CLEAN_STATS_SET =
+            new String[] {TOPIC_PUT_NUMS, TOPIC_PUT_SIZE, GROUP_GET_NUMS, GROUP_GET_SIZE, SNDBCK_PUT_NUMS, GROUP_GET_LATENCY};
 
     /**
      * read disk follow stats
@@ -134,6 +138,7 @@ public class BrokerStatsManager {
     private ScheduledExecutorService scheduledExecutorService;
     private ScheduledExecutorService commercialExecutor;
     private ScheduledExecutorService accountExecutor;
+    private ScheduledExecutorService cleanResourceExecutor;
 
     private final HashMap<String, StatsItemSet> statsTable = new HashMap<>();
     private final String clusterName;
@@ -142,7 +147,7 @@ public class BrokerStatsManager {
     private MomentStatsItemSet momentStatsItemSetFallTime;
 
     private final StatisticsManager accountStatManager = new StatisticsManager();
-    private StateGetter produerStateGetter;
+    private StateGetter producerStateGetter;
     private StateGetter consumerStateGetter;
 
     private BrokerConfig brokerConfig;
@@ -179,10 +184,10 @@ public void init() {
         this.statsTable.put(Stats.TOPIC_PUT_SIZE, new StatsItemSet(Stats.TOPIC_PUT_SIZE, this.scheduledExecutorService, log));
         this.statsTable.put(Stats.GROUP_GET_NUMS, new StatsItemSet(Stats.GROUP_GET_NUMS, this.scheduledExecutorService, log));
         this.statsTable.put(Stats.GROUP_GET_SIZE, new StatsItemSet(Stats.GROUP_GET_SIZE, this.scheduledExecutorService, log));
-        this.statsTable.put(GROUP_ACK_NUMS, new StatsItemSet(GROUP_ACK_NUMS, this.scheduledExecutorService, log));
-        this.statsTable.put(GROUP_CK_NUMS, new StatsItemSet(GROUP_CK_NUMS, this.scheduledExecutorService, log));
+        this.statsTable.put(Stats.GROUP_ACK_NUMS, new StatsItemSet(Stats.GROUP_ACK_NUMS, this.scheduledExecutorService, log));
+        this.statsTable.put(Stats.GROUP_CK_NUMS, new StatsItemSet(Stats.GROUP_CK_NUMS, this.scheduledExecutorService, log));
         this.statsTable.put(Stats.GROUP_GET_LATENCY, new StatsItemSet(Stats.GROUP_GET_LATENCY, this.scheduledExecutorService, log));
-        this.statsTable.put(TOPIC_PUT_LATENCY, new StatsItemSet(TOPIC_PUT_LATENCY, this.scheduledExecutorService, log));
+        this.statsTable.put(Stats.TOPIC_PUT_LATENCY, new StatsItemSet(Stats.TOPIC_PUT_LATENCY, this.scheduledExecutorService, log));
         this.statsTable.put(Stats.SNDBCK_PUT_NUMS, new StatsItemSet(Stats.SNDBCK_PUT_NUMS, this.scheduledExecutorService, log));
         this.statsTable.put(DLQ_PUT_NUMS, new StatsItemSet(DLQ_PUT_NUMS, this.scheduledExecutorService, log));
         this.statsTable.put(Stats.BROKER_PUT_NUMS, new StatsItemSet(Stats.BROKER_PUT_NUMS, this.scheduledExecutorService, log));
@@ -270,13 +275,19 @@ public boolean online(StatisticsItem item) {
 
                 String kind = item.getStatKind();
                 if (ACCOUNT_SEND.equals(kind) || ACCOUNT_SEND_REJ.equals(kind)) {
-                    return produerStateGetter.online(instanceId, group, topic);
+                    return producerStateGetter.online(instanceId, group, topic);
                 } else if (ACCOUNT_RCV.equals(kind) || ACCOUNT_SEND_BACK.equals(kind) || ACCOUNT_SEND_BACK_TO_DLQ.equals(kind) || ACCOUNT_REV_REJ.equals(kind)) {
                     return consumerStateGetter.online(instanceId, group, topic);
                 }
                 return false;
             }
         });
+        cleanResourceExecutor.scheduleWithFixedDelay(new Runnable() {
+            @Override
+            public void run() {
+                cleanAllResource();
+            }
+        }, 10, 10, TimeUnit.MINUTES);
     }
 
     private void initScheduleService() {
@@ -286,6 +297,8 @@ private void initScheduleService() {
             ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig));
         this.accountExecutor =
             ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig));
+        this.cleanResourceExecutor =
+                ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CleanStatsResourceThread", true, brokerConfig));
     }
 
     public MomentStatsItemSet getMomentStatsItemSetFallSize() {
@@ -296,12 +309,12 @@ public MomentStatsItemSet getMomentStatsItemSetFallTime() {
         return momentStatsItemSetFallTime;
     }
 
-    public StateGetter getProduerStateGetter() {
-        return produerStateGetter;
+    public StateGetter getProducerStateGetter() {
+        return producerStateGetter;
     }
 
-    public void setProduerStateGetter(StateGetter produerStateGetter) {
-        this.produerStateGetter = produerStateGetter;
+    public void setProducerStateGetter(StateGetter producerStateGetter) {
+        this.producerStateGetter = producerStateGetter;
     }
 
     public StateGetter getConsumerStateGetter() {
@@ -318,6 +331,7 @@ public void start() {
     public void shutdown() {
         this.scheduledExecutorService.shutdown();
         this.commercialExecutor.shutdown();
+        this.cleanResourceExecutor.shutdown();
     }
 
     public StatsItem getStatsItem(final String statsName, final String statsKey) {
@@ -338,10 +352,13 @@ public void onTopicDeleted(final String topic) {
         }
         this.statsTable.get(Stats.GROUP_GET_NUMS).delValueByPrefixKey(topic, "@");
         this.statsTable.get(Stats.GROUP_GET_SIZE).delValueByPrefixKey(topic, "@");
+        this.statsTable.get(Stats.GROUP_CK_NUMS).delValueByPrefixKey(topic, "@");
+        this.statsTable.get(Stats.GROUP_ACK_NUMS).delValueByPrefixKey(topic, "@");
         this.statsTable.get(Stats.QUEUE_GET_NUMS).delValueByPrefixKey(topic, "@");
         this.statsTable.get(Stats.QUEUE_GET_SIZE).delValueByPrefixKey(topic, "@");
         this.statsTable.get(Stats.SNDBCK_PUT_NUMS).delValueByPrefixKey(topic, "@");
         this.statsTable.get(Stats.GROUP_GET_LATENCY).delValueByInfixKey(topic, "@");
+        this.statsTable.get(Stats.TOPIC_PUT_LATENCY).delValueBySuffixKey(topic, "@");
         this.momentStatsItemSetFallSize.delValueByInfixKey(topic, "@");
         this.momentStatsItemSetFallTime.delValueByInfixKey(topic, "@");
     }
@@ -349,6 +366,8 @@ public void onTopicDeleted(final String topic) {
     public void onGroupDeleted(final String group) {
         this.statsTable.get(Stats.GROUP_GET_NUMS).delValueBySuffixKey(group, "@");
         this.statsTable.get(Stats.GROUP_GET_SIZE).delValueBySuffixKey(group, "@");
+        this.statsTable.get(Stats.GROUP_CK_NUMS).delValueBySuffixKey(group, "@");
+        this.statsTable.get(Stats.GROUP_ACK_NUMS).delValueBySuffixKey(group, "@");
         if (enableQueueStat) {
             this.statsTable.get(Stats.QUEUE_GET_NUMS).delValueBySuffixKey(group, "@");
             this.statsTable.get(Stats.QUEUE_GET_SIZE).delValueBySuffixKey(group, "@");
@@ -434,12 +453,12 @@ public void incGroupGetNums(final String group, final String topic, final int in
 
     public void incGroupCkNums(final String group, final String topic, final int incValue) {
         final String statsKey = buildStatsKey(topic, group);
-        this.statsTable.get(GROUP_CK_NUMS).addValue(statsKey, incValue, 1);
+        this.statsTable.get(Stats.GROUP_CK_NUMS).addValue(statsKey, incValue, 1);
     }
 
     public void incGroupAckNums(final String group, final String topic, final int incValue) {
         final String statsKey = buildStatsKey(topic, group);
-        this.statsTable.get(GROUP_ACK_NUMS).addValue(statsKey, incValue, 1);
+        this.statsTable.get(Stats.GROUP_ACK_NUMS).addValue(statsKey, incValue, 1);
     }
 
     public String buildStatsKey(String topic, String group) {
@@ -509,9 +528,8 @@ public void incTopicPutLatency(final String topic, final int queueId, final int
             statsKey = new StringBuilder(6);
         }
         statsKey.append(queueId).append("@").append(topic);
-        this.statsTable.get(TOPIC_PUT_LATENCY).addValue(statsKey.toString(), incValue, 1);
+        this.statsTable.get(Stats.TOPIC_PUT_LATENCY).addValue(statsKey.toString(), incValue, 1);
     }
-
     public void incBrokerPutNums() {
         this.statsTable.get(Stats.BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(1);
     }
@@ -577,6 +595,10 @@ public void incSendBackNums(final String group, final String topic) {
         this.statsTable.get(Stats.SNDBCK_PUT_NUMS).addValue(statsKey, 1, 1);
     }
 
+    public double tpsTopicPutNums(final String topic) {
+        return this.statsTable.get(TOPIC_PUT_NUMS).getStatsDataInMinute(topic).getTps();
+    }
+
     public double tpsGroupGetNums(final String group, final String topic) {
         final String statsKey = buildStatsKey(topic, group);
         return this.statsTable.get(Stats.GROUP_GET_NUMS).getStatsDataInMinute(statsKey).getTps();
@@ -585,13 +607,13 @@ public double tpsGroupGetNums(final String group, final String topic) {
     public void recordDiskFallBehindTime(final String group, final String topic, final int queueId,
         final long fallBehind) {
         final String statsKey = buildStatsKey(queueId, topic, group);
-        this.momentStatsItemSetFallTime.getAndCreateStatsItem(statsKey).getValue().set(fallBehind);
+        this.momentStatsItemSetFallTime.setValue(statsKey, fallBehind);
     }
 
     public void recordDiskFallBehindSize(final String group, final String topic, final int queueId,
         final long fallBehind) {
         final String statsKey = buildStatsKey(queueId, topic, group);
-        this.momentStatsItemSetFallSize.getAndCreateStatsItem(statsKey).getValue().set(fallBehind);
+        this.momentStatsItemSetFallSize.setValue(statsKey, fallBehind);
     }
 
     public void incDLQStatValue(final String key, final String owner, final String group,
@@ -760,6 +782,31 @@ public interface StateGetter {
         boolean online(String instanceId, String group, String topic);
     }
 
+
+    private void cleanAllResource() {
+        try {
+            int maxStatsIdleTimeInMinutes = brokerConfig != null ? brokerConfig.getMaxStatsIdleTimeInMinutes() : -1;
+            if (maxStatsIdleTimeInMinutes < 0) {
+                COMMERCIAL_LOG.info("[BrokerStatsManager#cleanAllResource] maxStatsIdleTimeInMinutes={}, no need to clean resource", maxStatsIdleTimeInMinutes);
+                return;
+            }
+            if (maxStatsIdleTimeInMinutes <= 10 && maxStatsIdleTimeInMinutes >= 0) {
+                maxStatsIdleTimeInMinutes = 30;
+            }
+            for (String statsKind : NEED_CLEAN_STATS_SET) {
+                StatsItemSet statsItemSet = this.statsTable.get(statsKind);
+                if (null == statsItemSet) {
+                    continue;
+                }
+                statsItemSet.cleanResource(maxStatsIdleTimeInMinutes);
+            }
+            momentStatsItemSetFallSize.cleanResource(maxStatsIdleTimeInMinutes);
+            momentStatsItemSetFallTime.cleanResource(maxStatsIdleTimeInMinutes);
+        } catch (Throwable throwable) {
+            COMMERCIAL_LOG.error("[BrokerStatsManager#cleanAllResource] clean resource error", throwable);
+        }
+    }
+
     public enum StatsType {
         SEND_SUCCESS,
         SEND_FAILURE,
diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java
index f0e23fe6388..4caea19f567 100644
--- a/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java
+++ b/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java
@@ -16,105 +16,71 @@
  */
 package org.apache.rocketmq.store.stats;
 
+import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.MixAll;
 
 public class LmqBrokerStatsManager extends BrokerStatsManager {
 
-    public LmqBrokerStatsManager(String clusterName, boolean enableQueueStat) {
-        super(clusterName, enableQueueStat);
+    private final BrokerConfig brokerConfig;
+
+    public LmqBrokerStatsManager(BrokerConfig brokerConfig) {
+        super(brokerConfig.getBrokerClusterName(), brokerConfig.isEnableDetailStat());
+        this.brokerConfig = brokerConfig;
     }
 
     @Override
     public void incGroupGetNums(final String group, final String topic, final int incValue) {
-        String lmqGroup = group;
-        String lmqTopic = topic;
-        if (MixAll.isLmq(group)) {
-            lmqGroup = MixAll.LMQ_PREFIX;
-        }
-        if (MixAll.isLmq(topic)) {
-            lmqTopic = MixAll.LMQ_PREFIX;
-        }
-        super.incGroupGetNums(lmqGroup, lmqTopic, incValue);
+        super.incGroupGetNums(getAdjustedGroup(group), getAdjustedTopic(topic), incValue);
     }
 
     @Override
     public void incGroupGetSize(final String group, final String topic, final int incValue) {
-        String lmqGroup = group;
-        String lmqTopic = topic;
-        if (MixAll.isLmq(group)) {
-            lmqGroup = MixAll.LMQ_PREFIX;
-        }
-        if (MixAll.isLmq(topic)) {
-            lmqTopic = MixAll.LMQ_PREFIX;
-        }
-        super.incGroupGetSize(lmqGroup, lmqTopic, incValue);
+        super.incGroupGetSize(getAdjustedGroup(group), getAdjustedTopic(topic), incValue);
+    }
+
+    @Override
+    public void incGroupAckNums(final String group, final String topic, final int incValue) {
+        super.incGroupAckNums(getAdjustedGroup(group), getAdjustedTopic(topic), incValue);
+    }
+
+    @Override
+    public void incGroupCkNums(final String group, final String topic, final int incValue) {
+        super.incGroupCkNums(getAdjustedGroup(group), getAdjustedTopic(topic), incValue);
     }
 
     @Override
     public void incGroupGetLatency(final String group, final String topic, final int queueId, final int incValue) {
-        String lmqGroup = group;
-        String lmqTopic = topic;
-        if (MixAll.isLmq(group)) {
-            lmqGroup = MixAll.LMQ_PREFIX;
-        }
-        if (MixAll.isLmq(topic)) {
-            lmqTopic = MixAll.LMQ_PREFIX;
-        }
-        super.incGroupGetLatency(lmqGroup, lmqTopic, queueId, incValue);
+        super.incGroupGetLatency(getAdjustedGroup(group), getAdjustedTopic(topic), queueId, incValue);
     }
 
     @Override
     public void incSendBackNums(final String group, final String topic) {
-        String lmqGroup = group;
-        String lmqTopic = topic;
-        if (MixAll.isLmq(group)) {
-            lmqGroup = MixAll.LMQ_PREFIX;
-        }
-        if (MixAll.isLmq(topic)) {
-            lmqTopic = MixAll.LMQ_PREFIX;
-        }
-        super.incSendBackNums(lmqGroup, lmqTopic);
+        super.incSendBackNums(getAdjustedGroup(group), getAdjustedTopic(topic));
     }
 
     @Override
     public double tpsGroupGetNums(final String group, final String topic) {
-        String lmqGroup = group;
-        String lmqTopic = topic;
-        if (MixAll.isLmq(group)) {
-            lmqGroup = MixAll.LMQ_PREFIX;
-        }
-        if (MixAll.isLmq(topic)) {
-            lmqTopic = MixAll.LMQ_PREFIX;
-        }
-        return super.tpsGroupGetNums(lmqGroup, lmqTopic);
+        return super.tpsGroupGetNums(getAdjustedGroup(group), getAdjustedTopic(topic));
     }
 
     @Override
     public void recordDiskFallBehindTime(final String group, final String topic, final int queueId,
         final long fallBehind) {
-        String lmqGroup = group;
-        String lmqTopic = topic;
-        if (MixAll.isLmq(group)) {
-            lmqGroup = MixAll.LMQ_PREFIX;
-        }
-        if (MixAll.isLmq(topic)) {
-            lmqTopic = MixAll.LMQ_PREFIX;
-        }
-        super.recordDiskFallBehindTime(lmqGroup, lmqTopic, queueId, fallBehind);
+        super.recordDiskFallBehindTime(getAdjustedGroup(group), getAdjustedTopic(topic), queueId, fallBehind);
     }
 
     @Override
     public void recordDiskFallBehindSize(final String group, final String topic, final int queueId,
         final long fallBehind) {
-        String lmqGroup = group;
-        String lmqTopic = topic;
-        if (MixAll.isLmq(group)) {
-            lmqGroup = MixAll.LMQ_PREFIX;
-        }
-        if (MixAll.isLmq(topic)) {
-            lmqTopic = MixAll.LMQ_PREFIX;
-        }
-        super.recordDiskFallBehindSize(lmqGroup, lmqTopic, queueId, fallBehind);
+        super.recordDiskFallBehindSize(getAdjustedGroup(group), getAdjustedTopic(topic), queueId, fallBehind);
+    }
+
+    private String getAdjustedGroup(String group) {
+        return !brokerConfig.isEnableLmqStats() && MixAll.isLmq(group) ? MixAll.LMQ_PREFIX : group;
+    }
+
+    private String getAdjustedTopic(String topic) {
+        return !brokerConfig.isEnableLmqStats() && MixAll.isLmq(topic) ? MixAll.LMQ_PREFIX : topic;
     }
 
 }
diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerLog.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerLog.java
index e0836fef183..8c93d3d5269 100644
--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerLog.java
+++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerLog.java
@@ -117,7 +117,7 @@ public void shutdown() {
 
     // be careful.
     // if the format of timerlog changed, this offset has to be changed too
-    // so dose the batch writing
+    // so does the batch writing
     public int getOffsetForLastUnit() {
 
         return fileSize - (fileSize - MIN_BLANK_LEN) % UNIT_SIZE - MIN_BLANK_LEN - UNIT_SIZE;
diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
index 819b3e96a43..d6af7b84e79 100644
--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
+++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java
@@ -293,6 +293,19 @@ public void recover() {
         }
         currQueueOffset = Math.min(currQueueOffset, timerCheckpoint.getMasterTimerQueueOffset());
 
+        ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, 0);
+
+        // Correction based consume queue
+        if (cq != null && currQueueOffset < cq.getMinOffsetInQueue()) {
+            LOGGER.warn("Timer currQueueOffset:{} is smaller than minOffsetInQueue:{}",
+                currQueueOffset, cq.getMinOffsetInQueue());
+            currQueueOffset = cq.getMinOffsetInQueue();
+        } else if (cq != null && currQueueOffset > cq.getMaxOffsetInQueue()) {
+            LOGGER.warn("Timer currQueueOffset:{} is larger than maxOffsetInQueue:{}",
+                currQueueOffset, cq.getMaxOffsetInQueue());
+            currQueueOffset = cq.getMaxOffsetInQueue();
+        }
+
         //check timer wheel
         currReadTimeMs = timerCheckpoint.getLastReadTimeMs();
         long nextReadTimeMs = formatTimeMs(
@@ -614,7 +627,7 @@ public void addMetric(MessageExt msg, int value) {
                 return;
             }
             if (msg.getProperty(TIMER_ENQUEUE_MS) != null
-                    && NumberUtils.toLong(msg.getProperty(TIMER_ENQUEUE_MS)) == Long.MAX_VALUE) {
+                && NumberUtils.toLong(msg.getProperty(TIMER_ENQUEUE_MS)) == Long.MAX_VALUE) {
                 return;
             }
             // pass msg into addAndGet, for further more judgement extension.
@@ -1084,46 +1097,44 @@ public int doPut(MessageExtBrokerInner message, boolean roll) throws Exception {
             putMessageResult = messageStore.putMessage(message);
         }
 
-        int retryNum = 0;
-        while (retryNum < 3) {
-            if (null == putMessageResult || null == putMessageResult.getPutMessageStatus()) {
-                retryNum++;
-            } else {
-                switch (putMessageResult.getPutMessageStatus()) {
-                    case PUT_OK:
-                        if (brokerStatsManager != null) {
-                            this.brokerStatsManager.incTopicPutNums(message.getTopic(), 1, 1);
-                            if (putMessageResult.getAppendMessageResult() != null) {
-                                this.brokerStatsManager.incTopicPutSize(message.getTopic(),
-                                        putMessageResult.getAppendMessageResult().getWroteBytes());
-                            }
-                            this.brokerStatsManager.incBrokerPutNums(message.getTopic(), 1);
+        if (putMessageResult != null && putMessageResult.getPutMessageStatus() != null) {
+            switch (putMessageResult.getPutMessageStatus()) {
+                case PUT_OK:
+                    if (brokerStatsManager != null) {
+                        brokerStatsManager.incTopicPutNums(message.getTopic(), 1, 1);
+                        if (putMessageResult.getAppendMessageResult() != null) {
+                            brokerStatsManager.incTopicPutSize(message.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());
                         }
-                        return PUT_OK;
-                    case SERVICE_NOT_AVAILABLE:
-                        return PUT_NEED_RETRY;
-                    case MESSAGE_ILLEGAL:
-                    case PROPERTIES_SIZE_EXCEEDED:
+                        brokerStatsManager.incBrokerPutNums(message.getTopic(), 1);
+                    }
+                    return PUT_OK;
+
+                case MESSAGE_ILLEGAL:
+                case PROPERTIES_SIZE_EXCEEDED:
+                case WHEEL_TIMER_NOT_ENABLE:
+                case WHEEL_TIMER_MSG_ILLEGAL:
+                    return PUT_NO_RETRY;
+
+                case SERVICE_NOT_AVAILABLE:
+                case FLUSH_DISK_TIMEOUT:
+                case FLUSH_SLAVE_TIMEOUT:
+                case OS_PAGE_CACHE_BUSY:
+                case CREATE_MAPPED_FILE_FAILED:
+                case SLAVE_NOT_AVAILABLE:
+                    return PUT_NEED_RETRY;
+
+                case UNKNOWN_ERROR:
+                default:
+                    if (storeConfig.isTimerSkipUnknownError()) {
+                        LOGGER.warn("Skipping message due to unknown error, msg: {}", message);
                         return PUT_NO_RETRY;
-                    case CREATE_MAPPED_FILE_FAILED:
-                    case FLUSH_DISK_TIMEOUT:
-                    case FLUSH_SLAVE_TIMEOUT:
-                    case OS_PAGE_CACHE_BUSY:
-                    case SLAVE_NOT_AVAILABLE:
-                    case UNKNOWN_ERROR:
-                    default:
-                        retryNum++;
-                }
-            }
-            Thread.sleep(50);
-            if (escapeBridgeHook != null) {
-                putMessageResult = escapeBridgeHook.apply(message);
-            } else {
-                putMessageResult = messageStore.putMessage(message);
+                    } else {
+                        holdMomentForUnknownError();
+                        return PUT_NEED_RETRY;
+                    }
             }
-            LOGGER.warn("Retrying to do put timer msg retryNum:{} putRes:{} msg:{}", retryNum, putMessageResult, message);
         }
-        return PUT_NO_RETRY;
+        return PUT_NEED_RETRY;
     }
 
     public MessageExtBrokerInner convertMessage(MessageExt msgExt, boolean needRoll) {
@@ -1458,7 +1469,6 @@ protected boolean isState(int state) {
     }
 
     public class TimerDequeuePutMessageService extends AbstractStateService {
-
         @Override
         public String getServiceName() {
             return getServiceThreadName() + this.getClass().getSimpleName();
@@ -1468,6 +1478,7 @@ public String getServiceName() {
         public void run() {
             setState(AbstractStateService.START);
             TimerMessageStore.LOGGER.info(this.getServiceName() + " service start");
+
             while (!this.isStopped() || dequeuePutQueue.size() != 0) {
                 try {
                     setState(AbstractStateService.WAITING);
@@ -1475,41 +1486,63 @@ public void run() {
                     if (null == tr) {
                         continue;
                     }
+
                     setState(AbstractStateService.RUNNING);
-                    boolean doRes = false;
                     boolean tmpDequeueChangeFlag = false;
+
                     try {
-                        while (!isStopped() && !doRes) {
+                        while (!isStopped()) {
                             if (!isRunningDequeue()) {
                                 dequeueStatusChangeFlag = true;
                                 tmpDequeueChangeFlag = true;
                                 break;
                             }
+
                             try {
                                 perfCounterTicks.startTick(DEQUEUE_PUT);
+
                                 MessageExt msgExt = tr.getMsg();
                                 DefaultStoreMetricsManager.incTimerDequeueCount(getRealTopic(msgExt));
+
                                 if (tr.getEnqueueTime() == Long.MAX_VALUE) {
-                                    // never enqueue, mark it.
+                                    // Never enqueue, mark it.
                                     MessageAccessor.putProperty(msgExt, TIMER_ENQUEUE_MS, String.valueOf(Long.MAX_VALUE));
                                 }
+
                                 addMetric(msgExt, -1);
                                 MessageExtBrokerInner msg = convert(msgExt, tr.getEnqueueTime(), needRoll(tr.getMagic()));
-                                doRes = PUT_NEED_RETRY != doPut(msg, needRoll(tr.getMagic()));
-                                while (!doRes && !isStopped()) {
-                                    if (!isRunningDequeue()) {
-                                        dequeueStatusChangeFlag = true;
-                                        tmpDequeueChangeFlag = true;
-                                        break;
+
+                                boolean processed = false;
+                                int retryCount = 0;
+
+                                while (!processed && !isStopped()) {
+                                    int result = doPut(msg, needRoll(tr.getMagic()));
+
+                                    if (result == PUT_OK) {
+                                        processed = true;
+                                    } else if (result == PUT_NO_RETRY) {
+                                        TimerMessageStore.LOGGER.warn("Skipping message due to unrecoverable error. Msg: {}", msg);
+                                        processed = true;
+                                    } else {
+                                        retryCount++;
+                                        // Without enabling TimerEnableRetryUntilSuccess, messages will retry up to 3 times before being discarded
+                                        if (!storeConfig.isTimerEnableRetryUntilSuccess() && retryCount >= 3) {
+                                            TimerMessageStore.LOGGER.error("Message processing failed after {} retries. Msg: {}", retryCount, msg);
+                                            processed = true;
+                                        } else {
+                                            Thread.sleep(500L * precisionMs / 1000);
+                                            TimerMessageStore.LOGGER.warn("Retrying to process message. Retry count: {}, Msg: {}", retryCount, msg);
+                                        }
                                     }
-                                    doRes = PUT_NEED_RETRY != doPut(msg, needRoll(tr.getMagic()));
-                                    Thread.sleep(500L * precisionMs / 1000);
                                 }
+
                                 perfCounterTicks.endTick(DEQUEUE_PUT);
+                                break;
+
                             } catch (Throwable t) {
-                                LOGGER.info("Unknown error", t);
+                                TimerMessageStore.LOGGER.info("Unknown error", t);
                                 if (storeConfig.isTimerSkipUnknownError()) {
-                                    doRes = true;
+                                    break;
                                 } else {
                                     holdMomentForUnknownError();
                                 }
@@ -1518,7 +1551,6 @@ public void run() {
                     } finally {
                         tr.idempotentRelease(!tmpDequeueChangeFlag);
                     }
-
                 } catch (Throwable e) {
                     TimerMessageStore.LOGGER.error("Error occurred in " + getServiceName(), e);
                 }
@@ -1539,6 +1571,9 @@ public String getServiceName() {
         public void run() {
             setState(AbstractStateService.START);
             TimerMessageStore.LOGGER.info(this.getServiceName() + " service start");
+            //Mark different rounds
+            boolean isRound = true;
+            Map<String ,MessageExt> avoidDeleteLose = new HashMap<>();
             while (!this.isStopped()) {
                 try {
                     setState(AbstractStateService.WAITING);
@@ -1555,7 +1590,18 @@ public void run() {
                             MessageExt msgExt = getMessageByCommitOffset(tr.getOffsetPy(), tr.getSizePy());
                             if (null != msgExt) {
                                 if (needDelete(tr.getMagic()) && !needRoll(tr.getMagic())) {
+                                    //Clearing is performed once in each round.
+                                    //The deletion message is received first and the common message is received once
+                                    if (!isRound) {
+                                        isRound = true;
+                                        for (MessageExt messageExt: avoidDeleteLose.values()) {
+                                            addMetric(messageExt, 1);
+                                        }
+                                        avoidDeleteLose.clear();
+                                    }
                                     if (msgExt.getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY) != null && tr.getDeleteList() != null) {
+
+                                        avoidDeleteLose.put(msgExt.getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY), msgExt);
                                         tr.getDeleteList().add(msgExt.getProperty(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY));
                                     }
                                     tr.idempotentRelease();
@@ -1565,7 +1611,13 @@ public void run() {
                                     if (null == uniqueKey) {
                                         LOGGER.warn("No uniqueKey for msg:{}", msgExt);
                                     }
-                                    if (null != uniqueKey && tr.getDeleteList() != null && tr.getDeleteList().size() > 0 && tr.getDeleteList().contains(uniqueKey)) {
+                                    //Mark ready for next round
+                                    if (isRound) {
+                                        isRound = false;
+                                    }
+                                    if (null != uniqueKey && tr.getDeleteList() != null && tr.getDeleteList().size() > 0
+                                        && tr.getDeleteList().contains(buildDeleteKey(getRealTopic(msgExt), uniqueKey))) {
+                                        avoidDeleteLose.remove(uniqueKey);
                                         doRes = true;
                                         tr.idempotentRelease();
                                         perfCounterTicks.getCounter("dequeue_delete").flow(1);
@@ -1720,7 +1772,7 @@ public long getEnqueueBehindMessages() {
 
     public long getEnqueueBehindMillis() {
         if (System.currentTimeMillis() - lastEnqueueButExpiredTime < 2000) {
-            return (System.currentTimeMillis() - lastEnqueueButExpiredStoreTime) / 1000;
+            return System.currentTimeMillis() - lastEnqueueButExpiredStoreTime;
         }
         return 0;
     }
@@ -1873,4 +1925,9 @@ public void setFrequency(AtomicInteger frequency) {
     public TimerCheckpoint getTimerCheckpoint() {
         return timerCheckpoint;
     }
+
+    // identify a message by topic + uk, like query operation
+    public static String buildDeleteKey(String realTopic, String uniqueKey) {
+        return realTopic + "+" + uniqueKey;
+    }
 }
diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java
index 7f8fedd8a5b..0d80dae3e38 100644
--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java
+++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java
@@ -37,6 +37,7 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import org.apache.rocketmq.common.ConfigManager;
+import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageExt;
@@ -184,7 +185,8 @@ public void cleanMetrics(Set<String> topics) {
         while (iterator.hasNext()) {
             Map.Entry<String, Metric> entry = iterator.next();
             final String topic = entry.getKey();
-            if (topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX)) {
+            if (topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX)
+                    || topic.startsWith(MixAll.LMQ_PREFIX)) {
                 continue;
             }
             if (topics.contains(topic)) {
@@ -196,6 +198,16 @@ public void cleanMetrics(Set<String> topics) {
         }
     }
 
+    public boolean removeTimingCount(String topic) {
+        try {
+            timingCount.remove(topic);
+        } catch (Exception e) {
+            log.error("removeTimingCount error", e);
+            return false;
+        }
+        return true;
+    }
+
     public static class TimerMetricsSerializeWrapper extends RemotingSerializable {
         private ConcurrentMap<String, Metric> timingCount =
                 new ConcurrentHashMap<>(1024);
diff --git a/store/src/main/java/org/apache/rocketmq/store/util/PerfCounter.java b/store/src/main/java/org/apache/rocketmq/store/util/PerfCounter.java
index e2a55d63994..99649398a83 100644
--- a/store/src/main/java/org/apache/rocketmq/store/util/PerfCounter.java
+++ b/store/src/main/java/org/apache/rocketmq/store/util/PerfCounter.java
@@ -356,7 +356,7 @@ public void run() {
                     }
 
                 } catch (Exception e) {
-                    logger.error("{} get unknown errror", getServiceName(), e);
+                    logger.error("{} get unknown error", getServiceName(), e);
                     try {
                         Thread.sleep(1000);
                     } catch (Throwable ignored) {
diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
index 1d09ca86ecb..eee38e0a8f4 100644
--- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
@@ -54,6 +54,7 @@
 import org.apache.rocketmq.store.config.FlushDiskType;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
 import org.apache.rocketmq.store.queue.CqUnit;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
@@ -374,7 +375,7 @@ public void testGetStoreTime_PhyOffsetIsLessThanCommitLogMinOffset() {
     }
 
     @Test
-    public void testPutMessage_whenMessagePropertyIsTooLong() {
+    public void testPutMessage_whenMessagePropertyIsTooLong() throws ConsumeQueueException {
         String topicName = "messagePropertyIsTooLongTest";
         MessageExtBrokerInner illegalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, Short.MAX_VALUE + 1);
         assertEquals(messageStore.putMessage(illegalMessage).getPutMessageStatus(), PutMessageStatus.PROPERTIES_SIZE_EXCEEDED);
@@ -539,7 +540,7 @@ public void testGroupCommit() throws Exception {
     }
 
     @Test
-    public void testMaxOffset() throws InterruptedException {
+    public void testMaxOffset() throws InterruptedException, ConsumeQueueException {
         int firstBatchMessages = 3;
         int queueId = 0;
         messageBody = storeMessage.getBytes();
diff --git a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
deleted file mode 100644
index eae5eaa07a2..00000000000
--- a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.store;
-
-import java.io.File;
-import java.net.InetSocketAddress;
-import java.nio.charset.Charset;
-
-import java.util.concurrent.ConcurrentHashMap;
-import org.apache.rocketmq.common.BrokerConfig;
-import org.apache.rocketmq.common.UtilAll;
-import org.apache.rocketmq.common.message.MessageConst;
-import org.apache.rocketmq.common.message.MessageDecoder;
-import org.apache.rocketmq.common.message.MessageExtBrokerInner;
-import org.apache.rocketmq.store.config.MessageStoreConfig;
-import org.apache.rocketmq.store.queue.MultiDispatchUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.rocksdb.RocksDBException;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class MultiDispatchTest {
-
-    private MultiDispatch multiDispatch;
-
-    private DefaultMessageStore messageStore;
-
-    @Before
-    public void init() throws Exception {
-        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
-        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8);
-        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4);
-        messageStoreConfig.setMaxHashSlotNum(100);
-        messageStoreConfig.setMaxIndexNum(100 * 10);
-        messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir") + File.separator + "unitteststore1");
-        messageStoreConfig.setStorePathCommitLog(
-            System.getProperty("java.io.tmpdir") + File.separator + "unitteststore1" + File.separator + "commitlog");
-
-        messageStoreConfig.setEnableLmq(true);
-        messageStoreConfig.setEnableMultiDispatch(true);
-        BrokerConfig brokerConfig = new BrokerConfig();
-        //too much reference
-        messageStore = new DefaultMessageStore(messageStoreConfig, null, null, brokerConfig, new ConcurrentHashMap<>());
-        multiDispatch = new MultiDispatch(messageStore);
-    }
-
-    @After
-    public void destroy() {
-        UtilAll.deleteFile(new File(System.getProperty("java.io.tmpdir") + File.separator + "unitteststore1"));
-    }
-
-    @Test
-    public void lmqQueueKey() {
-        MessageExtBrokerInner messageExtBrokerInner = mock(MessageExtBrokerInner.class);
-        when(messageExtBrokerInner.getQueueId()).thenReturn(2);
-        String ret = MultiDispatchUtils.lmqQueueKey("%LMQ%lmq123");
-        assertEquals(ret, "%LMQ%lmq123-0");
-    }
-
-    @Test
-    public void wrapMultiDispatch() throws RocksDBException {
-        MessageExtBrokerInner messageExtBrokerInner = buildMessageMultiQueue();
-        multiDispatch.wrapMultiDispatch(messageExtBrokerInner);
-        assertEquals(messageExtBrokerInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET), "0,0");
-    }
-
-    private MessageExtBrokerInner buildMessageMultiQueue() {
-        MessageExtBrokerInner msg = new MessageExtBrokerInner();
-        msg.setTopic("test");
-        msg.setTags("TAG1");
-        msg.setKeys("Hello");
-        msg.setBody("aaa".getBytes(Charset.forName("UTF-8")));
-        msg.setKeys(String.valueOf(System.currentTimeMillis()));
-        msg.setQueueId(0);
-        msg.setSysFlag(0);
-        msg.setBornTimestamp(System.currentTimeMillis());
-        msg.setStoreHost(new InetSocketAddress("127.0.0.1", 54270));
-        msg.setBornHost(new InetSocketAddress("127.0.0.1", 10911));
-        for (int i = 0; i < 1; i++) {
-            msg.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH, "%LMQ%123,%LMQ%456");
-        }
-        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));
-
-        return msg;
-    }
-}
\ No newline at end of file
diff --git a/store/src/test/java/org/apache/rocketmq/store/ReputMessageServiceTest.java b/store/src/test/java/org/apache/rocketmq/store/ReputMessageServiceTest.java
new file mode 100644
index 00000000000..d1ce0953331
--- /dev/null
+++ b/store/src/test/java/org/apache/rocketmq/store/ReputMessageServiceTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store;
+
+import java.lang.reflect.Field;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.UUID;
+import java.io.File;
+import java.util.concurrent.CompletableFuture;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.message.MessageDecoder;
+
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.config.FlushDiskType;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.ArgumentMatchers.any;
+
+public class ReputMessageServiceTest {
+    private DefaultMessageStore syncFlushMessageStore;
+    private DefaultMessageStore asyncFlushMessageStore;
+    private final String topic = "FooBar";
+    private final String tmpdir = System.getProperty("java.io.tmpdir");
+    private final String storePathRootParentDir = (StringUtils.endsWith(tmpdir, File.separator) ? tmpdir : tmpdir + File.separator) + UUID.randomUUID();
+    private SocketAddress bornHost;
+    private SocketAddress storeHost;
+
+    @Before
+    public void init() throws Exception {
+        File file = new File(storePathRootParentDir);
+        UtilAll.deleteFile(file);
+        syncFlushMessageStore = buildMessageStore(FlushDiskType.SYNC_FLUSH);
+        asyncFlushMessageStore = buildMessageStore(FlushDiskType.ASYNC_FLUSH);
+        assertTrue(syncFlushMessageStore.load());
+        assertTrue(asyncFlushMessageStore.load());
+        syncFlushMessageStore.start();
+        asyncFlushMessageStore.start();
+        storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);
+        bornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0);
+    }
+
+    private DefaultMessageStore buildMessageStore(FlushDiskType flushDiskType) throws Exception {
+        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+        messageStoreConfig.setHaListenPort(0);
+        messageStoreConfig.setFlushDiskType(flushDiskType);
+        messageStoreConfig.setStorePathRootDir(storePathRootParentDir + File.separator + flushDiskType);
+        messageStoreConfig.setStorePathCommitLog(storePathRootParentDir + File.separator + flushDiskType + File.separator + "commitlog");
+        BrokerConfig brokerConfig = new BrokerConfig();
+        brokerConfig.setLongPollingEnable(false);
+        DefaultMessageStore messageStore = new DefaultMessageStore(messageStoreConfig, mock(BrokerStatsManager.class), null, brokerConfig, null);
+        // Mock flush disk service
+        Field field = CommitLog.class.getDeclaredField("flushManager");
+        field.setAccessible(true);
+        FlushManager flushManager = mock(FlushManager.class);
+        CompletableFuture<PutMessageStatus> completableFuture = new CompletableFuture<>();
+        completableFuture.complete(PutMessageStatus.PUT_OK);
+        when(flushManager.handleDiskFlush(any(AppendMessageResult.class), any(MessageExt.class))).thenReturn(completableFuture);
+        field.set(messageStore.getCommitLog(), flushManager);
+        return messageStore;
+    }
+
+    @Test
+    public void testReputEndOffset_whenSyncFlush() throws Exception {
+        for (int i = 0; i < 10; i++) {
+            assertEquals(PutMessageStatus.PUT_OK, syncFlushMessageStore.putMessage(buildMessage()).getPutMessageStatus());
+        }
+        assertEquals(1580, syncFlushMessageStore.getMaxPhyOffset());
+        assertEquals(0, syncFlushMessageStore.getCommitLog().getFlushedWhere());
+        // wait for cq dispatch
+        Thread.sleep(3000);
+        assertEquals(0, syncFlushMessageStore.getCommitLog().getFlushedWhere());
+        assertEquals(0, syncFlushMessageStore.getMaxOffsetInQueue(topic, 0));
+        GetMessageResult getMessageResult = syncFlushMessageStore.getMessage("testGroup", topic, 0, 0, 32, null);
+        assertEquals(GetMessageStatus.NO_MESSAGE_IN_QUEUE, getMessageResult.getStatus());
+    }
+
+    @Test
+    public void testReputEndOffset_whenAsyncFlush() throws Exception {
+        for (int i = 0; i < 10; i++) {
+            assertEquals(PutMessageStatus.PUT_OK, asyncFlushMessageStore.putMessage(buildMessage()).getPutMessageStatus());
+        }
+        assertEquals(1580, asyncFlushMessageStore.getMaxPhyOffset());
+        assertEquals(0, asyncFlushMessageStore.getCommitLog().getFlushedWhere());
+        // wait for cq dispatch
+        Thread.sleep(3000);
+        assertEquals(0, asyncFlushMessageStore.getCommitLog().getFlushedWhere());
+        assertEquals(10, asyncFlushMessageStore.getMaxOffsetInQueue(topic, 0));
+        GetMessageResult getMessageResult = asyncFlushMessageStore.getMessage("testGroup", topic, 0, 0, 32, null);
+        assertEquals(10, getMessageResult.getMessageCount());
+    }
+
+    private MessageExtBrokerInner buildMessage() {
+        MessageExtBrokerInner msg = new MessageExtBrokerInner();
+        msg.setTopic(topic);
+        msg.setTags("TAG1");
+        msg.setBody("Once, there was a chance for me!".getBytes());
+        msg.setKeys(String.valueOf(System.currentTimeMillis()));
+        msg.setQueueId(0);
+        msg.setSysFlag(0);
+        msg.setBornTimestamp(System.currentTimeMillis());
+        msg.setStoreHost(storeHost);
+        msg.setBornHost(bornHost);
+        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));
+        return msg;
+    }
+
+    @After
+    public void destroy() throws Exception {
+        if (this.syncFlushMessageStore != null) {
+            syncFlushMessageStore.shutdown();
+            syncFlushMessageStore.destroy();
+        }
+        if (this.asyncFlushMessageStore != null) {
+            asyncFlushMessageStore.shutdown();
+            asyncFlushMessageStore.destroy();
+        }
+        File file = new File(storePathRootParentDir);
+        UtilAll.deleteFile(file);
+    }
+}
diff --git a/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java
index 2af07197a50..f934f803641 100644
--- a/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java
@@ -56,6 +56,7 @@
 import org.apache.rocketmq.store.config.FlushDiskType;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
 import org.apache.rocketmq.store.queue.CqUnit;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
@@ -434,7 +435,7 @@ public void testGetStoreTime_PhyOffsetIsLessThanCommitLogMinOffset() {
     }
 
     @Test
-    public void testPutMessage_whenMessagePropertyIsTooLong() {
+    public void testPutMessage_whenMessagePropertyIsTooLong() throws ConsumeQueueException {
         if (notExecuted()) {
             return;
         }
@@ -603,7 +604,7 @@ public void testGroupCommit() {
     }
 
     @Test
-    public void testMaxOffset() {
+    public void testMaxOffset() throws ConsumeQueueException {
         if (notExecuted()) {
             return;
         }
diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java
index 1e4bbf21bd2..386cb1f6787 100644
--- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java
@@ -41,6 +41,7 @@
 import org.apache.rocketmq.store.StoreCheckpoint;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.Assume;
 import org.apache.rocketmq.common.MixAll;
@@ -51,6 +52,12 @@
 
 public class DLedgerCommitlogTest extends MessageStoreTestBase {
 
+    @BeforeClass
+    public static void beforeClass() {
+        // Temporarily skip those tests on the macOS as they are flaky
+        Assume.assumeFalse(MixAll.isMac());
+    }
+
     @Test
     public void testTruncateCQ() throws Exception {
         String base = createBaseDir();
diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java
index 5eb83207322..9de4e4820ed 100644
--- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java
@@ -39,6 +39,7 @@ public class DLedgerMultiPathTest extends MessageStoreTestBase {
 
     @Test
     public void multiDirsStorageTest() throws Exception {
+        Assume.assumeFalse(MixAll.isMac());
         Assume.assumeFalse(MixAll.isWindows());
         String base = createBaseDir();
         String topic = UUID.randomUUID().toString();
diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java
index a21806ffcf6..c4d9f0727b9 100644
--- a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java
+++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java
@@ -16,6 +16,7 @@
  */
 package org.apache.rocketmq.store.dledger;
 
+import com.google.common.util.concurrent.RateLimiter;
 import io.openmessaging.storage.dledger.DLedgerConfig;
 import io.openmessaging.storage.dledger.DLedgerServer;
 import java.io.File;
@@ -122,7 +123,13 @@ protected DefaultMessageStore createMessageStore(String base, boolean createAbor
     }
 
     protected void doPutMessages(MessageStore messageStore, String topic, int queueId, int num, long beginLogicsOffset) throws UnknownHostException {
+        RateLimiter rateLimiter = RateLimiter.create(100);
+        MessageStoreConfig storeConfig = messageStore.getMessageStoreConfig();
+        boolean limitAppendRate = storeConfig.isEnableDLegerCommitLog();
         for (int i = 0; i < num; i++) {
+            if (limitAppendRate) {
+                rateLimiter.acquire();
+            }
             MessageExtBrokerInner msgInner = buildMessage();
             msgInner.setTopic(topic);
             msgInner.setQueueId(queueId);
diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java
index db7b594a73b..1bfc6f72eaa 100644
--- a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java
@@ -34,6 +34,7 @@ public class MixCommitlogTest extends MessageStoreTestBase {
     @Test
     public void testFallBehindCQ() throws Exception {
         Assume.assumeFalse(MixAll.isWindows());
+        Assume.assumeFalse(MixAll.isMac());
         String base = createBaseDir();
         String topic = UUID.randomUUID().toString();
         String peers = String.format("n0-localhost:%d", nextPort());
@@ -75,6 +76,7 @@ public void testFallBehindCQ() throws Exception {
 
     @Test
     public void testPutAndGet() throws Exception {
+        Assume.assumeFalse(MixAll.isMac());
         String base = createBaseDir();
         String topic = UUID.randomUUID().toString();
         String peers = String.format("n0-localhost:%d", nextPort());
@@ -138,6 +140,7 @@ public void testPutAndGet() throws Exception {
 
     @Test
     public void testDeleteExpiredFiles() throws Exception {
+        Assume.assumeFalse(MixAll.isMac());
         String base = createBaseDir();
         String topic = UUID.randomUUID().toString();
         String peers = String.format("n0-localhost:%d", nextPort());
diff --git a/store/src/test/java/org/apache/rocketmq/store/index/IndexServiceTest.java b/store/src/test/java/org/apache/rocketmq/store/index/IndexServiceTest.java
new file mode 100644
index 00000000000..057bbfd0e1e
--- /dev/null
+++ b/store/src/test/java/org/apache/rocketmq/store/index/IndexServiceTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store.index;
+
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.junit.Test;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+
+
+public class IndexServiceTest {
+
+    @Test
+    public void testQueryOffsetThrow() throws Exception {
+        assertDoesNotThrow(() -> {
+            DefaultMessageStore store = new DefaultMessageStore(
+                    new MessageStoreConfig(),
+                    new BrokerStatsManager(new BrokerConfig()),
+                    null,
+                    new BrokerConfig(),
+                    new ConcurrentHashMap<>()
+            );
+
+            IndexService indexService = new IndexService(store);
+            indexService.queryOffset("test", "", Integer.MAX_VALUE, 10, 100);
+        });
+    }
+
+}
diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java
new file mode 100644
index 00000000000..c24a1dccaed
--- /dev/null
+++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.lock;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class AdaptiveLockTest {
+
+    AdaptiveBackOffSpinLockImpl adaptiveLock;
+
+    @Before
+    public void init() {
+        adaptiveLock = new AdaptiveBackOffSpinLockImpl();
+    }
+
+    @Test
+    public void testAdaptiveLock() throws InterruptedException {
+        assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffSpinLock);
+        CountDownLatch countDownLatch = new CountDownLatch(1);
+        adaptiveLock.lock();
+        new Thread(new Runnable() {
+            @Override
+            public void run() {
+                adaptiveLock.lock();
+                try {
+                    Thread.sleep(1);
+                } catch (InterruptedException e) {
+                    //ignore
+                }
+                adaptiveLock.unlock();
+                countDownLatch.countDown();
+            }
+        }).start();
+        Thread.sleep(1000);
+        adaptiveLock.unlock();
+        assertEquals(2000, ((BackOffSpinLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree());
+        countDownLatch.await();
+
+        for (int i = 0; i <= 5; i++) {
+            CountDownLatch countDownLatch1 = new CountDownLatch(1);
+            adaptiveLock.lock();
+            new Thread(new Runnable() {
+                @Override
+                public void run() {
+                    adaptiveLock.lock();
+                    try {
+                        Thread.sleep(1);
+                    } catch (InterruptedException e) {
+                        //ignore
+                    }
+                    adaptiveLock.unlock();
+                    countDownLatch1.countDown();
+                }
+            }).start();
+            Thread.sleep(1000);
+            adaptiveLock.unlock();
+            countDownLatch1.await();
+        }
+        assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffReentrantLock);
+
+        for (int i = 0; i <= 2; i++) {
+            adaptiveLock.lock();
+            adaptiveLock.unlock();
+            Thread.sleep(1000);
+        }
+        assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffSpinLock);
+    }
+}
diff --git a/store/src/test/java/org/apache/rocketmq/store/queue/ConsumeQueueTest.java b/store/src/test/java/org/apache/rocketmq/store/queue/ConsumeQueueTest.java
index c3c8be52ddd..bf3b1eeca83 100644
--- a/store/src/test/java/org/apache/rocketmq/store/queue/ConsumeQueueTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/queue/ConsumeQueueTest.java
@@ -22,6 +22,7 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.attribute.CQType;
 import org.apache.rocketmq.common.message.MessageDecoder;
@@ -31,6 +32,7 @@
 import org.apache.rocketmq.store.DispatchRequest;
 import org.apache.rocketmq.store.MessageFilter;
 import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.RocksDBMessageStore;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 import org.junit.Assert;
@@ -84,7 +86,26 @@ messageStoreConfig, new BrokerStatsManager(brokerConfig),
         return master;
     }
 
-    protected void putMsg(DefaultMessageStore messageStore) throws Exception {
+    protected RocksDBMessageStore genRocksdbMessageStore() throws Exception {
+        MessageStoreConfig messageStoreConfig = buildStoreConfig(
+            COMMIT_LOG_FILE_SIZE, CQ_FILE_SIZE, true, CQ_EXT_FILE_SIZE
+        );
+
+        BrokerConfig brokerConfig = new BrokerConfig();
+
+        RocksDBMessageStore master = new RocksDBMessageStore(
+            messageStoreConfig, new BrokerStatsManager(brokerConfig),
+            (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> {
+            }, brokerConfig, new ConcurrentHashMap<>());
+
+        assertThat(master.load()).isTrue();
+
+        master.start();
+
+        return master;
+    }
+
+    protected void putMsg(MessageStore messageStore) {
         int totalMsgs = 200;
         for (int i = 0; i < totalMsgs; i++) {
             MessageExtBrokerInner message = buildMessage();
@@ -184,9 +205,33 @@ public void testIterator() throws Exception {
 
     @Test
     public void testEstimateMessageCountInEmptyConsumeQueue() {
-        DefaultMessageStore master = null;
+        DefaultMessageStore messageStore = null;
+        try {
+            messageStore = gen();
+            doTestEstimateMessageCountInEmptyConsumeQueue(messageStore);
+        } catch (Exception e) {
+            e.printStackTrace();
+            assertThat(Boolean.FALSE).isTrue();
+        }
+    }
+
+    @Test
+    public void testEstimateRocksdbMessageCountInEmptyConsumeQueue() {
+        if (notExecuted()) {
+            return;
+        }
+        DefaultMessageStore messageStore = null;
+        try {
+            messageStore = genRocksdbMessageStore();
+            doTestEstimateMessageCountInEmptyConsumeQueue(messageStore);
+        } catch (Exception e) {
+            e.printStackTrace();
+            assertThat(Boolean.FALSE).isTrue();
+        }
+    }
+
+    public void doTestEstimateMessageCountInEmptyConsumeQueue(MessageStore master) {
         try {
-            master = gen();
             ConsumeQueueInterface consumeQueue = master.findConsumeQueue(TOPIC, QUEUE_ID);
             MessageFilter filter = new MessageFilter() {
                 @Override
@@ -219,16 +264,34 @@ public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> pr
         }
     }
 
+    @Test
+    public void testEstimateRocksdbMessageCount() {
+        if (notExecuted()) {
+            return;
+        }
+        DefaultMessageStore messageStore = null;
+        try {
+            messageStore = genRocksdbMessageStore();
+            doTestEstimateMessageCount(messageStore);
+        } catch (Exception e) {
+            e.printStackTrace();
+            assertThat(Boolean.FALSE).isTrue();
+        }
+    }
+
     @Test
     public void testEstimateMessageCount() {
         DefaultMessageStore messageStore = null;
         try {
             messageStore = gen();
+            doTestEstimateMessageCount(messageStore);
         } catch (Exception e) {
             e.printStackTrace();
             assertThat(Boolean.FALSE).isTrue();
         }
+    }
 
+    public void doTestEstimateMessageCount(MessageStore messageStore) {
         try {
             try {
                 putMsg(messageStore);
@@ -265,15 +328,34 @@ public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> pr
         }
     }
 
+    @Test
+    public void testEstimateRocksdbMessageCountSample() {
+        if (notExecuted()) {
+            return;
+        }
+        DefaultMessageStore messageStore = null;
+        try {
+            messageStore = genRocksdbMessageStore();
+            doTestEstimateMessageCountSample(messageStore);
+        } catch (Exception e) {
+            e.printStackTrace();
+            assertThat(Boolean.FALSE).isTrue();
+        }
+    }
+
     @Test
     public void testEstimateMessageCountSample() {
         DefaultMessageStore messageStore = null;
         try {
             messageStore = gen();
+            doTestEstimateMessageCountSample(messageStore);
         } catch (Exception e) {
             e.printStackTrace();
             assertThat(Boolean.FALSE).isTrue();
         }
+    }
+
+    public void doTestEstimateMessageCountSample(MessageStore messageStore) {
 
         try {
             try {
@@ -303,4 +385,8 @@ public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> pr
             UtilAll.deleteFile(new File(STORE_PATH));
         }
     }
+
+    private boolean notExecuted() {
+        return MixAll.isMac();
+    }
 }
diff --git a/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTableTest.java b/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTableTest.java
new file mode 100644
index 00000000000..b1e12d49468
--- /dev/null
+++ b/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTableTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store.queue;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.queue.offset.OffsetEntryType;
+import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.rocksdb.Options;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteOptions;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RocksDBConsumeQueueOffsetTableTest {
+
+    private RocksDBConsumeQueueOffsetTable offsetTable;
+
+    @Mock
+    private ConsumeQueueRocksDBStorage rocksDBStorage;
+
+    @Mock
+    private RocksDBConsumeQueueTable consumeQueueTable;
+
+    @Mock
+    private DefaultMessageStore messageStore;
+
+    private static RocksDB db;
+
+    private static File dbPath;
+
+    private static String topicName;
+
+    @BeforeClass
+    public static void initDB() throws IOException, RocksDBException {
+        TemporaryFolder tempFolder = new TemporaryFolder();
+        tempFolder.create();
+        dbPath = tempFolder.newFolder();
+
+        db = RocksDB.open(dbPath.getAbsolutePath());
+        StringBuilder topicBuilder = new StringBuilder();
+        for (int i = 0; i < 100; i++) {
+            topicBuilder.append("topic");
+        }
+        topicName = topicBuilder.toString();
+        byte[] topicInBytes = topicName.getBytes(StandardCharsets.UTF_8);
+
+        ByteBuffer keyBuffer = ByteBuffer.allocateDirect(RocksDBConsumeQueueOffsetTable.OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicInBytes.length);
+        RocksDBConsumeQueueOffsetTable.buildOffsetKeyByteBuffer(keyBuffer, topicInBytes, 1, true);
+        Assert.assertEquals(0, keyBuffer.position());
+        Assert.assertEquals(RocksDBConsumeQueueOffsetTable.OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicInBytes.length, keyBuffer.limit());
+
+        ByteBuffer valueBuffer = ByteBuffer.allocateDirect(Long.BYTES + Long.BYTES);
+        valueBuffer.putLong(100);
+        valueBuffer.putLong(2);
+        valueBuffer.flip();
+
+        try (WriteBatch writeBatch = new WriteBatch();
+             WriteOptions writeOptions = new WriteOptions()) {
+            writeOptions.setDisableWAL(false);
+            writeOptions.setSync(true);
+            writeBatch.put(keyBuffer, valueBuffer);
+            db.write(writeOptions, writeBatch);
+        }
+
+    }
+
+    @AfterClass
+    public static void tearDownDB() throws RocksDBException {
+        db.closeE();
+        RocksDB.destroyDB(dbPath.getAbsolutePath(), new Options());
+    }
+
+    @Before
+    public void setUp() {
+        RocksIterator iterator = db.newIterator();
+        Mockito.doReturn(iterator).when(rocksDBStorage).seekOffsetCF();
+        offsetTable = new RocksDBConsumeQueueOffsetTable(consumeQueueTable, rocksDBStorage, messageStore);
+    }
+
+    /**
+     * Verify forEach can expand key-buffer properly and works well for long topic names.
+     *
+     * @throws RocksDBException If there is an RocksDB error.
+     */
+    @Test
+    public void testForEach() throws RocksDBException {
+        AtomicBoolean called = new AtomicBoolean(false);
+        offsetTable.forEach(entry -> true, entry -> {
+            called.set(true);
+            Assert.assertEquals(topicName, entry.topic);
+            Assert.assertTrue(topicName.length() > 256);
+            Assert.assertEquals(1, entry.queueId);
+            Assert.assertEquals(100, entry.commitLogOffset);
+            Assert.assertEquals(2, entry.offset);
+            Assert.assertEquals(OffsetEntryType.MAXIMUM, entry.type);
+        });
+        Assert.assertTrue(called.get());
+    }
+}
diff --git a/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTableTest.java b/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTableTest.java
new file mode 100644
index 00000000000..d06b6da2fbd
--- /dev/null
+++ b/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTableTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.queue;
+
+import org.apache.rocketmq.common.BoundaryType;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;
+import org.junit.Test;
+import org.mockito.stubbing.Answer;
+import org.rocksdb.RocksDBException;
+
+import java.nio.ByteBuffer;
+
+import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueTable.CQ_UNIT_SIZE;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+
+public class RocksDBConsumeQueueTableTest {
+
+    @Test
+    public void testBinarySearchInCQByTime() throws RocksDBException {
+        if (MixAll.isMac()) {
+            return;
+        }
+        ConsumeQueueRocksDBStorage rocksDBStorage = mock(ConsumeQueueRocksDBStorage.class);
+        DefaultMessageStore store = mock(DefaultMessageStore.class);
+        RocksDBConsumeQueueTable table = new RocksDBConsumeQueueTable(rocksDBStorage, store);
+        doAnswer((Answer<byte[]>) mock -> {
+            /*
+             * queueOffset timestamp
+             * 100         1000
+             * 200         2000
+             * 201         2010
+             * 1000        10000
+             */
+            byte[] keyBytes = mock.getArgument(0);
+            ByteBuffer keyBuffer = ByteBuffer.wrap(keyBytes);
+            int len = keyBuffer.getInt(0);
+            long offset = keyBuffer.getLong(4 + 1 + len + 1 + 4 + 1);
+            long phyOffset = offset;
+            long timestamp = offset * 10;
+            final ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_UNIT_SIZE);
+            byteBuffer.putLong(phyOffset);
+            byteBuffer.putInt(1);
+            byteBuffer.putLong(0);
+            byteBuffer.putLong(timestamp);
+            return byteBuffer.array();
+        }).when(rocksDBStorage).getCQ(any());
+        assertEquals(1001, table.binarySearchInCQByTime("topic", 0, 1000, 100, 20000, 0, BoundaryType.LOWER));
+        assertEquals(1000, table.binarySearchInCQByTime("topic", 0, 1000, 100, 20000, 0, BoundaryType.UPPER));
+        assertEquals(100, table.binarySearchInCQByTime("topic", 0, 1000, 100, 1, 0, BoundaryType.LOWER));
+        assertEquals(0, table.binarySearchInCQByTime("topic", 0, 1000, 100, 1, 0, BoundaryType.UPPER));
+        assertEquals(201, table.binarySearchInCQByTime("topic", 0, 1000, 100, 2001, 0, BoundaryType.LOWER));
+        assertEquals(200, table.binarySearchInCQByTime("topic", 0, 1000, 100, 2001, 0, BoundaryType.UPPER));
+        assertEquals(200, table.binarySearchInCQByTime("topic", 0, 1000, 100, 2000, 0, BoundaryType.LOWER));
+        assertEquals(200, table.binarySearchInCQByTime("topic", 0, 1000, 100, 2000, 0, BoundaryType.UPPER));
+    }
+}
\ No newline at end of file
diff --git a/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTest.java b/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTest.java
new file mode 100644
index 00000000000..b907ce59519
--- /dev/null
+++ b/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.queue;
+
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.nio.ByteBuffer;
+
+import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueTable.CQ_UNIT_SIZE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class RocksDBConsumeQueueTest extends QueueTestBase {
+
+    @Test
+    public void testIterator() throws Exception {
+        if (MixAll.isMac()) {
+            return;
+        }
+        DefaultMessageStore messageStore = mock(DefaultMessageStore.class);
+        RocksDBConsumeQueueStore rocksDBConsumeQueueStore = mock(RocksDBConsumeQueueStore.class);
+        when(messageStore.getQueueStore()).thenReturn(rocksDBConsumeQueueStore);
+        when(rocksDBConsumeQueueStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(10000L);
+        when(rocksDBConsumeQueueStore.get(anyString(), anyInt(), anyLong())).then(new Answer<ByteBuffer>() {
+            @Override
+            public ByteBuffer answer(InvocationOnMock mock) throws Throwable {
+                long startIndex = mock.getArgument(2);
+                final ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_UNIT_SIZE);
+                long phyOffset = startIndex * 10;
+                byteBuffer.putLong(phyOffset);
+                byteBuffer.putInt(1);
+                byteBuffer.putLong(0);
+                byteBuffer.putLong(0);
+                byteBuffer.flip();
+                return byteBuffer;
+            }
+        });
+
+        RocksDBConsumeQueue consumeQueue = new RocksDBConsumeQueue(messageStore, "topic", 0);
+        ReferredIterator<CqUnit> it = consumeQueue.iterateFrom(9000);
+        for (int i = 0; i < 1000; i++) {
+            assertTrue(it.hasNext());
+            CqUnit next = it.next();
+            assertEquals(9000 + i, next.getQueueOffset());
+            assertEquals(10 * (9000 + i), next.getPos());
+        }
+        assertFalse(it.hasNext());
+    }
+}
\ No newline at end of file
diff --git a/store/src/test/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactoryTest.java b/store/src/test/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactoryTest.java
new file mode 100644
index 00000000000..1d7273968f6
--- /dev/null
+++ b/store/src/test/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactoryTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store.rocksdb;
+
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.Assert;
+import org.junit.Test;
+import org.rocksdb.CompressionType;
+
+public class RocksDBOptionsFactoryTest {
+
+    @Test
+    public void testBottomMostCompressionType() {
+        MessageStoreConfig config = new MessageStoreConfig();
+        Assert.assertEquals(CompressionType.ZSTD_COMPRESSION,
+            CompressionType.getCompressionType(config.getBottomMostCompressionTypeForConsumeQueueStore()));
+        Assert.assertEquals(CompressionType.LZ4_COMPRESSION, CompressionType.getCompressionType("lz4"));
+    }
+}
diff --git a/store/src/test/java/org/apache/rocketmq/store/stats/BrokerStatsManagerTest.java b/store/src/test/java/org/apache/rocketmq/store/stats/BrokerStatsManagerTest.java
index a602da09395..058ad0b0208 100644
--- a/store/src/test/java/org/apache/rocketmq/store/stats/BrokerStatsManagerTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/stats/BrokerStatsManagerTest.java
@@ -24,6 +24,8 @@
 import org.junit.Test;
 
 import static org.apache.rocketmq.common.stats.Stats.BROKER_PUT_NUMS;
+import static org.apache.rocketmq.common.stats.Stats.GROUP_ACK_NUMS;
+import static org.apache.rocketmq.common.stats.Stats.GROUP_CK_NUMS;
 import static org.apache.rocketmq.common.stats.Stats.GROUP_GET_FALL_SIZE;
 import static org.apache.rocketmq.common.stats.Stats.GROUP_GET_FALL_TIME;
 import static org.apache.rocketmq.common.stats.Stats.GROUP_GET_LATENCY;
@@ -34,6 +36,7 @@
 import static org.apache.rocketmq.common.stats.Stats.QUEUE_PUT_NUMS;
 import static org.apache.rocketmq.common.stats.Stats.QUEUE_PUT_SIZE;
 import static org.apache.rocketmq.common.stats.Stats.SNDBCK_PUT_NUMS;
+import static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_LATENCY;
 import static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_NUMS;
 import static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_SIZE;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -139,8 +142,11 @@ public void testOnTopicDeleted() {
         brokerStatsManager.incTopicPutSize(TOPIC, 100);
         brokerStatsManager.incQueuePutNums(TOPIC, QUEUE_ID);
         brokerStatsManager.incQueuePutSize(TOPIC, QUEUE_ID, 100);
+        brokerStatsManager.incTopicPutLatency(TOPIC, QUEUE_ID, 10);
         brokerStatsManager.incGroupGetNums(GROUP_NAME, TOPIC, 1);
         brokerStatsManager.incGroupGetSize(GROUP_NAME, TOPIC, 100);
+        brokerStatsManager.incGroupCkNums(GROUP_NAME, TOPIC, 1);
+        brokerStatsManager.incGroupAckNums(GROUP_NAME, TOPIC, 1);
         brokerStatsManager.incQueueGetNums(GROUP_NAME, TOPIC, QUEUE_ID, 1);
         brokerStatsManager.incQueueGetSize(GROUP_NAME, TOPIC, QUEUE_ID, 100);
         brokerStatsManager.incSendBackNums(GROUP_NAME, TOPIC);
@@ -162,6 +168,9 @@ public void testOnTopicDeleted() {
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_LATENCY, "1@" + TOPIC + "@" + GROUP_NAME));
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_SIZE, "1@" + TOPIC + "@" + GROUP_NAME));
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_TIME, "1@" + TOPIC + "@" + GROUP_NAME));
+        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_CK_NUMS, TOPIC + "@" + GROUP_NAME));
+        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_ACK_NUMS, TOPIC + "@" + GROUP_NAME));
+        Assert.assertNull(brokerStatsManager.getStatsItem(TOPIC_PUT_LATENCY, QUEUE_ID + "@" + TOPIC));
     }
 
     @Test
@@ -174,6 +183,8 @@ public void testOnGroupDeleted() {
         brokerStatsManager.incGroupGetLatency(GROUP_NAME, TOPIC, 1, 1);
         brokerStatsManager.recordDiskFallBehindTime(GROUP_NAME, TOPIC, 1, 11L);
         brokerStatsManager.recordDiskFallBehindSize(GROUP_NAME, TOPIC, 1, 11L);
+        brokerStatsManager.incGroupCkNums(GROUP_NAME, TOPIC, 1);
+        brokerStatsManager.incGroupAckNums(GROUP_NAME, TOPIC, 1);
 
         brokerStatsManager.onGroupDeleted(GROUP_NAME);
 
@@ -185,6 +196,8 @@ public void testOnGroupDeleted() {
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_LATENCY, "1@" + TOPIC + "@" + GROUP_NAME));
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_SIZE, "1@" + TOPIC + "@" + GROUP_NAME));
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_TIME, "1@" + TOPIC + "@" + GROUP_NAME));
+        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_CK_NUMS, TOPIC + "@" + GROUP_NAME));
+        Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_ACK_NUMS, TOPIC + "@" + GROUP_NAME));
     }
 
     @Test
diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java
index 4ce3985f6c9..a014e77b90e 100644
--- a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java
+++ b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java
@@ -19,6 +19,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.Field;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
@@ -30,6 +31,7 @@
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.rocketmq.common.BrokerConfig;
@@ -40,23 +42,26 @@
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
-import org.apache.rocketmq.store.ConsumeQueue;
-import org.apache.rocketmq.store.DefaultMessageStore;
-import org.apache.rocketmq.store.GetMessageResult;
-import org.apache.rocketmq.store.GetMessageStatus;
-import org.apache.rocketmq.store.MessageArrivingListener;
 import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.DefaultMessageStore;
 import org.apache.rocketmq.store.PutMessageStatus;
 import org.apache.rocketmq.store.config.FlushDiskType;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.apache.rocketmq.store.ConsumeQueue;
+import org.apache.rocketmq.store.AppendMessageStatus;
+import org.apache.rocketmq.store.AppendMessageResult;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+import org.apache.rocketmq.store.MessageArrivingListener;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.awaitility.Awaitility.await;
@@ -65,10 +70,16 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.times;
 
 public class TimerMessageStoreTest {
     private final byte[] msgBody = new byte[1024];
     private static MessageStore messageStore;
+    private MessageStore mockMessageStore;
     private SocketAddress bornHost;
     private SocketAddress storeHost;
 
@@ -100,21 +111,23 @@ public void init() throws Exception {
         storeConfig.setTimerInterceptDelayLevel(true);
         storeConfig.setTimerPrecisionMs(precisionMs);
 
+        mockMessageStore = Mockito.mock(MessageStore.class);
         messageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("TimerTest",false), new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>());
         boolean load = messageStore.load();
         assertTrue(load);
         messageStore.start();
     }
 
-    public TimerMessageStore createTimerMessageStore(String rootDir) throws IOException {
+    public TimerMessageStore createTimerMessageStore(String rootDir , boolean needMock) throws IOException {
         if (null == rootDir) {
             rootDir = StoreTestUtils.createBaseDir();
         }
 
         TimerCheckpoint timerCheckpoint = new TimerCheckpoint(rootDir + File.separator + "config" + File.separator + "timercheck");
         TimerMetrics timerMetrics = new TimerMetrics(rootDir + File.separator + "config" + File.separator + "timermetrics");
-        TimerMessageStore timerMessageStore = new TimerMessageStore(messageStore, storeConfig, timerCheckpoint, timerMetrics, null);
-        messageStore.setTimerMessageStore(timerMessageStore);
+        MessageStore ms = needMock ? mockMessageStore : messageStore;
+        TimerMessageStore timerMessageStore = new TimerMessageStore(ms, storeConfig, timerCheckpoint, timerMetrics, null);
+        ms.setTimerMessageStore(timerMessageStore);
 
         baseDirs.add(rootDir);
         timerStores.add(timerMessageStore);
@@ -170,7 +183,7 @@ public void testPutTimerMessage() throws Exception {
         Assume.assumeFalse(MixAll.isWindows());
         String topic = "TimerTest_testPutTimerMessage";
 
-        final TimerMessageStore timerMessageStore = createTimerMessageStore(null);
+        final TimerMessageStore timerMessageStore = createTimerMessageStore(null , false);
         timerMessageStore.load();
         timerMessageStore.start(true);
 
@@ -212,12 +225,51 @@ public Boolean call() {
         }
     }
 
+    @Test
+    public void testRetryUntilSuccess() throws Exception {
+        storeConfig.setTimerEnableRetryUntilSuccess(true);
+        TimerMessageStore timerMessageStore = createTimerMessageStore(null , true);
+        timerMessageStore.load();
+        timerMessageStore.setShouldRunningDequeue(true);
+        Field stateField = TimerMessageStore.class.getDeclaredField("state");
+        stateField.setAccessible(true);
+        stateField.set(timerMessageStore, TimerMessageStore.RUNNING);
+
+        MessageExtBrokerInner msg = buildMessage(3000L, "TestRetry", true);
+        transformTimerMessage(timerMessageStore, msg);
+        TimerRequest timerRequest = new TimerRequest(100, 200, 3000, System.currentTimeMillis(), 0, msg);
+        boolean offered = timerMessageStore.dequeuePutQueue.offer(timerRequest);
+        assertTrue(offered);
+        assertFalse(timerMessageStore.dequeuePutQueue.isEmpty());
+
+        // If enableRetryUntilSuccess is set and putMessage return NEED_RETRY type, the message should be retried until success.
+        when(mockMessageStore.putMessage(any(MessageExtBrokerInner.class)))
+                .thenReturn(new PutMessageResult(PutMessageStatus.FLUSH_DISK_TIMEOUT, null))
+                .thenReturn(new PutMessageResult(PutMessageStatus.FLUSH_SLAVE_TIMEOUT, null))
+                .thenReturn(new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, null))
+                .thenReturn(new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, null))
+                .thenReturn(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null))
+                .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        new Thread(() -> {
+            try {
+                timerMessageStore.getDequeuePutMessageServices()[0].run();
+            } finally {
+                latch.countDown();
+            }
+        }).start();
+        latch.await(5, TimeUnit.SECONDS);
+        assertTrue(timerMessageStore.dequeuePutQueue.isEmpty());
+        verify(mockMessageStore, times(6)).putMessage(any(MessageExtBrokerInner.class));
+    }
+
     @Test
     public void testTimerFlowControl() throws Exception {
         String topic = "TimerTest_testTimerFlowControl";
 
         storeConfig.setTimerCongestNumEachSlot(100);
-        TimerMessageStore timerMessageStore = createTimerMessageStore(null);
+        TimerMessageStore timerMessageStore = createTimerMessageStore(null , false);
         timerMessageStore.load();
         timerMessageStore.start(true);
 
@@ -264,7 +316,7 @@ public void testPutExpiredTimerMessage() throws Exception {
 
         String topic = "TimerTest_testPutExpiredTimerMessage";
 
-        TimerMessageStore timerMessageStore = createTimerMessageStore(null);
+        TimerMessageStore timerMessageStore = createTimerMessageStore(null ,false);
         timerMessageStore.load();
         timerMessageStore.start(true);
 
@@ -288,7 +340,7 @@ public void testPutExpiredTimerMessage() throws Exception {
     public void testDeleteTimerMessage() throws Exception {
         String topic = "TimerTest_testDeleteTimerMessage";
 
-        TimerMessageStore timerMessageStore = createTimerMessageStore(null);
+        TimerMessageStore timerMessageStore = createTimerMessageStore(null ,false);
         timerMessageStore.load();
         timerMessageStore.start(true);
 
@@ -306,7 +358,7 @@ public void testDeleteTimerMessage() throws Exception {
 
         MessageExtBrokerInner delMsg = buildMessage(delayMs, topic, false);
         transformTimerMessage(timerMessageStore,delMsg);
-        MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, uniqKey);
+        MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, TimerMessageStore.buildDeleteKey(topic, uniqKey));
         delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties()));
         assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus());
 
@@ -321,11 +373,54 @@ public void testDeleteTimerMessage() throws Exception {
         assertNull(getOneMessage(topic, 0, 4, 500));
     }
 
+    @Test
+    public void testDeleteTimerMessage_ukCollision() throws Exception {
+        String topic = "TimerTest_testDeleteTimerMessage";
+        String collisionTopic = "TimerTest_testDeleteTimerMessage_collision";
+
+        TimerMessageStore timerMessageStore = createTimerMessageStore(null , false);
+        timerMessageStore.load();
+        timerMessageStore.start(true);
+
+        long curr = System.currentTimeMillis() / precisionMs * precisionMs;
+        long delayMs = curr + 1000;
+
+        MessageExtBrokerInner inner = buildMessage(delayMs, topic, false);
+        transformTimerMessage(timerMessageStore, inner);
+        String firstUniqKey = MessageClientIDSetter.getUniqID(inner);
+        assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(inner).getPutMessageStatus());
+
+        inner = buildMessage(delayMs, topic, false);
+        transformTimerMessage(timerMessageStore, inner);
+        String secondUniqKey = MessageClientIDSetter.getUniqID(inner);
+        assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(inner).getPutMessageStatus());
+
+        MessageExtBrokerInner delMsg = buildMessage(delayMs, "whatever", false);
+        transformTimerMessage(timerMessageStore, delMsg);
+        MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, TimerMessageStore.buildDeleteKey(topic, firstUniqKey));
+        delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties()));
+        assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus());
+
+        delMsg = buildMessage(delayMs, "whatever", false);
+        transformTimerMessage(timerMessageStore, delMsg);
+        MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, TimerMessageStore.buildDeleteKey(collisionTopic, secondUniqKey));
+        delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties()));
+        assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus());
+
+        // The first one should have been deleted, the second one should not be deleted.
+        ByteBuffer msgBuff = getOneMessage(topic, 0, 0, 3000);
+        assertNotNull(msgBuff);
+        MessageExt msgExt = MessageDecoder.decode(msgBuff);
+        assertNotNull(msgExt);
+        assertNotEquals(firstUniqKey, MessageClientIDSetter.getUniqID(msgExt));
+        assertEquals(secondUniqKey, MessageClientIDSetter.getUniqID(msgExt));
+    }
+
     @Test
     public void testPutDeleteTimerMessage() throws Exception {
         String topic = "TimerTest_testPutDeleteTimerMessage";
 
-        final TimerMessageStore timerMessageStore = createTimerMessageStore(null);
+        final TimerMessageStore timerMessageStore = createTimerMessageStore(null , false);
         timerMessageStore.load();
         timerMessageStore.start(true);
 
@@ -372,7 +467,7 @@ public void testStateAndRecover() throws Exception {
         final String topic = "TimerTest_testStateAndRecover";
 
         String base = StoreTestUtils.createBaseDir();
-        final TimerMessageStore first = createTimerMessageStore(base);
+        final TimerMessageStore first = createTimerMessageStore(base , false);
         first.load();
         first.start(true);
 
@@ -417,7 +512,7 @@ public Boolean call() {
         first.getTimerWheel().flush();
         first.shutdown();
 
-        final TimerMessageStore second = createTimerMessageStore(base);
+        final TimerMessageStore second = createTimerMessageStore(base , false);
         second.debug = true;
         assertTrue(second.load());
         assertEquals(msgNum, second.getQueueOffset());
@@ -446,7 +541,7 @@ public Boolean call() {
     public void testMaxDelaySec() throws Exception {
         String topic = "TimerTest_testMaxDelaySec";
 
-        TimerMessageStore first = createTimerMessageStore(null);
+        TimerMessageStore first = createTimerMessageStore(null , false);
         first.load();
         first.start(true);
 
@@ -468,7 +563,7 @@ public void testRollMessage() throws Exception {
         storeConfig.setTimerRollWindowSlot(2);
         String topic = "TimerTest_testRollMessage";
 
-        TimerMessageStore timerMessageStore = createTimerMessageStore(null);
+        TimerMessageStore timerMessageStore = createTimerMessageStore(null , false);
         timerMessageStore.load();
         timerMessageStore.start(true);
 
diff --git a/test/BUILD.bazel b/test/BUILD.bazel
index e6703d69a01..80bd06539e8 100644
--- a/test/BUILD.bazel
+++ b/test/BUILD.bazel
@@ -117,6 +117,8 @@ GenTestRules(
         "src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendIT",
         "src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendWithMQIT",
         "src/test/java/org/apache/rocketmq/test/offset/OffsetNotFoundIT",
+        "src/test/java/org/apache/rocketmq/test/recall/RecallWithTraceIT",
+        "src/test/java/org/apache/rocketmq/test/recall/SendAndRecallDelayMessageIT",
         "src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdIT",
         "src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendWithSelectorIT",
         "src/test/java/org/apache/rocketmq/test/smoke/NormalMessageSendAndRecvIT",
diff --git a/test/pom.xml b/test/pom.xml
index 168cbab0be2..993b883260f 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQBroadCastConsumer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQBroadCastConsumer.java
index 2a596197441..7ac5ec39786 100644
--- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQBroadCastConsumer.java
+++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQBroadCastConsumer.java
@@ -26,8 +26,8 @@ public class RMQBroadCastConsumer extends RMQNormalConsumer {
     private static Logger logger = LoggerFactory.getLogger(RMQBroadCastConsumer.class);
 
     public RMQBroadCastConsumer(String nsAddr, String topic, String subExpression,
-        String consumerGroup, AbstractListener listner) {
-        super(nsAddr, topic, subExpression, consumerGroup, listner);
+        String consumerGroup, AbstractListener listener) {
+        super(nsAddr, topic, subExpression, consumerGroup, listener);
     }
 
     @Override
diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopConsumer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopConsumer.java
index a7046bca7da..67a781aacc2 100644
--- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopConsumer.java
+++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopConsumer.java
@@ -58,7 +58,7 @@ public RMQPopConsumer(String nsAddr, String topic, String subExpression,
     @Override
     public void start() {
         client = ConsumerFactory.getRMQPopClient();
-        log.info(String.format("consumer[%s] started!", consumerGroup));
+        log.info("consumer[{}] started!", consumerGroup);
     }
 
     @Override
diff --git a/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQConsumer.java b/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQConsumer.java
index 5681ecc841a..22193bb4ba9 100644
--- a/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQConsumer.java
+++ b/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQConsumer.java
@@ -69,8 +69,8 @@ public AbstractListener getListener() {
         return listener;
     }
 
-    public void setListener(AbstractListener listner) {
-        this.listener = listner;
+    public void setListener(AbstractListener listener) {
+        this.listener = listener;
     }
 
     public String getNsAddr() {
diff --git a/test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java b/test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java
index d3d5de9e271..276d08d8061 100644
--- a/test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java
+++ b/test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java
@@ -117,8 +117,7 @@ public static boolean createSub(String nameSrvAddr, String clusterName, String c
             for (String addr : masterSet) {
                 try {
                     mqAdminExt.createAndUpdateSubscriptionGroupConfig(addr, config);
-                    log.info(String.format("create subscription group %s to %s success.\n", consumerId,
-                            addr));
+                    log.info("create subscription group {} to {} success.", consumerId, addr);
                 } catch (Exception e) {
                     e.printStackTrace();
                     Thread.sleep(1000 * 1);
@@ -194,7 +193,7 @@ public static boolean checkStaticTopic(String topic, DefaultMQAdminExt defaultMQ
         return  true;
     }
 
-    //should only be test, if some middle operation failed, it dose not backup the brokerConfigMap
+    //should only be test, if some middle operation failed, it does not backup the brokerConfigMap
     public static Map<String, TopicConfigAndQueueMapping> createStaticTopic(String topic, int queueNum, Set<String> targetBrokers, DefaultMQAdminExt defaultMQAdminExt) throws Exception {
         Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
         assert brokerConfigMap.isEmpty();
@@ -204,7 +203,7 @@ public static Map<String, TopicConfigAndQueueMapping> createStaticTopic(String t
         return brokerConfigMap;
     }
 
-    //should only be test, if some middle operation failed, it dose not backup the brokerConfigMap
+    //should only be test, if some middle operation failed, it does not backup the brokerConfigMap
     public static void remappingStaticTopic(String topic, Set<String> targetBrokers, DefaultMQAdminExt defaultMQAdminExt) throws Exception {
         Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt);
         assert !brokerConfigMap.isEmpty();
diff --git a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java
index bdd991a335f..b0a1ee3c6ac 100644
--- a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java
+++ b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java
@@ -39,19 +39,23 @@ public ListDataCollectorImpl(Collection<Object> datas) {
         }
     }
 
+    @Override
     public Collection<Object> getAllData() {
         return datas;
     }
 
+    @Override
     public synchronized void resetData() {
         datas.clear();
         unlockIncrement();
     }
 
+    @Override
     public long getDataSizeWithoutDuplicate() {
         return getAllDataWithoutDuplicate().size();
     }
 
+    @Override
     public synchronized void addData(Object data) {
         if (lock) {
             return;
@@ -59,18 +63,22 @@ public synchronized void addData(Object data) {
         datas.add(data);
     }
 
+    @Override
     public long getDataSize() {
         return datas.size();
     }
 
+    @Override
     public boolean isRepeatedData(Object data) {
         return Collections.frequency(datas, data) == 1;
     }
 
+    @Override
     public synchronized Collection<Object> getAllDataWithoutDuplicate() {
         return new HashSet<Object>(datas);
     }
 
+    @Override
     public int getRepeatedTimeForData(Object data) {
         int res = 0;
         for (Object obj : datas) {
@@ -81,14 +89,17 @@ public int getRepeatedTimeForData(Object data) {
         return res;
     }
 
+    @Override
     public synchronized void removeData(Object data) {
         datas.remove(data);
     }
 
+    @Override
     public void lockIncrement() {
         lock = true;
     }
 
+    @Override
     public void unlockIncrement() {
         lock = false;
     }
diff --git a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/MapDataCollectorImpl.java b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/MapDataCollectorImpl.java
index 899bb855585..7c51af75282 100644
--- a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/MapDataCollectorImpl.java
+++ b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/MapDataCollectorImpl.java
@@ -41,6 +41,7 @@ public MapDataCollectorImpl(Collection<Object> datas) {
         }
     }
 
+    @Override
     public synchronized void addData(Object data) {
         if (lock) {
             return;
@@ -52,6 +53,7 @@ public synchronized void addData(Object data) {
         }
     }
 
+    @Override
     public Collection<Object> getAllData() {
         List<Object> lst = new ArrayList<Object>();
         for (Entry<Object, AtomicInteger> entry : datas.entrySet()) {
@@ -62,15 +64,18 @@ public Collection<Object> getAllData() {
         return lst;
     }
 
+    @Override
     public long getDataSizeWithoutDuplicate() {
         return datas.keySet().size();
     }
 
+    @Override
     public void resetData() {
         datas.clear();
         unlockIncrement();
     }
 
+    @Override
     public long getDataSize() {
         long sum = 0;
         for (AtomicInteger count : datas.values()) {
@@ -79,6 +84,7 @@ public long getDataSize() {
         return sum;
     }
 
+    @Override
     public boolean isRepeatedData(Object data) {
         if (datas.containsKey(data)) {
             return datas.get(data).get() == 1;
@@ -86,10 +92,12 @@ public boolean isRepeatedData(Object data) {
         return false;
     }
 
+    @Override
     public Collection<Object> getAllDataWithoutDuplicate() {
         return datas.keySet();
     }
 
+    @Override
     public int getRepeatedTimeForData(Object data) {
         if (datas.containsKey(data)) {
             return datas.get(data).intValue();
@@ -97,14 +105,17 @@ public int getRepeatedTimeForData(Object data) {
         return 0;
     }
 
+    @Override
     public void removeData(Object data) {
         datas.remove(data);
     }
 
+    @Override
     public void lockIncrement() {
         lock = true;
     }
 
+    @Override
     public void unlockIncrement() {
         lock = false;
     }
diff --git a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java
index d2713919509..472e106ce35 100644
--- a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java
+++ b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java
@@ -100,8 +100,8 @@ public class BaseConf {
             brokerController2.getBrokerConfig().getListenPort());
 
         brokerController3 = IntegrationTestBase.createAndStartBroker(NAMESRV_ADDR);
-        log.debug("Broker {} started, listening: {}", brokerController2.getBrokerConfig().getBrokerName(),
-            brokerController2.getBrokerConfig().getListenPort());
+        log.debug("Broker {} started, listening: {}", brokerController3.getBrokerConfig().getBrokerName(),
+            brokerController3.getBrokerConfig().getListenPort());
 
         CLUSTER_NAME = brokerController1.getBrokerConfig().getBrokerClusterName();
         BROKER1_NAME = brokerController1.getBrokerConfig().getBrokerName();
@@ -287,8 +287,7 @@ public static RMQNormalConsumer getConsumer(String nsAddr, String consumerGroup,
             consumer.setDebug();
         }
         mqClients.add(consumer);
-        log.info(String.format("consumer[%s] start,topic[%s],subExpression[%s]", consumerGroup,
-                topic, subExpression));
+        log.info("consumer[{}] start,topic[{}],subExpression[{}]", consumerGroup, topic, subExpression);
         return consumer;
     }
 
diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java
index 2217936929c..287e54d5617 100644
--- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java
+++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java
@@ -136,6 +136,9 @@ public static BrokerController createAndStartBroker(String nsAddr) {
         brokerConfig.setNamesrvAddr(nsAddr);
         brokerConfig.setEnablePropertyFilter(true);
         brokerConfig.setEnableCalcFilterBitMap(true);
+        brokerConfig.setAppendAckAsync(true);
+        brokerConfig.setAppendCkAsync(true);
+        brokerConfig.setRecallMessageEnable(true);
         storeConfig.setEnableConsumeQueueExt(true);
         brokerConfig.setLoadBalancePollNameServerInterval(500);
         storeConfig.setStorePathRootDir(baseDir);
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java
index 684b718ae5d..7408a092c4b 100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java
@@ -96,6 +96,8 @@ public void test3ConsumerAndCrashOne() {
         MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer1.getListener(),
             consumer2.getListener(), consumer3.getListener());
         consumer3.shutdown();
+        TestUtils.waitForSeconds(WAIT_TIME);
+
         producer.clearMsg();
         consumer1.clearMsg();
         consumer2.clearMsg();
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadcastIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadcastIT.java
index 679fdd4f381..d1e2c0ddaf3 100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadcastIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadcastIT.java
@@ -36,7 +36,7 @@
 import static com.google.common.truth.Truth.assertThat;
 
 /**
- * Currently, dose not support the ordered broadcast message
+ * Currently, does not support the ordered broadcast message
  */
 @Ignore
 public class OrderMsgBroadcastIT extends BaseBroadcast {
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java
index 072159599ff..3a6ad060020 100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java
@@ -30,6 +30,7 @@
 import org.assertj.core.util.Lists;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.Ignore;
 
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
 
@@ -53,17 +54,18 @@ public void setUp() {
     }
 
     @Test
+    @Ignore
     public void testNotification() throws Exception {
         long pollTime = 500;
         CompletableFuture<Boolean> future1 = client.notification(brokerAddr, topic, group, messageQueue.getQueueId(), pollTime, System.currentTimeMillis(), 5000);
         CompletableFuture<Boolean> future2 = client.notification(brokerAddr, topic, group, messageQueue.getQueueId(), pollTime, System.currentTimeMillis(), 5000);
         sendMessage(1);
-        Boolean result1 = future1.get();
-        assertThat(result1).isTrue();
-        client.popMessageAsync(brokerAddr, messageQueue, 10000, 1, group, 1000, false,
-            ConsumeInitMode.MIN, false, null, null);
         Boolean result2 = future2.get();
-        assertThat(result2).isFalse();
+        assertThat(result2).isTrue();
+        client.popMessageAsync(brokerAddr, messageQueue, 10000, 1, group, 1000, false,
+            ConsumeInitMode.MIN, false, null, null).get();
+        Boolean result1 = future1.get();
+        assertThat(result1).isFalse();
     }
 
     @Test
@@ -76,7 +78,7 @@ public void testNotificationOrderly() throws Exception {
         Boolean result1 = future1.get();
         assertThat(result1).isTrue();
         client.popMessageAsync(brokerAddr, messageQueue, 10000, 1, group, 1000, false,
-            ConsumeInitMode.MIN, true, null, null, attemptId);
+            ConsumeInitMode.MIN, true, null, null, attemptId).get();
         Boolean result2 = future2.get();
         assertThat(result2).isTrue();
 
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java
index e2a657f4343..3034876846f 100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java
@@ -64,7 +64,7 @@ public void tearDown() {
     @Test
     public void testNormalPopAck() throws Exception {
         String topic = initTopic();
-        log.info(String.format("use topic: %s; group: %s !", topic, group));
+        log.info("use topic: {}; group: {} !", topic, group);
 
         RMQNormalProducer producer = getProducer(NAMESRV_ADDR, topic);
         producer.getProducer().setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE);
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/producer/batch/BatchSendIT.java b/test/src/test/java/org/apache/rocketmq/test/client/producer/batch/BatchSendIT.java
index a1fdaeafce0..38176c83ede 100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/producer/batch/BatchSendIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/producer/batch/BatchSendIT.java
@@ -92,7 +92,7 @@ public void testBatchSend_ViewMessage() throws Exception {
         Thread.sleep(2000);
 
         for (int i = 0; i < 3; i++) {
-            producer.viewMessage(offsetIds[random.nextInt(batchNum)]);
+            producer.viewMessage(topic, offsetIds[random.nextInt(batchNum)]);
         }
         for (int i = 0; i < 3; i++) {
             producer.viewMessage(topic, msgIds[random.nextInt(batchNum)]);
@@ -240,7 +240,7 @@ public void testBatchSend_CheckProperties() throws Exception {
 
         Thread.sleep(2000);
 
-        Message messageByOffset = producer.viewMessage(offsetIds[0]);
+        Message messageByOffset = producer.viewMessage(topic, offsetIds[0]);
         Message messageByMsgId = producer.viewMessage(topic, msgIds[0]);
 
         Assert.assertEquals(message.getTopic(), messageByMsgId.getTopic());
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdExceptionIT.java b/test/src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdExceptionIT.java
index c9fbee07a13..69cddbca92f 100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdExceptionIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdExceptionIT.java
@@ -56,7 +56,7 @@ public void testQueryMsgByErrorMsgId() {
 
         MessageExt queryMsg = null;
         try {
-            queryMsg = producer.getProducer().viewMessage(errorMsgId);
+            queryMsg = producer.getProducer().viewMessage("topic", errorMsgId);
         } catch (Exception e) {
         }
 
@@ -73,7 +73,7 @@ public void testQueryMsgByNullMsgId() {
 
         MessageExt queryMsg = null;
         try {
-            queryMsg = producer.getProducer().viewMessage(errorMsgId);
+            queryMsg = producer.getProducer().viewMessage("topic", errorMsgId);
         } catch (Exception e) {
         }
 
diff --git a/test/src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdIT.java b/test/src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdIT.java
index 74f1109efee..b14b399164c 100644
--- a/test/src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdIT.java
@@ -66,7 +66,7 @@ public void testQueryMsg() {
         MessageExt queryMsg = null;
         try {
             TestUtils.waitForMoment(3000);
-            queryMsg = producer.getProducer().viewMessage(((MessageClientExt) recvMsg).getOffsetMsgId());
+            queryMsg = producer.getProducer().viewMessage(recvMsg.getTopic(), ((MessageClientExt) recvMsg).getOffsetMsgId());
         } catch (Exception e) {
         }
 
diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java
index 33c3aa2fb89..b754466a916 100644
--- a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java
@@ -30,6 +30,7 @@
 import org.junit.Before;
 import org.junit.FixMethodOrder;
 import org.junit.Test;
+import org.junit.Ignore;
 import org.junit.runners.MethodSorters;
 
 import static org.awaitility.Awaitility.await;
@@ -87,10 +88,16 @@ public void testTransactionCheckThenCommit() {
     }
 
     @Test
+    @Ignore
     public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception {
         super.testSimpleConsumerSendAndRecvDelayMessage();
     }
 
+    @Test
+    public void testSimpleConsumerSendAndRecallDelayMessage() throws Exception {
+        super.testSimpleConsumerSendAndRecallDelayMessage();
+    }
+
     @Test
     public void testSimpleConsumerSendAndRecvBigMessage() throws Exception {
         super.testSimpleConsumerSendAndRecvBigMessage();
diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java
index 5d8aec822a1..534108c2805 100644
--- a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java
@@ -41,6 +41,8 @@
 import apache.rocketmq.v2.QueryAssignmentResponse;
 import apache.rocketmq.v2.QueryRouteRequest;
 import apache.rocketmq.v2.QueryRouteResponse;
+import apache.rocketmq.v2.RecallMessageRequest;
+import apache.rocketmq.v2.RecallMessageResponse;
 import apache.rocketmq.v2.ReceiveMessageRequest;
 import apache.rocketmq.v2.ReceiveMessageResponse;
 import apache.rocketmq.v2.RecoverOrphanedTransactionCommand;
@@ -100,7 +102,7 @@
 import org.apache.rocketmq.proxy.config.ConfigurationManager;
 import org.apache.rocketmq.proxy.grpc.interceptor.ContextInterceptor;
 import org.apache.rocketmq.proxy.grpc.interceptor.HeaderInterceptor;
-import org.apache.rocketmq.proxy.grpc.interceptor.InterceptorConstants;
+import org.apache.rocketmq.common.constant.GrpcConstants;
 import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder;
 import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
 import org.apache.rocketmq.test.base.BaseConf;
@@ -137,8 +139,8 @@ public void setUp() throws Exception {
         brokerController2.getBrokerConfig().setTransactionCheckInterval(3 * 1000);
         brokerController3.getBrokerConfig().setTransactionCheckInterval(3 * 1000);
 
-        header.put(InterceptorConstants.CLIENT_ID, "client-id" + UUID.randomUUID());
-        header.put(InterceptorConstants.LANGUAGE, "JAVA");
+        header.put(GrpcConstants.CLIENT_ID, "client-id" + UUID.randomUUID());
+        header.put(GrpcConstants.LANGUAGE, "JAVA");
 
         String mockProxyHome = "/mock/rmq/proxy/home";
         URL mockProxyHomeURL = getClass().getClassLoader().getResource("rmq-proxy-home");
@@ -393,6 +395,69 @@ public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception {
         assertThat(Math.abs(recvTime.get() - sendTime - delayTime) < 2 * 1000).isTrue();
     }
 
+    public void testSimpleConsumerSendAndRecallDelayMessage() throws Exception {
+        String topic = initTopicOnSampleTopicBroker(BROKER1_NAME, TopicMessageType.DELAY);
+        String group = MQRandomUtils.getRandomConsumerGroup();
+        long delayTime = TimeUnit.SECONDS.toMillis(5);
+
+        // init consumer offset
+        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();
+        receiveMessage(blockingStub, topic, group, 1);
+
+        this.sendClientSettings(stub, buildProducerClientSettings(topic)).get();
+        String messageId = createUniqID();
+        SendMessageResponse sendResponse = blockingStub.sendMessage(SendMessageRequest.newBuilder()
+            .addMessages(Message.newBuilder()
+                .setTopic(Resource.newBuilder()
+                    .setName(topic)
+                    .build())
+                .setSystemProperties(SystemProperties.newBuilder()
+                    .setMessageId(messageId)
+                    .setQueueId(0)
+                    .setMessageType(MessageType.DELAY)
+                    .setBodyEncoding(Encoding.GZIP)
+                    .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis()))
+                    .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), "127.0.0.1:1234"))
+                    .setDeliveryTimestamp(Timestamps.fromMillis(System.currentTimeMillis() + delayTime))
+                    .build())
+                .setBody(ByteString.copyFromUtf8("hello"))
+                .build())
+            .build());
+        long sendTime = System.currentTimeMillis();
+        assertSendMessage(sendResponse, messageId);
+        String recallHandle = sendResponse.getEntries(0).getRecallHandle();
+        assertThat(recallHandle).isNotEmpty();
+
+        RecallMessageRequest recallRequest = RecallMessageRequest.newBuilder()
+            .setRecallHandle(recallHandle)
+            .setTopic(Resource.newBuilder().setResourceNamespace("").setName(topic).build())
+            .build();
+        RecallMessageResponse recallResponse =
+            blockingStub.withDeadlineAfter(2, TimeUnit.SECONDS).recallMessage(recallRequest);
+        assertThat(recallResponse.getStatus()).isEqualTo(
+            ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name()));
+        assertThat(recallResponse.getMessageId()).isEqualTo(messageId);
+
+        this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();
+
+        AtomicLong recvTime = new AtomicLong();
+        AtomicReference<Message> recvMessage = new AtomicReference<>();
+        try {
+            await().atMost(java.time.Duration.ofSeconds(10)).until(() -> {
+                List<Message> messageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group));
+                if (messageList.isEmpty()) {
+                    return false;
+                }
+                recvTime.set(System.currentTimeMillis());
+                recvMessage.set(messageList.get(0));
+                return messageList.get(0).getSystemProperties().getMessageId().equals(messageId);
+            });
+        } catch (Exception e) {
+        }
+        assertThat(recvTime.get()).isEqualTo(0L);
+        assertThat(recvMessage.get()).isNull();
+    }
+
     public void testSimpleConsumerSendAndRecvBigMessage() throws Exception {
         String topic = initTopicOnSampleTopicBroker(BROKER1_NAME);
         String group = MQRandomUtils.getRandomConsumerGroup();
@@ -427,6 +492,7 @@ public void testSimpleConsumerSendAndRecv() throws Exception {
         String messageId = createUniqID();
         SendMessageResponse sendResponse = blockingStub.sendMessage(buildSendMessageRequest(topic, messageId));
         assertSendMessage(sendResponse, messageId);
+        assertThat(sendResponse.getEntries(0).getRecallHandle()).isNullOrEmpty();
 
         this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get();
 
diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java
index 7f837adebe5..5dd06f53420 100644
--- a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java
@@ -26,6 +26,7 @@
 import org.junit.Before;
 import org.junit.FixMethodOrder;
 import org.junit.Test;
+import org.junit.Ignore;
 import org.junit.runners.MethodSorters;
 
 @FixMethodOrder(value = MethodSorters.NAME_ASCENDING)
@@ -75,10 +76,16 @@ public void testTransactionCheckThenCommit() {
     }
 
     @Test
+    @Ignore
     public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception {
         super.testSimpleConsumerSendAndRecvDelayMessage();
     }
 
+    @Test
+    public void testSimpleConsumerSendAndRecallDelayMessage() throws Exception {
+        super.testSimpleConsumerSendAndRecallDelayMessage();
+    }
+
     @Test
     public void testSimpleConsumerSendAndRecvBigMessage() throws Exception {
         super.testSimpleConsumerSendAndRecvBigMessage();
diff --git a/test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java b/test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java
index ad521440e9e..bfed96e8cdf 100644
--- a/test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java
@@ -36,6 +36,7 @@
 import org.apache.rocketmq.remoting.protocol.filter.FilterAPI;
 import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
 import org.apache.rocketmq.store.DefaultMessageFilter;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
 import org.apache.rocketmq.test.base.BaseConf;
 import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;
 import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
@@ -80,7 +81,7 @@ public void tearDown() {
         shutdown();
     }
 
-    private Pair<Long, Long> getLag(List<MessageQueue> mqs) {
+    private Pair<Long, Long> getLag(List<MessageQueue> mqs) throws ConsumeQueueException {
         long lag = 0;
         long pullLag = 0;
         for (BrokerController controller : brokerControllerList) {
@@ -120,7 +121,7 @@ public void waitForFullyDispatched() {
     }
 
     @Test
-    public void testCalculateLag() {
+    public void testCalculateLag() throws ConsumeQueueException {
         int msgSize = 10;
         List<MessageQueue> mqs = producer.getMessageQueue();
         MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize);
diff --git a/test/src/test/java/org/apache/rocketmq/test/recall/RecallWithTraceIT.java b/test/src/test/java/org/apache/rocketmq/test/recall/RecallWithTraceIT.java
new file mode 100644
index 00000000000..d52c7002548
--- /dev/null
+++ b/test/src/test/java/org/apache/rocketmq/test/recall/RecallWithTraceIT.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.test.recall;
+
+import org.apache.rocketmq.client.consumer.PopResult;
+import org.apache.rocketmq.client.consumer.PopStatus;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.trace.TraceContext;
+import org.apache.rocketmq.client.trace.TraceDataEncoder;
+import org.apache.rocketmq.client.trace.TraceType;
+import org.apache.rocketmq.common.attribute.CQType;
+import org.apache.rocketmq.common.attribute.TopicMessageType;
+import org.apache.rocketmq.common.message.MessageClientIDSetter;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.test.base.BaseConf;
+import org.apache.rocketmq.test.base.IntegrationTestBase;
+import org.apache.rocketmq.test.client.rmq.RMQPopConsumer;
+import org.apache.rocketmq.test.factory.ConsumerFactory;
+import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
+import org.apache.rocketmq.test.util.MQRandomUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.awaitility.Awaitility.await;
+
+public class RecallWithTraceIT extends BaseConf {
+    private static String topic;
+    private static String group;
+    private static DefaultMQProducer producer;
+    private static RMQPopConsumer popConsumer;
+
+    @BeforeClass
+    public static void init() throws MQClientException {
+        System.setProperty("com.rocketmq.recall.default.trace.enable", Boolean.TRUE.toString());
+        topic = MQRandomUtils.getRandomTopic();
+        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.NORMAL);
+        group = initConsumerGroup();
+        producer = new DefaultMQProducer(group, true, topic);
+        producer.setNamesrvAddr(NAMESRV_ADDR);
+        producer.start();
+        popConsumer = ConsumerFactory.getRMQPopConsumer(NAMESRV_ADDR, group, topic, "*", new RMQNormalListener());
+        mqClients.add(popConsumer);
+        mqClients.add(producer);
+    }
+
+    @AfterClass
+    public static void tearDown() {
+        shutdown();
+    }
+
+    @Test
+    public void testRecallTrace() throws MQBrokerException, RemotingException, InterruptedException, MQClientException {
+        String msgId = MessageClientIDSetter.createUniqID();
+        String recallHandle = RecallMessageHandle.HandleV1.buildHandle(topic, BROKER1_NAME,
+            String.valueOf(System.currentTimeMillis() + 30000), msgId);
+        producer.recallMessage(topic, recallHandle);
+
+        MessageQueue messageQueue = new MessageQueue(topic, BROKER1_NAME, 0);
+        String brokerAddress = brokerController1.getBrokerAddr();
+        AtomicReference<MessageExt> traceMessage = new AtomicReference();
+        await()
+            .pollInterval(1, TimeUnit.SECONDS)
+            .atMost(15, TimeUnit.SECONDS)
+            .until(() -> {
+                PopResult popResult = popConsumer.pop(brokerAddress, messageQueue, 60 * 1000, -1);
+                boolean found = popResult.getPopStatus().equals(PopStatus.FOUND);
+                traceMessage.set(found ? popResult.getMsgFoundList().get(0) : null);
+                return found;
+            });
+
+        Assert.assertNotNull(traceMessage.get());
+        TraceContext context =
+            TraceDataEncoder.decoderFromTraceDataString(new String(traceMessage.get().getBody())).get(0);
+        Assert.assertEquals(TraceType.Recall, context.getTraceType());
+        Assert.assertEquals(group, context.getGroupName());
+        Assert.assertTrue(context.isSuccess());
+        Assert.assertEquals(msgId, context.getTraceBeans().get(0).getMsgId());
+    }
+}
diff --git a/test/src/test/java/org/apache/rocketmq/test/recall/SendAndRecallDelayMessageIT.java b/test/src/test/java/org/apache/rocketmq/test/recall/SendAndRecallDelayMessageIT.java
new file mode 100644
index 00000000000..2fb9e023712
--- /dev/null
+++ b/test/src/test/java/org/apache/rocketmq/test/recall/SendAndRecallDelayMessageIT.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.test.recall;
+
+import org.apache.rocketmq.client.consumer.PopResult;
+import org.apache.rocketmq.client.consumer.PopStatus;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.attribute.CQType;
+import org.apache.rocketmq.common.attribute.TopicMessageType;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
+import org.apache.rocketmq.test.base.BaseConf;
+import org.apache.rocketmq.test.base.IntegrationTestBase;
+import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
+import org.apache.rocketmq.test.client.rmq.RMQPopConsumer;
+import org.apache.rocketmq.test.factory.ConsumerFactory;
+import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener;
+import org.apache.rocketmq.test.util.MQRandomUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.awaitility.Awaitility.await;
+
+public class SendAndRecallDelayMessageIT extends BaseConf {
+
+    private static String initTopic;
+    private static String consumerGroup;
+    private static RMQNormalProducer producer;
+    private static RMQPopConsumer popConsumer;
+
+    @BeforeClass
+    public static void init() {
+        initTopic = initTopic();
+        consumerGroup = initConsumerGroup();
+        producer = getProducer(NAMESRV_ADDR, initTopic);
+        popConsumer = ConsumerFactory.getRMQPopConsumer(NAMESRV_ADDR, consumerGroup, initTopic, "*", new RMQNormalListener());
+        mqClients.add(popConsumer);
+    }
+
+    @AfterClass
+    public static void tearDown() {
+        shutdown();
+    }
+
+    @Test
+    public void testSendAndRecv() throws Exception {
+        int delaySecond = 1;
+        String topic = MQRandomUtils.getRandomTopic();
+        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.DELAY);
+        MessageQueue messageQueue = new MessageQueue(topic, BROKER1_NAME, 0);
+        String brokerAddress = brokerController1.getBrokerAddr();
+
+        List<Message> sendList = buildSendMessageList(topic, delaySecond);
+        List<Message> recvList = new ArrayList<>();
+
+        for (Message message : sendList) {
+            producer.getProducer().send(message);
+        }
+
+        await()
+            .pollInterval(1, TimeUnit.SECONDS)
+            .atMost(delaySecond + 15, TimeUnit.SECONDS)
+            .until(() -> {
+                PopResult popResult = popConsumer.pop(brokerAddress, messageQueue, 60 * 1000, -1);
+                processPopResult(recvList, popResult);
+                return recvList.size() == sendList.size();
+            });
+    }
+
+    @Test
+    public void testSendAndRecall() throws Exception {
+        int delaySecond = 5;
+        String topic = MQRandomUtils.getRandomTopic();
+        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.DELAY);
+        MessageQueue messageQueue = new MessageQueue(topic, BROKER1_NAME, 0);
+        String brokerAddress = brokerController1.getBrokerAddr();
+
+        List<Message> sendList = buildSendMessageList(topic, delaySecond);
+        List<Message> recvList = new ArrayList<>();
+        int recallCount = 0;
+
+        for (Message message : sendList) {
+            SendResult sendResult = producer.getProducer().send(message);
+            if (sendResult.getRecallHandle() != null) {
+                String messageId = producer.getProducer().recallMessage(topic, sendResult.getRecallHandle());
+                assertEquals(sendResult.getMsgId(), messageId);
+                recallCount += 1;
+            }
+        }
+        assertEquals(sendList.size() - 2, recallCount); // one normal and one delay-level message
+        try {
+            await()
+                .pollInterval(1, TimeUnit.SECONDS)
+                .atMost(delaySecond + 15, TimeUnit.SECONDS)
+                .until(() -> {
+                    PopResult popResult = popConsumer.pop(brokerAddress, messageQueue, 60 * 1000, -1);
+                    processPopResult(recvList, popResult);
+                    return recvList.size() == sendList.size();
+                });
+        } catch (Exception e) {
+        }
+        assertEquals(sendList.size() - recallCount, recvList.size());
+    }
+
+    @Test
+    public void testSendAndRecall_ukCollision() throws Exception {
+        int delaySecond = 5;
+        String topic = MQRandomUtils.getRandomTopic();
+        String collisionTopic = MQRandomUtils.getRandomTopic();
+        IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.DELAY);
+        IntegrationTestBase.initTopic(collisionTopic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.DELAY);
+        MessageQueue messageQueue = new MessageQueue(topic, BROKER1_NAME, 0);
+        String brokerAddress = brokerController1.getBrokerAddr();
+
+        List<Message> sendList = buildSendMessageList(topic, delaySecond);
+        List<Message> recvList = new ArrayList<>();
+        int recallCount = 0;
+
+        for (Message message : sendList) {
+            SendResult sendResult = producer.getProducer().send(message);
+            if (sendResult.getRecallHandle() != null) {
+                RecallMessageHandle.HandleV1 handleEntity =
+                    (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(sendResult.getRecallHandle());
+                String collisionHandle = RecallMessageHandle.HandleV1.buildHandle(collisionTopic,
+                    handleEntity.getBrokerName(), handleEntity.getTimestampStr(), handleEntity.getMessageId());
+                String messageId = producer.getProducer().recallMessage(collisionTopic, collisionHandle);
+                assertEquals(sendResult.getMsgId(), messageId);
+                recallCount += 1;
+            }
+        }
+        assertEquals(sendList.size() - 2, recallCount); // one normal and one delay-level message
+
+        try {
+            await()
+                .pollInterval(1, TimeUnit.SECONDS)
+                .atMost(delaySecond + 15, TimeUnit.SECONDS)
+                .until(() -> {
+                    PopResult popResult = popConsumer.pop(brokerAddress, messageQueue, 60 * 1000, -1);
+                    processPopResult(recvList, popResult);
+                    return recvList.size() == sendList.size();
+                });
+        } catch (Exception e) {
+        }
+        assertEquals(sendList.size(), recvList.size());
+    }
+
+    private void processPopResult(List<Message> recvList, PopResult popResult) {
+        if (popResult.getPopStatus() == PopStatus.FOUND && popResult.getMsgFoundList() != null) {
+            recvList.addAll(popResult.getMsgFoundList());
+        }
+    }
+
+    private List<Message> buildSendMessageList(String topic, int delaySecond) {
+        Message msg0 = new Message(topic, "tag", "Hello RocketMQ".getBytes()); // not supported
+
+        Message msg1 = new Message(topic, "tag", "Hello RocketMQ".getBytes()); // not supported
+        msg1.setDelayTimeLevel(2);
+
+        Message msg2 = new Message(topic, "tag", "Hello RocketMQ".getBytes());
+        msg2.setDelayTimeMs(delaySecond * 1000L);
+
+        Message msg3 = new Message(topic, "tag", "Hello RocketMQ".getBytes());
+        msg3.setDelayTimeSec(delaySecond);
+
+        Message msg4 = new Message(topic, "tag", "Hello RocketMQ".getBytes());
+        msg4.setDeliverTimeMs(System.currentTimeMillis() + delaySecond * 1000L);
+
+        return Arrays.asList(msg0, msg1, msg2, msg3, msg4);
+    }
+}
diff --git a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java
index 9004b91db39..9e9afb1ed2c 100644
--- a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java
+++ b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java
@@ -17,13 +17,16 @@
 
 package org.apache.rocketmq.test.route;
 
+import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
 import org.apache.rocketmq.test.base.BaseConf;
 import org.apache.rocketmq.test.util.MQAdminTestUtils;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
 
 public class CreateAndUpdateTopicIT extends BaseConf {
 
@@ -47,6 +50,8 @@ public void testCreateOrUpdateTopic_EnableSingleTopicRegistration() {
 
     }
 
+    // Temporarily ignore the fact that this test cannot pass in the integration test pipeline due to unknown reasons
+    @Ignore
     @Test
     public void testDeleteTopicFromNameSrvWithBrokerRegistration() {
         namesrvController.getNamesrvConfig().setDeleteTopicWithBrokerRegistration(true);
@@ -60,11 +65,9 @@ public void testDeleteTopicFromNameSrvWithBrokerRegistration() {
         boolean createResult = MQAdminTestUtils.createTopic(NAMESRV_ADDR, CLUSTER_NAME, testTopic1, 8, null);
         assertThat(createResult).isTrue();
 
-
         createResult = MQAdminTestUtils.createTopic(NAMESRV_ADDR, CLUSTER_NAME, testTopic2, 8, null);
         assertThat(createResult).isTrue();
 
-
         TopicRouteData route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic2);
         assertThat(route.getBrokerDatas()).hasSize(3);
 
@@ -73,11 +76,13 @@ public void testDeleteTopicFromNameSrvWithBrokerRegistration() {
         // Deletion is lazy, trigger broker registration
         brokerController1.registerBrokerAll(false, false, true);
 
-        // The route info of testTopic2 will be removed from broker1 after the registration
-        route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic2);
-        assertThat(route.getBrokerDatas()).hasSize(2);
-        assertThat(route.getQueueDatas().get(0).getBrokerName()).isEqualTo(BROKER2_NAME);
-        assertThat(route.getQueueDatas().get(1).getBrokerName()).isEqualTo(BROKER3_NAME);
+        await().atMost(10, TimeUnit.SECONDS).until(() -> {
+            // The route info of testTopic2 will be removed from broker1 after the registration
+            TopicRouteData finalRoute = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic2);
+            return finalRoute.getBrokerDatas().size() == 2
+                && finalRoute.getQueueDatas().get(0).getBrokerName().equals(BROKER2_NAME)
+                && finalRoute.getQueueDatas().get(1).getBrokerName().equals(BROKER3_NAME);
+        });
 
         brokerController1.getBrokerConfig().setEnableSingleTopicRegister(false);
         brokerController2.getBrokerConfig().setEnableSingleTopicRegister(false);
diff --git a/test/src/test/resources/schema/api/tools.admin.DefaultMQAdminExt.schema b/test/src/test/resources/schema/api/tools.admin.DefaultMQAdminExt.schema
index a9095049c40..026c1975dfe 100644
--- a/test/src/test/resources/schema/api/tools.admin.DefaultMQAdminExt.schema
+++ b/test/src/test/resources/schema/api/tools.admin.DefaultMQAdminExt.schema
@@ -51,7 +51,7 @@ Method cloneGroupOffset(boolean,java.lang.String,java.lang.String,java.lang.Stri
 Method consumeMessageDirectly(java.lang.String,java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult)
 Method consumeMessageDirectly(java.lang.String,java.lang.String,java.lang.String,java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult)
 Method createAndUpdateKvConfig(java.lang.String,java.lang.String,java.lang.String) : public throws (void)
-Method createAndUpdatePlainAccessConfig(java.lang.String,org.apache.rocketmq.common.PlainAccessConfig) : public throws (void)
+Method createAndUpdatePlainAccessConfig(java.lang.String,org.apache.rocketmq.auth.migration.plain.PlainAccessConfig) : public throws (void)
 Method createAndUpdateSubscriptionGroupConfig(java.lang.String,org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig) : public throws (void)
 Method createAndUpdateTopicConfig(java.lang.String,org.apache.rocketmq.common.TopicConfig) : public throws (void)
 Method createOrUpdateOrderConf(boolean,java.lang.String,java.lang.String) : public throws (void)
@@ -66,7 +66,7 @@ Method deleteSubscriptionGroup(java.lang.String,java.lang.String) : public throw
 Method deleteTopicInBroker(java.lang.String,java.util.Set) : public throws (void)
 Method deleteTopicInNameServer(java.lang.String,java.lang.String,java.util.Set) : public throws (void)
 Method earliestMsgStoreTime(org.apache.rocketmq.common.message.MessageQueue) : public throws (long)
-Method examineBrokerClusterAclConfig(java.lang.String) : public throws (org.apache.rocketmq.common.AclConfig)
+Method examineBrokerClusterAclConfig(java.lang.String) : public throws (org.apache.rocketmq.auth.migration.plain.AclConfig)
 Method examineBrokerClusterAclVersionInfo(java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo)
 Method examineBrokerClusterInfo() : public throws (org.apache.rocketmq.remoting.protocol.body.ClusterInfo)
 Method examineConsumeStats(java.lang.String) : public throws (org.apache.rocketmq.remoting.protocol.admin.ConsumeStats)
diff --git a/tieredstore/BUILD.bazel b/tieredstore/BUILD.bazel
index e16fca90d07..8822280ff87 100644
--- a/tieredstore/BUILD.bazel
+++ b/tieredstore/BUILD.bazel
@@ -42,6 +42,7 @@ java_library(
         "@maven//:com_alibaba_fastjson",
         "@maven//:org_apache_rocketmq_rocketmq_rocksdb",
         "@maven//:commons_collections_commons_collections",
+        "@maven//:org_slf4j_slf4j_api",
     ],
 )
 
@@ -68,8 +69,9 @@ java_library(
         "@maven//:org_apache_commons_commons_lang3",
         "@maven//:com_google_guava_guava",
         "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
-        "@maven//:io_github_aliyunmq_rocketmq_shaded_slf4j_api_bridge",
+        "@maven//:io_github_aliyunmq_rocketmq_logback_classic",
         "@maven//:net_java_dev_jna_jna",
+        "@maven//:org_slf4j_slf4j_api",
     ],
 )
 
diff --git a/tieredstore/README.md b/tieredstore/README.md
index 9c8ea6b8aa3..41e7458a2a6 100644
--- a/tieredstore/README.md
+++ b/tieredstore/README.md
@@ -13,26 +13,26 @@ This article is a cookbook for RocketMQ tiered storage.
 Use the following steps to easily use tiered storage
 
 1. Change `messageStorePlugIn` to `org.apache.rocketmq.tieredstore.TieredMessageStore` in your `broker.conf`.
-2. Configure your backend service provider. change `tieredBackendServiceProvider` to your storage medium implement. We give a default implement: POSIX provider, and you need to change `tieredStoreFilepath` to the mount point of storage medium for tiered storage.
+2. Configure your backend service provider. change `tieredBackendServiceProvider` to your storage medium implement. We give a default implement: POSIX provider, and you need to change `tieredStoreFilePath` to the mount point of storage medium for tiered storage.
 3. Start the broker and enjoy!
 
 ## Configuration
 
 The following are some core configurations, for more details, see [TieredMessageStoreConfig](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredMessageStoreConfig.java)
 
-| Configuration                   | Default value                                                   | Unit        | Function                                                                        |
-| ------------------------------- | --------------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------- |
-| messageStorePlugIn              |                                                                 |             | Set to org.apache.rocketmq.tieredstore.TieredMessageStore to use tiered storage |
-| tieredMetadataServiceProvider   | org.apache.rocketmq.tieredstore.metadata.TieredMetadataManager  |             | Select your metadata provider                                                   |
-| tieredBackendServiceProvider    | org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment |             | Select your backend service provider                                            |
-| tieredStoreFilepath             |                                                                 |             | Select the directory using for tiered storage, only for POSIX provider.         |
-| tieredStorageLevel              | NOT_IN_DISK                                                     |             | The options are DISABLE, NOT_IN_DISK, NOT_IN_MEM, FORCE                         |
-| tieredStoreFileReservedTime     | 72                                                              | hour        | Default topic TTL in tiered storage                                             |
-| tieredStoreGroupCommitCount     | 2500                                                            |             | The number of messages that trigger one batch transfer                          |
-| tieredStoreGroupCommitSize      | 33554432                                                        | byte        | The size of messages that trigger one batch transfer, 32M by default            |
-| tieredStoreMaxGroupCommitCount  | 10000                                                           |             | The maximum number of messages waiting to be transfered per queue               |
-| readAheadCacheExpireDuration    | 1000                                                            | millisecond | Read-ahead cache expiration time                                                |
-| readAheadCacheSizeThresholdRate | 0.3                                                             |             | The maximum heap space occupied by the read-ahead cache                         |
+| Configuration                   | Default value                                                 | Unit        | Function                                                                        |
+| ------------------------------- |---------------------------------------------------------------| ----------- | ------------------------------------------------------------------------------- |
+| messageStorePlugIn              |                                                               |             | Set to org.apache.rocketmq.tieredstore.TieredMessageStore to use tiered storage |
+| tieredMetadataServiceProvider   | org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore |             | Select your metadata provider                                                   |
+| tieredBackendServiceProvider    | org.apache.rocketmq.tieredstore.provider.PosixFileSegment     |             | Select your backend service provider                                            |
+| tieredStoreFilePath             |                                                               |             | Select the directory using for tiered storage, only for POSIX provider.         |
+| tieredStorageLevel              | NOT_IN_DISK                                                   |             | The options are DISABLE, NOT_IN_DISK, NOT_IN_MEM, FORCE                         |
+| tieredStoreFileReservedTime     | 72                                                            | hour        | Default topic TTL in tiered storage                                             |
+| tieredStoreGroupCommitCount     | 2500                                                          |             | The number of messages that trigger one batch transfer                          |
+| tieredStoreGroupCommitSize      | 33554432                                                      | byte        | The size of messages that trigger one batch transfer, 32M by default            |
+| tieredStoreMaxGroupCommitCount  | 10000                                                         |             | The maximum number of messages waiting to be transfered per queue               |
+| readAheadCacheExpireDuration    | 1000                                                          | millisecond | Read-ahead cache expiration time                                                |
+| readAheadCacheSizeThresholdRate | 0.3                                                           |             | The maximum heap space occupied by the read-ahead cache                         |
 
 ## Metrics
 
@@ -57,8 +57,8 @@ Tiered storage provides some useful metrics, see [RIP-46](https://github.com/apa
 
 ## How to contribute
 
-We need community participation to add more backend service providers for tiered storage. [PosixFileSegment](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java), the implementation provided by default is just an example. People who want to contribute can follow it to implement their own providers, such as S3FileSegment, OSSFileSegment, and MinIOFileSegment. Here are some guidelines:
+We need community participation to add more backend service providers for tiered storage. [PosixFileSegment](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java), the implementation provided by default is just an example. People who want to contribute can follow it to implement their own providers, such as S3FileSegment, OSSFileSegment, and MinIOFileSegment. Here are some guidelines:
 
-1. Extend [TieredFileSegment](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java) and implement the methods of [TieredStoreProvider](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java) interface.
+1. Extend [FileSegment](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegment.java) and implement the methods of [FileSegmentProvider](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentProvider.java) interface.
 2. Record metrics where appropriate. See `rocketmq_tiered_store_provider_rpc_latency`, `rocketmq_tiered_store_provider_upload_bytes`, and `rocketmq_tiered_store_provider_download_bytes`
 3. No need to maintain your own cache and avoid polluting the page cache. It is already having the read-ahead cache.
diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml
index 9f2a8bf2283..5692b41632a 100644
--- a/tieredstore/pom.xml
+++ b/tieredstore/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredMessageStoreConfig.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreConfig.java
similarity index 80%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredMessageStoreConfig.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreConfig.java
index b0750e55094..10667566aa0 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredMessageStoreConfig.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreConfig.java
@@ -14,13 +14,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.tieredstore.common;
+package org.apache.rocketmq.tieredstore;
 
 import java.io.File;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.time.Duration;
+
+public class MessageStoreConfig {
 
-public class TieredMessageStoreConfig {
     private String brokerName = localHostName();
     private String brokerClusterName = "DefaultCluster";
     private TieredStorageLevel tieredStorageLevel = TieredStorageLevel.NOT_IN_DISK;
@@ -58,6 +60,7 @@ public int getValue() {
             return value;
         }
 
+        @SuppressWarnings("DuplicatedCode")
         public static TieredStorageLevel valueOf(int value) {
             switch (value) {
                 case 1:
@@ -90,40 +93,41 @@ public boolean check(TieredStorageLevel targetLevel) {
     private long tieredStoreConsumeQueueMaxSize = 100 * 1024 * 1024;
     private int tieredStoreIndexFileMaxHashSlotNum = 5000000;
     private int tieredStoreIndexFileMaxIndexNum = 5000000 * 4;
-    // index file will force rolling to next file after idle specified time, default is 3h
-    private int tieredStoreIndexFileRollingIdleInterval = 3 * 60 * 60 * 1000;
-    private String tieredMetadataServiceProvider = "org.apache.rocketmq.tieredstore.metadata.TieredMetadataManager";
-    private String tieredBackendServiceProvider = "org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment";
+
+    private String tieredMetadataServiceProvider = "org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore";
+    private String tieredBackendServiceProvider = "org.apache.rocketmq.tieredstore.provider.MemoryFileSegment";
+
     // file reserved time, default is 72 hour
+    private boolean tieredStoreDeleteFileEnable = true;
     private int tieredStoreFileReservedTime = 72;
+    private long tieredStoreDeleteFileInterval = Duration.ofHours(1).toMillis();
+
     // time of forcing commitLog to roll to next file, default is 24 hour
     private int commitLogRollingInterval = 24;
-    // rolling will only happen if file segment size is larger than commitLogRollingMinimumSize, default is 128M
-    private int commitLogRollingMinimumSize = 128 * 1024 * 1024;
-    // default is 100, unit is millisecond
-    private int maxCommitJitter = 100;
-    // Cached message count larger than this value will trigger async commit. default is 1000
-    private int tieredStoreGroupCommitCount = 2500;
-    // Cached message size larger than this value will trigger async commit. default is 32M
-    private int tieredStoreGroupCommitSize = 32 * 1024 * 1024;
-    // Cached message count larger than this value will suspend append. default is 2000
+    private int commitLogRollingMinimumSize = 16 * 1024 * 1024;
+
+    private boolean tieredStoreGroupCommit = true;
+    private int tieredStoreGroupCommitTimeout = 30 * 1000;
+    // Cached message count larger than this value will trigger async commit. default is 4096
+    private int tieredStoreGroupCommitCount = 4 * 1024;
+    // Cached message size larger than this value will trigger async commit. default is 4M
+    private int tieredStoreGroupCommitSize = 4 * 1024 * 1024;
+    // Cached message count larger than this value will suspend append. default is 10000
     private int tieredStoreMaxGroupCommitCount = 10000;
-    private int readAheadMinFactor  = 2;
-    private int readAheadMaxFactor = 24;
-    private int readAheadBatchSizeFactorThreshold = 8;
-    private int readAheadMessageCountThreshold = 2048;
-    private int readAheadMessageSizeThreshold = 128 * 1024 * 1024;
-    private long readAheadCacheExpireDuration = 10 * 1000;
+
+    private boolean readAheadCacheEnable = true;
+    private int readAheadMessageCountThreshold = 4096;
+    private int readAheadMessageSizeThreshold = 16 * 1024 * 1024;
+    private long readAheadCacheExpireDuration = 15 * 1000;
     private double readAheadCacheSizeThresholdRate = 0.3;
 
-    private String tieredStoreFilePath = "";
+    private int tieredStoreMaxPendingLimit = 10000;
+    private boolean tieredStoreCrcCheckEnable = false;
 
+    private String tieredStoreFilePath = "";
     private String objectStoreEndpoint = "";
-
     private String objectStoreBucket = "";
-
     private String objectStoreAccessKey = "";
-
     private String objectStoreSecretKey = "";
 
     public static String localHostName() {
@@ -223,14 +227,6 @@ public void setTieredStoreIndexFileMaxIndexNum(int tieredStoreIndexFileMaxIndexN
         this.tieredStoreIndexFileMaxIndexNum = tieredStoreIndexFileMaxIndexNum;
     }
 
-    public int getTieredStoreIndexFileRollingIdleInterval() {
-        return tieredStoreIndexFileRollingIdleInterval;
-    }
-
-    public void setTieredStoreIndexFileRollingIdleInterval(int tieredStoreIndexFileRollingIdleInterval) {
-        this.tieredStoreIndexFileRollingIdleInterval = tieredStoreIndexFileRollingIdleInterval;
-    }
-
     public String getTieredMetadataServiceProvider() {
         return tieredMetadataServiceProvider;
     }
@@ -247,6 +243,14 @@ public void setTieredBackendServiceProvider(String tieredBackendServiceProvider)
         this.tieredBackendServiceProvider = tieredBackendServiceProvider;
     }
 
+    public boolean isTieredStoreDeleteFileEnable() {
+        return tieredStoreDeleteFileEnable;
+    }
+
+    public void setTieredStoreDeleteFileEnable(boolean tieredStoreDeleteFileEnable) {
+        this.tieredStoreDeleteFileEnable = tieredStoreDeleteFileEnable;
+    }
+
     public int getTieredStoreFileReservedTime() {
         return tieredStoreFileReservedTime;
     }
@@ -255,6 +259,14 @@ public void setTieredStoreFileReservedTime(int tieredStoreFileReservedTime) {
         this.tieredStoreFileReservedTime = tieredStoreFileReservedTime;
     }
 
+    public long getTieredStoreDeleteFileInterval() {
+        return tieredStoreDeleteFileInterval;
+    }
+
+    public void setTieredStoreDeleteFileInterval(long tieredStoreDeleteFileInterval) {
+        this.tieredStoreDeleteFileInterval = tieredStoreDeleteFileInterval;
+    }
+
     public int getCommitLogRollingInterval() {
         return commitLogRollingInterval;
     }
@@ -271,12 +283,20 @@ public void setCommitLogRollingMinimumSize(int commitLogRollingMinimumSize) {
         this.commitLogRollingMinimumSize = commitLogRollingMinimumSize;
     }
 
-    public int getMaxCommitJitter() {
-        return maxCommitJitter;
+    public boolean isTieredStoreGroupCommit() {
+        return tieredStoreGroupCommit;
+    }
+
+    public void setTieredStoreGroupCommit(boolean tieredStoreGroupCommit) {
+        this.tieredStoreGroupCommit = tieredStoreGroupCommit;
     }
 
-    public void setMaxCommitJitter(int maxCommitJitter) {
-        this.maxCommitJitter = maxCommitJitter;
+    public int getTieredStoreGroupCommitTimeout() {
+        return tieredStoreGroupCommitTimeout;
+    }
+
+    public void setTieredStoreGroupCommitTimeout(int tieredStoreGroupCommitTimeout) {
+        this.tieredStoreGroupCommitTimeout = tieredStoreGroupCommitTimeout;
     }
 
     public int getTieredStoreGroupCommitCount() {
@@ -303,28 +323,12 @@ public void setTieredStoreMaxGroupCommitCount(int tieredStoreMaxGroupCommitCount
         this.tieredStoreMaxGroupCommitCount = tieredStoreMaxGroupCommitCount;
     }
 
-    public int getReadAheadMinFactor() {
-        return readAheadMinFactor;
-    }
-
-    public void setReadAheadMinFactor(int readAheadMinFactor) {
-        this.readAheadMinFactor = readAheadMinFactor;
-    }
-
-    public int getReadAheadMaxFactor() {
-        return readAheadMaxFactor;
-    }
-
-    public int getReadAheadBatchSizeFactorThreshold() {
-        return readAheadBatchSizeFactorThreshold;
+    public boolean isReadAheadCacheEnable() {
+        return readAheadCacheEnable;
     }
 
-    public void setReadAheadBatchSizeFactorThreshold(int readAheadBatchSizeFactorThreshold) {
-        this.readAheadBatchSizeFactorThreshold = readAheadBatchSizeFactorThreshold;
-    }
-
-    public void setReadAheadMaxFactor(int readAheadMaxFactor) {
-        this.readAheadMaxFactor = readAheadMaxFactor;
+    public void setReadAheadCacheEnable(boolean readAheadCacheEnable) {
+        this.readAheadCacheEnable = readAheadCacheEnable;
     }
 
     public int getReadAheadMessageCountThreshold() {
@@ -359,6 +363,22 @@ public void setReadAheadCacheSizeThresholdRate(double rate) {
         this.readAheadCacheSizeThresholdRate = rate;
     }
 
+    public int getTieredStoreMaxPendingLimit() {
+        return tieredStoreMaxPendingLimit;
+    }
+
+    public void setTieredStoreMaxPendingLimit(int tieredStoreMaxPendingLimit) {
+        this.tieredStoreMaxPendingLimit = tieredStoreMaxPendingLimit;
+    }
+
+    public boolean isTieredStoreCrcCheckEnable() {
+        return tieredStoreCrcCheckEnable;
+    }
+
+    public void setTieredStoreCrcCheckEnable(boolean tieredStoreCrcCheckEnable) {
+        this.tieredStoreCrcCheckEnable = tieredStoreCrcCheckEnable;
+    }
+
     public String getTieredStoreFilePath() {
         return tieredStoreFilePath;
     }
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreExecutor.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreExecutor.java
new file mode 100644
index 00000000000..56f564e7d2d
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreExecutor.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.common.ThreadFactoryImpl;
+import org.apache.rocketmq.common.utils.ThreadUtils;
+
+public class MessageStoreExecutor {
+
+    public final BlockingQueue<Runnable> bufferCommitThreadPoolQueue;
+    public final BlockingQueue<Runnable> bufferFetchThreadPoolQueue;
+    public final BlockingQueue<Runnable> fileRecyclingThreadPoolQueue;
+
+    public final ScheduledExecutorService commonExecutor;
+    public final ExecutorService bufferCommitExecutor;
+    public final ExecutorService bufferFetchExecutor;
+    public final ExecutorService fileRecyclingExecutor;
+
+    private static class SingletonHolder {
+        private static final MessageStoreExecutor INSTANCE = new MessageStoreExecutor();
+    }
+
+    public static MessageStoreExecutor getInstance() {
+        return SingletonHolder.INSTANCE;
+    }
+
+    public MessageStoreExecutor() {
+        this(10000);
+    }
+
+    public MessageStoreExecutor(int maxQueueCapacity) {
+
+        this.commonExecutor = ThreadUtils.newScheduledThreadPool(
+            Math.max(4, Runtime.getRuntime().availableProcessors()),
+            new ThreadFactoryImpl("TieredCommonExecutor_"));
+
+        this.bufferCommitThreadPoolQueue = new LinkedBlockingQueue<>(maxQueueCapacity);
+        this.bufferCommitExecutor = ThreadUtils.newThreadPoolExecutor(
+            Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
+            Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
+            TimeUnit.MINUTES.toMillis(1), TimeUnit.MILLISECONDS,
+            this.bufferCommitThreadPoolQueue,
+            new ThreadFactoryImpl("BufferCommitExecutor_"));
+
+        this.bufferFetchThreadPoolQueue = new LinkedBlockingQueue<>(maxQueueCapacity);
+        this.bufferFetchExecutor = ThreadUtils.newThreadPoolExecutor(
+            Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
+            Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
+            TimeUnit.MINUTES.toMillis(1), TimeUnit.MILLISECONDS,
+            this.bufferFetchThreadPoolQueue,
+            new ThreadFactoryImpl("BufferFetchExecutor_"));
+
+        this.fileRecyclingThreadPoolQueue = new LinkedBlockingQueue<>(maxQueueCapacity);
+        this.fileRecyclingExecutor = ThreadUtils.newThreadPoolExecutor(
+            Math.max(4, Runtime.getRuntime().availableProcessors()),
+            Math.max(4, Runtime.getRuntime().availableProcessors()),
+            TimeUnit.MINUTES.toMillis(1), TimeUnit.MILLISECONDS,
+            this.fileRecyclingThreadPoolQueue,
+            new ThreadFactoryImpl("BufferFetchExecutor_"));
+    }
+
+    private void shutdownExecutor(ExecutorService executor) {
+        if (executor != null) {
+            executor.shutdown();
+        }
+    }
+
+    public void shutdown() {
+        this.shutdownExecutor(this.commonExecutor);
+        this.shutdownExecutor(this.bufferCommitExecutor);
+        this.shutdownExecutor(this.bufferFetchExecutor);
+        this.shutdownExecutor(this.fileRecyclingExecutor);
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java
deleted file mode 100644
index 766c559e9c8..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java
+++ /dev/null
@@ -1,607 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore;
-
-import io.opentelemetry.api.common.Attributes;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Consumer;
-import org.apache.rocketmq.common.ServiceThread;
-import org.apache.rocketmq.common.message.MessageConst;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.store.CommitLogDispatcher;
-import org.apache.rocketmq.store.ConsumeQueue;
-import org.apache.rocketmq.store.DispatchRequest;
-import org.apache.rocketmq.store.MessageStore;
-import org.apache.rocketmq.store.SelectMappedBufferResult;
-import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.file.CompositeQueueFlatFile;
-import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager;
-import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant;
-import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager;
-import org.apache.rocketmq.tieredstore.provider.TieredStoreTopicBlackListFilter;
-import org.apache.rocketmq.tieredstore.provider.TieredStoreTopicFilter;
-import org.apache.rocketmq.tieredstore.util.CQItemBufferUtil;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-
-public class TieredDispatcher extends ServiceThread implements CommitLogDispatcher {
-
-    private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
-
-    private TieredStoreTopicFilter topicFilter;
-    private final String brokerName;
-    private final MessageStore defaultStore;
-    private final TieredMessageStoreConfig storeConfig;
-    private final TieredFlatFileManager tieredFlatFileManager;
-    private final ReentrantLock dispatchTaskLock;
-    private final ReentrantLock dispatchWriteLock;
-
-    private ConcurrentMap<CompositeQueueFlatFile, List<DispatchRequest>> dispatchRequestReadMap;
-    private ConcurrentMap<CompositeQueueFlatFile, List<DispatchRequest>> dispatchRequestWriteMap;
-
-    public TieredDispatcher(MessageStore defaultStore, TieredMessageStoreConfig storeConfig) {
-        this.defaultStore = defaultStore;
-        this.storeConfig = storeConfig;
-        this.brokerName = storeConfig.getBrokerName();
-        this.topicFilter = new TieredStoreTopicBlackListFilter();
-        this.tieredFlatFileManager = TieredFlatFileManager.getInstance(storeConfig);
-        this.dispatchRequestReadMap = new ConcurrentHashMap<>();
-        this.dispatchRequestWriteMap = new ConcurrentHashMap<>();
-        this.dispatchTaskLock = new ReentrantLock();
-        this.dispatchWriteLock = new ReentrantLock();
-    }
-
-    protected void initScheduleTask() {
-        TieredStoreExecutor.commonScheduledExecutor.scheduleWithFixedDelay(() ->
-            tieredFlatFileManager.deepCopyFlatFileToList().forEach(flatFile -> {
-                if (!flatFile.getCompositeFlatFileLock().isLocked()) {
-                    dispatchFlatFileAsync(flatFile);
-                }
-            }), 30, 10, TimeUnit.SECONDS);
-    }
-
-    public TieredStoreTopicFilter getTopicFilter() {
-        return topicFilter;
-    }
-
-    public void setTopicFilter(TieredStoreTopicFilter topicFilter) {
-        this.topicFilter = topicFilter;
-    }
-
-    @Override
-    public void dispatch(DispatchRequest request) {
-        if (stopped) {
-            return;
-        }
-
-        String topic = request.getTopic();
-        if (topicFilter != null && topicFilter.filterTopic(topic)) {
-            return;
-        }
-
-        CompositeQueueFlatFile flatFile = tieredFlatFileManager.getOrCreateFlatFileIfAbsent(
-            new MessageQueue(topic, brokerName, request.getQueueId()));
-
-        if (flatFile == null) {
-            logger.error("[Bug] TieredDispatcher#dispatch: get or create flat file failed, skip this request. ",
-                "topic: {}, queueId: {}", request.getTopic(), request.getQueueId());
-            return;
-        }
-
-        if (detectFallBehind(flatFile)) {
-            return;
-        }
-
-        // Set cq offset as commitlog first dispatch offset if flat file first init
-        if (flatFile.getDispatchOffset() == -1) {
-            flatFile.initOffset(request.getConsumeQueueOffset());
-        }
-
-        if (request.getConsumeQueueOffset() == flatFile.getDispatchOffset()) {
-
-            // In order to ensure the efficiency of dispatch operation and avoid high dispatch delay,
-            // it is not allowed to block for a long time here.
-            try {
-                // Acquired flat file write lock to append commitlog
-                if (flatFile.getCompositeFlatFileLock().isLocked()
-                    || !flatFile.getCompositeFlatFileLock().tryLock(3, TimeUnit.MILLISECONDS)) {
-                    return;
-                }
-            } catch (Exception e) {
-                logger.warn("Temporarily skip dispatch request because we can not acquired write lock. " +
-                    "topic: {}, queueId: {}", request.getTopic(), request.getQueueId(), e);
-                if (flatFile.getCompositeFlatFileLock().isLocked()) {
-                    flatFile.getCompositeFlatFileLock().unlock();
-                }
-                return;
-            }
-
-            // double check whether the offset matches
-            if (request.getConsumeQueueOffset() != flatFile.getDispatchOffset()) {
-                flatFile.getCompositeFlatFileLock().unlock();
-                return;
-            }
-
-            // obtain message
-            SelectMappedBufferResult message =
-                defaultStore.selectOneMessageByOffset(request.getCommitLogOffset(), request.getMsgSize());
-
-            if (message == null) {
-                logger.error("TieredDispatcher#dispatch: dispatch failed, " +
-                        "can not get message from next store: topic: {}, queueId: {}, commitLog offset: {}, size: {}",
-                    request.getTopic(), request.getQueueId(), request.getCommitLogOffset(), request.getMsgSize());
-                flatFile.getCompositeFlatFileLock().unlock();
-                return;
-            }
-
-            // drop expired request
-            try {
-                if (request.getConsumeQueueOffset() < flatFile.getDispatchOffset()) {
-                    return;
-                }
-                AppendResult result = flatFile.appendCommitLog(message.getByteBuffer());
-                long newCommitLogOffset = flatFile.getCommitLogMaxOffset() - message.getByteBuffer().remaining();
-                doRedispatchRequestToWriteMap(result, flatFile, request.getConsumeQueueOffset(),
-                    newCommitLogOffset, request.getMsgSize(), request.getTagsCode(), message.getByteBuffer());
-
-                if (result == AppendResult.SUCCESS) {
-                    Attributes attributes = TieredStoreMetricsManager.newAttributesBuilder()
-                        .put(TieredStoreMetricsConstant.LABEL_TOPIC, request.getTopic())
-                        .put(TieredStoreMetricsConstant.LABEL_QUEUE_ID, request.getQueueId())
-                        .put(TieredStoreMetricsConstant.LABEL_FILE_TYPE,
-                            FileSegmentType.COMMIT_LOG.name().toLowerCase())
-                        .build();
-                    TieredStoreMetricsManager.messagesDispatchTotal.add(1, attributes);
-                }
-            } catch (Exception throwable) {
-                logger.error("TieredDispatcher#dispatch: dispatch has unexpected problem. " +
-                        "topic: {}, queueId: {}, queue offset: {}", request.getTopic(), request.getQueueId(),
-                    request.getConsumeQueueOffset(), throwable);
-            } finally {
-                message.release();
-                flatFile.getCompositeFlatFileLock().unlock();
-            }
-        }
-    }
-
-    // prevent consume queue and index file falling too far
-    private boolean detectFallBehind(CompositeQueueFlatFile flatFile) {
-        int groupCommitCount = storeConfig.getTieredStoreMaxGroupCommitCount();
-        return dispatchRequestWriteMap.getOrDefault(flatFile, Collections.emptyList()).size() > groupCommitCount
-            || dispatchRequestReadMap.getOrDefault(flatFile, Collections.emptyList()).size() > groupCommitCount;
-    }
-
-    public void dispatchFlatFileAsync(CompositeQueueFlatFile flatFile) {
-        this.dispatchFlatFileAsync(flatFile, null);
-    }
-
-    public void dispatchFlatFileAsync(CompositeQueueFlatFile flatFile, Consumer<Long> consumer) {
-        // Avoid dispatch tasks too much
-        if (TieredStoreExecutor.dispatchThreadPoolQueue.size() >
-            TieredStoreExecutor.QUEUE_CAPACITY * 0.75) {
-            return;
-        }
-        TieredStoreExecutor.dispatchExecutor.execute(() -> {
-            try {
-                dispatchFlatFile(flatFile);
-            } catch (Throwable throwable) {
-                logger.error("[Bug] TieredDispatcher#dispatchFlatFileAsync failed, topic: {}, queueId: {}",
-                    flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), throwable);
-            }
-
-            if (consumer != null) {
-                consumer.accept(flatFile.getDispatchOffset());
-            }
-        });
-    }
-
-    protected void dispatchFlatFile(CompositeQueueFlatFile flatFile) {
-        if (stopped) {
-            return;
-        }
-
-        if (topicFilter != null && topicFilter.filterTopic(flatFile.getMessageQueue().getTopic())) {
-            return;
-        }
-
-        if (flatFile.getDispatchOffset() == -1L) {
-            return;
-        }
-
-        if (detectFallBehind(flatFile)) {
-            return;
-        }
-
-        MessageQueue mq = flatFile.getMessageQueue();
-        String topic = mq.getTopic();
-        int queueId = mq.getQueueId();
-
-        long beforeOffset = flatFile.getDispatchOffset();
-        long minOffsetInQueue = defaultStore.getMinOffsetInQueue(topic, queueId);
-        long maxOffsetInQueue = defaultStore.getMaxOffsetInQueue(topic, queueId);
-
-        // perhaps it was caused by local cq file corruption or ha truncation
-        if (beforeOffset >= maxOffsetInQueue) {
-            return;
-        }
-
-        try {
-            if (!flatFile.getCompositeFlatFileLock().tryLock(200, TimeUnit.MILLISECONDS)) {
-                return;
-            }
-        } catch (Exception e) {
-            logger.warn("TieredDispatcher#dispatchFlatFile: can not acquire flatFile lock, " +
-                "topic: {}, queueId: {}", mq.getTopic(), mq.getQueueId(), e);
-            if (flatFile.getCompositeFlatFileLock().isLocked()) {
-                flatFile.getCompositeFlatFileLock().unlock();
-            }
-            return;
-        }
-
-        try {
-            long dispatchOffset = flatFile.getDispatchOffset();
-            if (dispatchOffset < minOffsetInQueue) {
-                // If the tiered storage feature is turned off midway,
-                // it may cause cq discontinuity, resulting in data loss here.
-                logger.warn("TieredDispatcher#dispatchFlatFile: dispatch offset is too small, " +
-                        "topic: {}, queueId: {}, dispatch offset: {}, local cq offset range {}-{}",
-                    topic, queueId, dispatchOffset, minOffsetInQueue, maxOffsetInQueue);
-
-                // when dispatch offset is smaller than min offset in local cq
-                // some earliest messages may be lost at this time
-                tieredFlatFileManager.destroyCompositeFile(flatFile.getMessageQueue());
-                CompositeQueueFlatFile newFlatFile =
-                    tieredFlatFileManager.getOrCreateFlatFileIfAbsent(new MessageQueue(topic, brokerName, queueId));
-                if (newFlatFile != null) {
-                    newFlatFile.initOffset(maxOffsetInQueue);
-                }
-                return;
-            }
-            beforeOffset = dispatchOffset;
-
-            // flow control by max count, also we could do flow control based on message size
-            long maxCount = storeConfig.getTieredStoreGroupCommitCount();
-            long upperBound = Math.min(dispatchOffset + maxCount, maxOffsetInQueue);
-            ConsumeQueue consumeQueue = (ConsumeQueue) defaultStore.getConsumeQueue(topic, queueId);
-
-            logger.debug("DispatchFlatFile race, topic={}, queueId={}, cq range={}-{}, dispatch offset={}-{}",
-                topic, queueId, minOffsetInQueue, maxOffsetInQueue, dispatchOffset, upperBound - 1);
-
-            for (; dispatchOffset < upperBound; dispatchOffset++) {
-                // get consume queue
-                SelectMappedBufferResult cqItem = consumeQueue.getIndexBuffer(dispatchOffset);
-                if (cqItem == null) {
-                    logger.error("[Bug] TieredDispatcher#dispatchFlatFile: cq item is null, " +
-                            "topic: {}, queueId: {}, dispatch offset: {}, local cq offset range {}-{}",
-                        topic, queueId, dispatchOffset, minOffsetInQueue, maxOffsetInQueue);
-                    return;
-                }
-                long commitLogOffset = CQItemBufferUtil.getCommitLogOffset(cqItem.getByteBuffer());
-                int size = CQItemBufferUtil.getSize(cqItem.getByteBuffer());
-                long tagCode = CQItemBufferUtil.getTagCode(cqItem.getByteBuffer());
-                cqItem.release();
-
-                // get message
-                SelectMappedBufferResult message = defaultStore.selectOneMessageByOffset(commitLogOffset, size);
-                if (message == null) {
-                    logger.error("TieredDispatcher#dispatchFlatFile: get message from next store failed, " +
-                            "topic: {}, queueId: {}, commitLog offset: {}, size: {}",
-                        topic, queueId, commitLogOffset, size);
-                    // not dispatch immediately
-                    return;
-                }
-
-                // append commitlog will increase dispatch offset here
-                AppendResult result = flatFile.appendCommitLog(message.getByteBuffer(), true);
-                long newCommitLogOffset = flatFile.getCommitLogMaxOffset() - message.getByteBuffer().remaining();
-                doRedispatchRequestToWriteMap(
-                    result, flatFile, dispatchOffset, newCommitLogOffset, size, tagCode, message.getByteBuffer());
-                message.release();
-
-                switch (result) {
-                    case SUCCESS:
-                        continue;
-                    case FILE_CLOSED:
-                        tieredFlatFileManager.destroyCompositeFile(flatFile.getMessageQueue());
-                        logger.info("File has been closed and destroy, topic: {}, queueId: {}", topic, queueId);
-                        return;
-                    default:
-                        dispatchOffset--;
-                        break;
-                }
-            }
-
-            Attributes attributes = TieredStoreMetricsManager.newAttributesBuilder()
-                .put(TieredStoreMetricsConstant.LABEL_TOPIC, mq.getTopic())
-                .put(TieredStoreMetricsConstant.LABEL_QUEUE_ID, mq.getQueueId())
-                .put(TieredStoreMetricsConstant.LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase())
-                .build();
-
-            TieredStoreMetricsManager.messagesDispatchTotal.add(dispatchOffset - beforeOffset, attributes);
-        } finally {
-            flatFile.getCompositeFlatFileLock().unlock();
-        }
-
-        // If this queue dispatch falls too far, dispatch again immediately
-        if (flatFile.getDispatchOffset() < maxOffsetInQueue && !flatFile.getCompositeFlatFileLock().isLocked()) {
-            dispatchFlatFileAsync(flatFile);
-        }
-    }
-
-    // Submit cq to write map if append commitlog success
-    public void doRedispatchRequestToWriteMap(AppendResult result, CompositeQueueFlatFile flatFile,
-        long queueOffset, long newCommitLogOffset, int size, long tagCode, ByteBuffer message) {
-
-        MessageQueue mq = flatFile.getMessageQueue();
-        String topic = mq.getTopic();
-        int queueId = mq.getQueueId();
-
-        switch (result) {
-            case SUCCESS:
-                long offset = MessageBufferUtil.getQueueOffset(message);
-                if (queueOffset != offset) {
-                    logger.warn("Message cq offset in commitlog does not meet expectations, " +
-                            "result={}, topic={}, queueId={}, cq offset={}, msg offset={}",
-                        AppendResult.OFFSET_INCORRECT, topic, queueId, queueOffset, offset);
-                }
-                break;
-            case BUFFER_FULL:
-                logger.debug("Commitlog buffer full, result={}, topic={}, queueId={}, offset={}",
-                    result, topic, queueId, queueOffset);
-                return;
-            default:
-                logger.info("Commitlog append failed, result={}, topic={}, queueId={}, offset={}",
-                    result, topic, queueId, queueOffset);
-                return;
-        }
-
-        dispatchWriteLock.lock();
-        try {
-            Map<String, String> properties = MessageBufferUtil.getProperties(message);
-            DispatchRequest dispatchRequest = new DispatchRequest(
-                topic,
-                queueId,
-                newCommitLogOffset,
-                size,
-                tagCode,
-                MessageBufferUtil.getStoreTimeStamp(message),
-                queueOffset,
-                properties.getOrDefault(MessageConst.PROPERTY_KEYS, ""),
-                properties.getOrDefault(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, ""),
-                0, 0, new HashMap<>());
-            dispatchRequest.setOffsetId(MessageBufferUtil.getOffsetId(message));
-            List<DispatchRequest> requestList =
-                dispatchRequestWriteMap.computeIfAbsent(flatFile, k -> new ArrayList<>());
-            requestList.add(dispatchRequest);
-            if (requestList.get(0).getConsumeQueueOffset() >= flatFile.getConsumeQueueMaxOffset()) {
-                wakeup();
-            }
-        } finally {
-            dispatchWriteLock.unlock();
-        }
-    }
-
-    public void swapDispatchRequestList() {
-        dispatchWriteLock.lock();
-        try {
-            dispatchRequestReadMap = dispatchRequestWriteMap;
-            dispatchRequestWriteMap = new ConcurrentHashMap<>();
-        } finally {
-            dispatchWriteLock.unlock();
-        }
-    }
-
-    public void copySurvivorObject() {
-        if (dispatchRequestReadMap.isEmpty()) {
-            return;
-        }
-
-        try {
-            dispatchWriteLock.lock();
-            dispatchRequestReadMap.forEach((flatFile, requestList) -> {
-                String topic = flatFile.getMessageQueue().getTopic();
-                int queueId = flatFile.getMessageQueue().getQueueId();
-                if (requestList.isEmpty()) {
-                    logger.warn("Copy survivor object failed, dispatch request list is empty, " +
-                        "topic: {}, queueId: {}", topic, queueId);
-                    return;
-                }
-
-                List<DispatchRequest> requestListToWrite =
-                    dispatchRequestWriteMap.computeIfAbsent(flatFile, k -> new ArrayList<>());
-
-                if (!requestListToWrite.isEmpty()) {
-                    long readOffset = requestList.get(requestList.size() - 1).getConsumeQueueOffset();
-                    long writeOffset = requestListToWrite.get(0).getConsumeQueueOffset();
-                    if (readOffset > writeOffset) {
-                        logger.warn("Copy survivor object failed, offset in request list are not continuous. " +
-                                "topic: {}, queueId: {}, read offset: {}, write offset: {}",
-                            topic, queueId, readOffset, writeOffset);
-
-                        // sort request list according cq offset
-                        requestList.sort(Comparator.comparingLong(DispatchRequest::getConsumeQueueOffset));
-                    }
-                }
-
-                requestList.addAll(requestListToWrite);
-                dispatchRequestWriteMap.put(flatFile, requestList);
-            });
-            dispatchRequestReadMap = new ConcurrentHashMap<>();
-        } finally {
-            dispatchWriteLock.unlock();
-        }
-    }
-
-    protected void buildConsumeQueueAndIndexFile() {
-        swapDispatchRequestList();
-        Map<MessageQueue, Long> cqMetricsMap = new HashMap<>();
-        Map<MessageQueue, Long> ifMetricsMap = new HashMap<>();
-
-        for (Map.Entry<CompositeQueueFlatFile, List<DispatchRequest>> entry : dispatchRequestReadMap.entrySet()) {
-            CompositeQueueFlatFile flatFile = entry.getKey();
-            List<DispatchRequest> requestList = entry.getValue();
-            if (flatFile.isClosed()) {
-                requestList.clear();
-            }
-
-            MessageQueue messageQueue = flatFile.getMessageQueue();
-            Iterator<DispatchRequest> iterator = requestList.iterator();
-            while (iterator.hasNext()) {
-                DispatchRequest request = iterator.next();
-
-                // remove expired request
-                if (request.getConsumeQueueOffset() < flatFile.getConsumeQueueMaxOffset()) {
-                    iterator.remove();
-                    continue;
-                }
-
-                // wait uploading commitLog
-                if (flatFile.getCommitLogDispatchCommitOffset() < request.getConsumeQueueOffset()) {
-                    break;
-                }
-
-                // build consume queue
-                AppendResult result = flatFile.appendConsumeQueue(request, true);
-
-                // handle build cq result
-                if (AppendResult.SUCCESS.equals(result)) {
-                    long cqCount = cqMetricsMap.computeIfAbsent(messageQueue, key -> 0L);
-                    cqMetricsMap.put(messageQueue, cqCount + 1);
-
-                    // build index
-                    if (storeConfig.isMessageIndexEnable()) {
-                        result = flatFile.appendIndexFile(request);
-                        if (AppendResult.SUCCESS.equals(result)) {
-                            long ifCount = ifMetricsMap.computeIfAbsent(messageQueue, key -> 0L);
-                            ifMetricsMap.put(messageQueue, ifCount + 1);
-                            iterator.remove();
-                        } else {
-                            logger.warn("Build index failed, skip this message, " +
-                                    "result: {}, topic: {}, queue: {}, request offset: {}",
-                                result, request.getTopic(), request.getQueueId(), request.getConsumeQueueOffset());
-                        }
-                    }
-                    continue;
-                }
-
-                if (AppendResult.OFFSET_INCORRECT.equals(result)) {
-                    logger.error("Consume queue offset incorrect, try to recreated consume queue, " +
-                            "result: {}, topic: {}, queue: {}, request offset: {}, current cq offset: {}",
-                        result, request.getTopic(), request.getQueueId(),
-                        request.getConsumeQueueOffset(), flatFile.getConsumeQueueMaxOffset());
-
-                    try {
-                        flatFile.getCompositeFlatFileLock().lock();
-
-                        // reset dispatch offset, this operation will cause duplicate message in commitLog
-                        long minOffsetInQueue =
-                            defaultStore.getMinOffsetInQueue(request.getTopic(), request.getQueueId());
-
-                        // when dispatch offset is smaller than min offset in local cq
-                        // some messages may be lost at this time
-                        if (flatFile.getConsumeQueueMaxOffset() < minOffsetInQueue) {
-                            // if we use flatFile.destroy() directly will cause manager reference leak.
-                            tieredFlatFileManager.destroyCompositeFile(flatFile.getMessageQueue());
-                            logger.warn("Found cq max offset is smaller than local cq min offset, " +
-                                    "so destroy tiered flat file to recreated, topic: {}, queueId: {}",
-                                request.getTopic(), request.getQueueId());
-                        } else {
-                            flatFile.initOffset(flatFile.getConsumeQueueMaxOffset());
-                        }
-
-                        // clean invalid dispatch request
-                        dispatchRequestWriteMap.remove(flatFile);
-                        requestList.clear();
-                    } finally {
-                        flatFile.getCompositeFlatFileLock().unlock();
-                    }
-                    break;
-                }
-
-                // other append result
-                logger.warn("Append consume queue failed, result: {}, topic: {}, queue: {}, request offset: {}",
-                    result, request.getTopic(), request.getQueueId(), request.getConsumeQueueOffset());
-            }
-
-            // remove empty list, prevent send back
-            if (requestList.isEmpty()) {
-                dispatchRequestReadMap.remove(flatFile);
-            }
-        }
-
-        cqMetricsMap.forEach((messageQueue, count) -> {
-            Attributes attributes = TieredStoreMetricsManager.newAttributesBuilder()
-                .put(TieredStoreMetricsConstant.LABEL_TOPIC, messageQueue.getTopic())
-                .put(TieredStoreMetricsConstant.LABEL_QUEUE_ID, messageQueue.getQueueId())
-                .put(TieredStoreMetricsConstant.LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase())
-                .build();
-            TieredStoreMetricsManager.messagesDispatchTotal.add(count, attributes);
-        });
-
-        ifMetricsMap.forEach((messageQueue, count) -> {
-            Attributes attributes = TieredStoreMetricsManager.newAttributesBuilder()
-                .put(TieredStoreMetricsConstant.LABEL_TOPIC, messageQueue.getTopic())
-                .put(TieredStoreMetricsConstant.LABEL_QUEUE_ID, messageQueue.getQueueId())
-                .put(TieredStoreMetricsConstant.LABEL_FILE_TYPE, FileSegmentType.INDEX.name().toLowerCase())
-                .build();
-            TieredStoreMetricsManager.messagesDispatchTotal.add(count, attributes);
-        });
-
-        copySurvivorObject();
-    }
-
-    // Allow work-stealing
-    public void doDispatchTask() {
-        try {
-            dispatchTaskLock.lock();
-            buildConsumeQueueAndIndexFile();
-        } catch (Exception e) {
-            logger.error("Tiered storage do dispatch task failed", e);
-        } finally {
-            dispatchTaskLock.unlock();
-        }
-    }
-
-    @Override
-    public String getServiceName() {
-        return "TieredStoreDispatcherService";
-    }
-
-    @Override
-    public void run() {
-        while (!stopped) {
-            waitForRunning(1000);
-            doDispatchTask();
-        }
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java
deleted file mode 100644
index 7b0c47c592b..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java
+++ /dev/null
@@ -1,585 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore;
-
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.Scheduler;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Stopwatch;
-import io.opentelemetry.api.common.Attributes;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.rocketmq.common.BoundaryType;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.store.GetMessageResult;
-import org.apache.rocketmq.store.GetMessageStatus;
-import org.apache.rocketmq.store.MessageFilter;
-import org.apache.rocketmq.store.QueryMessageResult;
-import org.apache.rocketmq.store.SelectMappedBufferResult;
-import org.apache.rocketmq.tieredstore.common.GetMessageResultExt;
-import org.apache.rocketmq.tieredstore.common.InFlightRequestFuture;
-import org.apache.rocketmq.tieredstore.common.MessageCacheKey;
-import org.apache.rocketmq.tieredstore.common.SelectBufferResult;
-import org.apache.rocketmq.tieredstore.common.SelectBufferResultWrapper;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.exception.TieredStoreException;
-import org.apache.rocketmq.tieredstore.file.CompositeFlatFile;
-import org.apache.rocketmq.tieredstore.file.CompositeQueueFlatFile;
-import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue;
-import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager;
-import org.apache.rocketmq.tieredstore.index.IndexItem;
-import org.apache.rocketmq.tieredstore.index.IndexService;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
-import org.apache.rocketmq.tieredstore.metadata.TopicMetadata;
-import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant;
-import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager;
-import org.apache.rocketmq.tieredstore.util.CQItemBufferUtil;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-
-public class TieredMessageFetcher implements MessageStoreFetcher {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
-
-    private final String brokerName;
-    private final TieredMetadataStore metadataStore;
-    private final TieredMessageStoreConfig storeConfig;
-    private final TieredFlatFileManager flatFileManager;
-    private final Cache<MessageCacheKey, SelectBufferResultWrapper> readAheadCache;
-
-    public TieredMessageFetcher(TieredMessageStoreConfig storeConfig) {
-        this.storeConfig = storeConfig;
-        this.brokerName = storeConfig.getBrokerName();
-        this.metadataStore = TieredStoreUtil.getMetadataStore(storeConfig);
-        this.flatFileManager = TieredFlatFileManager.getInstance(storeConfig);
-        this.readAheadCache = this.initCache(storeConfig);
-    }
-
-    private Cache<MessageCacheKey, SelectBufferResultWrapper> initCache(TieredMessageStoreConfig storeConfig) {
-        long memoryMaxSize =
-            (long) (Runtime.getRuntime().maxMemory() * storeConfig.getReadAheadCacheSizeThresholdRate());
-
-        return Caffeine.newBuilder()
-            .scheduler(Scheduler.systemScheduler())
-            .expireAfterWrite(storeConfig.getReadAheadCacheExpireDuration(), TimeUnit.MILLISECONDS)
-            .maximumWeight(memoryMaxSize)
-            // Using the buffer size of messages to calculate memory usage
-            .weigher((MessageCacheKey key, SelectBufferResultWrapper msg) -> msg.getBufferSize())
-            .recordStats()
-            .build();
-    }
-
-    @VisibleForTesting
-    public Cache<MessageCacheKey, SelectBufferResultWrapper> getMessageCache() {
-        return readAheadCache;
-    }
-
-    protected void putMessageToCache(CompositeFlatFile flatFile, SelectBufferResultWrapper result) {
-        readAheadCache.put(new MessageCacheKey(flatFile, result.getOffset()), result);
-    }
-
-    protected SelectBufferResultWrapper getMessageFromCache(CompositeFlatFile flatFile, long offset) {
-        return readAheadCache.getIfPresent(new MessageCacheKey(flatFile, offset));
-    }
-
-    protected void recordCacheAccess(CompositeFlatFile flatFile,
-        String group, long offset, List<SelectBufferResultWrapper> resultWrapperList) {
-        if (!resultWrapperList.isEmpty()) {
-            offset = resultWrapperList.get(resultWrapperList.size() - 1).getOffset();
-        }
-        flatFile.recordGroupAccess(group, offset);
-        resultWrapperList.forEach(wrapper -> {
-            if (wrapper.incrementAndGet() >= flatFile.getActiveGroupCount()) {
-                readAheadCache.invalidate(new MessageCacheKey(flatFile, wrapper.getOffset()));
-            }
-        });
-    }
-
-    private void prefetchMessage(CompositeQueueFlatFile flatFile, String group, int maxCount, long nextBeginOffset) {
-        if (maxCount == 1 || flatFile.getReadAheadFactor() == 1) {
-            return;
-        }
-
-        // make sure there is only one request per group and request range
-        int prefetchBatchSize = Math.min(maxCount * flatFile.getReadAheadFactor(), storeConfig.getReadAheadMessageCountThreshold());
-        InFlightRequestFuture inflightRequest = flatFile.getInflightRequest(group, nextBeginOffset, prefetchBatchSize);
-        if (!inflightRequest.isAllDone()) {
-            return;
-        }
-
-        synchronized (flatFile) {
-            inflightRequest = flatFile.getInflightRequest(nextBeginOffset, maxCount);
-            if (!inflightRequest.isAllDone()) {
-                return;
-            }
-
-            long maxOffsetOfLastRequest = inflightRequest.getLastFuture().join();
-            boolean lastRequestIsExpired = getMessageFromCache(flatFile, nextBeginOffset) == null;
-
-            if (lastRequestIsExpired ||
-                maxOffsetOfLastRequest != -1L && nextBeginOffset >= inflightRequest.getStartOffset()) {
-
-                long queueOffset;
-                if (lastRequestIsExpired) {
-                    queueOffset = nextBeginOffset;
-                    flatFile.decreaseReadAheadFactor();
-                } else {
-                    queueOffset = maxOffsetOfLastRequest + 1;
-                    flatFile.increaseReadAheadFactor();
-                }
-
-                int factor = Math.min(flatFile.getReadAheadFactor(), storeConfig.getReadAheadMessageCountThreshold() / maxCount);
-                int flag = 0;
-                int concurrency = 1;
-                if (factor > storeConfig.getReadAheadBatchSizeFactorThreshold()) {
-                    flag = factor % storeConfig.getReadAheadBatchSizeFactorThreshold() == 0 ? 0 : 1;
-                    concurrency = factor / storeConfig.getReadAheadBatchSizeFactorThreshold() + flag;
-                }
-                int requestBatchSize = maxCount * Math.min(factor, storeConfig.getReadAheadBatchSizeFactorThreshold());
-
-                List<Pair<Integer, CompletableFuture<Long>>> futureList = new ArrayList<>();
-                long nextQueueOffset = queueOffset;
-                if (flag == 1) {
-                    int firstBatchSize = factor % storeConfig.getReadAheadBatchSizeFactorThreshold() * maxCount;
-                    CompletableFuture<Long> future = prefetchMessageThenPutToCache(flatFile, nextQueueOffset, firstBatchSize);
-                    futureList.add(Pair.of(firstBatchSize, future));
-                    nextQueueOffset += firstBatchSize;
-                }
-                for (long i = 0; i < concurrency - flag; i++) {
-                    CompletableFuture<Long> future = prefetchMessageThenPutToCache(flatFile, nextQueueOffset + i * requestBatchSize, requestBatchSize);
-                    futureList.add(Pair.of(requestBatchSize, future));
-                }
-                flatFile.putInflightRequest(group, queueOffset, maxCount * factor, futureList);
-                LOGGER.debug("TieredMessageFetcher#preFetchMessage: try to prefetch messages for later requests: next begin offset: {}, request offset: {}, factor: {}, flag: {}, request batch: {}, concurrency: {}",
-                    nextBeginOffset, queueOffset, factor, flag, requestBatchSize, concurrency);
-            }
-        }
-    }
-
-    private CompletableFuture<Long> prefetchMessageThenPutToCache(
-        CompositeQueueFlatFile flatFile, long queueOffset, int batchSize) {
-
-        MessageQueue mq = flatFile.getMessageQueue();
-        return getMessageFromTieredStoreAsync(flatFile, queueOffset, batchSize)
-            .thenApply(result -> {
-                if (result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_ONE ||
-                    result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_BADLY) {
-                    return -1L;
-                }
-                if (result.getStatus() != GetMessageStatus.FOUND) {
-                    LOGGER.warn("MessageFetcher prefetch message then put to cache failed, result: {}, " +
-                            "topic: {}, queue: {}, queue offset: {}, batch size: {}",
-                        result.getStatus(), mq.getTopic(), mq.getQueueId(), queueOffset, batchSize);
-                    return -1L;
-                }
-                try {
-                    List<Long> offsetList = result.getMessageQueueOffset();
-                    List<Long> tagCodeList = result.getTagCodeList();
-                    List<SelectMappedBufferResult> msgList = result.getMessageMapedList();
-                    for (int i = 0; i < offsetList.size(); i++) {
-                        SelectMappedBufferResult msg = msgList.get(i);
-                        SelectBufferResultWrapper bufferResult = new SelectBufferResultWrapper(
-                            msg, offsetList.get(i), tagCodeList.get(i), false);
-                        this.putMessageToCache(flatFile, bufferResult);
-                    }
-                    return offsetList.get(offsetList.size() - 1);
-                } catch (Exception e) {
-                    LOGGER.error("MessageFetcher prefetch message then put to cache failed, " +
-                            "topic: {}, queue: {}, queue offset: {}, batch size: {}",
-                        mq.getTopic(), mq.getQueueId(), queueOffset, batchSize, e);
-                }
-                return -1L;
-            });
-    }
-
-    public CompletableFuture<GetMessageResultExt> getMessageFromCacheAsync(CompositeQueueFlatFile flatFile,
-        String group, long queueOffset, int maxCount, boolean waitInflightRequest) {
-
-        MessageQueue mq = flatFile.getMessageQueue();
-
-        long lastGetOffset = queueOffset - 1;
-        List<SelectBufferResultWrapper> resultWrapperList = new ArrayList<>(maxCount);
-        for (int i = 0; i < maxCount; i++) {
-            lastGetOffset++;
-            SelectBufferResultWrapper wrapper = getMessageFromCache(flatFile, lastGetOffset);
-            if (wrapper == null) {
-                lastGetOffset--;
-                break;
-            }
-            resultWrapperList.add(wrapper);
-        }
-
-        // only record cache access count once
-        if (waitInflightRequest) {
-            Attributes attributes = TieredStoreMetricsManager.newAttributesBuilder()
-                .put(TieredStoreMetricsConstant.LABEL_TOPIC, mq.getTopic())
-                .put(TieredStoreMetricsConstant.LABEL_GROUP, group)
-                .build();
-            TieredStoreMetricsManager.cacheAccess.add(maxCount, attributes);
-            TieredStoreMetricsManager.cacheHit.add(resultWrapperList.size(), attributes);
-        }
-
-        // If there are no messages in the cache and there are currently requests being pulled.
-        // We need to wait for the request to return before continuing.
-        if (resultWrapperList.isEmpty() && waitInflightRequest) {
-            CompletableFuture<Long> future =
-                flatFile.getInflightRequest(group, queueOffset, maxCount).getFuture(queueOffset);
-            if (!future.isDone()) {
-                Stopwatch stopwatch = Stopwatch.createStarted();
-                // to prevent starvation issues, only allow waiting for processing request once
-                return future.thenComposeAsync(v -> {
-                    LOGGER.debug("MessageFetcher#getMessageFromCacheAsync: wait for response cost: {}ms",
-                        stopwatch.elapsed(TimeUnit.MILLISECONDS));
-                    return getMessageFromCacheAsync(flatFile, group, queueOffset, maxCount, false);
-                }, TieredStoreExecutor.fetchDataExecutor);
-            }
-        }
-
-        // try to get message from cache again when prefetch request is done
-        for (int i = 0; i < maxCount - resultWrapperList.size(); i++) {
-            lastGetOffset++;
-            SelectBufferResultWrapper wrapper = getMessageFromCache(flatFile, lastGetOffset);
-            if (wrapper == null) {
-                lastGetOffset--;
-                break;
-            }
-            resultWrapperList.add(wrapper);
-        }
-
-        recordCacheAccess(flatFile, group, queueOffset, resultWrapperList);
-
-        if (resultWrapperList.isEmpty()) {
-            // If cache miss, pull messages immediately
-            LOGGER.info("MessageFetcher cache miss, group: {}, topic: {}, queueId: {}, offset: {}, maxCount: {}",
-                group, mq.getTopic(), mq.getQueueId(), queueOffset, maxCount);
-        } else {
-            // If cache hit, return buffer result immediately and asynchronously prefetch messages
-            LOGGER.debug("MessageFetcher cache hit, group: {}, topic: {}, queueId: {}, offset: {}, maxCount: {}, resultSize: {}",
-                group, mq.getTopic(), mq.getQueueId(), queueOffset, maxCount, resultWrapperList.size());
-
-            GetMessageResultExt result = new GetMessageResultExt();
-            result.setStatus(GetMessageStatus.FOUND);
-            result.setMinOffset(flatFile.getConsumeQueueMinOffset());
-            result.setMaxOffset(flatFile.getConsumeQueueCommitOffset());
-            result.setNextBeginOffset(queueOffset + resultWrapperList.size());
-            resultWrapperList.forEach(wrapper -> result.addMessageExt(
-                wrapper.getDuplicateResult(), wrapper.getOffset(), wrapper.getTagCode()));
-
-            if (lastGetOffset < result.getMaxOffset()) {
-                this.prefetchMessage(flatFile, group, maxCount, lastGetOffset + 1);
-            }
-            return CompletableFuture.completedFuture(result);
-        }
-
-        CompletableFuture<GetMessageResultExt> resultFuture;
-        synchronized (flatFile) {
-            int batchSize = maxCount * storeConfig.getReadAheadMinFactor();
-            resultFuture = getMessageFromTieredStoreAsync(flatFile, queueOffset, batchSize)
-                .thenApply(result -> {
-                    if (result.getStatus() != GetMessageStatus.FOUND) {
-                        return result;
-                    }
-
-                    GetMessageResultExt newResult = new GetMessageResultExt();
-                    List<Long> offsetList = result.getMessageQueueOffset();
-                    List<Long> tagCodeList = result.getTagCodeList();
-                    List<SelectMappedBufferResult> msgList = result.getMessageMapedList();
-
-                    for (int i = 0; i < offsetList.size(); i++) {
-                        SelectMappedBufferResult msg = msgList.get(i);
-                        SelectBufferResultWrapper bufferResult = new SelectBufferResultWrapper(
-                            msg, offsetList.get(i), tagCodeList.get(i), true);
-                        this.putMessageToCache(flatFile, bufferResult);
-                        if (newResult.getMessageMapedList().size() < maxCount) {
-                            newResult.addMessageExt(msg, offsetList.get(i), tagCodeList.get(i));
-                        }
-                    }
-
-                    newResult.setStatus(GetMessageStatus.FOUND);
-                    newResult.setMinOffset(flatFile.getConsumeQueueMinOffset());
-                    newResult.setMaxOffset(flatFile.getConsumeQueueCommitOffset());
-                    newResult.setNextBeginOffset(queueOffset + newResult.getMessageMapedList().size());
-                    return newResult;
-                });
-
-            List<Pair<Integer, CompletableFuture<Long>>> futureList = new ArrayList<>();
-            CompletableFuture<Long> inflightRequestFuture = resultFuture.thenApply(result ->
-                result.getStatus() == GetMessageStatus.FOUND ?
-                    result.getMessageQueueOffset().get(result.getMessageQueueOffset().size() - 1) : -1L);
-            futureList.add(Pair.of(batchSize, inflightRequestFuture));
-            flatFile.putInflightRequest(group, queueOffset, batchSize, futureList);
-        }
-        return resultFuture;
-    }
-
-    public CompletableFuture<GetMessageResultExt> getMessageFromTieredStoreAsync(
-        CompositeQueueFlatFile flatFile, long queueOffset, int batchSize) {
-
-        GetMessageResultExt result = new GetMessageResultExt();
-        result.setMinOffset(flatFile.getConsumeQueueMinOffset());
-        result.setMaxOffset(flatFile.getConsumeQueueCommitOffset());
-
-        if (queueOffset < result.getMaxOffset()) {
-            batchSize = Math.min(batchSize, (int) Math.min(result.getMaxOffset() - queueOffset, Integer.MAX_VALUE));
-        } else if (queueOffset == result.getMaxOffset()) {
-            result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE);
-            result.setNextBeginOffset(queueOffset);
-            return CompletableFuture.completedFuture(result);
-        } else if (queueOffset > result.getMaxOffset()) {
-            result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY);
-            result.setNextBeginOffset(result.getMaxOffset());
-            return CompletableFuture.completedFuture(result);
-        }
-
-        LOGGER.info("MessageFetcher#getMessageFromTieredStoreAsync, " +
-                "topic: {}, queueId: {}, broker offset: {}-{}, offset: {}, expect: {}",
-            flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(),
-            result.getMinOffset(), result.getMaxOffset(), queueOffset, batchSize);
-
-        CompletableFuture<ByteBuffer> readConsumeQueueFuture;
-        try {
-            readConsumeQueueFuture = flatFile.getConsumeQueueAsync(queueOffset, batchSize);
-        } catch (TieredStoreException e) {
-            switch (e.getErrorCode()) {
-                case NO_NEW_DATA:
-                    result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE);
-                    result.setNextBeginOffset(queueOffset);
-                    return CompletableFuture.completedFuture(result);
-                case ILLEGAL_PARAM:
-                case ILLEGAL_OFFSET:
-                default:
-                    result.setStatus(GetMessageStatus.OFFSET_FOUND_NULL);
-                    result.setNextBeginOffset(queueOffset);
-                    return CompletableFuture.completedFuture(result);
-            }
-        }
-
-        CompletableFuture<ByteBuffer> readCommitLogFuture = readConsumeQueueFuture.thenCompose(cqBuffer -> {
-            long firstCommitLogOffset = CQItemBufferUtil.getCommitLogOffset(cqBuffer);
-            cqBuffer.position(cqBuffer.remaining() - TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-            long lastCommitLogOffset = CQItemBufferUtil.getCommitLogOffset(cqBuffer);
-            if (lastCommitLogOffset < firstCommitLogOffset) {
-                LOGGER.error("MessageFetcher#getMessageFromTieredStoreAsync, " +
-                        "last offset is smaller than first offset, " +
-                        "topic: {} queueId: {}, offset: {}, firstOffset: {}, lastOffset: {}",
-                    flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), queueOffset,
-                    firstCommitLogOffset, lastCommitLogOffset);
-                return CompletableFuture.completedFuture(ByteBuffer.allocate(0));
-            }
-
-            // Get the total size of the data by reducing the length limit of cq to prevent OOM
-            long length = lastCommitLogOffset - firstCommitLogOffset + CQItemBufferUtil.getSize(cqBuffer);
-            while (cqBuffer.limit() > TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE &&
-                length > storeConfig.getReadAheadMessageSizeThreshold()) {
-                cqBuffer.limit(cqBuffer.position());
-                cqBuffer.position(cqBuffer.limit() - TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-                length = CQItemBufferUtil.getCommitLogOffset(cqBuffer)
-                    - firstCommitLogOffset + CQItemBufferUtil.getSize(cqBuffer);
-            }
-
-            return flatFile.getCommitLogAsync(firstCommitLogOffset, (int) length);
-        });
-
-        int finalBatchSize = batchSize;
-        return readConsumeQueueFuture.thenCombine(readCommitLogFuture, (cqBuffer, msgBuffer) -> {
-            List<SelectBufferResult> bufferList = MessageBufferUtil.splitMessageBuffer(cqBuffer, msgBuffer);
-            int requestSize = cqBuffer.remaining() / TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE;
-            if (bufferList.isEmpty()) {
-                result.setStatus(GetMessageStatus.NO_MATCHED_MESSAGE);
-                result.setNextBeginOffset(queueOffset + requestSize);
-            } else {
-                result.setStatus(GetMessageStatus.FOUND);
-                result.setNextBeginOffset(queueOffset + requestSize);
-
-                for (SelectBufferResult bufferResult : bufferList) {
-                    ByteBuffer slice = bufferResult.getByteBuffer().slice();
-                    slice.limit(bufferResult.getSize());
-                    SelectMappedBufferResult msg = new SelectMappedBufferResult(bufferResult.getStartOffset(),
-                        bufferResult.getByteBuffer(), bufferResult.getSize(), null);
-                    result.addMessageExt(msg, MessageBufferUtil.getQueueOffset(slice), bufferResult.getTagCode());
-                }
-            }
-            return result;
-        }).exceptionally(e -> {
-            MessageQueue mq = flatFile.getMessageQueue();
-            LOGGER.warn("MessageFetcher#getMessageFromTieredStoreAsync failed, " +
-                "topic: {} queueId: {}, offset: {}, batchSize: {}", mq.getTopic(), mq.getQueueId(), queueOffset, finalBatchSize, e);
-            result.setStatus(GetMessageStatus.OFFSET_FOUND_NULL);
-            result.setNextBeginOffset(queueOffset);
-            return result;
-        });
-    }
-
-    @Override
-    public CompletableFuture<GetMessageResult> getMessageAsync(
-        String group, String topic, int queueId, long queueOffset, int maxCount, final MessageFilter messageFilter) {
-
-        GetMessageResult result = new GetMessageResult();
-        CompositeQueueFlatFile flatFile = flatFileManager.getFlatFile(new MessageQueue(topic, brokerName, queueId));
-
-        if (flatFile == null) {
-            result.setNextBeginOffset(queueOffset);
-            result.setStatus(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE);
-            return CompletableFuture.completedFuture(result);
-        }
-
-        // Max queue offset means next message put position
-        result.setMinOffset(flatFile.getConsumeQueueMinOffset());
-        result.setMaxOffset(flatFile.getConsumeQueueCommitOffset());
-
-        // Fill result according file offset.
-        // Offset range  | Result           | Fix to
-        // (-oo, 0]      | no message       | current offset
-        // (0, min)      | too small        | min offset
-        // [min, max)    | correct          |
-        // [max, max]    | overflow one     | max offset
-        // (max, +oo)    | overflow badly   | max offset
-
-        if (result.getMaxOffset() <= 0) {
-            result.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);
-            result.setNextBeginOffset(queueOffset);
-            return CompletableFuture.completedFuture(result);
-        } else if (queueOffset < result.getMinOffset()) {
-            result.setStatus(GetMessageStatus.OFFSET_TOO_SMALL);
-            result.setNextBeginOffset(result.getMinOffset());
-            return CompletableFuture.completedFuture(result);
-        } else if (queueOffset == result.getMaxOffset()) {
-            result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE);
-            result.setNextBeginOffset(result.getMaxOffset());
-            return CompletableFuture.completedFuture(result);
-        } else if (queueOffset > result.getMaxOffset()) {
-            result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY);
-            result.setNextBeginOffset(result.getMaxOffset());
-            return CompletableFuture.completedFuture(result);
-        }
-
-        return getMessageFromCacheAsync(flatFile, group, queueOffset, maxCount, true)
-            .thenApply(messageResultExt -> messageResultExt.doFilterMessage(messageFilter));
-    }
-
-    @Override
-    public CompletableFuture<Long> getEarliestMessageTimeAsync(String topic, int queueId) {
-        CompositeFlatFile flatFile = flatFileManager.getFlatFile(new MessageQueue(topic, brokerName, queueId));
-        if (flatFile == null) {
-            return CompletableFuture.completedFuture(-1L);
-        }
-
-        // read from timestamp to timestamp + length
-        int length = MessageBufferUtil.STORE_TIMESTAMP_POSITION + 8;
-        return flatFile.getCommitLogAsync(flatFile.getCommitLogMinOffset(), length)
-            .thenApply(MessageBufferUtil::getStoreTimeStamp);
-    }
-
-    @Override
-    public CompletableFuture<Long> getMessageStoreTimeStampAsync(String topic, int queueId, long queueOffset) {
-        CompositeFlatFile flatFile = flatFileManager.getFlatFile(new MessageQueue(topic, brokerName, queueId));
-        if (flatFile == null) {
-            return CompletableFuture.completedFuture(-1L);
-        }
-
-        return flatFile.getConsumeQueueAsync(queueOffset)
-            .thenComposeAsync(cqItem -> {
-                long commitLogOffset = CQItemBufferUtil.getCommitLogOffset(cqItem);
-                int size = CQItemBufferUtil.getSize(cqItem);
-                return flatFile.getCommitLogAsync(commitLogOffset, size);
-            }, TieredStoreExecutor.fetchDataExecutor)
-            .thenApply(MessageBufferUtil::getStoreTimeStamp)
-            .exceptionally(e -> {
-                LOGGER.error("TieredMessageFetcher#getMessageStoreTimeStampAsync: " +
-                    "get or decode message failed: topic: {}, queue: {}, offset: {}", topic, queueId, queueOffset, e);
-                return -1L;
-            });
-    }
-
-    @Override
-    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType type) {
-        CompositeFlatFile flatFile = flatFileManager.getFlatFile(new MessageQueue(topic, brokerName, queueId));
-        if (flatFile == null) {
-            return -1L;
-        }
-
-        try {
-            return flatFile.getOffsetInConsumeQueueByTime(timestamp, type);
-        } catch (Exception e) {
-            LOGGER.error("TieredMessageFetcher#getOffsetInQueueByTime: " +
-                    "get offset in queue by time failed: topic: {}, queue: {}, timestamp: {}, type: {}",
-                topic, queueId, timestamp, type, e);
-        }
-        return -1L;
-    }
-
-    @Override
-    public CompletableFuture<QueryMessageResult> queryMessageAsync(
-        String topic, String key, int maxCount, long begin, long end) {
-
-        IndexService indexStoreService = TieredFlatFileManager.getTieredIndexService(storeConfig);
-
-        long topicId;
-        try {
-            TopicMetadata topicMetadata = metadataStore.getTopic(topic);
-            if (topicMetadata == null) {
-                LOGGER.info("MessageFetcher#queryMessageAsync, topic metadata not found, topic: {}", topic);
-                return CompletableFuture.completedFuture(new QueryMessageResult());
-            }
-            topicId = topicMetadata.getTopicId();
-        } catch (Exception e) {
-            LOGGER.error("MessageFetcher#queryMessageAsync, get topic id failed, topic: {}", topic, e);
-            return CompletableFuture.completedFuture(new QueryMessageResult());
-        }
-
-        CompletableFuture<List<IndexItem>> future = indexStoreService.queryAsync(topic, key, maxCount, begin, end);
-
-        return future.thenCompose(indexItemList -> {
-            QueryMessageResult result = new QueryMessageResult();
-            List<CompletableFuture<Void>> futureList = new ArrayList<>(maxCount);
-            for (IndexItem indexItem : indexItemList) {
-                if (topicId != indexItem.getTopicId()) {
-                    continue;
-                }
-                CompositeFlatFile flatFile =
-                    flatFileManager.getFlatFile(new MessageQueue(topic, brokerName, indexItem.getQueueId()));
-                if (flatFile == null) {
-                    continue;
-                }
-                CompletableFuture<Void> getMessageFuture = flatFile
-                    .getCommitLogAsync(indexItem.getOffset(), indexItem.getSize())
-                    .thenAccept(messageBuffer -> result.addMessage(
-                        new SelectMappedBufferResult(
-                            indexItem.getOffset(), messageBuffer, indexItem.getSize(), null)));
-                futureList.add(getMessageFuture);
-                if (futureList.size() >= maxCount) {
-                    break;
-                }
-            }
-            return CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).thenApply(v -> result);
-        }).whenComplete((result, throwable) -> {
-            if (result != null) {
-                LOGGER.info("MessageFetcher#queryMessageAsync, " +
-                        "query result: {}, topic: {}, topicId: {}, key: {}, maxCount: {}, timestamp: {}-{}",
-                    result.getMessageBufferList().size(), topic, topicId, key, maxCount, begin, end);
-            }
-        });
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
index 015c27efae1..f1c935d00b7 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java
@@ -16,109 +16,162 @@
  */
 package org.apache.rocketmq.tieredstore;
 
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Sets;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.common.AttributesBuilder;
+import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.sdk.metrics.InstrumentSelector;
+import io.opentelemetry.sdk.metrics.ViewBuilder;
+import java.lang.reflect.Constructor;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
-
-import com.google.common.base.Stopwatch;
-
-import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.BoundaryType;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.Pair;
-import org.apache.rocketmq.common.PopAckConstants;
-import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.topic.TopicValidator;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.store.GetMessageResult;
 import org.apache.rocketmq.store.GetMessageStatus;
 import org.apache.rocketmq.store.MessageFilter;
 import org.apache.rocketmq.store.MessageStore;
-import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.QueryMessageResult;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
 import org.apache.rocketmq.store.plugin.AbstractPluginMessageStore;
 import org.apache.rocketmq.store.plugin.MessageStorePluginContext;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.file.CompositeFlatFile;
-import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
+import org.apache.rocketmq.tieredstore.core.MessageStoreDispatcher;
+import org.apache.rocketmq.tieredstore.core.MessageStoreDispatcherImpl;
+import org.apache.rocketmq.tieredstore.core.MessageStoreFetcher;
+import org.apache.rocketmq.tieredstore.core.MessageStoreFetcherImpl;
+import org.apache.rocketmq.tieredstore.core.MessageStoreFilter;
+import org.apache.rocketmq.tieredstore.core.MessageStoreTopicFilter;
+import org.apache.rocketmq.tieredstore.file.FlatFileStore;
+import org.apache.rocketmq.tieredstore.file.FlatMessageFile;
+import org.apache.rocketmq.tieredstore.index.IndexService;
+import org.apache.rocketmq.tieredstore.index.IndexStoreService;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
 import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant;
 import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-
-import io.opentelemetry.api.common.Attributes;
-import io.opentelemetry.api.common.AttributesBuilder;
-import io.opentelemetry.api.metrics.Meter;
-import io.opentelemetry.sdk.metrics.InstrumentSelector;
-import io.opentelemetry.sdk.metrics.ViewBuilder;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TieredMessageStore extends AbstractPluginMessageStore {
 
-    protected static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
+    protected static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
+    protected static final long MIN_STORE_TIME = -1L;
 
     protected final String brokerName;
-    protected final TieredMessageStoreConfig storeConfig;
-    protected final TieredMetadataStore metadataStore;
-
-    protected final TieredDispatcher dispatcher;
-    protected final TieredMessageFetcher fetcher;
-    protected final TieredFlatFileManager flatFileManager;
+    protected final MessageStore defaultStore;
+    protected final MessageStoreConfig storeConfig;
+    protected final MessageStorePluginContext context;
+
+    protected final MetadataStore metadataStore;
+    protected final MessageStoreExecutor storeExecutor;
+    protected final IndexService indexService;
+    protected final FlatFileStore flatFileStore;
+    protected final MessageStoreFilter topicFilter;
+    protected final MessageStoreFetcher fetcher;
+    protected final MessageStoreDispatcher dispatcher;
 
     public TieredMessageStore(MessageStorePluginContext context, MessageStore next) {
         super(context, next);
-        this.storeConfig = new TieredMessageStoreConfig();
-        context.registerConfiguration(storeConfig);
-        this.brokerName = storeConfig.getBrokerName();
-        TieredStoreUtil.addSystemTopic(storeConfig.getBrokerClusterName());
-        TieredStoreUtil.addSystemTopic(brokerName);
-
-        TieredStoreExecutor.init();
-        this.metadataStore = TieredStoreUtil.getMetadataStore(storeConfig);
-        this.fetcher = new TieredMessageFetcher(storeConfig);
-        this.dispatcher = new TieredDispatcher(next, storeConfig);
-
-        this.flatFileManager = TieredFlatFileManager.getInstance(storeConfig);
+
+        this.storeConfig = new MessageStoreConfig();
+        this.context = context;
+        this.context.registerConfiguration(this.storeConfig);
+        this.brokerName = this.storeConfig.getBrokerName();
+        this.defaultStore = next;
+
+        this.metadataStore = this.getMetadataStore(this.storeConfig);
+        this.topicFilter = new MessageStoreTopicFilter(this.storeConfig);
+        this.storeExecutor = new MessageStoreExecutor();
+        this.flatFileStore = new FlatFileStore(this.storeConfig, this.metadataStore, this.storeExecutor);
+        this.indexService = new IndexStoreService(this.flatFileStore.getFlatFileFactory(),
+            MessageStoreUtil.getIndexFilePath(this.storeConfig.getBrokerName()));
+        this.fetcher = new MessageStoreFetcherImpl(this);
+        this.dispatcher = new MessageStoreDispatcherImpl(this);
         next.addDispatcher(dispatcher);
     }
 
     @Override
     public boolean load() {
-        boolean loadFlatFile = flatFileManager.load();
+        boolean loadFlatFile = flatFileStore.load();
         boolean loadNextStore = next.load();
         boolean result = loadFlatFile && loadNextStore;
         if (result) {
-            dispatcher.initScheduleTask();
+            indexService.start();
             dispatcher.start();
+            storeExecutor.commonExecutor.scheduleWithFixedDelay(
+                flatFileStore::scheduleDeleteExpireFile, storeConfig.getTieredStoreDeleteFileInterval(),
+                storeConfig.getTieredStoreDeleteFileInterval(), TimeUnit.MILLISECONDS);
         }
         return result;
     }
 
-    public TieredMessageStoreConfig getStoreConfig() {
+    public String getBrokerName() {
+        return brokerName;
+    }
+
+    public MessageStoreConfig getStoreConfig() {
         return storeConfig;
     }
 
+    public MessageStore getDefaultStore() {
+        return defaultStore;
+    }
+
+    private MetadataStore getMetadataStore(MessageStoreConfig storeConfig) {
+        try {
+            Class<? extends MetadataStore> clazz =
+                Class.forName(storeConfig.getTieredMetadataServiceProvider()).asSubclass(MetadataStore.class);
+            Constructor<? extends MetadataStore> constructor = clazz.getConstructor(MessageStoreConfig.class);
+            return constructor.newInstance(storeConfig);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public MetadataStore getMetadataStore() {
+        return metadataStore;
+    }
+
+    public MessageStoreFilter getTopicFilter() {
+        return topicFilter;
+    }
+
+    public MessageStoreExecutor getStoreExecutor() {
+        return storeExecutor;
+    }
+
+    public FlatFileStore getFlatFileStore() {
+        return flatFileStore;
+    }
+
+    public IndexService getIndexService() {
+        return indexService;
+    }
+
     public boolean fetchFromCurrentStore(String topic, int queueId, long offset) {
         return fetchFromCurrentStore(topic, queueId, offset, 1);
     }
 
+    @SuppressWarnings("all")
     public boolean fetchFromCurrentStore(String topic, int queueId, long offset, int batchSize) {
-        TieredMessageStoreConfig.TieredStorageLevel deepStorageLevel = storeConfig.getTieredStorageLevel();
+        MessageStoreConfig.TieredStorageLevel storageLevel = storeConfig.getTieredStorageLevel();
 
-        if (deepStorageLevel.check(TieredMessageStoreConfig.TieredStorageLevel.FORCE)) {
+        if (storageLevel.check(MessageStoreConfig.TieredStorageLevel.FORCE)) {
             return true;
         }
 
-        if (!deepStorageLevel.isEnable()) {
+        if (!storageLevel.isEnable()) {
             return false;
         }
 
-        CompositeFlatFile flatFile = flatFileManager.getFlatFile(new MessageQueue(topic, brokerName, queueId));
+        FlatMessageFile flatFile = flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, queueId));
         if (flatFile == null) {
             return false;
         }
@@ -128,12 +181,18 @@ public boolean fetchFromCurrentStore(String topic, int queueId, long offset, int
         }
 
         // determine whether tiered storage path conditions are met
-        if (deepStorageLevel.check(TieredMessageStoreConfig.TieredStorageLevel.NOT_IN_DISK)
-            && !next.checkInStoreByConsumeOffset(topic, queueId, offset)) {
-            return true;
+        if (storageLevel.check(MessageStoreConfig.TieredStorageLevel.NOT_IN_DISK)) {
+            // return true to read from tiered storage if the CommitLog is empty
+            if (next != null && next.getCommitLog() != null &&
+                next.getCommitLog().getMinOffset() < 0L) {
+                return true;
+            }
+            if (!next.checkInStoreByConsumeOffset(topic, queueId, offset)) {
+                return true;
+            }
         }
 
-        if (deepStorageLevel.check(TieredMessageStoreConfig.TieredStorageLevel.NOT_IN_MEM)
+        if (storageLevel.check(MessageStoreConfig.TieredStorageLevel.NOT_IN_MEM)
             && !next.checkInMemByConsumeOffset(topic, queueId, offset, batchSize)) {
             return true;
         }
@@ -150,15 +209,17 @@ public GetMessageResult getMessage(String group, String topic, int queueId, long
     public CompletableFuture<GetMessageResult> getMessageAsync(String group, String topic,
         int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) {
 
-        // For system topic, force reading from local store
-        if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) {
+        // for system topic, force reading from local store
+        if (topicFilter.filterTopic(topic)) {
             return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter);
         }
 
         if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) {
-            logger.trace("GetMessageAsync from current store, topic: {}, queue: {}, offset: {}", topic, queueId, offset);
+            log.trace("GetMessageAsync from remote store, " +
+                "topic: {}, queue: {}, offset: {}, maxCount: {}", topic, queueId, offset, maxMsgNums);
         } else {
-            logger.trace("GetMessageAsync from next store, topic: {}, queue: {}, offset: {}", topic, queueId, offset);
+            log.trace("GetMessageAsync from next store, " +
+                "topic: {}, queue: {}, offset: {}, maxCount: {}", topic, queueId, offset, maxMsgNums);
             return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter);
         }
 
@@ -179,7 +240,7 @@ public CompletableFuture<GetMessageResult> getMessageAsync(String group, String
 
                     if (next.checkInStoreByConsumeOffset(topic, queueId, offset)) {
                         TieredStoreMetricsManager.fallbackTotal.add(1, latencyAttributes);
-                        logger.debug("GetMessageAsync not found, then back to next store, result: {}, " +
+                        log.debug("GetMessageAsync not found, then back to next store, result: {}, " +
                                 "topic: {}, queue: {}, queue offset: {}, offset range: {}-{}",
                             result.getStatus(), topic, queueId, offset, result.getMinOffset(), result.getMaxOffset());
                         return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter);
@@ -187,10 +248,12 @@ public CompletableFuture<GetMessageResult> getMessageAsync(String group, String
                 }
 
                 if (result.getStatus() != GetMessageStatus.FOUND &&
+                    result.getStatus() != GetMessageStatus.NO_MESSAGE_IN_QUEUE &&
                     result.getStatus() != GetMessageStatus.NO_MATCHED_LOGIC_QUEUE &&
+                    result.getStatus() != GetMessageStatus.OFFSET_TOO_SMALL &&
                     result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_ONE &&
                     result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_BADLY) {
-                    logger.warn("GetMessageAsync not found and message is not in next store, result: {}, " +
+                    log.warn("GetMessageAsync not found and message is not in next store, result: {}, " +
                             "topic: {}, queue: {}, queue offset: {}, offset range: {}-{}",
                         result.getStatus(), topic, queueId, offset, result.getMinOffset(), result.getMaxOffset());
                 }
@@ -201,6 +264,10 @@ public CompletableFuture<GetMessageResult> getMessageAsync(String group, String
                         .put(TieredStoreMetricsConstant.LABEL_GROUP, group)
                         .build();
                     TieredStoreMetricsManager.messagesOutTotal.add(result.getMessageCount(), messagesOutAttributes);
+
+                    if (next.getStoreStatsService() != null) {
+                        next.getStoreStatsService().getGetMessageTransferredMsgCount().add(result.getMessageCount());
+                    }
                 }
 
                 // Fix min or max offset according next store at last
@@ -211,29 +278,24 @@ public CompletableFuture<GetMessageResult> getMessageAsync(String group, String
 
                 // In general, the local cq offset is slightly greater than the commit offset in read message,
                 // so there is no need to update the maximum offset to the local cq offset here,
-                // otherwise it will cause repeated consumption after next begin offset over commit offset.
+                // otherwise it will cause repeated consumption after next start offset over commit offset.
 
                 if (storeConfig.isRecordGetMessageResult()) {
-                    logger.info("GetMessageAsync result, {}, group: {}, topic: {}, queueId: {}, offset: {}, count:{}",
+                    log.info("GetMessageAsync result, {}, group: {}, topic: {}, queueId: {}, offset: {}, count:{}",
                         result, group, topic, queueId, offset, maxMsgNums);
                 }
 
                 return result;
             }).exceptionally(e -> {
-                logger.error("GetMessageAsync from tiered store failed", e);
+                log.error("GetMessageAsync from tiered store failed", e);
                 return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter);
             });
     }
 
-    @Override
-    public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner msg) {
-        return super.asyncPutMessage(msg);
-    }
-
     @Override
     public long getMinOffsetInQueue(String topic, int queueId) {
         long minOffsetInNextStore = next.getMinOffsetInQueue(topic, queueId);
-        CompositeFlatFile flatFile = flatFileManager.getFlatFile(new MessageQueue(topic, brokerName, queueId));
+        FlatMessageFile flatFile = flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, queueId));
         if (flatFile == null) {
             return minOffsetInNextStore;
         }
@@ -249,24 +311,21 @@ public long getEarliestMessageTime(String topic, int queueId) {
         return getEarliestMessageTimeAsync(topic, queueId).join();
     }
 
+    /**
+     * In the original design, getting the earliest time of the first message
+     * would generate two RPC requests. However, using the timestamp stored in the metadata
+     * avoids these requests, although this approach might introduce some level of inaccuracy.
+     */
     @Override
     public CompletableFuture<Long> getEarliestMessageTimeAsync(String topic, int queueId) {
-        long nextEarliestMessageTime = next.getEarliestMessageTime(topic, queueId);
-        long finalNextEarliestMessageTime = nextEarliestMessageTime > 0 ? nextEarliestMessageTime : Long.MAX_VALUE;
-        Stopwatch stopwatch = Stopwatch.createStarted();
+        long localMinTime = next.getEarliestMessageTime(topic, queueId);
         return fetcher.getEarliestMessageTimeAsync(topic, queueId)
-            .thenApply(time -> {
-                Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder()
-                    .put(TieredStoreMetricsConstant.LABEL_OPERATION, TieredStoreMetricsConstant.OPERATION_API_GET_EARLIEST_MESSAGE_TIME)
-                    .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic)
-                    .build();
-                TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes);
-                if (time < 0) {
-                    logger.debug("GetEarliestMessageTimeAsync failed, try to get earliest message time from next store: topic: {}, queue: {}",
-                        topic, queueId);
-                    return finalNextEarliestMessageTime != Long.MAX_VALUE ? finalNextEarliestMessageTime : -1;
+            .thenApply(remoteMinTime -> {
+                if (localMinTime > MIN_STORE_TIME && remoteMinTime > MIN_STORE_TIME) {
+                    return Math.min(localMinTime, remoteMinTime);
                 }
-                return Math.min(finalNextEarliestMessageTime, time);
+                return localMinTime > MIN_STORE_TIME ? localMinTime :
+                    (remoteMinTime > MIN_STORE_TIME ? remoteMinTime : MIN_STORE_TIME);
             });
     }
 
@@ -278,12 +337,13 @@ public CompletableFuture<Long> getMessageStoreTimeStampAsync(String topic, int q
             return fetcher.getMessageStoreTimeStampAsync(topic, queueId, consumeQueueOffset)
                 .thenApply(time -> {
                     Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder()
-                        .put(TieredStoreMetricsConstant.LABEL_OPERATION, TieredStoreMetricsConstant.OPERATION_API_GET_TIME_BY_OFFSET)
+                        .put(TieredStoreMetricsConstant.LABEL_OPERATION,
+                                TieredStoreMetricsConstant.OPERATION_API_GET_TIME_BY_OFFSET)
                         .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic)
                         .build();
                     TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes);
                     if (time == -1) {
-                        logger.debug("GetEarliestMessageTimeAsync failed, try to get message time from next store, topic: {}, queue: {}, queue offset: {}",
+                        log.debug("GetEarliestMessageTimeAsync failed, try to get message time from next store, topic: {}, queue: {}, queue offset: {}",
                             topic, queueId, consumeQueueOffset);
                         return next.getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset);
                     }
@@ -300,13 +360,8 @@ public long getOffsetInQueueByTime(String topic, int queueId, long timestamp) {
 
     @Override
     public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) {
-        long earliestTimeInNextStore = next.getEarliestMessageTime();
-        if (earliestTimeInNextStore <= 0) {
-            logger.warn("TieredMessageStore#getOffsetInQueueByTimeAsync: get earliest message time in next store failed: {}", earliestTimeInNextStore);
-            return next.getOffsetInQueueByTime(topic, queueId, timestamp);
-        }
-        boolean isForce = storeConfig.getTieredStorageLevel() == TieredMessageStoreConfig.TieredStorageLevel.FORCE;
-        if (timestamp < earliestTimeInNextStore || isForce) {
+        boolean isForce = storeConfig.getTieredStorageLevel() == MessageStoreConfig.TieredStorageLevel.FORCE;
+        if (timestamp < next.getEarliestMessageTime() || isForce) {
             Stopwatch stopwatch = Stopwatch.createStarted();
             long offsetInTieredStore = fetcher.getOffsetInQueueByTime(topic, queueId, timestamp, boundaryType);
             Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder()
@@ -314,7 +369,7 @@ public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, Bo
                 .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic)
                 .build();
             TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes);
-            if (offsetInTieredStore == -1 && !isForce) {
+            if (offsetInTieredStore == -1L && !isForce) {
                 return next.getOffsetInQueueByTime(topic, queueId, timestamp);
             }
             return offsetInTieredStore;
@@ -332,9 +387,9 @@ public CompletableFuture<QueryMessageResult> queryMessageAsync(String topic, Str
         int maxNum, long begin, long end) {
         long earliestTimeInNextStore = next.getEarliestMessageTime();
         if (earliestTimeInNextStore <= 0) {
-            logger.warn("TieredMessageStore#queryMessageAsync: get earliest message time in next store failed: {}", earliestTimeInNextStore);
+            log.warn("TieredMessageStore#queryMessageAsync: get earliest message time in next store failed: {}", earliestTimeInNextStore);
         }
-        boolean isForce = storeConfig.getTieredStorageLevel() == TieredMessageStoreConfig.TieredStorageLevel.FORCE;
+        boolean isForce = storeConfig.getTieredStorageLevel() == MessageStoreConfig.TieredStorageLevel.FORCE;
         QueryMessageResult result = end < earliestTimeInNextStore || isForce ?
             new QueryMessageResult() :
             next.queryMessage(topic, key, maxNum, begin, end);
@@ -355,7 +410,7 @@ public CompletableFuture<QueryMessageResult> queryMessageAsync(String topic, Str
                         return result;
                     });
             } catch (Exception e) {
-                logger.error("TieredMessageStore#queryMessageAsync: query message in tiered store failed", e);
+                log.error("TieredMessageStore#queryMessageAsync: query message in tiered store failed", e);
                 return CompletableFuture.completedFuture(result);
             }
         }
@@ -372,69 +427,67 @@ public List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {
     @Override
     public void initMetrics(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier) {
         super.initMetrics(meter, attributesBuilderSupplier);
-        TieredStoreMetricsManager.init(meter, attributesBuilderSupplier, storeConfig, fetcher, next);
+        TieredStoreMetricsManager.init(meter, attributesBuilderSupplier, storeConfig, fetcher, flatFileStore, next);
     }
 
     @Override
-    public void shutdown() {
-        next.shutdown();
-
-        dispatcher.shutdown();
-        TieredFlatFileManager.getInstance(storeConfig).shutdown();
-        TieredStoreExecutor.shutdown();
+    public int cleanUnusedTopic(Set<String> retainTopics) {
+        metadataStore.iterateTopic(topicMetadata -> {
+            String topic = topicMetadata.getTopic();
+            if (retainTopics.contains(topic) ||
+                TopicValidator.isSystemTopic(topic) ||
+                MixAll.isLmq(topic)) {
+                return;
+            }
+            this.deleteTopics(Sets.newHashSet(topicMetadata.getTopic()));
+        });
+        return next.cleanUnusedTopic(retainTopics);
     }
 
     @Override
-    public void destroy() {
-        next.destroy();
-
-        TieredFlatFileManager.getInstance(storeConfig).destroy();
-        try {
-            metadataStore.destroy();
-        } catch (Exception e) {
-            logger.error("TieredMessageStore#destroy: destroy metadata store failed", e);
+    public int deleteTopics(Set<String> deleteTopics) {
+        for (String topic : deleteTopics) {
+            metadataStore.iterateQueue(topic, queueMetadata -> {
+                flatFileStore.destroyFile(queueMetadata.getQueue());
+            });
+            metadataStore.deleteTopic(topic);
+            log.info("MessageStore delete topic success, topicName={}", topic);
         }
+        return next.deleteTopics(deleteTopics);
     }
 
     @Override
-    public int cleanUnusedTopic(Set<String> retainTopics) {
-        try {
-            metadataStore.iterateTopic(topicMetadata -> {
-                String topic = topicMetadata.getTopic();
-                if (retainTopics.contains(topic) ||
-                    TopicValidator.isSystemTopic(topic) ||
-                    MixAll.isLmq(topic)) {
-                    return;
-                }
-                this.destroyCompositeFlatFile(topicMetadata.getTopic());
-            });
-        } catch (Exception e) {
-            logger.error("TieredMessageStore#cleanUnusedTopic: iterate topic metadata failed", e);
+    public synchronized void shutdown() {
+        if (next != null) {
+            next.shutdown();
+        }
+        if (dispatcher != null) {
+            dispatcher.shutdown();
+        }
+        if (indexService != null) {
+            indexService.shutdown();
+        }
+        if (flatFileStore != null) {
+            flatFileStore.shutdown();
+        }
+        if (storeExecutor != null) {
+            storeExecutor.shutdown();
         }
-        return next.cleanUnusedTopic(retainTopics);
     }
 
     @Override
-    public int deleteTopics(Set<String> deleteTopics) {
-        for (String topic : deleteTopics) {
-            this.destroyCompositeFlatFile(topic);
+    public void destroy() {
+        if (next != null) {
+            next.destroy();
         }
-        return next.deleteTopics(deleteTopics);
-    }
-
-    public void destroyCompositeFlatFile(String topic) {
-        try {
-            if (StringUtils.isBlank(topic)) {
-                return;
-            }
-            metadataStore.iterateQueue(topic, queueMetadata -> {
-                flatFileManager.destroyCompositeFile(queueMetadata.getQueue());
-            });
-            // delete topic metadata
-            metadataStore.deleteTopic(topic);
-            logger.info("Destroy composite flat file in message store, topic={}", topic);
-        } catch (Exception e) {
-            logger.error("Destroy composite flat file in message store failed, topic={}", topic, e);
+        if (indexService != null) {
+            indexService.destroy();
+        }
+        if (flatFileStore != null) {
+            flatFileStore.destroy();
+        }
+        if (metadataStore != null) {
+            metadataStore.destroy();
         }
     }
 }
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/AppendResult.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/AppendResult.java
index 4482cb79be2..97cfe4d4247 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/AppendResult.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/AppendResult.java
@@ -23,11 +23,6 @@ public enum AppendResult {
      */
     SUCCESS,
 
-    /**
-     * The offset provided for the append operation is incorrect.
-     */
-    OFFSET_INCORRECT,
-
     /**
      * The buffer used for the append operation is full.
      */
@@ -38,11 +33,6 @@ public enum AppendResult {
      */
     FILE_FULL,
 
-    /**
-     * There was an I/O error during the append operation.
-     */
-    IO_ERROR,
-
     /**
      * The file is closed and cannot accept more data.
      */
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/FileSegmentType.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/FileSegmentType.java
index a370bec00bd..d7b3c9af87b 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/FileSegmentType.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/FileSegmentType.java
@@ -16,31 +16,30 @@
  */
 package org.apache.rocketmq.tieredstore.common;
 
+import java.util.Arrays;
+
 public enum FileSegmentType {
+
     COMMIT_LOG(0),
+
     CONSUME_QUEUE(1),
+
     INDEX(2);
 
-    private final int type;
+    private final int code;
 
-    FileSegmentType(int type) {
-        this.type = type;
+    FileSegmentType(int code) {
+        this.code = code;
     }
 
-    public int getType() {
-        return type;
+    public int getCode() {
+        return code;
     }
 
-    public static FileSegmentType valueOf(int type) {
-        switch (type) {
-            case 0:
-                return COMMIT_LOG;
-            case 1:
-                return CONSUME_QUEUE;
-            case 2:
-                return INDEX;
-            default:
-                throw new IllegalStateException("Unexpected value: " + type);
-        }
+    public static FileSegmentType valueOf(int fileType) {
+        return Arrays.stream(FileSegmentType.values())
+            .filter(segmentType -> segmentType.getCode() == fileType)
+            .findFirst()
+            .orElse(COMMIT_LOG);
     }
 }
\ No newline at end of file
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/GetMessageResultExt.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/GetMessageResultExt.java
index 2e294c1c7dc..b6016f25a37 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/GetMessageResultExt.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/GetMessageResultExt.java
@@ -41,6 +41,10 @@ public List<Long> getTagCodeList() {
         return tagCodeList;
     }
 
+    /**
+     * Due to the message fetched from the object storage is sequential,
+     * do message filtering occurs after the data retrieval.
+     */
     public GetMessageResult doFilterMessage(MessageFilter messageFilter) {
         if (GetMessageStatus.FOUND != super.getStatus() || messageFilter == null) {
             return this;
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/GroupCommitContext.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/GroupCommitContext.java
new file mode 100644
index 00000000000..f677e7c934e
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/GroupCommitContext.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.tieredstore.common;
+
+import java.util.List;
+import org.apache.rocketmq.store.DispatchRequest;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+
+public class GroupCommitContext {
+
+    private long endOffset;
+
+    private List<SelectMappedBufferResult> bufferList;
+
+    private List<DispatchRequest> dispatchRequests;
+
+    public long getEndOffset() {
+        return endOffset;
+    }
+
+    public void setEndOffset(long endOffset) {
+        this.endOffset = endOffset;
+    }
+
+    public List<SelectMappedBufferResult> getBufferList() {
+        return bufferList;
+    }
+
+    public void setBufferList(List<SelectMappedBufferResult> bufferList) {
+        this.bufferList = bufferList;
+    }
+
+    public List<DispatchRequest> getDispatchRequests() {
+        return dispatchRequests;
+    }
+
+    public void setDispatchRequests(List<DispatchRequest> dispatchRequests) {
+        this.dispatchRequests = dispatchRequests;
+    }
+
+    public void release() {
+        if (bufferList != null) {
+            for (SelectMappedBufferResult bufferResult : bufferList) {
+                bufferResult.release();
+            }
+            bufferList.clear();
+            bufferList = null;
+        }
+        if (dispatchRequests != null) {
+            dispatchRequests.clear();
+            dispatchRequests = null;
+        }
+
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/InFlightRequestFuture.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/InFlightRequestFuture.java
deleted file mode 100644
index fb872833a84..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/InFlightRequestFuture.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.common;
-
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.stream.Collectors;
-import javax.annotation.Nonnull;
-import org.apache.commons.lang3.tuple.Pair;
-
-public class InFlightRequestFuture {
-
-    private final long startOffset;
-    private final List<Pair<Integer, CompletableFuture<Long>>> futureList;
-
-    public InFlightRequestFuture(long startOffset, @Nonnull List<Pair<Integer, CompletableFuture<Long>>> futureList) {
-        this.startOffset = startOffset;
-        this.futureList = futureList;
-    }
-
-    public long getStartOffset() {
-        return startOffset;
-    }
-
-    public CompletableFuture<Long> getFirstFuture() {
-        return futureList.isEmpty() ? CompletableFuture.completedFuture(-1L) : futureList.get(0).getRight();
-    }
-
-    public CompletableFuture<Long> getFuture(long queueOffset) {
-        if (queueOffset < startOffset) {
-            return CompletableFuture.completedFuture(-1L);
-        }
-        long nextRequestOffset = startOffset;
-        for (Pair<Integer, CompletableFuture<Long>> pair : futureList) {
-            nextRequestOffset += pair.getLeft();
-            if (queueOffset < nextRequestOffset) {
-                return pair.getRight();
-            }
-        }
-        return CompletableFuture.completedFuture(-1L);
-    }
-
-    public CompletableFuture<Long> getLastFuture() {
-        return futureList.isEmpty() ?
-            CompletableFuture.completedFuture(-1L) : futureList.get(futureList.size() - 1).getRight();
-    }
-
-    public boolean isFirstDone() {
-        if (!futureList.isEmpty()) {
-            return futureList.get(0).getRight().isDone();
-        }
-        return true;
-    }
-
-    public boolean isAllDone() {
-        for (Pair<Integer, CompletableFuture<Long>> pair : futureList) {
-            if (!pair.getRight().isDone()) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    public List<CompletableFuture<Long>> getAllFuture() {
-        return futureList.stream().map(Pair::getValue).collect(Collectors.toList());
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/InFlightRequestKey.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/InFlightRequestKey.java
deleted file mode 100644
index 0e461a83072..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/InFlightRequestKey.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.common;
-
-import com.google.common.base.Objects;
-
-public class InFlightRequestKey {
-
-    private final String group;
-    private long offset;
-    private int batchSize;
-    private final long requestTime = System.currentTimeMillis();
-
-    public InFlightRequestKey(String group) {
-        this.group = group;
-    }
-
-    public InFlightRequestKey(String group, long offset, int batchSize) {
-        this.group = group;
-        this.offset = offset;
-        this.batchSize = batchSize;
-    }
-
-    public String getGroup() {
-        return group;
-    }
-
-    public long getOffset() {
-        return offset;
-    }
-
-    public int getBatchSize() {
-        return batchSize;
-    }
-
-    public long getRequestTime() {
-        return requestTime;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o)
-            return true;
-        if (o == null || getClass() != o.getClass())
-            return false;
-        InFlightRequestKey key = (InFlightRequestKey) o;
-        return Objects.equal(group, key.group);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hashCode(group);
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/MessageCacheKey.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/MessageCacheKey.java
deleted file mode 100644
index ab06aa64d2e..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/MessageCacheKey.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.common;
-
-import java.util.Objects;
-import org.apache.rocketmq.tieredstore.file.CompositeFlatFile;
-
-public class MessageCacheKey {
-
-    private final CompositeFlatFile flatFile;
-    private final long offset;
-
-    public MessageCacheKey(CompositeFlatFile flatFile, long offset) {
-        this.flatFile = flatFile;
-        this.offset = offset;
-    }
-
-    public CompositeFlatFile getFlatFile() {
-        return flatFile;
-    }
-
-    public long getOffset() {
-        return offset;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        MessageCacheKey that = (MessageCacheKey) o;
-        return offset == that.offset && Objects.equals(flatFile, that.flatFile);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(flatFile, offset);
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/SelectBufferResult.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/SelectBufferResult.java
index d265ed0fc42..cad37c7bc43 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/SelectBufferResult.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/SelectBufferResult.java
@@ -18,6 +18,7 @@
 package org.apache.rocketmq.tieredstore.common;
 
 import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicLong;
 
 public class SelectBufferResult {
 
@@ -25,12 +26,14 @@ public class SelectBufferResult {
     private final long startOffset;
     private final int size;
     private final long tagCode;
+    private final AtomicLong accessCount;
 
     public SelectBufferResult(ByteBuffer byteBuffer, long startOffset, int size, long tagCode) {
         this.startOffset = startOffset;
         this.byteBuffer = byteBuffer;
         this.size = size;
         this.tagCode = tagCode;
+        this.accessCount = new AtomicLong();
     }
 
     public ByteBuffer getByteBuffer() {
@@ -48,4 +51,8 @@ public int getSize() {
     public long getTagCode() {
         return tagCode;
     }
+
+    public AtomicLong getAccessCount() {
+        return accessCount;
+    }
 }
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/SelectBufferResultWrapper.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/SelectBufferResultWrapper.java
deleted file mode 100644
index 4f9f00a074c..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/SelectBufferResultWrapper.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.common;
-
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.rocketmq.store.SelectMappedBufferResult;
-
-public class SelectBufferResultWrapper {
-
-    private final SelectMappedBufferResult result;
-    private final long offset;
-    private final long tagCode;
-    private final AtomicInteger accessCount;
-
-    public SelectBufferResultWrapper(SelectMappedBufferResult result, long offset, long tagCode, boolean used) {
-        this.result = result;
-        this.offset = offset;
-        this.tagCode = tagCode;
-        this.accessCount = new AtomicInteger(used ? 1 : 0);
-    }
-
-    public SelectMappedBufferResult getDuplicateResult() {
-
-        return new SelectMappedBufferResult(
-            result.getStartOffset(),
-            result.getByteBuffer().asReadOnlyBuffer(),
-            result.getSize(),
-            result.getMappedFile());
-    }
-
-    public long getOffset() {
-        return offset;
-    }
-
-    public int getBufferSize() {
-        return this.result.getSize();
-    }
-
-    public long getTagCode() {
-        return tagCode;
-    }
-
-    public int incrementAndGet() {
-        return accessCount.incrementAndGet();
-    }
-
-    public int getAccessCount() {
-        return accessCount.get();
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java
deleted file mode 100644
index 65d586f43dd..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.common;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import org.apache.rocketmq.common.ThreadFactoryImpl;
-import org.apache.rocketmq.common.utils.ThreadUtils;
-
-public class TieredStoreExecutor {
-
-    public static final int QUEUE_CAPACITY = 10000;
-
-    // Visible for monitor
-    public static BlockingQueue<Runnable> dispatchThreadPoolQueue;
-    public static BlockingQueue<Runnable> fetchDataThreadPoolQueue;
-    public static BlockingQueue<Runnable> compactIndexFileThreadPoolQueue;
-
-    public static ScheduledExecutorService commonScheduledExecutor;
-    public static ScheduledExecutorService commitExecutor;
-    public static ScheduledExecutorService cleanExpiredFileExecutor;
-
-    public static ExecutorService dispatchExecutor;
-    public static ExecutorService fetchDataExecutor;
-    public static ExecutorService compactIndexFileExecutor;
-
-    public static void init() {
-        commonScheduledExecutor = ThreadUtils.newScheduledThreadPool(
-            Math.max(4, Runtime.getRuntime().availableProcessors()),
-            new ThreadFactoryImpl("TieredCommonExecutor_"));
-
-        commitExecutor = ThreadUtils.newScheduledThreadPool(
-            Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
-            new ThreadFactoryImpl("TieredCommitExecutor_"));
-
-        cleanExpiredFileExecutor = ThreadUtils.newScheduledThreadPool(
-            Math.max(4, Runtime.getRuntime().availableProcessors()),
-            new ThreadFactoryImpl("TieredCleanFileExecutor_"));
-
-        dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY);
-        dispatchExecutor = ThreadUtils.newThreadPoolExecutor(
-            Math.max(2, Runtime.getRuntime().availableProcessors()),
-            Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
-            1000 * 60,
-            TimeUnit.MILLISECONDS,
-            dispatchThreadPoolQueue,
-            new ThreadFactoryImpl("TieredDispatchExecutor_"),
-            new ThreadPoolExecutor.DiscardOldestPolicy());
-
-        fetchDataThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY);
-        fetchDataExecutor = ThreadUtils.newThreadPoolExecutor(
-            Math.max(16, Runtime.getRuntime().availableProcessors() * 4),
-            Math.max(64, Runtime.getRuntime().availableProcessors() * 8),
-            1000 * 60,
-            TimeUnit.MILLISECONDS,
-            fetchDataThreadPoolQueue,
-            new ThreadFactoryImpl("TieredFetchExecutor_"));
-
-        compactIndexFileThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY);
-        compactIndexFileExecutor = ThreadUtils.newThreadPoolExecutor(
-            1,
-            1,
-            1000 * 60,
-            TimeUnit.MILLISECONDS,
-            compactIndexFileThreadPoolQueue,
-            new ThreadFactoryImpl("TieredCompactIndexFileExecutor_"));
-    }
-
-    public static void shutdown() {
-        shutdownExecutor(dispatchExecutor);
-        shutdownExecutor(commonScheduledExecutor);
-        shutdownExecutor(commitExecutor);
-        shutdownExecutor(cleanExpiredFileExecutor);
-        shutdownExecutor(fetchDataExecutor);
-        shutdownExecutor(compactIndexFileExecutor);
-    }
-
-    private static void shutdownExecutor(ExecutorService executor) {
-        if (executor != null) {
-            executor.shutdown();
-            try {
-                if (!executor.awaitTermination(5, TimeUnit.SECONDS)) {
-                    executor.shutdownNow();
-                }
-            } catch (InterruptedException e) {
-                executor.shutdownNow();
-            }
-        }
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcher.java
new file mode 100644
index 00000000000..e1e142ad236
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcher.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.tieredstore.core;
+
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.store.CommitLogDispatcher;
+import org.apache.rocketmq.tieredstore.file.FlatFileInterface;
+
+public interface MessageStoreDispatcher extends CommitLogDispatcher {
+
+    void start();
+
+    void shutdown();
+
+    CompletableFuture<Boolean> doScheduleDispatch(FlatFileInterface flatFile, boolean force);
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java
new file mode 100644
index 00000000000..bcc4e225da2
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.core;
+
+import com.google.common.annotations.VisibleForTesting;
+import io.opentelemetry.api.common.Attributes;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.store.DispatchRequest;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
+import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
+import org.apache.rocketmq.store.queue.CqUnit;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
+import org.apache.rocketmq.tieredstore.TieredMessageStore;
+import org.apache.rocketmq.tieredstore.common.AppendResult;
+import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.common.GroupCommitContext;
+import org.apache.rocketmq.tieredstore.file.FlatFileInterface;
+import org.apache.rocketmq.tieredstore.file.FlatFileStore;
+import org.apache.rocketmq.tieredstore.index.IndexService;
+import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant;
+import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtil;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MessageStoreDispatcherImpl extends ServiceThread implements MessageStoreDispatcher {
+
+    protected static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
+
+    protected final String brokerName;
+    protected final MessageStore defaultStore;
+    protected final MessageStoreConfig storeConfig;
+    protected final TieredMessageStore messageStore;
+    protected final FlatFileStore flatFileStore;
+    protected final MessageStoreExecutor storeExecutor;
+    protected final MessageStoreFilter topicFilter;
+    protected final Semaphore semaphore;
+    protected final IndexService indexService;
+    protected final Map<FlatFileInterface, GroupCommitContext> failedGroupCommitMap;
+
+    public MessageStoreDispatcherImpl(TieredMessageStore messageStore) {
+        this.messageStore = messageStore;
+        this.storeConfig = messageStore.getStoreConfig();
+        this.defaultStore = messageStore.getDefaultStore();
+        this.brokerName = storeConfig.getBrokerName();
+        this.semaphore = new Semaphore(
+            this.storeConfig.getTieredStoreMaxPendingLimit() / 4);
+        this.topicFilter = messageStore.getTopicFilter();
+        this.flatFileStore = messageStore.getFlatFileStore();
+        this.storeExecutor = messageStore.getStoreExecutor();
+        this.indexService = messageStore.getIndexService();
+        this.failedGroupCommitMap = new ConcurrentHashMap<>();
+    }
+
+    @Override
+    public String getServiceName() {
+        return MessageStoreDispatcher.class.getSimpleName();
+    }
+
+    @VisibleForTesting
+    public Map<FlatFileInterface, GroupCommitContext> getFailedGroupCommitMap() {
+        return failedGroupCommitMap;
+    }
+
+    public void dispatchWithSemaphore(FlatFileInterface flatFile) {
+        try {
+            if (stopped) {
+                return;
+            }
+            semaphore.acquire();
+            this.doScheduleDispatch(flatFile, false)
+                .whenComplete((future, throwable) -> semaphore.release());
+        } catch (Throwable t) {
+            semaphore.release();
+            log.error("MessageStore dispatch error, topic={}, queueId={}",
+                flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), t);
+        }
+    }
+
+    @Override
+    public void dispatch(DispatchRequest request) {
+        if (stopped || topicFilter != null && topicFilter.filterTopic(request.getTopic())) {
+            return;
+        }
+        flatFileStore.computeIfAbsent(
+            new MessageQueue(request.getTopic(), brokerName, request.getQueueId()));
+    }
+
+    @Override
+    public CompletableFuture<Boolean> doScheduleDispatch(FlatFileInterface flatFile, boolean force) {
+        if (stopped) {
+            return CompletableFuture.completedFuture(true);
+        }
+
+        String topic = flatFile.getMessageQueue().getTopic();
+        int queueId = flatFile.getMessageQueue().getQueueId();
+
+        // For test scenarios, we set the 'force' variable to true to
+        // ensure that the data in the cache is directly committed successfully.
+        force = !storeConfig.isTieredStoreGroupCommit() || force;
+        if (force) {
+            flatFile.getFileLock().lock();
+        } else {
+            if (!flatFile.getFileLock().tryLock()) {
+                return CompletableFuture.completedFuture(false);
+            }
+        }
+
+        try {
+            if (topicFilter != null && topicFilter.filterTopic(flatFile.getMessageQueue().getTopic())) {
+                flatFileStore.destroyFile(flatFile.getMessageQueue());
+                return CompletableFuture.completedFuture(false);
+            }
+
+            long currentOffset = flatFile.getConsumeQueueMaxOffset();
+            long commitOffset = flatFile.getConsumeQueueCommitOffset();
+            long minOffsetInQueue = defaultStore.getMinOffsetInQueue(topic, queueId);
+            long maxOffsetInQueue = defaultStore.getMaxOffsetInQueue(topic, queueId);
+
+            // If set to max offset here, some written messages may be lost
+            if (!flatFile.isFlatFileInit()) {
+                currentOffset = defaultStore.getOffsetInQueueByTime(
+                    topic, queueId, System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(2));
+                currentOffset = Math.max(currentOffset, minOffsetInQueue);
+                currentOffset = Math.min(currentOffset, maxOffsetInQueue);
+                flatFile.initOffset(currentOffset);
+                log.warn("MessageDispatcher#dispatch init, topic={}, queueId={}, offset={}-{}, current={}",
+                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset);
+                return CompletableFuture.completedFuture(true);
+            }
+
+            // If the previous commit fails, attempt to trigger a commit directly.
+            if (commitOffset < currentOffset) {
+                this.commitAsync(flatFile).whenComplete((result, throwable) -> {
+                    if (throwable != null) {
+                        log.error("MessageDispatcher#flatFile commitOffset less than currentOffset, commitAsync again failed. topic: {}, queueId: {} ", topic, queueId, throwable);
+                    }
+                });
+                return CompletableFuture.completedFuture(false);
+            }
+
+            if (failedGroupCommitMap.containsKey(flatFile)) {
+                GroupCommitContext failedCommit = failedGroupCommitMap.get(flatFile);
+                if (failedCommit.getEndOffset() <= commitOffset) {
+                    failedGroupCommitMap.remove(flatFile);
+                    constructIndexFile(flatFile.getTopicId(), failedCommit);
+                }
+            }
+
+            if (currentOffset < minOffsetInQueue) {
+                log.warn("MessageDispatcher#dispatch, current offset is too small, topic={}, queueId={}, offset={}-{}, current={}",
+                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset);
+                flatFileStore.destroyFile(flatFile.getMessageQueue());
+                flatFileStore.computeIfAbsent(new MessageQueue(topic, brokerName, queueId));
+                return CompletableFuture.completedFuture(true);
+            }
+
+            if (currentOffset > maxOffsetInQueue) {
+                log.warn("MessageDispatcher#dispatch, current offset is too large, topic={}, queueId={}, offset={}-{}, current={}",
+                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset);
+                return CompletableFuture.completedFuture(false);
+            }
+
+            long interval = TimeUnit.HOURS.toMillis(storeConfig.getCommitLogRollingInterval());
+            if (flatFile.rollingFile(interval)) {
+                log.info("MessageDispatcher#dispatch, rolling file, topic={}, queueId={}, offset={}-{}, current={}",
+                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset);
+            }
+
+            if (currentOffset == maxOffsetInQueue) {
+                return CompletableFuture.completedFuture(false);
+            }
+
+            long bufferSize = 0L;
+            long groupCommitSize = storeConfig.getTieredStoreGroupCommitSize();
+            long groupCommitCount = storeConfig.getTieredStoreGroupCommitCount();
+            long targetOffset = Math.min(currentOffset + groupCommitCount, maxOffsetInQueue);
+
+            ConsumeQueueInterface consumeQueue = defaultStore.getConsumeQueue(topic, queueId);
+            CqUnit cqUnit = consumeQueue.get(currentOffset);
+            if (cqUnit == null) {
+                log.warn("MessageDispatcher#dispatch cq not found, topic={}, queueId={}, offset={}-{}, current={}, remain={}",
+                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset);
+                return CompletableFuture.completedFuture(false);
+            }
+
+            SelectMappedBufferResult message =
+                defaultStore.selectOneMessageByOffset(cqUnit.getPos(), cqUnit.getSize());
+            if (message == null) {
+                log.warn("MessageDispatcher#dispatch message not found, topic={}, queueId={}, offset={}-{}, current={}, remain={}",
+                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset);
+                return CompletableFuture.completedFuture(false);
+            }
+
+            boolean timeout = MessageFormatUtil.getStoreTimeStamp(message.getByteBuffer()) +
+                storeConfig.getTieredStoreGroupCommitTimeout() < System.currentTimeMillis();
+            boolean bufferFull = maxOffsetInQueue - currentOffset > storeConfig.getTieredStoreGroupCommitCount();
+
+            if (!timeout && !bufferFull && !force) {
+                log.debug("MessageDispatcher#dispatch hold, topic={}, queueId={}, offset={}-{}, current={}, remain={}",
+                    topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset);
+                message.release();
+                return CompletableFuture.completedFuture(false);
+            } else {
+                if (MessageFormatUtil.getStoreTimeStamp(message.getByteBuffer()) +
+                    TimeUnit.MINUTES.toMillis(5) < System.currentTimeMillis()) {
+                    log.warn("MessageDispatcher#dispatch behind too much, topic={}, queueId={}, offset={}-{}, current={}, remain={}",
+                        topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset);
+                } else {
+                    log.info("MessageDispatcher#dispatch success, topic={}, queueId={}, offset={}-{}, current={}, remain={}",
+                        topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset);
+                }
+                message.release();
+            }
+
+            long offset = currentOffset;
+            List<SelectMappedBufferResult> appendingBufferList = new ArrayList<>();
+            List<DispatchRequest> dispatchRequestList = new ArrayList<>();
+            for (; offset < targetOffset; offset++) {
+                cqUnit = consumeQueue.get(offset);
+                bufferSize += cqUnit.getSize();
+                if (bufferSize >= groupCommitSize) {
+                    break;
+                }
+                message = defaultStore.selectOneMessageByOffset(cqUnit.getPos(), cqUnit.getSize());
+                appendingBufferList.add(message);
+
+                ByteBuffer byteBuffer = message.getByteBuffer();
+                AppendResult result = flatFile.appendCommitLog(message);
+                if (!AppendResult.SUCCESS.equals(result)) {
+                    break;
+                }
+
+                long mappedCommitLogOffset = flatFile.getCommitLogMaxOffset() - byteBuffer.remaining();
+                Map<String, String> properties = MessageFormatUtil.getProperties(byteBuffer);
+
+                DispatchRequest dispatchRequest = new DispatchRequest(topic, queueId, mappedCommitLogOffset,
+                    cqUnit.getSize(), cqUnit.getTagsCode(), MessageFormatUtil.getStoreTimeStamp(byteBuffer),
+                    cqUnit.getQueueOffset(), properties.getOrDefault(MessageConst.PROPERTY_KEYS, ""),
+                    properties.getOrDefault(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, ""),
+                    0, 0, new HashMap<>());
+                dispatchRequest.setOffsetId(MessageFormatUtil.getOffsetId(byteBuffer));
+
+                result = flatFile.appendConsumeQueue(dispatchRequest);
+                if (!AppendResult.SUCCESS.equals(result)) {
+                    break;
+                } else {
+                    dispatchRequestList.add(dispatchRequest);
+                }
+            }
+
+            GroupCommitContext groupCommitContext = new GroupCommitContext();
+            groupCommitContext.setEndOffset(offset);
+            groupCommitContext.setBufferList(appendingBufferList);
+            groupCommitContext.setDispatchRequests(dispatchRequestList);
+
+            // If there are many messages waiting to be uploaded, call the upload logic immediately.
+            boolean repeat = timeout || maxOffsetInQueue - offset > storeConfig.getTieredStoreGroupCommitCount();
+
+            if (!dispatchRequestList.isEmpty()) {
+                Attributes attributes = TieredStoreMetricsManager.newAttributesBuilder()
+                    .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic)
+                    .put(TieredStoreMetricsConstant.LABEL_QUEUE_ID, queueId)
+                    .put(TieredStoreMetricsConstant.LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase())
+                    .build();
+                TieredStoreMetricsManager.messagesDispatchTotal.add(offset - currentOffset, attributes);
+
+                this.commitAsync(flatFile).whenComplete((success, throwable) -> {
+                        if (success) {
+                            constructIndexFile(flatFile.getTopicId(), groupCommitContext);
+                        }
+                        else {
+                            //next commit async,execute constructIndexFile.
+                            GroupCommitContext oldCommit = failedGroupCommitMap.put(flatFile, groupCommitContext);
+                            if (oldCommit != null) {
+                                log.warn("MessageDispatcher#commitAsync failed,flatFile old failed commit context not release, topic={}, queueId={}  ", topic, queueId);
+                                oldCommit.release();
+                            }
+                        }
+                        if (success && repeat) {
+                            storeExecutor.commonExecutor.submit(() -> dispatchWithSemaphore(flatFile));
+                        }
+                    }
+                );
+            }
+        } catch (ConsumeQueueException e) {
+            CompletableFuture<Boolean> future = new CompletableFuture<>();
+            future.completeExceptionally(e);
+            return future;
+        } finally {
+            flatFile.getFileLock().unlock();
+        }
+        return CompletableFuture.completedFuture(false);
+    }
+
+    public CompletableFuture<Boolean> commitAsync(FlatFileInterface flatFile) {
+        return flatFile.commitAsync();
+    }
+
+    public void constructIndexFile(long topicId, GroupCommitContext groupCommitContext) {
+        MessageStoreExecutor.getInstance().bufferCommitExecutor.submit(() -> {
+            if (storeConfig.isMessageIndexEnable()) {
+                try {
+                    groupCommitContext.getDispatchRequests().forEach(request -> constructIndexFile0(topicId, request));
+                }
+                catch (Throwable e) {
+                    log.error("constructIndexFile error {}", topicId, e);
+                }
+            }
+            groupCommitContext.release();
+        });
+    }
+
+    /**
+     * Building indexes with offsetId is no longer supported because offsetId has changed in tiered storage
+     */
+    public void constructIndexFile0(long topicId, DispatchRequest request) {
+        Set<String> keySet = new HashSet<>();
+        if (StringUtils.isNotBlank(request.getUniqKey())) {
+            keySet.add(request.getUniqKey());
+        }
+        if (StringUtils.isNotBlank(request.getKeys())) {
+            keySet.addAll(Arrays.asList(request.getKeys().split(MessageConst.KEY_SEPARATOR)));
+        }
+        indexService.putKey(request.getTopic(), (int) topicId, request.getQueueId(), keySet,
+            request.getCommitLogOffset(), request.getMsgSize(), request.getStoreTimestamp());
+    }
+
+    public void releaseClosedPendingGroupCommit() {
+        Iterator<Map.Entry<FlatFileInterface, GroupCommitContext>> iterator = failedGroupCommitMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<FlatFileInterface, GroupCommitContext> entry = iterator.next();
+            if (entry.getKey().isClosed()) {
+                entry.getValue().release();
+                iterator.remove();
+            }
+        }
+    }
+
+
+    @Override
+    public void run() {
+        log.info("{} service started", this.getServiceName());
+        while (!this.isStopped()) {
+            try {
+                flatFileStore.deepCopyFlatFileToList().forEach(this::dispatchWithSemaphore);
+
+                releaseClosedPendingGroupCommit();
+
+                this.waitForRunning(Duration.ofSeconds(20).toMillis());
+            } catch (Throwable t) {
+                log.error("MessageStore dispatch error", t);
+            }
+        }
+        log.info("{} service shutdown", this.getServiceName());
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreFetcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcher.java
similarity index 98%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreFetcher.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcher.java
index 8ae4dc7f9ef..8e2e8bdef59 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreFetcher.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcher.java
@@ -15,13 +15,13 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.tieredstore;
+package org.apache.rocketmq.tieredstore.core;
 
 import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.common.BoundaryType;
 import org.apache.rocketmq.store.GetMessageResult;
 import org.apache.rocketmq.store.MessageFilter;
 import org.apache.rocketmq.store.QueryMessageResult;
-import org.apache.rocketmq.common.BoundaryType;
 
 public interface MessageStoreFetcher {
 
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImpl.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImpl.java
new file mode 100644
index 00000000000..9e5ab01d3b8
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImpl.java
@@ -0,0 +1,463 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.core;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.Scheduler;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.common.BoundaryType;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+import org.apache.rocketmq.store.MessageFilter;
+import org.apache.rocketmq.store.QueryMessageResult;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.TieredMessageStore;
+import org.apache.rocketmq.tieredstore.common.GetMessageResultExt;
+import org.apache.rocketmq.tieredstore.common.SelectBufferResult;
+import org.apache.rocketmq.tieredstore.exception.TieredStoreException;
+import org.apache.rocketmq.tieredstore.file.FlatFileStore;
+import org.apache.rocketmq.tieredstore.file.FlatMessageFile;
+import org.apache.rocketmq.tieredstore.index.IndexItem;
+import org.apache.rocketmq.tieredstore.index.IndexService;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtil;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MessageStoreFetcherImpl implements MessageStoreFetcher {
+
+    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
+
+    protected static final String CACHE_KEY_FORMAT = "%s@%d@%d";
+
+    private final String brokerName;
+    private final MetadataStore metadataStore;
+    private final MessageStoreConfig storeConfig;
+    private final TieredMessageStore messageStore;
+    private final IndexService indexService;
+    private final FlatFileStore flatFileStore;
+    private final long memoryMaxSize;
+    private final Cache<String /* topic@queueId@offset */, SelectBufferResult> fetcherCache;
+
+    public MessageStoreFetcherImpl(TieredMessageStore messageStore) {
+        this(messageStore, messageStore.getStoreConfig(),
+            messageStore.getFlatFileStore(), messageStore.getIndexService());
+    }
+
+    public MessageStoreFetcherImpl(TieredMessageStore messageStore, MessageStoreConfig storeConfig,
+        FlatFileStore flatFileStore, IndexService indexService) {
+
+        this.storeConfig = storeConfig;
+        this.brokerName = storeConfig.getBrokerName();
+        this.flatFileStore = flatFileStore;
+        this.messageStore = messageStore;
+        this.indexService = indexService;
+        this.metadataStore = flatFileStore.getMetadataStore();
+        this.memoryMaxSize =
+            (long) (Runtime.getRuntime().maxMemory() * storeConfig.getReadAheadCacheSizeThresholdRate());
+        this.fetcherCache = this.initCache(storeConfig);
+        log.info("MessageStoreFetcher init success, brokerName={}", storeConfig.getBrokerName());
+    }
+
+    private Cache<String, SelectBufferResult> initCache(MessageStoreConfig storeConfig) {
+
+        return Caffeine.newBuilder()
+            .scheduler(Scheduler.systemScheduler())
+            // Clients may repeatedly request messages at the same offset in tiered storage,
+            // causing the request queue to become full. Using expire after read or write policy
+            // to refresh the cache expiration time.
+            .expireAfterAccess(storeConfig.getReadAheadCacheExpireDuration(), TimeUnit.MILLISECONDS)
+            .maximumWeight(memoryMaxSize)
+            // Using the buffer size of messages to calculate memory usage
+            .weigher((String key, SelectBufferResult buffer) -> buffer.getSize())
+            .recordStats()
+            .build();
+    }
+
+    public Cache<String, SelectBufferResult> getFetcherCache() {
+        return fetcherCache;
+    }
+
+    protected void putMessageToCache(FlatMessageFile flatFile, long offset, SelectBufferResult result) {
+        MessageQueue mq = flatFile.getMessageQueue();
+        this.fetcherCache.put(String.format(CACHE_KEY_FORMAT, mq.getTopic(), mq.getQueueId(), offset), result);
+    }
+
+    protected SelectBufferResult getMessageFromCache(FlatMessageFile flatFile, long offset) {
+        MessageQueue mq = flatFile.getMessageQueue();
+        SelectBufferResult buffer = this.fetcherCache.getIfPresent(
+            String.format(CACHE_KEY_FORMAT, mq.getTopic(), mq.getQueueId(), offset));
+        // return duplicate buffer here
+        if (buffer == null) {
+            return null;
+        }
+        long count = buffer.getAccessCount().incrementAndGet();
+        if (count % 1000L == 0L) {
+            log.warn("MessageFetcher fetch same offset message too many times, " +
+                "topic={}, queueId={}, offset={}, count={}", mq.getTopic(), mq.getQueueId(), offset, count);
+        }
+        return new SelectBufferResult(
+            buffer.getByteBuffer().asReadOnlyBuffer(), buffer.getStartOffset(), buffer.getSize(), buffer.getTagCode());
+    }
+
+    protected GetMessageResultExt getMessageFromCache(
+        FlatMessageFile flatFile, long offset, int maxCount, MessageFilter messageFilter) {
+        GetMessageResultExt result = new GetMessageResultExt();
+        int interval = storeConfig.getReadAheadMessageCountThreshold();
+        for (long current = offset, end = offset + interval; current < end; current++) {
+            SelectBufferResult buffer = getMessageFromCache(flatFile, current);
+            if (buffer == null) {
+                result.setNextBeginOffset(current);
+                break;
+            }
+            result.setNextBeginOffset(current + 1);
+            if (messageFilter != null) {
+                if (!messageFilter.isMatchedByConsumeQueue(buffer.getTagCode(), null)) {
+                    continue;
+                }
+                if (!messageFilter.isMatchedByCommitLog(buffer.getByteBuffer().slice(), null)) {
+                    continue;
+                }
+            }
+            SelectMappedBufferResult bufferResult = new SelectMappedBufferResult(
+                buffer.getStartOffset(), buffer.getByteBuffer(), buffer.getSize(), null);
+            result.addMessageExt(bufferResult, current, buffer.getTagCode());
+            if (result.getMessageCount() == maxCount) {
+                break;
+            }
+            long maxTransferBytes = messageStore.getMessageStoreConfig().getMaxTransferBytesOnMessageInMemory();
+            if (result.getBufferTotalSize() >= maxTransferBytes) {
+                break;
+            }
+        }
+        result.setStatus(result.getMessageCount() > 0 ?
+            GetMessageStatus.FOUND : GetMessageStatus.NO_MATCHED_MESSAGE);
+        result.setMinOffset(flatFile.getConsumeQueueMinOffset());
+        result.setMaxOffset(flatFile.getConsumeQueueCommitOffset());
+        return result;
+    }
+
+    protected CompletableFuture<Long> fetchMessageThenPutToCache(
+        FlatMessageFile flatFile, long queueOffset, int batchSize) {
+
+        MessageQueue mq = flatFile.getMessageQueue();
+        return this.getMessageFromTieredStoreAsync(flatFile, queueOffset, batchSize)
+            .thenApply(result -> {
+                if (result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_ONE ||
+                    result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_BADLY) {
+                    return -1L;
+                }
+                if (result.getStatus() != GetMessageStatus.FOUND) {
+                    log.warn("MessageFetcher prefetch message then put to cache failed, result={}, " +
+                            "topic={}, queue={}, queue offset={}, batch size={}",
+                        result.getStatus(), mq.getTopic(), mq.getQueueId(), queueOffset, batchSize);
+                    return -1L;
+                }
+                List<Long> offsetList = result.getMessageQueueOffset();
+                List<Long> tagCodeList = result.getTagCodeList();
+                List<SelectMappedBufferResult> msgList = result.getMessageMapedList();
+                for (int i = 0; i < offsetList.size(); i++) {
+                    SelectMappedBufferResult msg = msgList.get(i);
+                    SelectBufferResult bufferResult = new SelectBufferResult(
+                        msg.getByteBuffer(), msg.getStartOffset(), msg.getSize(), tagCodeList.get(i));
+                    this.putMessageToCache(flatFile, queueOffset + i, bufferResult);
+                }
+                return offsetList.get(offsetList.size() - 1);
+            });
+    }
+
+    public CompletableFuture<GetMessageResult> getMessageFromCacheAsync(
+        FlatMessageFile flatFile, String group, long queueOffset, int maxCount, MessageFilter messageFilter) {
+
+        MessageQueue mq = flatFile.getMessageQueue();
+        GetMessageResultExt result = getMessageFromCache(flatFile, queueOffset, maxCount, messageFilter);
+
+        if (GetMessageStatus.FOUND.equals(result.getStatus())) {
+            log.debug("MessageFetcher cache hit, group={}, topic={}, queueId={}, offset={}, maxCount={}, resultSize={}, lag={}",
+                group, mq.getTopic(), mq.getQueueId(), queueOffset, maxCount,
+                result.getMessageCount(), result.getMaxOffset() - result.getNextBeginOffset());
+            return CompletableFuture.completedFuture(result);
+        }
+
+        // If cache miss, pull messages immediately
+        log.debug("MessageFetcher cache miss, group={}, topic={}, queueId={}, offset={}, maxCount={}, lag={}",
+            group, mq.getTopic(), mq.getQueueId(), queueOffset, maxCount, result.getMaxOffset() - result.getNextBeginOffset());
+
+        // To optimize the performance of pop consumption
+        // Pop revive will cause a large number of random reads,
+        // so the amount of pre-fetch message num needs to be reduced.
+        int fetchSize = maxCount == 1 ? 32 : storeConfig.getReadAheadMessageCountThreshold();
+        return fetchMessageThenPutToCache(flatFile, queueOffset, fetchSize)
+            .thenApply(maxOffset -> getMessageFromCache(flatFile, queueOffset, maxCount, messageFilter));
+    }
+
+    public CompletableFuture<GetMessageResultExt> getMessageFromTieredStoreAsync(
+        FlatMessageFile flatFile, long queueOffset, int batchSize) {
+
+        GetMessageResultExt result = new GetMessageResultExt();
+        result.setMinOffset(flatFile.getConsumeQueueMinOffset());
+        result.setMaxOffset(flatFile.getConsumeQueueCommitOffset());
+
+        if (queueOffset < result.getMinOffset()) {
+            result.setStatus(GetMessageStatus.OFFSET_TOO_SMALL);
+            result.setNextBeginOffset(result.getMinOffset());
+            return CompletableFuture.completedFuture(result);
+        } else if (queueOffset == result.getMaxOffset()) {
+            result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE);
+            result.setNextBeginOffset(queueOffset);
+            return CompletableFuture.completedFuture(result);
+        } else if (queueOffset > result.getMaxOffset()) {
+            result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY);
+            result.setNextBeginOffset(result.getMaxOffset());
+            return CompletableFuture.completedFuture(result);
+        }
+
+        if (queueOffset < result.getMaxOffset()) {
+            batchSize = Math.min(batchSize, (int) Math.min(
+                result.getMaxOffset() - queueOffset, storeConfig.getReadAheadMessageCountThreshold()));
+        }
+
+        CompletableFuture<ByteBuffer> readConsumeQueueFuture;
+        try {
+            readConsumeQueueFuture = flatFile.getConsumeQueueAsync(queueOffset, batchSize);
+        } catch (TieredStoreException e) {
+            switch (e.getErrorCode()) {
+                case ILLEGAL_PARAM:
+                case ILLEGAL_OFFSET:
+                default:
+                    result.setStatus(GetMessageStatus.OFFSET_FOUND_NULL);
+                    result.setNextBeginOffset(queueOffset);
+                    return CompletableFuture.completedFuture(result);
+            }
+        }
+
+        int finalBatchSize = batchSize;
+        CompletableFuture<ByteBuffer> readCommitLogFuture = readConsumeQueueFuture.thenCompose(cqBuffer -> {
+
+            long firstCommitLogOffset = MessageFormatUtil.getCommitLogOffsetFromItem(cqBuffer);
+            cqBuffer.position(cqBuffer.remaining() - MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);
+            long lastCommitLogOffset = MessageFormatUtil.getCommitLogOffsetFromItem(cqBuffer);
+            if (lastCommitLogOffset < firstCommitLogOffset) {
+                log.error("MessageFetcher#getMessageFromTieredStoreAsync, last offset is smaller than first offset, " +
+                        "topic={} queueId={}, offset={}, firstOffset={}, lastOffset={}",
+                    flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), queueOffset,
+                    firstCommitLogOffset, lastCommitLogOffset);
+                return CompletableFuture.completedFuture(ByteBuffer.allocate(0));
+            }
+
+            // Get at least one message
+            // Reducing the length limit of cq to prevent OOM
+            long length = lastCommitLogOffset - firstCommitLogOffset + MessageFormatUtil.getSizeFromItem(cqBuffer);
+            while (cqBuffer.limit() > MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE &&
+                length > storeConfig.getReadAheadMessageSizeThreshold()) {
+                cqBuffer.limit(cqBuffer.position());
+                cqBuffer.position(cqBuffer.limit() - MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);
+                length = MessageFormatUtil.getCommitLogOffsetFromItem(cqBuffer)
+                    - firstCommitLogOffset + MessageFormatUtil.getSizeFromItem(cqBuffer);
+            }
+            int messageCount = cqBuffer.position() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE + 1;
+
+            log.info("MessageFetcher#getMessageFromTieredStoreAsync, " +
+                    "topic={}, queueId={}, broker offset={}-{}, offset={}, expect={}, actually={}, lag={}",
+                flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(),
+                result.getMinOffset(), result.getMaxOffset(), queueOffset, finalBatchSize,
+                messageCount, result.getMaxOffset() - queueOffset);
+
+            return flatFile.getCommitLogAsync(firstCommitLogOffset, (int) length);
+        });
+
+        return readConsumeQueueFuture.thenCombine(readCommitLogFuture, (cqBuffer, msgBuffer) -> {
+            List<SelectBufferResult> bufferList = MessageFormatUtil.splitMessageBuffer(cqBuffer, msgBuffer);
+            int requestSize = cqBuffer.remaining() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE;
+
+            // not use buffer list size to calculate next offset to prevent split error
+            if (bufferList.isEmpty()) {
+                result.setStatus(GetMessageStatus.NO_MATCHED_MESSAGE);
+                result.setNextBeginOffset(queueOffset + requestSize);
+            } else {
+                result.setStatus(GetMessageStatus.FOUND);
+                result.setNextBeginOffset(queueOffset + requestSize);
+
+                for (SelectBufferResult bufferResult : bufferList) {
+                    ByteBuffer slice = bufferResult.getByteBuffer().slice();
+                    slice.limit(bufferResult.getSize());
+                    SelectMappedBufferResult msg = new SelectMappedBufferResult(bufferResult.getStartOffset(),
+                        bufferResult.getByteBuffer(), bufferResult.getSize(), null);
+                    result.addMessageExt(msg, MessageFormatUtil.getQueueOffset(slice), bufferResult.getTagCode());
+                }
+            }
+            return result;
+        }).exceptionally(e -> {
+            MessageQueue mq = flatFile.getMessageQueue();
+            log.warn("MessageFetcher#getMessageFromTieredStoreAsync failed, " +
+                "topic={} queueId={}, offset={}, batchSize={}", mq.getTopic(), mq.getQueueId(), queueOffset, finalBatchSize, e);
+            result.setStatus(GetMessageStatus.OFFSET_FOUND_NULL);
+            result.setNextBeginOffset(queueOffset);
+            return result;
+        });
+    }
+
+    @Override
+    public CompletableFuture<GetMessageResult> getMessageAsync(
+        String group, String topic, int queueId, long queueOffset, int maxCount, final MessageFilter messageFilter) {
+
+        GetMessageResult result = new GetMessageResult();
+        FlatMessageFile flatFile = flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, queueId));
+
+        if (flatFile == null) {
+            result.setNextBeginOffset(queueOffset);
+            result.setStatus(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE);
+            return CompletableFuture.completedFuture(result);
+        }
+
+        // Max queue offset means next message put position
+        result.setMinOffset(flatFile.getConsumeQueueMinOffset());
+        result.setMaxOffset(flatFile.getConsumeQueueCommitOffset());
+
+        // Fill result according file offset.
+        // Offset range  | Result           | Fix to
+        // (-oo, 0]      | no message       | current offset
+        // (0, min)      | too small        | min offset
+        // [min, max)    | correct          |
+        // [max, max]    | overflow one     | max offset
+        // (max, +oo)    | overflow badly   | max offset
+
+        if (result.getMaxOffset() <= 0) {
+            result.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);
+            result.setNextBeginOffset(queueOffset);
+            return CompletableFuture.completedFuture(result);
+        } else if (queueOffset < result.getMinOffset()) {
+            result.setStatus(GetMessageStatus.OFFSET_TOO_SMALL);
+            result.setNextBeginOffset(result.getMinOffset());
+            return CompletableFuture.completedFuture(result);
+        } else if (queueOffset == result.getMaxOffset()) {
+            result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE);
+            result.setNextBeginOffset(result.getMaxOffset());
+            return CompletableFuture.completedFuture(result);
+        } else if (queueOffset > result.getMaxOffset()) {
+            result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY);
+            result.setNextBeginOffset(result.getMaxOffset());
+            return CompletableFuture.completedFuture(result);
+        }
+
+        boolean cacheBusy = fetcherCache.estimatedSize() > memoryMaxSize * 0.8;
+        if (storeConfig.isReadAheadCacheEnable() && !cacheBusy) {
+            return getMessageFromCacheAsync(flatFile, group, queueOffset, maxCount, messageFilter);
+        } else {
+            return getMessageFromTieredStoreAsync(flatFile, queueOffset, maxCount)
+                .thenApply(messageResultExt -> messageResultExt.doFilterMessage(messageFilter));
+        }
+    }
+
+    @Override
+    public CompletableFuture<Long> getEarliestMessageTimeAsync(String topic, int queueId) {
+        FlatMessageFile flatFile = flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, queueId));
+        return CompletableFuture.completedFuture(flatFile == null ? -1L : flatFile.getMinStoreTimestamp());
+    }
+
+    @Override
+    public CompletableFuture<Long> getMessageStoreTimeStampAsync(String topic, int queueId, long queueOffset) {
+        FlatMessageFile flatFile = flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, queueId));
+        if (flatFile == null) {
+            return CompletableFuture.completedFuture(-1L);
+        }
+
+        return flatFile.getConsumeQueueAsync(queueOffset)
+            .thenComposeAsync(cqItem -> {
+                long commitLogOffset = MessageFormatUtil.getCommitLogOffsetFromItem(cqItem);
+                int size = MessageFormatUtil.getSizeFromItem(cqItem);
+                return flatFile.getCommitLogAsync(commitLogOffset, size);
+            }, messageStore.getStoreExecutor().bufferFetchExecutor)
+            .thenApply(MessageFormatUtil::getStoreTimeStamp)
+            .exceptionally(e -> {
+                log.error("MessageStoreFetcherImpl#getMessageStoreTimeStampAsync: " +
+                    "get or decode message failed, topic={}, queue={}, offset={}", topic, queueId, queueOffset, e);
+                return -1L;
+            });
+    }
+
+    @Override
+    public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType type) {
+        FlatMessageFile flatFile = flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, queueId));
+        if (flatFile == null) {
+            return -1L;
+        }
+        return flatFile.getQueueOffsetByTimeAsync(timestamp, type).join();
+    }
+
+    @Override
+    public CompletableFuture<QueryMessageResult> queryMessageAsync(
+        String topic, String key, int maxCount, long begin, long end) {
+
+        long topicId;
+        try {
+            TopicMetadata topicMetadata = metadataStore.getTopic(topic);
+            if (topicMetadata == null) {
+                log.info("MessageFetcher#queryMessageAsync, topic metadata not found, topic={}", topic);
+                return CompletableFuture.completedFuture(new QueryMessageResult());
+            }
+            topicId = topicMetadata.getTopicId();
+        } catch (Exception e) {
+            log.error("MessageFetcher#queryMessageAsync, get topic id failed, topic={}", topic, e);
+            return CompletableFuture.completedFuture(new QueryMessageResult());
+        }
+
+        CompletableFuture<List<IndexItem>> future = indexService.queryAsync(topic, key, maxCount, begin, end);
+
+        return future.thenCompose(indexItemList -> {
+            List<CompletableFuture<SelectMappedBufferResult>> futureList = new ArrayList<>(maxCount);
+            for (IndexItem indexItem : indexItemList) {
+                if (topicId != indexItem.getTopicId()) {
+                    continue;
+                }
+                FlatMessageFile flatFile =
+                    flatFileStore.getFlatFile(new MessageQueue(topic, brokerName, indexItem.getQueueId()));
+                if (flatFile == null) {
+                    continue;
+                }
+                CompletableFuture<SelectMappedBufferResult> getMessageFuture = flatFile
+                    .getCommitLogAsync(indexItem.getOffset(), indexItem.getSize())
+                    .thenApply(messageBuffer -> new SelectMappedBufferResult(
+                        indexItem.getOffset(), messageBuffer, indexItem.getSize(), null));
+                futureList.add(getMessageFuture);
+                if (futureList.size() >= maxCount) {
+                    break;
+                }
+            }
+            return CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).thenApply(v -> {
+                QueryMessageResult result = new QueryMessageResult();
+                futureList.forEach(f -> f.thenAccept(result::addMessage));
+                return result;
+            });
+        }).whenComplete((result, throwable) -> {
+            if (result != null) {
+                log.info("MessageFetcher#queryMessageAsync, " +
+                        "query result={}, topic={}, topicId={}, key={}, maxCount={}, timestamp={}-{}",
+                    result.getMessageBufferList().size(), topic, topicId, key, maxCount, begin, end);
+            }
+        });
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFilter.java
similarity index 90%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFilter.java
index f983ed6e961..524761fd2b1 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFilter.java
@@ -15,9 +15,9 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.tieredstore.provider;
+package org.apache.rocketmq.tieredstore.core;
 
-public interface TieredStoreTopicFilter {
+public interface MessageStoreFilter {
 
     boolean filterTopic(String topicName);
 
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreTopicFilter.java
similarity index 63%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreTopicFilter.java
index f8bf165bc11..b64f163eb23 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreTopicFilter.java
@@ -15,19 +15,24 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.tieredstore.provider;
+package org.apache.rocketmq.tieredstore.core;
 
 import java.util.HashSet;
 import java.util.Set;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.PopAckConstants;
+import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
 
-public class TieredStoreTopicBlackListFilter implements TieredStoreTopicFilter {
+public class MessageStoreTopicFilter implements MessageStoreFilter {
 
     private final Set<String> topicBlackSet;
 
-    public TieredStoreTopicBlackListFilter() {
+    public MessageStoreTopicFilter(MessageStoreConfig storeConfig) {
         this.topicBlackSet = new HashSet<>();
+        this.topicBlackSet.add(storeConfig.getBrokerClusterName());
+        this.topicBlackSet.add(storeConfig.getBrokerName());
     }
 
     @Override
@@ -35,7 +40,10 @@ public boolean filterTopic(String topicName) {
         if (StringUtils.isBlank(topicName)) {
             return true;
         }
-        return TieredStoreUtil.isSystemTopic(topicName) || topicBlackSet.contains(topicName);
+        return TopicValidator.isSystemTopic(topicName) ||
+            PopAckConstants.isStartWithRevivePrefix(topicName) ||
+            this.topicBlackSet.contains(topicName) ||
+            MixAll.isLmq(topicName);
     }
 
     @Override
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/exception/TieredStoreException.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/exception/TieredStoreException.java
index 1c85181329c..3841643299b 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/exception/TieredStoreException.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/exception/TieredStoreException.java
@@ -18,28 +18,25 @@
 
 public class TieredStoreException extends RuntimeException {
 
-    private TieredStoreErrorCode errorCode;
-    private long position = -1;
-
+    private final TieredStoreErrorCode errorCode;
     private String requestId;
+    private long position = -1L;
 
     public TieredStoreException(TieredStoreErrorCode errorCode, String errorMessage) {
         super(errorMessage);
         this.errorCode = errorCode;
     }
 
-    public TieredStoreException(TieredStoreErrorCode errorCode, String errorMessage, String requestId) {
-        super(errorMessage);
-        this.errorCode = errorCode;
-        this.requestId = requestId;
-    }
-
     public TieredStoreErrorCode getErrorCode() {
         return errorCode;
     }
 
-    public void setErrorCode(TieredStoreErrorCode errorCode) {
-        this.errorCode = errorCode;
+    public String getRequestId() {
+        return requestId;
+    }
+
+    public void setRequestId(String requestId) {
+        this.requestId = requestId;
     }
 
     public long getPosition() {
@@ -52,13 +49,13 @@ public void setPosition(long position) {
 
     @Override
     public String toString() {
-        String errStr = super.toString();
+        StringBuilder errorStringBuilder = new StringBuilder(super.toString());
         if (requestId != null) {
-            errStr += " requestId: " + requestId;
+            errorStringBuilder.append(" requestId: ").append(requestId);
         }
-        if (position != -1) {
-            errStr += ", position: " + position;
+        if (position != -1L) {
+            errorStringBuilder.append(", position: ").append(position);
         }
-        return errStr;
+        return errorStringBuilder.toString();
     }
 }
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java
deleted file mode 100644
index 5ad3a6ff320..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java
+++ /dev/null
@@ -1,495 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.file;
-
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.RemovalCause;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.store.DispatchRequest;
-import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.InFlightRequestFuture;
-import org.apache.rocketmq.tieredstore.common.InFlightRequestKey;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
-import org.apache.rocketmq.tieredstore.util.CQItemBufferUtil;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-import org.apache.rocketmq.common.BoundaryType;
-
-public class CompositeFlatFile implements CompositeAccess {
-
-    protected static final Logger LOGGER = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
-
-    protected volatile boolean closed = false;
-    protected int readAheadFactor;
-
-    /**
-     * Dispatch offset represents the offset of the messages that have been
-     * dispatched to the current chunk, indicating the progress of the message distribution.
-     * It's consume queue current offset.
-     */
-    protected final AtomicLong dispatchOffset;
-
-    protected final ReentrantLock compositeFlatFileLock;
-    protected final TieredMessageStoreConfig storeConfig;
-    protected final TieredMetadataStore metadataStore;
-
-    protected final String filePath;
-    protected final TieredCommitLog commitLog;
-    protected final TieredConsumeQueue consumeQueue;
-    protected final Cache<String, Long> groupOffsetCache;
-    protected final ConcurrentMap<InFlightRequestKey, InFlightRequestFuture> inFlightRequestMap;
-
-    public CompositeFlatFile(TieredFileAllocator fileQueueFactory, String filePath) {
-        this.filePath = filePath;
-        this.storeConfig = fileQueueFactory.getStoreConfig();
-        this.readAheadFactor = this.storeConfig.getReadAheadMinFactor();
-        this.metadataStore = TieredStoreUtil.getMetadataStore(this.storeConfig);
-        this.compositeFlatFileLock = new ReentrantLock();
-        this.inFlightRequestMap = new ConcurrentHashMap<>();
-        this.commitLog = new TieredCommitLog(fileQueueFactory, filePath);
-        this.consumeQueue = new TieredConsumeQueue(fileQueueFactory, filePath);
-        this.dispatchOffset = new AtomicLong(
-            this.consumeQueue.isInitialized() ? this.getConsumeQueueCommitOffset() : -1L);
-        this.groupOffsetCache = this.initOffsetCache();
-    }
-
-    private Cache<String, Long> initOffsetCache() {
-        return Caffeine.newBuilder()
-            .expireAfterWrite(2, TimeUnit.MINUTES)
-            .removalListener((key, value, cause) -> {
-                if (cause.equals(RemovalCause.EXPIRED)) {
-                    inFlightRequestMap.remove(new InFlightRequestKey((String) key));
-                }
-            }).build();
-    }
-
-    public boolean isClosed() {
-        return closed;
-    }
-
-    public ReentrantLock getCompositeFlatFileLock() {
-        return compositeFlatFileLock;
-    }
-
-    public long getCommitLogMinOffset() {
-        return commitLog.getMinOffset();
-    }
-
-    public long getCommitLogMaxOffset() {
-        return commitLog.getMaxOffset();
-    }
-
-    public long getCommitLogBeginTimestamp() {
-        return commitLog.getBeginTimestamp();
-    }
-
-    @Override
-    public long getCommitLogDispatchCommitOffset() {
-        return commitLog.getDispatchCommitOffset();
-    }
-
-    public long getConsumeQueueBaseOffset() {
-        return consumeQueue.getBaseOffset();
-    }
-
-    public long getConsumeQueueMinOffset() {
-        long cqOffset = consumeQueue.getMinOffset() / TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE;
-        long effectiveOffset = this.commitLog.getMinConsumeQueueOffset();
-        return Math.max(cqOffset, effectiveOffset);
-    }
-
-    public long getConsumeQueueCommitOffset() {
-        return consumeQueue.getCommitOffset() / TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE;
-    }
-
-    public long getConsumeQueueMaxOffset() {
-        return consumeQueue.getMaxOffset() / TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE;
-    }
-
-    public long getConsumeQueueEndTimestamp() {
-        return consumeQueue.getEndTimestamp();
-    }
-
-    public long getDispatchOffset() {
-        return dispatchOffset.get();
-    }
-
-    @Override
-    public CompletableFuture<ByteBuffer> getMessageAsync(long queueOffset) {
-        return getConsumeQueueAsync(queueOffset).thenComposeAsync(cqBuffer -> {
-            long commitLogOffset = CQItemBufferUtil.getCommitLogOffset(cqBuffer);
-            int length = CQItemBufferUtil.getSize(cqBuffer);
-            return getCommitLogAsync(commitLogOffset, length);
-        });
-    }
-
-    @Override
-    public long getOffsetInConsumeQueueByTime(long timestamp, BoundaryType boundaryType) {
-        Pair<Long, Long> pair = consumeQueue.getQueueOffsetInFileByTime(timestamp, boundaryType);
-        long minQueueOffset = pair.getLeft();
-        long maxQueueOffset = pair.getRight();
-
-        if (maxQueueOffset == -1 || maxQueueOffset < minQueueOffset) {
-            return -1L;
-        }
-
-        long low = minQueueOffset;
-        long high = maxQueueOffset;
-
-        long offset = 0;
-
-        // Handle the following corner cases first:
-        // 1. store time of (high) < timestamp
-        // 2. store time of (low) > timestamp
-        long storeTime;
-        // Handle case 1
-        ByteBuffer message = getMessageAsync(maxQueueOffset).join();
-        storeTime = MessageBufferUtil.getStoreTimeStamp(message);
-        if (storeTime < timestamp) {
-            switch (boundaryType) {
-                case LOWER:
-                    return maxQueueOffset + 1;
-                case UPPER:
-                    return maxQueueOffset;
-                default:
-                    LOGGER.warn("CompositeFlatFile#getQueueOffsetByTime: unknown boundary boundaryType");
-                    break;
-            }
-        }
-
-        // Handle case 2
-        message = getMessageAsync(minQueueOffset).join();
-        storeTime = MessageBufferUtil.getStoreTimeStamp(message);
-        if (storeTime > timestamp) {
-            switch (boundaryType) {
-                case LOWER:
-                    return minQueueOffset;
-                case UPPER:
-                    return 0L;
-                default:
-                    LOGGER.warn("CompositeFlatFile#getQueueOffsetByTime: unknown boundary boundaryType");
-                    break;
-            }
-        }
-
-        // Perform binary search
-        long midOffset = -1;
-        long targetOffset = -1;
-        long leftOffset = -1;
-        long rightOffset = -1;
-        while (high >= low) {
-            midOffset = (low + high) / 2;
-            message = getMessageAsync(midOffset).join();
-            storeTime = MessageBufferUtil.getStoreTimeStamp(message);
-            if (storeTime == timestamp) {
-                targetOffset = midOffset;
-                break;
-            } else if (storeTime > timestamp) {
-                high = midOffset - 1;
-                rightOffset = midOffset;
-            } else {
-                low = midOffset + 1;
-                leftOffset = midOffset;
-            }
-        }
-
-        if (targetOffset != -1) {
-            // We just found ONE matched record. These next to it might also share the same store-timestamp.
-            offset = targetOffset;
-            long previousAttempt = targetOffset;
-            switch (boundaryType) {
-                case LOWER:
-                    while (true) {
-                        long attempt = previousAttempt - 1;
-                        if (attempt < minQueueOffset) {
-                            break;
-                        }
-                        message = getMessageAsync(attempt).join();
-                        storeTime = MessageBufferUtil.getStoreTimeStamp(message);
-                        if (storeTime == timestamp) {
-                            previousAttempt = attempt;
-                            continue;
-                        }
-                        break;
-                    }
-                    offset = previousAttempt;
-                    break;
-                case UPPER:
-                    while (true) {
-                        long attempt = previousAttempt + 1;
-                        if (attempt > maxQueueOffset) {
-                            break;
-                        }
-
-                        message = getMessageAsync(attempt).join();
-                        storeTime = MessageBufferUtil.getStoreTimeStamp(message);
-                        if (storeTime == timestamp) {
-                            previousAttempt = attempt;
-                            continue;
-                        }
-                        break;
-                    }
-                    offset = previousAttempt;
-                    break;
-                default:
-                    LOGGER.warn("CompositeFlatFile#getQueueOffsetByTime: unknown boundary boundaryType");
-                    break;
-            }
-        } else {
-            // Given timestamp does not have any message records. But we have a range enclosing the
-            // timestamp.
-            /*
-             * Consider the follow case: t2 has no consume queue entry and we are searching offset of
-             * t2 for lower and upper boundaries.
-             *  --------------------------
-             *   timestamp   Consume Queue
-             *       t1          1
-             *       t1          2
-             *       t1          3
-             *       t3          4
-             *       t3          5
-             *   --------------------------
-             * Now, we return 3 as upper boundary of t2 and 4 as its lower boundary. It looks
-             * contradictory at first sight, but it does make sense when performing range queries.
-             */
-            switch (boundaryType) {
-                case LOWER: {
-                    offset = rightOffset;
-                    break;
-                }
-
-                case UPPER: {
-                    offset = leftOffset;
-                    break;
-                }
-                default: {
-                    LOGGER.warn("CompositeFlatFile#getQueueOffsetByTime: unknown boundary boundaryType");
-                    break;
-                }
-            }
-        }
-        return offset;
-    }
-
-    @Override
-    public void initOffset(long offset) {
-        if (consumeQueue.isInitialized()) {
-            dispatchOffset.set(this.getConsumeQueueCommitOffset());
-        } else {
-            consumeQueue.setBaseOffset(offset * TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-            dispatchOffset.set(offset);
-        }
-    }
-
-    @Override
-    public AppendResult appendCommitLog(ByteBuffer message) {
-        return appendCommitLog(message, false);
-    }
-
-    @Override
-    public AppendResult appendCommitLog(ByteBuffer message, boolean commit) {
-        if (closed) {
-            return AppendResult.FILE_CLOSED;
-        }
-
-        AppendResult result = commitLog.append(message, commit);
-        if (result == AppendResult.SUCCESS) {
-            dispatchOffset.incrementAndGet();
-        }
-        return result;
-    }
-
-    @Override
-    public AppendResult appendConsumeQueue(DispatchRequest request) {
-        return appendConsumeQueue(request, false);
-    }
-
-    @Override
-    public AppendResult appendConsumeQueue(DispatchRequest request, boolean commit) {
-        if (closed) {
-            return AppendResult.FILE_CLOSED;
-        }
-
-        if (request.getConsumeQueueOffset() != getConsumeQueueMaxOffset()) {
-            return AppendResult.OFFSET_INCORRECT;
-        }
-
-        return consumeQueue.append(request.getCommitLogOffset(),
-            request.getMsgSize(), request.getTagsCode(), request.getStoreTimestamp(), commit);
-    }
-
-    @Override
-    public CompletableFuture<ByteBuffer> getCommitLogAsync(long offset, int length) {
-        return commitLog.readAsync(offset, length);
-    }
-
-    @Override
-    public CompletableFuture<ByteBuffer> getConsumeQueueAsync(long queueOffset) {
-        return getConsumeQueueAsync(queueOffset, 1);
-    }
-
-    @Override
-    public CompletableFuture<ByteBuffer> getConsumeQueueAsync(long queueOffset, int count) {
-        return consumeQueue.readAsync(queueOffset * TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE,
-            count * TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-    }
-
-    @Override
-    public void commitCommitLog() {
-        commitLog.commit(true);
-    }
-
-    @Override
-    public void commitConsumeQueue() {
-        consumeQueue.commit(true);
-    }
-
-    @Override
-    public void cleanExpiredFile(long expireTimestamp) {
-        commitLog.cleanExpiredFile(expireTimestamp);
-        consumeQueue.cleanExpiredFile(expireTimestamp);
-    }
-
-    @Override
-    public void destroyExpiredFile() {
-        commitLog.destroyExpiredFile();
-        consumeQueue.destroyExpiredFile();
-    }
-
-    @Override
-    public void commit(boolean sync) {
-        commitLog.commit(sync);
-        consumeQueue.commit(sync);
-    }
-
-    public void increaseReadAheadFactor() {
-        readAheadFactor = Math.min(readAheadFactor + 1, storeConfig.getReadAheadMaxFactor());
-    }
-
-    public void decreaseReadAheadFactor() {
-        readAheadFactor = Math.max(readAheadFactor - 1, storeConfig.getReadAheadMinFactor());
-    }
-
-    public void setNotReadAhead() {
-        readAheadFactor = 1;
-    }
-
-    public int getReadAheadFactor() {
-        return readAheadFactor;
-    }
-
-    public void recordGroupAccess(String group, long offset) {
-        groupOffsetCache.put(group, offset);
-    }
-
-    public long getActiveGroupCount(long minOffset, long maxOffset) {
-        return groupOffsetCache.asMap()
-            .values()
-            .stream()
-            .filter(offset -> offset >= minOffset && offset <= maxOffset)
-            .count();
-    }
-
-    public long getActiveGroupCount() {
-        return groupOffsetCache.estimatedSize();
-    }
-
-    public InFlightRequestFuture getInflightRequest(long offset, int batchSize) {
-        Optional<InFlightRequestFuture> optional = inFlightRequestMap.entrySet()
-            .stream()
-            .filter(entry -> {
-                InFlightRequestKey key = entry.getKey();
-                return Math.max(key.getOffset(), offset) <= Math.min(key.getOffset() + key.getBatchSize(), offset + batchSize);
-            })
-            .max(Comparator.comparing(entry -> entry.getKey().getRequestTime()))
-            .map(Map.Entry::getValue);
-        return optional.orElseGet(() -> new InFlightRequestFuture(Long.MAX_VALUE, new ArrayList<>()));
-    }
-
-    public InFlightRequestFuture getInflightRequest(String group, long offset, int batchSize) {
-        InFlightRequestFuture future = inFlightRequestMap.get(new InFlightRequestKey(group));
-        if (future != null && !future.isAllDone()) {
-            return future;
-        }
-        return getInflightRequest(offset, batchSize);
-    }
-
-    public void putInflightRequest(String group, long offset, int requestMsgCount,
-        List<Pair<Integer, CompletableFuture<Long>>> futureList) {
-        InFlightRequestKey key = new InFlightRequestKey(group, offset, requestMsgCount);
-        inFlightRequestMap.remove(key);
-        inFlightRequestMap.putIfAbsent(key, new InFlightRequestFuture(offset, futureList));
-    }
-
-    @Override
-    public int hashCode() {
-        return filePath.hashCode();
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-        if (obj == null) {
-            return false;
-        }
-        if (getClass() != obj.getClass()) {
-            return false;
-        }
-        return StringUtils.equals(filePath, ((CompositeFlatFile) obj).filePath);
-    }
-
-    public void shutdown() {
-        closed = true;
-        commitLog.commit(true);
-        consumeQueue.commit(true);
-    }
-
-    public void destroy() {
-        try {
-            closed = true;
-            compositeFlatFileLock.lock();
-            commitLog.destroy();
-            consumeQueue.destroy();
-            metadataStore.deleteFileSegment(filePath, FileSegmentType.COMMIT_LOG);
-            metadataStore.deleteFileSegment(filePath, FileSegmentType.CONSUME_QUEUE);
-        } catch (Exception e) {
-            LOGGER.error("CompositeFlatFile#destroy: delete file failed", e);
-        } finally {
-            compositeFlatFileLock.unlock();
-        }
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java
deleted file mode 100644
index 67d2cf06462..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.tieredstore.file;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.common.message.MessageConst;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.store.DispatchRequest;
-import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.index.IndexService;
-import org.apache.rocketmq.tieredstore.metadata.QueueMetadata;
-import org.apache.rocketmq.tieredstore.metadata.TopicMetadata;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-
-public class CompositeQueueFlatFile extends CompositeFlatFile {
-
-    private final MessageQueue messageQueue;
-    private long topicSequenceNumber;
-    private QueueMetadata queueMetadata;
-    private final IndexService indexStoreService;
-
-    public CompositeQueueFlatFile(TieredFileAllocator fileQueueFactory, MessageQueue messageQueue) {
-        super(fileQueueFactory, TieredStoreUtil.toPath(messageQueue));
-        this.messageQueue = messageQueue;
-        this.recoverQueueMetadata();
-        this.indexStoreService = TieredFlatFileManager.getTieredIndexService(storeConfig);
-    }
-
-    @Override
-    public void initOffset(long offset) {
-        if (!consumeQueue.isInitialized()) {
-            queueMetadata.setMinOffset(offset);
-            queueMetadata.setMaxOffset(offset);
-            metadataStore.updateQueue(queueMetadata);
-        }
-        super.initOffset(offset);
-    }
-
-    public void recoverQueueMetadata() {
-        TopicMetadata topicMetadata = this.metadataStore.getTopic(messageQueue.getTopic());
-        if (topicMetadata == null) {
-            topicMetadata = this.metadataStore.addTopic(messageQueue.getTopic(), -1L);
-        }
-        this.topicSequenceNumber = topicMetadata.getTopicId();
-
-        queueMetadata = this.metadataStore.getQueue(messageQueue);
-        if (queueMetadata == null) {
-            queueMetadata = this.metadataStore.addQueue(messageQueue, -1);
-        }
-        if (queueMetadata.getMaxOffset() < queueMetadata.getMinOffset()) {
-            queueMetadata.setMaxOffset(queueMetadata.getMinOffset());
-        }
-    }
-
-    public void flushMetadata() {
-        try {
-            queueMetadata.setMinOffset(super.getConsumeQueueMinOffset());
-            queueMetadata.setMaxOffset(super.getConsumeQueueMaxOffset());
-            metadataStore.updateQueue(queueMetadata);
-        } catch (Exception e) {
-            LOGGER.error("CompositeFlatFile#flushMetadata error, topic: {}, queue: {}",
-                messageQueue.getTopic(), messageQueue.getQueueId(), e);
-        }
-    }
-
-    /**
-     * Building indexes with offsetId is no longer supported because offsetId has changed in tiered storage
-     */
-    public AppendResult appendIndexFile(DispatchRequest request) {
-        if (closed) {
-            return AppendResult.FILE_CLOSED;
-        }
-
-        Set<String> keySet = new HashSet<>(
-            Arrays.asList(request.getKeys().split(MessageConst.KEY_SEPARATOR)));
-        if (StringUtils.isNotBlank(request.getUniqKey())) {
-            keySet.add(request.getUniqKey());
-        }
-
-        return indexStoreService.putKey(
-            messageQueue.getTopic(), (int) topicSequenceNumber, messageQueue.getQueueId(), keySet,
-            request.getCommitLogOffset(), request.getMsgSize(), request.getStoreTimestamp());
-    }
-
-    public MessageQueue getMessageQueue() {
-        return messageQueue;
-    }
-
-    @Override
-    public void shutdown() {
-        super.shutdown();
-        this.flushMetadata();
-    }
-
-    @Override
-    public void destroy() {
-        super.destroy();
-        metadataStore.deleteQueue(messageQueue);
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java
new file mode 100644
index 00000000000..38e451d3ff0
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.file;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
+import org.apache.rocketmq.tieredstore.common.AppendResult;
+import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.entity.FileSegmentMetadata;
+import org.apache.rocketmq.tieredstore.provider.FileSegment;
+import org.apache.rocketmq.tieredstore.provider.FileSegmentFactory;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FlatAppendFile {
+
+    protected static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
+    public static final long GET_FILE_SIZE_ERROR = -1L;
+
+    protected final String filePath;
+    protected final FileSegmentType fileType;
+    protected final MetadataStore metadataStore;
+    protected final FileSegmentFactory fileSegmentFactory;
+    protected final ReentrantReadWriteLock fileSegmentLock;
+    protected final CopyOnWriteArrayList<FileSegment> fileSegmentTable;
+
+    protected FlatAppendFile(FileSegmentFactory fileSegmentFactory, FileSegmentType fileType, String filePath) {
+
+        this.fileType = fileType;
+        this.filePath = filePath;
+        this.metadataStore = fileSegmentFactory.getMetadataStore();
+        this.fileSegmentFactory = fileSegmentFactory;
+        this.fileSegmentLock = new ReentrantReadWriteLock();
+        this.fileSegmentTable = new CopyOnWriteArrayList<>();
+        this.recover();
+        this.recoverFileSize();
+    }
+
+    public void recover() {
+        List<FileSegment> fileSegmentList = new ArrayList<>();
+        this.metadataStore.iterateFileSegment(this.filePath, this.fileType, metadata -> {
+            FileSegment fileSegment = this.fileSegmentFactory.createSegment(
+                this.fileType, metadata.getPath(), metadata.getBaseOffset());
+            fileSegment.initPosition(metadata.getSize());
+            fileSegment.setMinTimestamp(metadata.getBeginTimestamp());
+            fileSegment.setMaxTimestamp(metadata.getEndTimestamp());
+            fileSegmentList.add(fileSegment);
+        });
+        this.fileSegmentTable.addAll(fileSegmentList.stream().sorted().collect(Collectors.toList()));
+    }
+
+    public void recoverFileSize() {
+        if (fileSegmentTable.isEmpty() || FileSegmentType.INDEX.equals(fileType)) {
+            return;
+        }
+        FileSegment fileSegment = fileSegmentTable.get(fileSegmentTable.size() - 1);
+        long fileSize = fileSegment.getSize();
+        if (fileSize == GET_FILE_SIZE_ERROR) {
+            log.warn("FlatAppendFile get last file size error, filePath: {}", this.filePath);
+            return;
+        }
+        if (fileSegment.getCommitPosition() != fileSize) {
+            fileSegment.initPosition(fileSize);
+            flushFileSegmentMeta(fileSegment);
+            log.warn("FlatAppendFile last file size not correct, filePath: {}", this.filePath);
+        }
+    }
+
+    public void initOffset(long offset) {
+        if (this.fileSegmentTable.isEmpty()) {
+            FileSegment fileSegment = fileSegmentFactory.createSegment(fileType, filePath, offset);
+            fileSegment.initPosition(fileSegment.getSize());
+            this.flushFileSegmentMeta(fileSegment);
+            this.fileSegmentTable.add(fileSegment);
+        }
+    }
+
+    public void flushFileSegmentMeta(FileSegment fileSegment) {
+        FileSegmentMetadata metadata = this.metadataStore.getFileSegment(
+            this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset());
+        if (metadata == null) {
+            metadata = new FileSegmentMetadata(
+                this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getCode());
+            metadata.setCreateTimestamp(System.currentTimeMillis());
+        }
+        metadata.setSize(fileSegment.getCommitPosition());
+        metadata.setBeginTimestamp(fileSegment.getMinTimestamp());
+        metadata.setEndTimestamp(fileSegment.getMaxTimestamp());
+        this.metadataStore.updateFileSegment(metadata);
+    }
+
+    public String getFilePath() {
+        return filePath;
+    }
+
+    public FileSegmentType getFileType() {
+        return fileType;
+    }
+
+    public List<FileSegment> getFileSegmentList() {
+        return fileSegmentTable;
+    }
+
+    public long getMinOffset() {
+        List<FileSegment> list = this.fileSegmentTable;
+        return list.isEmpty() ? 0L : list.get(0).getBaseOffset();
+    }
+
+    public long getCommitOffset() {
+        List<FileSegment> list = this.fileSegmentTable;
+        return list.isEmpty() ? 0L : list.get(list.size() - 1).getCommitOffset();
+    }
+
+    public long getAppendOffset() {
+        List<FileSegment> list = this.fileSegmentTable;
+        return list.isEmpty() ? 0L : list.get(list.size() - 1).getAppendOffset();
+    }
+
+    public long getMinTimestamp() {
+        List<FileSegment> list = this.fileSegmentTable;
+        return list.isEmpty() ? GET_FILE_SIZE_ERROR : list.get(0).getMinTimestamp();
+    }
+
+    public long getMaxTimestamp() {
+        List<FileSegment> list = this.fileSegmentTable;
+        return list.isEmpty() ? GET_FILE_SIZE_ERROR : list.get(list.size() - 1).getMaxTimestamp();
+    }
+
+    public FileSegment rollingNewFile(long offset) {
+        FileSegment fileSegment;
+        fileSegmentLock.writeLock().lock();
+        try {
+            fileSegment = this.fileSegmentFactory.createSegment(this.fileType, this.filePath, offset);
+            this.flushFileSegmentMeta(fileSegment);
+            this.fileSegmentTable.add(fileSegment);
+        } finally {
+            fileSegmentLock.writeLock().unlock();
+        }
+        return fileSegment;
+    }
+
+    public FileSegment getFileToWrite() {
+        List<FileSegment> fileSegmentList = this.fileSegmentTable;
+        if (fileSegmentList.isEmpty()) {
+            throw new IllegalStateException("Need to set base offset before create file segment");
+        } else {
+            return fileSegmentList.get(fileSegmentList.size() - 1);
+        }
+    }
+
+    public AppendResult append(ByteBuffer buffer, long timestamp) {
+        AppendResult result;
+        fileSegmentLock.writeLock().lock();
+        try {
+            FileSegment fileSegment = this.getFileToWrite();
+            result = fileSegment.append(buffer, timestamp);
+            if (result == AppendResult.FILE_FULL) {
+                boolean commitResult = fileSegment.commitAsync().join();
+                log.info("FlatAppendFile#append not successful for the file {} is full, commit result={}",
+                    fileSegment.getPath(), commitResult);
+                if (commitResult) {
+                    this.flushFileSegmentMeta(fileSegment);
+                    return this.rollingNewFile(this.getAppendOffset()).append(buffer, timestamp);
+                } else {
+                    return AppendResult.UNKNOWN_ERROR;
+                }
+            }
+        } finally {
+            fileSegmentLock.writeLock().unlock();
+        }
+        return result;
+    }
+
+    public CompletableFuture<Boolean> commitAsync() {
+        List<FileSegment> fileSegmentsList = this.fileSegmentTable;
+        if (fileSegmentsList.isEmpty()) {
+            return CompletableFuture.completedFuture(true);
+        }
+        FileSegment fileSegment = fileSegmentsList.get(fileSegmentsList.size() - 1);
+        return fileSegment.commitAsync().thenApply(success -> {
+            if (success) {
+                this.flushFileSegmentMeta(fileSegment);
+            }
+            return success;
+        });
+    }
+
+    public CompletableFuture<ByteBuffer> readAsync(long offset, int length) {
+        List<FileSegment> fileSegmentList = this.fileSegmentTable;
+        int index = fileSegmentList.size() - 1;
+        for (; index >= 0; index--) {
+            if (fileSegmentList.get(index).getBaseOffset() <= offset) {
+                break;
+            }
+        }
+
+        FileSegment fileSegment1 = fileSegmentList.get(index);
+        FileSegment fileSegment2 = offset + length > fileSegment1.getCommitOffset() &&
+            fileSegmentList.size() > index + 1 ? fileSegmentList.get(index + 1) : null;
+
+        if (fileSegment2 == null) {
+            return fileSegment1.readAsync(offset - fileSegment1.getBaseOffset(), length);
+        }
+
+        int segment1Length = (int) (fileSegment1.getCommitOffset() - offset);
+        return fileSegment1.readAsync(offset - fileSegment1.getBaseOffset(), segment1Length)
+            .thenCombine(fileSegment2.readAsync(0, length - segment1Length),
+                (buffer1, buffer2) -> {
+                    ByteBuffer buffer = ByteBuffer.allocate(buffer1.remaining() + buffer2.remaining());
+                    buffer.put(buffer1).put(buffer2);
+                    buffer.flip();
+                    return buffer;
+                });
+    }
+
+    public void shutdown() {
+        fileSegmentLock.writeLock().lock();
+        try {
+            fileSegmentTable.forEach(FileSegment::close);
+        } finally {
+            fileSegmentLock.writeLock().unlock();
+        }
+    }
+
+    public void destroyExpiredFile(long expireTimestamp) {
+        fileSegmentLock.writeLock().lock();
+        try {
+            while (!fileSegmentTable.isEmpty()) {
+
+                // first remove expired file from fileSegmentTable
+                // then close and delete expired file
+                FileSegment fileSegment = fileSegmentTable.get(0);
+
+                if (fileSegment.getMaxTimestamp() != Long.MAX_VALUE &&
+                    fileSegment.getMaxTimestamp() >= expireTimestamp) {
+                    log.debug("FileSegment has not expired, filePath={}, fileType={}, " +
+                            "offset={}, expireTimestamp={}, maxTimestamp={}", filePath, fileType,
+                        fileSegment.getBaseOffset(), expireTimestamp, fileSegment.getMaxTimestamp());
+                    break;
+                }
+
+                fileSegment.destroyFile();
+                if (!fileSegment.exists()) {
+                    fileSegmentTable.remove(0);
+                    metadataStore.deleteFileSegment(filePath, fileType, fileSegment.getBaseOffset());
+                }
+            }
+        } finally {
+            fileSegmentLock.writeLock().unlock();
+        }
+    }
+
+    public void destroy() {
+        this.destroyExpiredFile(Long.MAX_VALUE);
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFile.java
new file mode 100644
index 00000000000..bdf0bf375eb
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFile.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.file;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.provider.FileSegment;
+import org.apache.rocketmq.tieredstore.provider.FileSegmentFactory;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtil;
+
+public class FlatCommitLogFile extends FlatAppendFile {
+
+    private static final long GET_OFFSET_ERROR = -1L;
+
+    private final AtomicLong firstOffset = new AtomicLong(GET_OFFSET_ERROR);
+
+    public FlatCommitLogFile(FileSegmentFactory fileSegmentFactory, String filePath) {
+        super(fileSegmentFactory, FileSegmentType.COMMIT_LOG, filePath);
+        this.initOffset(0L);
+    }
+
+    /**
+     * Two rules are set here:
+     * 1. Single file must be saved for more than one day as default.
+     * 2. Single file must reach the minimum size before switching.
+     * When calculating storage space, due to the limitation of condition 2,
+     * the actual usage of storage space may be slightly higher than expected.
+     */
+    public boolean tryRollingFile(long interval) {
+        FileSegment fileSegment = this.getFileToWrite();
+        long timestamp = fileSegment.getMinTimestamp();
+        if (timestamp != Long.MAX_VALUE && timestamp + interval < System.currentTimeMillis() &&
+            fileSegment.getAppendPosition() >=
+                fileSegmentFactory.getStoreConfig().getCommitLogRollingMinimumSize()) {
+            this.rollingNewFile(this.getAppendOffset());
+            return true;
+        }
+        return false;
+    }
+
+    public long getMinOffsetFromFile() {
+        return firstOffset.get() == GET_OFFSET_ERROR ?
+            this.getMinOffsetFromFileAsync().join() : firstOffset.get();
+    }
+
+    public CompletableFuture<Long> getMinOffsetFromFileAsync() {
+        int length = MessageFormatUtil.QUEUE_OFFSET_POSITION + Long.BYTES;
+        if (this.fileSegmentTable.isEmpty() ||
+            this.getCommitOffset() - this.getMinOffset() < length) {
+            return CompletableFuture.completedFuture(GET_OFFSET_ERROR);
+        }
+        return this.readAsync(this.getMinOffset(), length)
+            .thenApply(buffer -> {
+                firstOffset.set(MessageFormatUtil.getQueueOffset(buffer));
+                return firstOffset.get();
+            });
+    }
+
+    @Override
+    public void destroyExpiredFile(long expireTimestamp) {
+        long beforeOffset = this.getMinOffset();
+        super.destroyExpiredFile(expireTimestamp);
+        long afterOffset = this.getMinOffset();
+
+        if (beforeOffset != afterOffset && afterOffset > 0) {
+            log.info("CommitLog min cq offset reset, filePath={}, offset={}, expireTimestamp={}, change={}-{}",
+                filePath, firstOffset.get(), expireTimestamp, beforeOffset, afterOffset);
+            firstOffset.set(GET_OFFSET_ERROR);
+        }
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatConsumeQueueFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatConsumeQueueFile.java
new file mode 100644
index 00000000000..caad4749b5d
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatConsumeQueueFile.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.file;
+
+import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.provider.FileSegmentFactory;
+
+public class FlatConsumeQueueFile extends FlatAppendFile {
+
+    public FlatConsumeQueueFile(FileSegmentFactory fileSegmentFactory, String filePath) {
+        super(fileSegmentFactory, FileSegmentType.CONSUME_QUEUE, filePath);
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileFactory.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileFactory.java
new file mode 100644
index 00000000000..d14ea7ffdb2
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileFactory.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.tieredstore.file;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
+import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.provider.FileSegmentFactory;
+
+public class FlatFileFactory {
+
+    private final MetadataStore metadataStore;
+    private final MessageStoreConfig storeConfig;
+    private final FileSegmentFactory fileSegmentFactory;
+
+    @VisibleForTesting
+    public FlatFileFactory(MetadataStore metadataStore, MessageStoreConfig storeConfig) {
+        this.metadataStore = metadataStore;
+        this.storeConfig = storeConfig;
+        this.fileSegmentFactory = new FileSegmentFactory(metadataStore, storeConfig, new MessageStoreExecutor());
+    }
+
+    public FlatFileFactory(MetadataStore metadataStore,
+        MessageStoreConfig storeConfig, MessageStoreExecutor executor) {
+
+        this.metadataStore = metadataStore;
+        this.storeConfig = storeConfig;
+        this.fileSegmentFactory = new FileSegmentFactory(metadataStore, storeConfig, executor);
+    }
+
+    public MessageStoreConfig getStoreConfig() {
+        return storeConfig;
+    }
+
+    public MetadataStore getMetadataStore() {
+        return metadataStore;
+    }
+
+    public FlatCommitLogFile createFlatFileForCommitLog(String filePath) {
+        return new FlatCommitLogFile(this.fileSegmentFactory, filePath);
+    }
+
+    public FlatConsumeQueueFile createFlatFileForConsumeQueue(String filePath) {
+        return new FlatConsumeQueueFile(this.fileSegmentFactory, filePath);
+    }
+
+    public FlatAppendFile createFlatFileForIndexFile(String filePath) {
+        return new FlatAppendFile(this.fileSegmentFactory, FileSegmentType.INDEX, filePath);
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeAccess.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileInterface.java
similarity index 55%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeAccess.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileInterface.java
index 3d962e40d65..01e7f25a467 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeAccess.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileInterface.java
@@ -18,35 +18,36 @@
 
 import java.nio.ByteBuffer;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.locks.Lock;
+import org.apache.rocketmq.common.BoundaryType;
+import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.store.DispatchRequest;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
 import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.common.BoundaryType;
 
-interface CompositeAccess {
+public interface FlatFileInterface {
+
+    long getTopicId();
+
+    Lock getFileLock();
+
+    MessageQueue getMessageQueue();
+
+    boolean isFlatFileInit();
 
-    /**
-     * Initializes the offset for the flat file.
-     * Will only affect the distribution site if the file has already been initialized.
-     *
-     * @param offset init offset for consume queue
-     */
     void initOffset(long offset);
 
-    /**
-     * Appends a message to the commit log file, but does not commit it immediately
-     *
-     * @param message the message to append
-     * @return append result
-     */
-    AppendResult appendCommitLog(ByteBuffer message);
+    boolean rollingFile(long interval);
 
     /**
      * Appends a message to the commit log file
      *
-     * @param message the message to append
+     * @param message thByteBuffere message to append
      * @return append result
      */
-    AppendResult appendCommitLog(ByteBuffer message, boolean commit);
+    AppendResult appendCommitLog(ByteBuffer message);
+
+    AppendResult appendCommitLog(SelectMappedBufferResult message);
 
     /**
      * Append message to consume queue file, but does not commit it immediately
@@ -56,29 +57,30 @@ interface CompositeAccess {
      */
     AppendResult appendConsumeQueue(DispatchRequest request);
 
-    /**
-     * Append message to consume queue file
-     *
-     * @param request the dispatch request
-     * @param commit  whether to commit
-     * @return append result
-     */
-    AppendResult appendConsumeQueue(DispatchRequest request, boolean commit);
+    void release();
 
-    /**
-     * Persist commit log file
-     */
-    void commitCommitLog();
+    long getMinStoreTimestamp();
 
-    /**
-     * Persist the consume queue file
-     */
-    void commitConsumeQueue();
+    long getMaxStoreTimestamp();
+
+    long getFirstMessageOffset();
+
+    long getCommitLogMinOffset();
+
+    long getCommitLogMaxOffset();
+
+    long getCommitLogCommitOffset();
+
+    long getConsumeQueueMinOffset();
+
+    long getConsumeQueueMaxOffset();
+
+    long getConsumeQueueCommitOffset();
 
     /**
      * Persist commit log file and consume queue file
      */
-    void commit(boolean sync);
+    CompletableFuture<Boolean> commitAsync();
 
     /**
      * Asynchronously retrieves the message at the specified consume queue offset
@@ -89,7 +91,7 @@ interface CompositeAccess {
     CompletableFuture<ByteBuffer> getMessageAsync(long consumeQueueOffset);
 
     /**
-     * Get message from commitlog file at specified offset and length
+     * Get message from commitLog file at specified offset and length
      *
      * @param offset the offset
      * @param length the length
@@ -115,37 +117,40 @@ interface CompositeAccess {
     CompletableFuture<ByteBuffer> getConsumeQueueAsync(long consumeQueueOffset, int count);
 
     /**
-     * Return the consensus queue site corresponding to the confirmed site in the commitlog
-     *
-     * @return the maximum offset
-     */
-    long getCommitLogDispatchCommitOffset();
+     * Gets the start offset in the consume queue based on the timestamp and boundary type.
+     * The consume queues consist of ordered units, and their storage times are non-decreasing
+     * sequence. If the specified message exists, it returns the offset of either the first
+     * or last message, depending on the boundary type. If the specified message does not exist,
+     * it returns the offset of the next message as the pull offset. For example:
+     * ------------------------------------------------------------
+     *   store time   : 40, 50, 50, 50, 60, 60, 70
+     *   queue offset : 10, 11, 12, 13, 14, 15, 16
+     * ------------------------------------------------------------
+     *   query timestamp | boundary | result (reason)
+     *         35        |    -     |   10 (minimum offset)
+     *         45        |    -     |   11 (next offset)
+     *         50        |   lower  |   11
+     *         50        |   upper  |   13
+     *         60        |    -     |   14 (default to lower)
+     *         75        |    -     |   17 (maximum offset + 1)
+     * ------------------------------------------------------------
+     * @param timestamp    The search time
+     * @param boundaryType 'lower' or 'upper' to determine the boundary
+     * @return Returns the offset of the message in the consume queue
+     */
+    CompletableFuture<Long> getQueueOffsetByTimeAsync(long timestamp, BoundaryType boundaryType);
+
+    boolean isClosed();
 
     /**
-     * Gets the offset in the consume queue by timestamp and boundary type
-     *
-     * @param timestamp    search time
-     * @param boundaryType lower or upper to decide boundary
-     * @return Returns the offset of the message
-     */
-    long getOffsetInConsumeQueueByTime(long timestamp, BoundaryType boundaryType);
-
-    /**
-     * Mark some commit log and consume file sealed and expired
-     *
-     * @param expireTimestamp expire timestamp, usually several days before the current time
+     * Shutdown process
      */
-    void cleanExpiredFile(long expireTimestamp);
+    void shutdown();
 
     /**
      * Destroys expired files
      */
-    void destroyExpiredFile();
-
-    /**
-     * Shutdown process
-     */
-    void shutdown();
+    void destroyExpiredFile(long timestamp);
 
     /**
      * Delete file
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java
new file mode 100644
index 00000000000..700f12d0d6e
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.file;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FlatFileStore {
+
+    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
+
+    private final MetadataStore metadataStore;
+    private final MessageStoreConfig storeConfig;
+    private final MessageStoreExecutor executor;
+    private final FlatFileFactory flatFileFactory;
+    private final ConcurrentMap<MessageQueue, FlatMessageFile> flatFileConcurrentMap;
+
+    public FlatFileStore(MessageStoreConfig storeConfig, MetadataStore metadataStore, MessageStoreExecutor executor) {
+        this.storeConfig = storeConfig;
+        this.metadataStore = metadataStore;
+        this.executor = executor;
+        this.flatFileFactory = new FlatFileFactory(metadataStore, storeConfig, executor);
+        this.flatFileConcurrentMap = new ConcurrentHashMap<>();
+    }
+
+    public boolean load() {
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        try {
+            this.flatFileConcurrentMap.clear();
+            this.recover();
+            log.info("FlatFileStore recover finished, total cost={}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
+        } catch (Exception e) {
+            long costTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+            log.info("FlatFileStore recover error, total cost={}ms", costTime);
+            LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME)
+                .error("FlatFileStore recover error, total cost={}ms", costTime, e);
+            return false;
+        }
+        return true;
+    }
+
+    public void recover() {
+        Semaphore semaphore = new Semaphore(storeConfig.getTieredStoreMaxPendingLimit() / 4);
+        List<CompletableFuture<Void>> futures = new ArrayList<>();
+        metadataStore.iterateTopic(topicMetadata -> {
+            semaphore.acquireUninterruptibly();
+            futures.add(this.recoverAsync(topicMetadata)
+                .whenComplete((unused, throwable) -> {
+                    if (throwable != null) {
+                        log.error("FlatFileStore recover file error, topic={}", topicMetadata.getTopic(), throwable);
+                    }
+                    semaphore.release();
+                }));
+        });
+        CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
+    }
+
+    public CompletableFuture<Void> recoverAsync(TopicMetadata topicMetadata) {
+        return CompletableFuture.runAsync(() -> {
+            Stopwatch stopwatch = Stopwatch.createStarted();
+            AtomicLong queueCount = new AtomicLong();
+            metadataStore.iterateQueue(topicMetadata.getTopic(), queueMetadata -> {
+                FlatMessageFile flatFile = this.computeIfAbsent(new MessageQueue(
+                    topicMetadata.getTopic(), storeConfig.getBrokerName(), queueMetadata.getQueue().getQueueId()));
+                queueCount.incrementAndGet();
+                log.debug("FlatFileStore recover file, topicId={}, topic={}, queueId={}, cost={}ms",
+                    flatFile.getTopicId(), flatFile.getMessageQueue().getTopic(),
+                    flatFile.getMessageQueue().getQueueId(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
+            });
+            log.info("FlatFileStore recover file, topic={}, total={}, cost={}ms",
+                topicMetadata.getTopic(), queueCount.get(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
+        }, executor.bufferCommitExecutor);
+    }
+
+    public void scheduleDeleteExpireFile() {
+        if (!storeConfig.isTieredStoreDeleteFileEnable()) {
+            return;
+        }
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        ImmutableList<FlatMessageFile> fileList = this.deepCopyFlatFileToList();
+        for (FlatMessageFile flatFile : fileList) {
+            flatFile.getFileLock().lock();
+            try {
+                flatFile.destroyExpiredFile(System.currentTimeMillis() -
+                    TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours()));
+            } catch (Exception e) {
+                log.error("FlatFileStore delete expire file error", e);
+            } finally {
+                flatFile.getFileLock().unlock();
+            }
+        }
+        log.info("FlatFileStore schedule delete expired file, count={}, cost={}ms",
+            fileList.size(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
+    }
+
+    public MetadataStore getMetadataStore() {
+        return metadataStore;
+    }
+
+    public MessageStoreConfig getStoreConfig() {
+        return storeConfig;
+    }
+
+    public FlatFileFactory getFlatFileFactory() {
+        return flatFileFactory;
+    }
+
+    public FlatMessageFile computeIfAbsent(MessageQueue messageQueue) {
+        return flatFileConcurrentMap.computeIfAbsent(messageQueue,
+            mq -> new FlatMessageFile(flatFileFactory, mq.getTopic(), mq.getQueueId()));
+    }
+
+    public FlatMessageFile getFlatFile(MessageQueue messageQueue) {
+        return flatFileConcurrentMap.get(messageQueue);
+    }
+
+    public ImmutableList<FlatMessageFile> deepCopyFlatFileToList() {
+        return ImmutableList.copyOf(flatFileConcurrentMap.values());
+    }
+
+    public void shutdown() {
+        flatFileConcurrentMap.values().forEach(FlatMessageFile::shutdown);
+    }
+
+    public void destroyFile(MessageQueue mq) {
+        if (mq == null) {
+            return;
+        }
+
+        FlatMessageFile flatFile = flatFileConcurrentMap.remove(mq);
+        if (flatFile != null) {
+            flatFile.shutdown();
+            flatFile.destroy();
+        }
+        log.info("FlatFileStore destroy file, topic={}, queueId={}", mq.getTopic(), mq.getQueueId());
+    }
+
+    public void destroy() {
+        this.shutdown();
+        flatFileConcurrentMap.values().forEach(FlatMessageFile::destroy);
+        flatFileConcurrentMap.clear();
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatMessageFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatMessageFile.java
new file mode 100644
index 00000000000..b0e4dd6e3b0
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatMessageFile.java
@@ -0,0 +1,426 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.file;
+
+import com.alibaba.fastjson.JSON;
+import com.google.common.annotations.VisibleForTesting;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.BoundaryType;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.store.DispatchRequest;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.common.AppendResult;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.entity.QueueMetadata;
+import org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata;
+import org.apache.rocketmq.tieredstore.provider.FileSegment;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtil;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FlatMessageFile implements FlatFileInterface {
+
+    protected static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
+    protected volatile boolean closed = false;
+
+    protected TopicMetadata topicMetadata;
+    protected QueueMetadata queueMetadata;
+
+    protected final String filePath;
+    protected final ReentrantLock fileLock;
+    protected final Semaphore commitLock = new Semaphore(1);
+    protected final MessageStoreConfig storeConfig;
+    protected final MetadataStore metadataStore;
+    protected final FlatCommitLogFile commitLog;
+    protected final FlatConsumeQueueFile consumeQueue;
+
+    protected final ConcurrentMap<String, CompletableFuture<?>> inFlightRequestMap;
+
+    public FlatMessageFile(FlatFileFactory fileFactory, String topic, int queueId) {
+        this(fileFactory, MessageStoreUtil.toFilePath(
+            new MessageQueue(topic, fileFactory.getStoreConfig().getBrokerName(), queueId)));
+        this.topicMetadata = this.recoverTopicMetadata(topic);
+        this.queueMetadata = this.recoverQueueMetadata(topic, queueId);
+    }
+
+    public FlatMessageFile(FlatFileFactory fileFactory, String filePath) {
+        this.filePath = filePath;
+        this.fileLock = new ReentrantLock(false);
+        this.storeConfig = fileFactory.getStoreConfig();
+        this.metadataStore = fileFactory.getMetadataStore();
+        this.commitLog = fileFactory.createFlatFileForCommitLog(filePath);
+        this.consumeQueue = fileFactory.createFlatFileForConsumeQueue(filePath);
+        this.inFlightRequestMap = new ConcurrentHashMap<>();
+    }
+
+    @Override
+    public long getTopicId() {
+        return topicMetadata.getTopicId();
+    }
+
+    @Override
+    public MessageQueue getMessageQueue() {
+        return queueMetadata != null ? queueMetadata.getQueue() : null;
+    }
+
+    @Override
+    public boolean isFlatFileInit() {
+        return !this.consumeQueue.fileSegmentTable.isEmpty();
+    }
+
+    public TopicMetadata recoverTopicMetadata(String topic) {
+        TopicMetadata topicMetadata = this.metadataStore.getTopic(topic);
+        if (topicMetadata == null) {
+            topicMetadata = this.metadataStore.addTopic(topic, -1L);
+        }
+        return topicMetadata;
+    }
+
+    public QueueMetadata recoverQueueMetadata(String topic, int queueId) {
+        MessageQueue mq = new MessageQueue(topic, storeConfig.getBrokerName(), queueId);
+        QueueMetadata queueMetadata = this.metadataStore.getQueue(mq);
+        if (queueMetadata == null) {
+            queueMetadata = this.metadataStore.addQueue(mq, -1L);
+        }
+        return queueMetadata;
+    }
+
+    public void flushMetadata() {
+        if (queueMetadata != null) {
+            queueMetadata.setMinOffset(this.getConsumeQueueMinOffset());
+            queueMetadata.setMaxOffset(this.getConsumeQueueCommitOffset());
+            queueMetadata.setUpdateTimestamp(System.currentTimeMillis());
+            metadataStore.updateQueue(queueMetadata);
+        }
+    }
+
+    @Override
+    public Lock getFileLock() {
+        return this.fileLock;
+    }
+
+    @VisibleForTesting
+    public Semaphore getCommitLock() {
+        return commitLock;
+    }
+
+    @Override
+    public boolean rollingFile(long interval) {
+        return this.commitLog.tryRollingFile(interval);
+    }
+
+    @Override
+    public void initOffset(long offset) {
+        fileLock.lock();
+        try {
+            this.commitLog.initOffset(0L);
+            this.consumeQueue.initOffset(offset * MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);
+        } finally {
+            fileLock.unlock();
+        }
+    }
+
+    @Override
+    public AppendResult appendCommitLog(ByteBuffer message) {
+        if (closed) {
+            return AppendResult.FILE_CLOSED;
+        }
+        return commitLog.append(message, MessageFormatUtil.getStoreTimeStamp(message));
+    }
+
+    @Override
+    public AppendResult appendCommitLog(SelectMappedBufferResult message) {
+        if (closed) {
+            return AppendResult.FILE_CLOSED;
+        }
+        return this.appendCommitLog(message.getByteBuffer());
+    }
+
+    @Override
+    public AppendResult appendConsumeQueue(DispatchRequest request) {
+        if (closed) {
+            return AppendResult.FILE_CLOSED;
+        }
+
+        ByteBuffer buffer = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);
+        buffer.putLong(request.getCommitLogOffset());
+        buffer.putInt(request.getMsgSize());
+        buffer.putLong(request.getTagsCode());
+        buffer.flip();
+
+        return consumeQueue.append(buffer, request.getStoreTimestamp());
+    }
+
+    @Override
+    public void release() {
+    }
+
+    @Override
+    public long getMinStoreTimestamp() {
+        long minStoreTime = -1L;
+        if (Long.MAX_VALUE != commitLog.getMinTimestamp()) {
+            minStoreTime = Math.max(minStoreTime, commitLog.getMinTimestamp());
+        }
+        if (Long.MAX_VALUE != consumeQueue.getMinTimestamp()) {
+            minStoreTime = Math.max(minStoreTime, consumeQueue.getMinTimestamp());
+        }
+        return minStoreTime;
+    }
+
+    @Override
+    public long getMaxStoreTimestamp() {
+        return commitLog.getMaxTimestamp();
+    }
+
+    @Override
+    public long getFirstMessageOffset() {
+        return commitLog.getMinOffsetFromFile();
+    }
+
+    @Override
+    public long getCommitLogMinOffset() {
+        return commitLog.getMinOffset();
+    }
+
+    @Override
+    public long getCommitLogMaxOffset() {
+        return commitLog.getAppendOffset();
+    }
+
+    @Override
+    public long getCommitLogCommitOffset() {
+        return commitLog.getCommitOffset();
+    }
+
+    @Override
+    public long getConsumeQueueMinOffset() {
+        long cqOffset = consumeQueue.getMinOffset() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE;
+        long effectiveOffset = this.commitLog.getMinOffsetFromFile();
+        return Math.max(cqOffset, effectiveOffset);
+    }
+
+    @Override
+    public long getConsumeQueueMaxOffset() {
+        return consumeQueue.getAppendOffset() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE;
+    }
+
+    @Override
+    public long getConsumeQueueCommitOffset() {
+        return consumeQueue.getCommitOffset() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE;
+    }
+
+    @Override
+    public CompletableFuture<Boolean> commitAsync() {
+        // acquire lock
+        if (commitLock.drainPermits() <= 0) {
+            return CompletableFuture.completedFuture(false);
+        }
+
+        return this.commitLog.commitAsync()
+            .thenCompose(result -> {
+                if (result) {
+                    return consumeQueue.commitAsync();
+                }
+                return CompletableFuture.completedFuture(false);
+            }).whenComplete((result, throwable) -> commitLock.release());
+    }
+
+    @Override
+    public CompletableFuture<ByteBuffer> getMessageAsync(long queueOffset) {
+        return getConsumeQueueAsync(queueOffset).thenCompose(cqBuffer -> {
+            long commitLogOffset = MessageFormatUtil.getCommitLogOffsetFromItem(cqBuffer);
+            int length = MessageFormatUtil.getSizeFromItem(cqBuffer);
+            return getCommitLogAsync(commitLogOffset, length);
+        });
+    }
+
+    @Override
+    public CompletableFuture<ByteBuffer> getCommitLogAsync(long offset, int length) {
+        return commitLog.readAsync(offset, length);
+    }
+
+    @Override
+    public CompletableFuture<ByteBuffer> getConsumeQueueAsync(long queueOffset) {
+        return this.getConsumeQueueAsync(queueOffset, 1);
+    }
+
+    @Override
+    public CompletableFuture<ByteBuffer> getConsumeQueueAsync(long queueOffset, int count) {
+        return consumeQueue.readAsync(
+            queueOffset * MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE,
+            count * MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);
+    }
+
+    @Override
+    public CompletableFuture<Long> getQueueOffsetByTimeAsync(long timestamp, BoundaryType boundaryType) {
+        long cqMin = getConsumeQueueMinOffset();
+        long cqMax = getConsumeQueueCommitOffset() - 1;
+        if (cqMax == -1 || cqMax < cqMin) {
+            return CompletableFuture.completedFuture(cqMin);
+        }
+
+        ByteBuffer buffer = getMessageAsync(cqMax).join();
+        long storeTime = MessageFormatUtil.getStoreTimeStamp(buffer);
+        if (storeTime < timestamp) {
+            log.info("FlatMessageFile getQueueOffsetByTimeAsync, exceeded maximum time, " +
+                "filePath={}, timestamp={}, result={}", filePath, timestamp, cqMax + 1);
+            return CompletableFuture.completedFuture(cqMax + 1);
+        }
+
+        buffer = getMessageAsync(cqMin).join();
+        storeTime = MessageFormatUtil.getStoreTimeStamp(buffer);
+        if (storeTime > timestamp) {
+            log.info("FlatMessageFile getQueueOffsetByTimeAsync, less than minimum time, " +
+                "filePath={}, timestamp={}, result={}", filePath, timestamp, cqMin);
+            return CompletableFuture.completedFuture(cqMin);
+        }
+
+        // get correct consume queue file by binary search
+        List<FileSegment> consumeQueueFileList = this.consumeQueue.getFileSegmentList();
+        int low = 0, high = consumeQueueFileList.size() - 1;
+        int mid = low + (high - low) / 2;
+        while (low <= high) {
+            FileSegment fileSegment = consumeQueueFileList.get(mid);
+            if (fileSegment.getMinTimestamp() <= timestamp && timestamp <= fileSegment.getMaxTimestamp()) {
+                break;
+            } else if (timestamp < fileSegment.getMinTimestamp()) {
+                high = mid - 1;
+            } else {
+                low = mid + 1;
+            }
+            mid = low + (high - low) / 2;
+        }
+        FileSegment target = consumeQueueFileList.get(mid);
+
+        // binary search lower bound index in a sorted array
+        long minOffset = target.getBaseOffset() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE;
+        long maxOffset = target.getCommitOffset() / MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE - 1;
+        List<String> queryLog = new ArrayList<>();
+        while (minOffset < maxOffset) {
+            long middle = minOffset + (maxOffset - minOffset) / 2;
+            buffer = this.getMessageAsync(middle).join();
+            storeTime = MessageFormatUtil.getStoreTimeStamp(buffer);
+            queryLog.add(String.format("(range=%d-%d, middle=%d, timestamp=%d, diff=%dms)",
+                minOffset, maxOffset, middle, storeTime, timestamp - storeTime));
+            if (storeTime < timestamp) {
+                minOffset = middle + 1;
+            } else {
+                maxOffset = middle;
+            }
+        }
+
+        long offset = minOffset;
+        if (boundaryType == BoundaryType.UPPER) {
+            while (true) {
+                long next = offset + 1;
+                if (next > cqMax) {
+                    break;
+                }
+                buffer = this.getMessageAsync(next).join();
+                storeTime = MessageFormatUtil.getStoreTimeStamp(buffer);
+                if (storeTime == timestamp) {
+                    offset = next;
+                } else {
+                    break;
+                }
+            }
+        }
+
+        log.info("FlatMessageFile getQueueOffsetByTimeAsync, filePath={}, timestamp={}, result={}, log={}",
+            filePath, timestamp, offset, JSON.toJSONString(queryLog));
+        return CompletableFuture.completedFuture(offset);
+    }
+
+    @Override
+    public int hashCode() {
+        return filePath.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        return StringUtils.equals(filePath, ((FlatMessageFile) obj).filePath);
+    }
+
+    @Override
+    public boolean isClosed() {
+        return closed;
+    }
+
+    @Override
+    public void shutdown() {
+        closed = true;
+        fileLock.lock();
+        try {
+            consumeQueue.shutdown();
+            commitLog.shutdown();
+        } finally {
+            fileLock.unlock();
+        }
+    }
+
+    @Override
+    public void destroyExpiredFile(long timestamp) {
+        fileLock.lock();
+        try {
+            consumeQueue.destroyExpiredFile(timestamp);
+            commitLog.destroyExpiredFile(timestamp);
+        } finally {
+            fileLock.unlock();
+        }
+    }
+
+    public void destroy() {
+        this.shutdown();
+        fileLock.lock();
+        try {
+            consumeQueue.destroyExpiredFile(Long.MAX_VALUE);
+            commitLog.destroyExpiredFile(Long.MAX_VALUE);
+            if (queueMetadata != null) {
+                metadataStore.deleteQueue(queueMetadata.getQueue());
+            }
+        } finally {
+            fileLock.unlock();
+        }
+    }
+
+    public long getFileReservedHours() {
+        if (topicMetadata.getReserveTime() > 0) {
+            return topicMetadata.getReserveTime();
+        }
+        return storeConfig.getTieredStoreFileReservedTime();
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java
deleted file mode 100644
index 0e5f79132f0..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.file;
-
-import com.google.common.annotations.VisibleForTesting;
-import java.nio.ByteBuffer;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-
-public class TieredCommitLog {
-
-    private static final Logger log = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
-    private static final Long NOT_EXIST_MIN_OFFSET = -1L;
-
-    /**
-     * item size: int, 4 bytes
-     * magic code: int, 4 bytes
-     * max store timestamp: long, 8 bytes
-     */
-    public static final int CODA_SIZE = 4 + 8 + 4;
-    public static final int BLANK_MAGIC_CODE = 0xBBCCDDEE ^ 1880681586 + 8;
-
-    private final TieredMessageStoreConfig storeConfig;
-    private final TieredFlatFile flatFile;
-    private final AtomicLong minConsumeQueueOffset;
-
-    public TieredCommitLog(TieredFileAllocator fileQueueFactory, String filePath) {
-        this.storeConfig = fileQueueFactory.getStoreConfig();
-        this.flatFile = fileQueueFactory.createFlatFileForCommitLog(filePath);
-        this.minConsumeQueueOffset = new AtomicLong(NOT_EXIST_MIN_OFFSET);
-        this.correctMinOffsetAsync();
-    }
-
-    @VisibleForTesting
-    public TieredFlatFile getFlatFile() {
-        return flatFile;
-    }
-
-    public long getMinOffset() {
-        return flatFile.getMinOffset();
-    }
-
-    public long getCommitOffset() {
-        return flatFile.getCommitOffset();
-    }
-
-    public long getMinConsumeQueueOffset() {
-        return minConsumeQueueOffset.get() != NOT_EXIST_MIN_OFFSET ? minConsumeQueueOffset.get() : correctMinOffset();
-    }
-
-    public long getDispatchCommitOffset() {
-        return flatFile.getDispatchCommitOffset();
-    }
-
-    public long getMaxOffset() {
-        return flatFile.getMaxOffset();
-    }
-
-    public long getBeginTimestamp() {
-        TieredFileSegment firstIndexFile = flatFile.getFileByIndex(0);
-        if (firstIndexFile == null) {
-            return -1L;
-        }
-        long beginTimestamp = firstIndexFile.getMinTimestamp();
-        return beginTimestamp != Long.MAX_VALUE ? beginTimestamp : -1;
-    }
-
-    public long getEndTimestamp() {
-        return flatFile.getFileToWrite().getMaxTimestamp();
-    }
-
-    public long correctMinOffset() {
-        try {
-            return correctMinOffsetAsync().get();
-        } catch (Exception e) {
-            log.error("Correct min offset failed in clean expired file", e);
-        }
-        return NOT_EXIST_MIN_OFFSET;
-    }
-
-    public synchronized CompletableFuture<Long> correctMinOffsetAsync() {
-        if (flatFile.getFileSegmentCount() == 0) {
-            this.minConsumeQueueOffset.set(NOT_EXIST_MIN_OFFSET);
-            return CompletableFuture.completedFuture(NOT_EXIST_MIN_OFFSET);
-        }
-
-        // queue offset field length is 8
-        int length = MessageBufferUtil.QUEUE_OFFSET_POSITION + 8;
-        if (flatFile.getCommitOffset() - flatFile.getMinOffset() < length) {
-            this.minConsumeQueueOffset.set(NOT_EXIST_MIN_OFFSET);
-            return CompletableFuture.completedFuture(NOT_EXIST_MIN_OFFSET);
-        }
-
-        try {
-            return this.flatFile.readAsync(this.flatFile.getMinOffset(), length)
-                .thenApply(buffer -> {
-                    long offset = MessageBufferUtil.getQueueOffset(buffer);
-                    minConsumeQueueOffset.set(offset);
-                    log.debug("Correct commitlog min cq offset success, " +
-                            "filePath={}, min cq offset={}, commitlog range={}-{}",
-                        flatFile.getFilePath(), offset, flatFile.getMinOffset(), flatFile.getCommitOffset());
-                    return offset;
-                })
-                .exceptionally(throwable -> {
-                    log.warn("Correct commitlog min cq offset error, filePath={}, range={}-{}",
-                        flatFile.getFilePath(), flatFile.getMinOffset(), flatFile.getCommitOffset(), throwable);
-                    return minConsumeQueueOffset.get();
-                });
-        } catch (Exception e) {
-            log.error("Correct commitlog min cq offset error, filePath={}", flatFile.getFilePath(), e);
-        }
-        return CompletableFuture.completedFuture(minConsumeQueueOffset.get());
-    }
-
-    public AppendResult append(ByteBuffer byteBuf) {
-        return flatFile.append(byteBuf, MessageBufferUtil.getStoreTimeStamp(byteBuf));
-    }
-
-    public AppendResult append(ByteBuffer byteBuf, boolean commit) {
-        return flatFile.append(byteBuf, MessageBufferUtil.getStoreTimeStamp(byteBuf), commit);
-    }
-
-    public CompletableFuture<ByteBuffer> readAsync(long offset, int length) {
-        return flatFile.readAsync(offset, length);
-    }
-
-    public void commit(boolean sync) {
-        flatFile.commit(sync);
-    }
-
-    public void cleanExpiredFile(long expireTimestamp) {
-        if (flatFile.cleanExpiredFile(expireTimestamp) > 0) {
-            correctMinOffset();
-        }
-    }
-
-    public void destroyExpiredFile() {
-        flatFile.destroyExpiredFile();
-        if (flatFile.getFileSegmentCount() == 0) {
-            return;
-        }
-        TieredFileSegment fileSegment = flatFile.getFileToWrite();
-        try {
-            if (System.currentTimeMillis() - fileSegment.getMaxTimestamp() >
-                TimeUnit.HOURS.toMillis(storeConfig.getCommitLogRollingInterval())
-                && fileSegment.getAppendPosition() > storeConfig.getCommitLogRollingMinimumSize()) {
-                flatFile.rollingNewFile();
-            }
-        } catch (Exception e) {
-            log.error("Rolling to next file failed", e);
-        }
-    }
-
-    public void destroy() {
-        flatFile.destroy();
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredConsumeQueue.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredConsumeQueue.java
deleted file mode 100644
index 6953db032d6..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredConsumeQueue.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.file;
-
-import com.google.common.annotations.VisibleForTesting;
-import java.nio.ByteBuffer;
-import java.util.concurrent.CompletableFuture;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.rocketmq.common.BoundaryType;
-import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
-
-public class TieredConsumeQueue {
-
-    /**
-     * commit log offset: long, 8 bytes
-     * message size: int, 4 bytes
-     * tag hash code: long, 8 bytes
-     */
-    public static final int CONSUME_QUEUE_STORE_UNIT_SIZE = 8 + 4 + 8;
-
-    private final TieredFlatFile flatFile;
-
-    public TieredConsumeQueue(TieredFileAllocator fileQueueFactory, String filePath) {
-        this.flatFile = fileQueueFactory.createFlatFileForConsumeQueue(filePath);
-    }
-
-    public boolean isInitialized() {
-        return flatFile.getBaseOffset() != -1;
-    }
-
-    @VisibleForTesting
-    public TieredFlatFile getFlatFile() {
-        return flatFile;
-    }
-
-    public long getBaseOffset() {
-        return flatFile.getBaseOffset();
-    }
-
-    public void setBaseOffset(long baseOffset) {
-        flatFile.setBaseOffset(baseOffset);
-    }
-
-    public long getMinOffset() {
-        return flatFile.getMinOffset();
-    }
-
-    public long getCommitOffset() {
-        return flatFile.getCommitOffset();
-    }
-
-    public long getMaxOffset() {
-        return flatFile.getMaxOffset();
-    }
-
-    public long getEndTimestamp() {
-        return flatFile.getFileToWrite().getMaxTimestamp();
-    }
-
-    public AppendResult append(final long offset, final int size, final long tagsCode, long timeStamp) {
-        return append(offset, size, tagsCode, timeStamp, false);
-    }
-
-    public AppendResult append(final long offset, final int size, final long tagsCode, long timeStamp, boolean commit) {
-        ByteBuffer cqItem = ByteBuffer.allocate(CONSUME_QUEUE_STORE_UNIT_SIZE);
-        cqItem.putLong(offset);
-        cqItem.putInt(size);
-        cqItem.putLong(tagsCode);
-        cqItem.flip();
-        return flatFile.append(cqItem, timeStamp, commit);
-    }
-
-    public CompletableFuture<ByteBuffer> readAsync(long offset, int length) {
-        return flatFile.readAsync(offset, length);
-    }
-
-    public void commit(boolean sync) {
-        flatFile.commit(sync);
-    }
-
-    public void cleanExpiredFile(long expireTimestamp) {
-        flatFile.cleanExpiredFile(expireTimestamp);
-    }
-
-    public void destroyExpiredFile() {
-        flatFile.destroyExpiredFile();
-    }
-
-    protected Pair<Long, Long> getQueueOffsetInFileByTime(long timestamp, BoundaryType boundaryType) {
-        TieredFileSegment fileSegment = flatFile.getFileByTime(timestamp, boundaryType);
-        if (fileSegment == null) {
-            return Pair.of(-1L, -1L);
-        }
-        return Pair.of(fileSegment.getBaseOffset() / TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE,
-            fileSegment.getCommitOffset() / TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE - 1);
-    }
-
-    public void destroy() {
-        flatFile.destroy();
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFileAllocator.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFileAllocator.java
deleted file mode 100644
index 51a88e57256..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFileAllocator.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.tieredstore.file;
-
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.provider.FileSegmentAllocator;
-
-public class TieredFileAllocator {
-
-    private final FileSegmentAllocator fileSegmentAllocator;
-    private final TieredMessageStoreConfig storeConfig;
-
-    public TieredFileAllocator(TieredMessageStoreConfig storeConfig)
-        throws ClassNotFoundException, NoSuchMethodException {
-
-        this.storeConfig = storeConfig;
-        this.fileSegmentAllocator = new FileSegmentAllocator(storeConfig);
-    }
-
-    public TieredMessageStoreConfig getStoreConfig() {
-        return storeConfig;
-    }
-
-    public TieredFlatFile createFlatFileForCommitLog(String filePath) {
-        TieredFlatFile tieredFlatFile =
-            new TieredFlatFile(fileSegmentAllocator, FileSegmentType.COMMIT_LOG, filePath);
-        if (tieredFlatFile.getBaseOffset() == -1L) {
-            tieredFlatFile.setBaseOffset(0L);
-        }
-        return tieredFlatFile;
-    }
-
-    public TieredFlatFile createFlatFileForConsumeQueue(String filePath) {
-        return new TieredFlatFile(fileSegmentAllocator, FileSegmentType.CONSUME_QUEUE, filePath);
-    }
-
-    public TieredFlatFile createFlatFileForIndexFile(String filePath) {
-        return new TieredFlatFile(fileSegmentAllocator, FileSegmentType.INDEX, filePath);
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java
deleted file mode 100644
index a41d562d108..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java
+++ /dev/null
@@ -1,590 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.file;
-
-import com.google.common.annotations.VisibleForTesting;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-import org.apache.rocketmq.common.BoundaryType;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.exception.TieredStoreErrorCode;
-import org.apache.rocketmq.tieredstore.exception.TieredStoreException;
-import org.apache.rocketmq.tieredstore.metadata.FileSegmentMetadata;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
-import org.apache.rocketmq.tieredstore.provider.FileSegmentAllocator;
-import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-
-public class TieredFlatFile {
-
-    private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
-
-    private final String filePath;
-    private final FileSegmentType fileType;
-    private final TieredMetadataStore tieredMetadataStore;
-
-    private volatile long baseOffset = -1L;
-    private final FileSegmentAllocator fileSegmentAllocator;
-    private final List<TieredFileSegment> fileSegmentList;
-    private final List<TieredFileSegment> needCommitFileSegmentList;
-    private final ReentrantReadWriteLock fileSegmentLock;
-
-    public TieredFlatFile(FileSegmentAllocator fileSegmentAllocator,
-        FileSegmentType fileType, String filePath) {
-
-        this.fileType = fileType;
-        this.filePath = filePath;
-        this.fileSegmentList = new LinkedList<>();
-        this.fileSegmentLock = new ReentrantReadWriteLock();
-        this.fileSegmentAllocator = fileSegmentAllocator;
-        this.needCommitFileSegmentList = new CopyOnWriteArrayList<>();
-        this.tieredMetadataStore = TieredStoreUtil.getMetadataStore(fileSegmentAllocator.getStoreConfig());
-        this.recoverMetadata();
-
-        if (fileType != FileSegmentType.INDEX) {
-            checkAndFixFileSize();
-        }
-    }
-
-    public long getBaseOffset() {
-        return baseOffset;
-    }
-
-    public void setBaseOffset(long baseOffset) {
-        if (fileSegmentList.size() > 0) {
-            throw new IllegalStateException("Can not set base offset after file segment has been created");
-        }
-        this.baseOffset = baseOffset;
-    }
-
-    public long getMinOffset() {
-        fileSegmentLock.readLock().lock();
-        try {
-            if (fileSegmentList.isEmpty()) {
-                return baseOffset;
-            }
-            return fileSegmentList.get(0).getBaseOffset();
-        } finally {
-            fileSegmentLock.readLock().unlock();
-        }
-    }
-
-    public long getCommitOffset() {
-        fileSegmentLock.readLock().lock();
-        try {
-            if (fileSegmentList.isEmpty()) {
-                return baseOffset;
-            }
-            return fileSegmentList.get(fileSegmentList.size() - 1).getCommitOffset();
-        } finally {
-            fileSegmentLock.readLock().unlock();
-        }
-    }
-
-    public long getMaxOffset() {
-        fileSegmentLock.readLock().lock();
-        try {
-            if (fileSegmentList.isEmpty()) {
-                return baseOffset;
-            }
-            return fileSegmentList.get(fileSegmentList.size() - 1).getMaxOffset();
-        } finally {
-            fileSegmentLock.readLock().unlock();
-        }
-    }
-
-    public long getDispatchCommitOffset() {
-        fileSegmentLock.readLock().lock();
-        try {
-            if (fileSegmentList.isEmpty()) {
-                return 0;
-            }
-            return fileSegmentList.get(fileSegmentList.size() - 1).getDispatchCommitOffset();
-        } finally {
-            fileSegmentLock.readLock().unlock();
-        }
-    }
-
-    public String getFilePath() {
-        return filePath;
-    }
-
-    public FileSegmentType getFileType() {
-        return fileType;
-    }
-
-    public List<TieredFileSegment> getFileSegmentList() {
-        return fileSegmentList;
-    }
-
-    protected void recoverMetadata() {
-        fileSegmentList.clear();
-        needCommitFileSegmentList.clear();
-
-        tieredMetadataStore.iterateFileSegment(filePath, fileType, metadata -> {
-            if (metadata.getStatus() == FileSegmentMetadata.STATUS_DELETED) {
-                return;
-            }
-
-            TieredFileSegment segment = this.newSegment(fileType, metadata.getBaseOffset(), false);
-            segment.initPosition(metadata.getSize());
-            segment.setMinTimestamp(metadata.getBeginTimestamp());
-            segment.setMaxTimestamp(metadata.getEndTimestamp());
-            if (metadata.getStatus() == FileSegmentMetadata.STATUS_SEALED) {
-                segment.setFull(false);
-            }
-
-            // TODO check coda/size
-            fileSegmentList.add(segment);
-        });
-
-        if (!fileSegmentList.isEmpty()) {
-            fileSegmentList.sort(Comparator.comparingLong(TieredFileSegment::getBaseOffset));
-            baseOffset = fileSegmentList.get(0).getBaseOffset();
-            needCommitFileSegmentList.addAll(
-                fileSegmentList.stream().filter(segment -> !segment.isFull()).collect(Collectors.toList()));
-        }
-    }
-
-    /**
-     * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full
-     */
-    public void updateFileSegment(TieredFileSegment fileSegment) {
-
-        FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment(
-            this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset());
-
-        // Note: file segment path may not the same as file base path, use base path here.
-        if (metadata == null) {
-            metadata = new FileSegmentMetadata(
-                this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType());
-            metadata.setCreateTimestamp(System.currentTimeMillis());
-        }
-
-        metadata.setSize(fileSegment.getCommitPosition());
-        metadata.setBeginTimestamp(fileSegment.getMinTimestamp());
-        metadata.setEndTimestamp(fileSegment.getMaxTimestamp());
-
-        if (fileSegment.isFull() && !fileSegment.needCommit()) {
-            if (metadata.getStatus() == FileSegmentMetadata.STATUS_NEW) {
-                metadata.markSealed();
-            }
-        }
-
-        if (fileSegment.isClosed()) {
-            metadata.setStatus(FileSegmentMetadata.STATUS_DELETED);
-        }
-
-        this.tieredMetadataStore.updateFileSegment(metadata);
-    }
-
-    private void checkAndFixFileSize() {
-        for (int i = 1; i < fileSegmentList.size(); i++) {
-            TieredFileSegment pre = fileSegmentList.get(i - 1);
-            TieredFileSegment cur = fileSegmentList.get(i);
-            if (pre.getCommitOffset() != cur.getBaseOffset()) {
-                logger.warn("TieredFlatFile#checkAndFixFileSize: file segment has incorrect size: " +
-                    "filePath:{}, file type: {}, base offset: {}", filePath, fileType, pre.getBaseOffset());
-                try {
-                    long actualSize = pre.getSize();
-                    if (pre.getBaseOffset() + actualSize != cur.getBaseOffset()) {
-                        logger.error("[Bug]TieredFlatFile#checkAndFixFileSize: " +
-                                "file segment has incorrect size and can not fix: " +
-                                "filePath:{}, file type: {}, base offset: {}, actual size: {}, next file offset: {}",
-                            filePath, fileType, pre.getBaseOffset(), actualSize, cur.getBaseOffset());
-                        continue;
-                    }
-                    pre.initPosition(actualSize);
-                    this.updateFileSegment(pre);
-                } catch (Exception e) {
-                    logger.error("TieredFlatFile#checkAndFixFileSize: " +
-                            "fix file segment size failed: filePath: {}, file type: {}, base offset: {}",
-                        filePath, fileType, pre.getBaseOffset());
-                }
-            }
-        }
-
-        if (!fileSegmentList.isEmpty()) {
-            TieredFileSegment lastFile = fileSegmentList.get(fileSegmentList.size() - 1);
-            long lastFileSize = lastFile.getSize();
-            if (lastFile.getCommitPosition() != lastFileSize) {
-                logger.warn("TieredFlatFile#checkAndFixFileSize: fix last file {} size: origin: {}, actual: {}",
-                    lastFile.getPath(), lastFile.getCommitOffset() - lastFile.getBaseOffset(), lastFileSize);
-                lastFile.initPosition(lastFileSize);
-                this.updateFileSegment(lastFile);
-            }
-        }
-    }
-
-    private TieredFileSegment newSegment(FileSegmentType fileType, long baseOffset, boolean createMetadata) {
-        TieredFileSegment segment = null;
-        try {
-            segment = fileSegmentAllocator.createSegment(fileType, filePath, baseOffset);
-            if (fileType != FileSegmentType.INDEX) {
-                segment.createFile();
-            }
-            if (createMetadata) {
-                this.updateFileSegment(segment);
-            }
-        } catch (Exception e) {
-            logger.error("create file segment failed: filePath:{}, file type: {}, base offset: {}",
-                filePath, fileType, baseOffset, e);
-        }
-        return segment;
-    }
-
-    public void rollingNewFile() {
-        TieredFileSegment segment = getFileToWrite();
-        segment.setFull();
-        // create new segment
-        getFileToWrite();
-    }
-
-    public int getFileSegmentCount() {
-        return fileSegmentList.size();
-    }
-
-    @Nullable
-    public TieredFileSegment getFileByIndex(int index) {
-        fileSegmentLock.readLock().lock();
-        try {
-            if (index < fileSegmentList.size()) {
-                return fileSegmentList.get(index);
-            }
-            return null;
-        } finally {
-            fileSegmentLock.readLock().unlock();
-        }
-    }
-
-    protected TieredFileSegment getFileToWrite() {
-        if (baseOffset == -1) {
-            throw new IllegalStateException("need to set base offset before create file segment");
-        }
-        fileSegmentLock.readLock().lock();
-        try {
-            if (!fileSegmentList.isEmpty()) {
-                TieredFileSegment fileSegment = fileSegmentList.get(fileSegmentList.size() - 1);
-                if (!fileSegment.isFull()) {
-                    return fileSegment;
-                }
-            }
-        } finally {
-            fileSegmentLock.readLock().unlock();
-        }
-        // Create new file segment
-        fileSegmentLock.writeLock().lock();
-        try {
-            long offset = baseOffset;
-            if (!fileSegmentList.isEmpty()) {
-                TieredFileSegment segment = fileSegmentList.get(fileSegmentList.size() - 1);
-                if (!segment.isFull()) {
-                    return segment;
-                }
-                if (segment.commit()) {
-                    try {
-                        this.updateFileSegment(segment);
-                    } catch (Exception e) {
-                        return segment;
-                    }
-                } else {
-                    return segment;
-                }
-
-                offset = segment.getMaxOffset();
-            }
-            TieredFileSegment fileSegment = this.newSegment(fileType, offset, true);
-            fileSegmentList.add(fileSegment);
-            needCommitFileSegmentList.add(fileSegment);
-            Collections.sort(fileSegmentList);
-            logger.debug("Create a new file segment: baseOffset: {}, file: {}, file type: {}",
-                offset, fileSegment.getPath(), fileType);
-            return fileSegment;
-        } finally {
-            fileSegmentLock.writeLock().unlock();
-        }
-    }
-
-    @Nullable
-    protected TieredFileSegment getFileByTime(long timestamp, BoundaryType boundaryType) {
-        fileSegmentLock.readLock().lock();
-        try {
-            List<TieredFileSegment> segmentList = fileSegmentList.stream()
-                .sorted(boundaryType == BoundaryType.UPPER ? Comparator.comparingLong(TieredFileSegment::getMaxTimestamp) : Comparator.comparingLong(TieredFileSegment::getMinTimestamp))
-                .filter(segment -> boundaryType == BoundaryType.UPPER ? segment.getMaxTimestamp() >= timestamp : segment.getMinTimestamp() <= timestamp)
-                .collect(Collectors.toList());
-            if (!segmentList.isEmpty()) {
-                return boundaryType == BoundaryType.UPPER ? segmentList.get(0) : segmentList.get(segmentList.size() - 1);
-            }
-            if (fileSegmentList.isEmpty()) {
-                return null;
-            }
-            return boundaryType == BoundaryType.UPPER ? fileSegmentList.get(fileSegmentList.size() - 1) : fileSegmentList.get(0);
-        } finally {
-            fileSegmentLock.readLock().unlock();
-        }
-    }
-
-    public List<TieredFileSegment> getFileListByTime(long beginTime, long endTime) {
-        fileSegmentLock.readLock().lock();
-        try {
-            return fileSegmentList.stream()
-                .filter(segment -> Math.max(beginTime, segment.getMinTimestamp()) <= Math.min(endTime, segment.getMaxTimestamp()))
-                .collect(Collectors.toList());
-        } finally {
-            fileSegmentLock.readLock().unlock();
-        }
-    }
-
-    protected int getSegmentIndexByOffset(long offset) {
-        fileSegmentLock.readLock().lock();
-        try {
-            if (fileSegmentList.size() == 0) {
-                return -1;
-            }
-
-            int left = 0;
-            int right = fileSegmentList.size() - 1;
-            int mid = (left + right) / 2;
-
-            long firstSegmentOffset = fileSegmentList.get(left).getBaseOffset();
-            long lastSegmentOffset = fileSegmentList.get(right).getCommitOffset();
-            long midSegmentOffset = fileSegmentList.get(mid).getBaseOffset();
-
-            if (offset < firstSegmentOffset || offset > lastSegmentOffset) {
-                return -1;
-            }
-
-            while (left < right - 1) {
-                if (offset == midSegmentOffset) {
-                    return mid;
-                }
-                if (offset < midSegmentOffset) {
-                    right = mid;
-                } else {
-                    left = mid;
-                }
-                mid = (left + right) / 2;
-                midSegmentOffset = fileSegmentList.get(mid).getBaseOffset();
-            }
-            return offset < fileSegmentList.get(right).getBaseOffset() ? mid : right;
-        } finally {
-            fileSegmentLock.readLock().unlock();
-        }
-    }
-
-    public AppendResult append(ByteBuffer byteBuf) {
-        return append(byteBuf, Long.MAX_VALUE, false);
-    }
-
-    public AppendResult append(ByteBuffer byteBuf, long timeStamp) {
-        return append(byteBuf, timeStamp, false);
-    }
-
-    public AppendResult append(ByteBuffer byteBuf, long timeStamp, boolean commit) {
-        TieredFileSegment fileSegment = getFileToWrite();
-        AppendResult result = fileSegment.append(byteBuf, timeStamp);
-        if (commit && result == AppendResult.BUFFER_FULL && fileSegment.commit()) {
-            result = fileSegment.append(byteBuf, timeStamp);
-        }
-        if (result == AppendResult.FILE_FULL) {
-            // write to new file
-            return getFileToWrite().append(byteBuf, timeStamp);
-        }
-        return result;
-    }
-
-    public int cleanExpiredFile(long expireTimestamp) {
-        Set<Long> needToDeleteSet = new HashSet<>();
-        try {
-            tieredMetadataStore.iterateFileSegment(filePath, fileType, metadata -> {
-                if (metadata.getEndTimestamp() < expireTimestamp) {
-                    needToDeleteSet.add(metadata.getBaseOffset());
-                }
-            });
-        } catch (Exception e) {
-            logger.error("Clean expired file, filePath: {}, file type: {}, expire timestamp: {}",
-                filePath, fileType, expireTimestamp);
-        }
-
-        if (needToDeleteSet.isEmpty()) {
-            return 0;
-        }
-
-        fileSegmentLock.writeLock().lock();
-        try {
-            for (int i = 0; i < fileSegmentList.size(); i++) {
-                TieredFileSegment fileSegment = fileSegmentList.get(i);
-                try {
-                    if (needToDeleteSet.contains(fileSegment.getBaseOffset())) {
-                        fileSegment.close();
-                        fileSegmentList.remove(fileSegment);
-                        needCommitFileSegmentList.remove(fileSegment);
-                        i--;
-                        this.updateFileSegment(fileSegment);
-                        logger.debug("Clean expired file, filePath: {}", fileSegment.getPath());
-                    } else {
-                        break;
-                    }
-                } catch (Exception e) {
-                    logger.error("Clean expired file failed: filePath: {}, file type: {}, expire timestamp: {}",
-                        fileSegment.getPath(), fileSegment.getFileType(), expireTimestamp, e);
-                }
-            }
-            if (fileSegmentList.size() > 0) {
-                baseOffset = fileSegmentList.get(0).getBaseOffset();
-            } else if (fileType == FileSegmentType.CONSUME_QUEUE) {
-                baseOffset = -1;
-            } else {
-                baseOffset = 0;
-            }
-        } finally {
-            fileSegmentLock.writeLock().unlock();
-        }
-        return needToDeleteSet.size();
-    }
-
-    @VisibleForTesting
-    protected List<TieredFileSegment> getNeedCommitFileSegmentList() {
-        return needCommitFileSegmentList;
-    }
-
-    public void destroyExpiredFile() {
-        try {
-            tieredMetadataStore.iterateFileSegment(filePath, fileType, metadata -> {
-                if (metadata.getStatus() == FileSegmentMetadata.STATUS_DELETED) {
-                    try {
-                        TieredFileSegment fileSegment =
-                            this.newSegment(fileType, metadata.getBaseOffset(), false);
-                        fileSegment.destroyFile();
-                        if (!fileSegment.exists()) {
-                            tieredMetadataStore.deleteFileSegment(filePath, fileType, metadata.getBaseOffset());
-                        }
-                    } catch (Exception e) {
-                        logger.error("Destroyed expired file failed, file path: {}, file type: {}",
-                            filePath, fileType, e);
-                    }
-                }
-            });
-        } catch (Exception e) {
-            logger.error("Destroyed expired file, file path: {}, file type: {}", filePath, fileType);
-        }
-    }
-
-    public void commit(boolean sync) {
-        ArrayList<CompletableFuture<Void>> futureList = new ArrayList<>();
-        try {
-            for (TieredFileSegment segment : needCommitFileSegmentList) {
-                if (segment.isClosed()) {
-                    continue;
-                }
-                futureList.add(segment
-                    .commitAsync()
-                    .thenAccept(success -> {
-                        try {
-                            this.updateFileSegment(segment);
-                        } catch (Exception e) {
-                            // TODO handle update segment metadata failed exception
-                            logger.error("Update file segment metadata failed: " +
-                                    "file path: {}, file type: {}, base offset: {}",
-                                filePath, fileType, segment.getBaseOffset(), e);
-                        }
-                        if (segment.isFull() && !segment.needCommit()) {
-                            needCommitFileSegmentList.remove(segment);
-                        }
-                    })
-                );
-            }
-        } catch (Exception e) {
-            logger.error("Commit file segment failed: topic: {}, queue: {}, file type: {}", filePath, fileType, e);
-        }
-        if (sync) {
-            CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join();
-        }
-    }
-
-    public CompletableFuture<ByteBuffer> readAsync(long offset, int length) {
-        int index = getSegmentIndexByOffset(offset);
-        if (index == -1) {
-            String errorMsg = String.format("TieredFlatFile#readAsync: offset is illegal, " +
-                    "file path: %s, file type: %s, start: %d, length: %d, file num: %d",
-                filePath, fileType, offset, length, fileSegmentList.size());
-            logger.error(errorMsg);
-            throw new TieredStoreException(TieredStoreErrorCode.ILLEGAL_OFFSET, errorMsg);
-        }
-        TieredFileSegment fileSegment1;
-        TieredFileSegment fileSegment2 = null;
-        fileSegmentLock.readLock().lock();
-        try {
-            fileSegment1 = fileSegmentList.get(index);
-            if (offset + length > fileSegment1.getCommitOffset()) {
-                if (fileSegmentList.size() > index + 1) {
-                    fileSegment2 = fileSegmentList.get(index + 1);
-                }
-            }
-        } finally {
-            fileSegmentLock.readLock().unlock();
-        }
-        if (fileSegment2 == null) {
-            return fileSegment1.readAsync(offset - fileSegment1.getBaseOffset(), length);
-        }
-        int segment1Length = (int) (fileSegment1.getCommitOffset() - offset);
-        return fileSegment1.readAsync(offset - fileSegment1.getBaseOffset(), segment1Length)
-            .thenCombine(fileSegment2.readAsync(0, length - segment1Length), (buffer1, buffer2) -> {
-                ByteBuffer compositeBuffer = ByteBuffer.allocate(buffer1.remaining() + buffer2.remaining());
-                compositeBuffer.put(buffer1).put(buffer2);
-                compositeBuffer.flip();
-                return compositeBuffer;
-            });
-    }
-
-    public void destroy() {
-        fileSegmentLock.writeLock().lock();
-        try {
-            for (TieredFileSegment fileSegment : fileSegmentList) {
-                fileSegment.close();
-                try {
-                    this.updateFileSegment(fileSegment);
-                } catch (Exception e) {
-                    logger.error("TieredFlatFile#destroy: mark file segment: {} is deleted failed", fileSegment.getPath(), e);
-                }
-                fileSegment.destroyFile();
-                if (!fileSegment.exists()) {
-                    tieredMetadataStore.deleteFileSegment(filePath, fileType, fileSegment.getBaseOffset());
-                }
-            }
-            fileSegmentList.clear();
-            needCommitFileSegmentList.clear();
-        } finally {
-            fileSegmentLock.writeLock().unlock();
-        }
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java
deleted file mode 100644
index ffe0836f126..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java
+++ /dev/null
@@ -1,300 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.file;
-
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.ImmutableList;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import javax.annotation.Nullable;
-import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.index.IndexService;
-import org.apache.rocketmq.tieredstore.index.IndexStoreService;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-
-public class TieredFlatFileManager {
-
-    private static final Logger BROKER_LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
-    private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
-
-    private static volatile TieredFlatFileManager instance;
-    private static volatile IndexStoreService indexStoreService;
-
-    private final TieredMetadataStore metadataStore;
-    private final TieredMessageStoreConfig storeConfig;
-    private final TieredFileAllocator tieredFileAllocator;
-    private final ConcurrentMap<MessageQueue, CompositeQueueFlatFile> flatFileConcurrentMap;
-
-    public TieredFlatFileManager(TieredMessageStoreConfig storeConfig)
-        throws ClassNotFoundException, NoSuchMethodException {
-
-        this.storeConfig = storeConfig;
-        this.metadataStore = TieredStoreUtil.getMetadataStore(storeConfig);
-        this.tieredFileAllocator = new TieredFileAllocator(storeConfig);
-        this.flatFileConcurrentMap = new ConcurrentHashMap<>();
-        this.doScheduleTask();
-    }
-
-    public static TieredFlatFileManager getInstance(TieredMessageStoreConfig storeConfig) {
-        if (storeConfig == null || instance != null) {
-            return instance;
-        }
-        synchronized (TieredFlatFileManager.class) {
-            if (instance == null) {
-                try {
-                    instance = new TieredFlatFileManager(storeConfig);
-                } catch (Exception e) {
-                    logger.error("Construct FlatFileManager instance error", e);
-                }
-            }
-        }
-        return instance;
-    }
-
-    public static IndexService getTieredIndexService(TieredMessageStoreConfig storeConfig) {
-        if (storeConfig == null) {
-            return indexStoreService;
-        }
-
-        if (indexStoreService == null) {
-            synchronized (TieredFlatFileManager.class) {
-                if (indexStoreService == null) {
-                    try {
-                        String filePath = TieredStoreUtil.toPath(new MessageQueue(
-                            TieredStoreUtil.RMQ_SYS_TIERED_STORE_INDEX_TOPIC, storeConfig.getBrokerName(), 0));
-                        indexStoreService = new IndexStoreService(new TieredFileAllocator(storeConfig), filePath);
-                        indexStoreService.start();
-                    } catch (Exception e) {
-                        logger.error("Construct FlatFileManager indexFile error", e);
-                    }
-                }
-            }
-        }
-        return indexStoreService;
-    }
-
-    public void doCommit() {
-        Random random = new Random();
-        for (CompositeQueueFlatFile flatFile : deepCopyFlatFileToList()) {
-            int delay = random.nextInt(storeConfig.getMaxCommitJitter());
-            TieredStoreExecutor.commitExecutor.schedule(() -> {
-                try {
-                    flatFile.commitCommitLog();
-                } catch (Throwable e) {
-                    MessageQueue mq = flatFile.getMessageQueue();
-                    logger.error("Commit commitLog periodically failed: topic: {}, queue: {}",
-                        mq.getTopic(), mq.getQueueId(), e);
-                }
-            }, delay, TimeUnit.MILLISECONDS);
-            TieredStoreExecutor.commitExecutor.schedule(() -> {
-                try {
-                    flatFile.commitConsumeQueue();
-                } catch (Throwable e) {
-                    MessageQueue mq = flatFile.getMessageQueue();
-                    logger.error("Commit consumeQueue periodically failed: topic: {}, queue: {}",
-                        mq.getTopic(), mq.getQueueId(), e);
-                }
-            }, delay, TimeUnit.MILLISECONDS);
-        }
-    }
-
-    public void doCleanExpiredFile() {
-        long expiredTimeStamp = System.currentTimeMillis() -
-            TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime());
-        for (CompositeQueueFlatFile flatFile : deepCopyFlatFileToList()) {
-            TieredStoreExecutor.cleanExpiredFileExecutor.submit(() -> {
-                try {
-                    flatFile.getCompositeFlatFileLock().lock();
-                    flatFile.cleanExpiredFile(expiredTimeStamp);
-                    flatFile.destroyExpiredFile();
-                } catch (Throwable t) {
-                    logger.error("Do Clean expired file error, topic={}, queueId={}",
-                        flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), t);
-                } finally {
-                    flatFile.getCompositeFlatFileLock().unlock();
-                }
-            });
-        }
-    }
-
-    private void doScheduleTask() {
-        TieredStoreExecutor.commonScheduledExecutor.scheduleWithFixedDelay(() -> {
-            try {
-                doCommit();
-            } catch (Throwable e) {
-                logger.error("Commit flat file periodically failed: ", e);
-            }
-        }, 60, 60, TimeUnit.SECONDS);
-
-        TieredStoreExecutor.commonScheduledExecutor.scheduleWithFixedDelay(() -> {
-            try {
-                doCleanExpiredFile();
-            } catch (Throwable e) {
-                logger.error("Clean expired flat file failed: ", e);
-            }
-        }, 30, 30, TimeUnit.SECONDS);
-    }
-
-    public boolean load() {
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        try {
-            flatFileConcurrentMap.clear();
-            this.recoverSequenceNumber();
-            this.recoverTieredFlatFile();
-            logger.info("Message store recover end, total cost={}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS));
-        } catch (Exception e) {
-            long costTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
-            logger.info("Message store recover error, total cost={}ms", costTime);
-            BROKER_LOG.error("Message store recover error, total cost={}ms", costTime, e);
-            return false;
-        }
-        return true;
-    }
-
-    public void recoverSequenceNumber() {
-        AtomicLong topicSequenceNumber = new AtomicLong();
-        metadataStore.iterateTopic(topicMetadata -> {
-            if (topicMetadata != null && topicMetadata.getTopicId() > 0) {
-                topicSequenceNumber.set(Math.max(topicSequenceNumber.get(), topicMetadata.getTopicId()));
-            }
-        });
-        metadataStore.setTopicSequenceNumber(topicSequenceNumber.incrementAndGet());
-    }
-
-    public void recoverTieredFlatFile() {
-        Semaphore semaphore = new Semaphore((int) (TieredStoreExecutor.QUEUE_CAPACITY * 0.75));
-        List<CompletableFuture<Void>> futures = new ArrayList<>();
-        metadataStore.iterateTopic(topicMetadata -> {
-            try {
-                semaphore.acquire();
-                CompletableFuture<Void> future = CompletableFuture.runAsync(() -> {
-                    try {
-                        Stopwatch subWatch = Stopwatch.createStarted();
-                        if (topicMetadata.getStatus() != 0) {
-                            return;
-                        }
-                        AtomicLong queueCount = new AtomicLong();
-                        metadataStore.iterateQueue(topicMetadata.getTopic(), queueMetadata -> {
-                            this.getOrCreateFlatFileIfAbsent(new MessageQueue(topicMetadata.getTopic(),
-                                storeConfig.getBrokerName(), queueMetadata.getQueue().getQueueId()));
-                            queueCount.incrementAndGet();
-                        });
-
-                        if (queueCount.get() == 0L) {
-                            metadataStore.deleteTopic(topicMetadata.getTopic());
-                        } else {
-                            logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms",
-                                topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS));
-                        }
-                    } catch (Exception e) {
-                        logger.error("Recover TopicFlatFile error, topic: {}", topicMetadata.getTopic(), e);
-                    } finally {
-                        semaphore.release();
-                    }
-                }, TieredStoreExecutor.commitExecutor);
-                futures.add(future);
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        });
-        CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
-    }
-
-    public void cleanup() {
-        flatFileConcurrentMap.clear();
-        cleanStaticReference();
-    }
-
-    private static void cleanStaticReference() {
-        instance = null;
-        indexStoreService = null;
-    }
-
-    @Nullable
-    public CompositeQueueFlatFile getOrCreateFlatFileIfAbsent(MessageQueue messageQueue) {
-        return flatFileConcurrentMap.computeIfAbsent(messageQueue, mq -> {
-            try {
-                logger.debug("Create new TopicFlatFile, topic: {}, queueId: {}",
-                    messageQueue.getTopic(), messageQueue.getQueueId());
-                return new CompositeQueueFlatFile(tieredFileAllocator, mq);
-            } catch (Exception e) {
-                logger.debug("Create new TopicFlatFile failed, topic: {}, queueId: {}",
-                    messageQueue.getTopic(), messageQueue.getQueueId(), e);
-            }
-            return null;
-        });
-    }
-
-    public CompositeQueueFlatFile getFlatFile(MessageQueue messageQueue) {
-        return flatFileConcurrentMap.get(messageQueue);
-    }
-
-    public ImmutableList<CompositeQueueFlatFile> deepCopyFlatFileToList() {
-        return ImmutableList.copyOf(flatFileConcurrentMap.values());
-    }
-
-    public void shutdown() {
-        if (indexStoreService != null) {
-            indexStoreService.shutdown();
-        }
-        for (CompositeFlatFile flatFile : deepCopyFlatFileToList()) {
-            flatFile.shutdown();
-        }
-    }
-
-    public void destroy() {
-        if (indexStoreService != null) {
-            indexStoreService.destroy();
-        }
-        ImmutableList<CompositeQueueFlatFile> flatFileList = deepCopyFlatFileToList();
-        cleanup();
-        for (CompositeFlatFile flatFile : flatFileList) {
-            flatFile.destroy();
-        }
-    }
-
-    public void destroyCompositeFile(MessageQueue mq) {
-        if (mq == null) {
-            return;
-        }
-
-        // delete memory reference
-        CompositeQueueFlatFile flatFile = flatFileConcurrentMap.remove(mq);
-        if (flatFile != null) {
-            MessageQueue messageQueue = flatFile.getMessageQueue();
-            logger.info("TieredFlatFileManager#destroyCompositeFile: " +
-                    "try to destroy composite flat file: topic: {}, queueId: {}",
-                messageQueue.getTopic(), messageQueue.getQueueId());
-
-            // delete queue metadata
-            flatFile.destroy();
-        }
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexFile.java
index d131b9b53ea..63d1193d6a9 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexFile.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexFile.java
@@ -29,6 +29,8 @@ enum IndexStatusEnum {
 
     long getTimestamp();
 
+    long getEndTimestamp();
+
     IndexStatusEnum getFileStatus();
 
     ByteBuffer doCompaction();
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexService.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexService.java
index d4eb854a2e8..a4ea7e78a85 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexService.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexService.java
@@ -24,6 +24,8 @@
 
 public interface IndexService {
 
+    void start();
+
     /**
      * Puts a key into the index.
      *
@@ -50,6 +52,9 @@ AppendResult putKey(
      */
     CompletableFuture<List<IndexItem>> queryAsync(String topic, String key, int maxCount, long beginTime, long endTime);
 
+    default void forceUpload() {
+    }
+
     /**
      * Shutdown the index service.
      */
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java
index def5c8f2d06..165c0b767f8 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java
@@ -35,15 +35,15 @@
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.store.logfile.DefaultMappedFile;
 import org.apache.rocketmq.store.logfile.MappedFile;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
 import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
+import org.apache.rocketmq.tieredstore.provider.FileSegment;
+import org.apache.rocketmq.tieredstore.provider.PosixFileSegment;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.apache.rocketmq.tieredstore.index.IndexFile.IndexStatusEnum.SEALED;
 import static org.apache.rocketmq.tieredstore.index.IndexFile.IndexStatusEnum.UNSEALED;
@@ -57,7 +57,7 @@
  */
 public class IndexStoreFile implements IndexFile {
 
-    private static final Logger log = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
+    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
 
     /**
      * header format:
@@ -93,9 +93,9 @@ public class IndexStoreFile implements IndexFile {
     private MappedFile mappedFile;
     private ByteBuffer byteBuffer;
     private MappedFile compactMappedFile;
-    private TieredFileSegment fileSegment;
+    private FileSegment fileSegment;
 
-    public IndexStoreFile(TieredMessageStoreConfig storeConfig, long timestamp) throws IOException {
+    public IndexStoreFile(MessageStoreConfig storeConfig, long timestamp) throws IOException {
         this.hashSlotMaxCount = storeConfig.getTieredStoreIndexFileMaxHashSlotNum();
         this.indexItemMaxCount = storeConfig.getTieredStoreIndexFileMaxIndexNum();
         this.fileStatus = new AtomicReference<>(UNSEALED);
@@ -112,7 +112,7 @@ public IndexStoreFile(TieredMessageStoreConfig storeConfig, long timestamp) thro
         this.flushNewMetadata(byteBuffer, indexItemMaxCount == this.indexItemCount.get() + 1);
     }
 
-    public IndexStoreFile(TieredMessageStoreConfig storeConfig, TieredFileSegment fileSegment) {
+    public IndexStoreFile(MessageStoreConfig storeConfig, FileSegment fileSegment) {
         this.fileSegment = fileSegment;
         this.fileStatus = new AtomicReference<>(UPLOAD);
         this.fileReadWriteLock = new ReentrantReadWriteLock();
@@ -130,6 +130,7 @@ public long getTimestamp() {
         return this.beginTimestamp.get();
     }
 
+    @Override
     public long getEndTimestamp() {
         return this.endTimestamp.get();
     }
@@ -176,6 +177,11 @@ protected int getItemPosition(int itemIndex) {
         return INDEX_HEADER_SIZE + hashSlotMaxCount * HASH_SLOT_SIZE + itemIndex * IndexItem.INDEX_ITEM_SIZE;
     }
 
+    @Override
+    public void start() {
+
+    }
+
     @Override
     public AppendResult putKey(
         String topic, int topicId, int queueId, Set<String> keySet, long offset, int size, long timestamp) {
@@ -254,54 +260,55 @@ public CompletableFuture<List<IndexItem>> queryAsync(
     protected CompletableFuture<List<IndexItem>> queryAsyncFromUnsealedFile(
         String key, int maxCount, long beginTime, long endTime) {
 
-        return CompletableFuture.supplyAsync(() -> {
-            List<IndexItem> result = new ArrayList<>();
-            try {
-                fileReadWriteLock.readLock().lock();
-                if (!UNSEALED.equals(this.fileStatus.get()) && !SEALED.equals(this.fileStatus.get())) {
-                    return result;
-                }
+        List<IndexItem> result = new ArrayList<>();
+        try {
+            fileReadWriteLock.readLock().lock();
+            if (!UNSEALED.equals(this.fileStatus.get()) && !SEALED.equals(this.fileStatus.get())) {
+                return CompletableFuture.completedFuture(result);
+            }
 
-                if (mappedFile == null || !mappedFile.hold()) {
-                    return result;
-                }
+            if (mappedFile == null || !mappedFile.hold()) {
+                return CompletableFuture.completedFuture(result);
+            }
 
-                int hashCode = this.hashCode(key);
-                int slotPosition = this.getSlotPosition(hashCode % this.hashSlotMaxCount);
-                int slotValue = this.getSlotValue(slotPosition);
+            int hashCode = this.hashCode(key);
+            int slotPosition = this.getSlotPosition(hashCode % this.hashSlotMaxCount);
+            int slotValue = this.getSlotValue(slotPosition);
 
-                int left = MAX_QUERY_COUNT;
-                while (left > 0 &&
-                    slotValue > INVALID_INDEX &&
-                    slotValue <= this.indexItemCount.get()) {
+            int left = MAX_QUERY_COUNT;
+            while (left > 0 &&
+                slotValue > INVALID_INDEX &&
+                slotValue <= this.indexItemCount.get()) {
 
-                    byte[] bytes = new byte[IndexItem.INDEX_ITEM_SIZE];
-                    ByteBuffer buffer = this.byteBuffer.duplicate();
-                    buffer.position(this.getItemPosition(slotValue));
-                    buffer.get(bytes);
-                    IndexItem indexItem = new IndexItem(bytes);
-                    if (hashCode == indexItem.getHashCode()) {
-                        result.add(indexItem);
-                        if (result.size() > maxCount) {
-                            break;
-                        }
+                byte[] bytes = new byte[IndexItem.INDEX_ITEM_SIZE];
+                ByteBuffer buffer = this.byteBuffer.duplicate();
+                buffer.position(this.getItemPosition(slotValue));
+                buffer.get(bytes);
+                IndexItem indexItem = new IndexItem(bytes);
+                long storeTimestamp = indexItem.getTimeDiff() * 1000L + beginTimestamp.get();
+                if (hashCode == indexItem.getHashCode() &&
+                    beginTime <= storeTimestamp && storeTimestamp <= endTime) {
+                    result.add(indexItem);
+                    if (result.size() > maxCount) {
+                        break;
                     }
-                    slotValue = indexItem.getItemIndex();
-                    left--;
                 }
-
-                log.debug("IndexStoreFile query from unsealed mapped file, timestamp: {}, result size: {}, " +
-                        "key: {}, hashCode: {}, maxCount: {}, timestamp={}-{}",
-                    getTimestamp(), result.size(), key, hashCode, maxCount, beginTime, endTime);
-            } catch (Exception e) {
-                log.error("IndexStoreFile query from unsealed mapped file error, timestamp: {}, " +
-                    "key: {}, maxCount: {}, timestamp={}-{}", getTimestamp(), key, maxCount, beginTime, endTime, e);
-            } finally {
-                fileReadWriteLock.readLock().unlock();
-                mappedFile.release();
+                slotValue = indexItem.getItemIndex();
+                left--;
             }
-            return result;
-        }, TieredStoreExecutor.fetchDataExecutor);
+
+            log.debug("IndexStoreFile query from unsealed mapped file, timestamp: {}, result size: {}, " +
+                    "key: {}, hashCode: {}, maxCount: {}, timestamp={}-{}",
+                getTimestamp(), result.size(), key, hashCode, maxCount, beginTime, endTime);
+        } catch (Exception e) {
+            log.error("IndexStoreFile query from unsealed mapped file error, timestamp: {}, " +
+                "key: {}, maxCount: {}, timestamp={}-{}", getTimestamp(), key, maxCount, beginTime, endTime, e);
+        } finally {
+            fileReadWriteLock.readLock().unlock();
+            mappedFile.release();
+        }
+
+        return CompletableFuture.completedFuture(result);
     }
 
     protected CompletableFuture<List<IndexItem>> queryAsyncFromSegmentFile(
@@ -346,7 +353,7 @@ protected CompletableFuture<List<IndexItem>> queryAsyncFromSegmentFile(
                 for (int i = 0; i < size; i++) {
                     itemBuffer.get(bytes);
                     IndexItem indexItem = new IndexItem(bytes);
-                    long storeTimestamp = indexItem.getTimeDiff() + beginTimestamp.get();
+                    long storeTimestamp = indexItem.getTimeDiff() * 1000L + beginTimestamp.get();
                     if (hashCode == indexItem.getHashCode() &&
                         beginTime <= storeTimestamp && storeTimestamp <= endTime &&
                         result.size() < maxCount) {
@@ -455,6 +462,9 @@ public void shutdown() {
         try {
             fileReadWriteLock.writeLock().lock();
             this.fileStatus.set(IndexStatusEnum.SHUTDOWN);
+            if (this.fileSegment != null && this.fileSegment instanceof PosixFileSegment) {
+                this.fileSegment.close();
+            }
             if (this.mappedFile != null) {
                 this.mappedFile.shutdown(TimeUnit.SECONDS.toMillis(10));
             }
@@ -483,7 +493,7 @@ public void destroy() {
                     if (this.compactMappedFile != null) {
                         this.compactMappedFile.destroy(TimeUnit.SECONDS.toMillis(10));
                     }
-                    log.info("IndexStoreService destroy local file, timestamp: {}, status: {}", this.getTimestamp(), fileStatus.get());
+                    log.debug("IndexStoreService destroy local file, timestamp: {}, status: {}", this.getTimestamp(), fileStatus.get());
                     break;
                 case UPLOAD:
                     log.warn("[BUG] IndexStoreService destroy remote file, timestamp: {}", this.getTimestamp());
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java
index e99ea0de182..f4f602a1056 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java
@@ -38,20 +38,20 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.ServiceThread;
 import org.apache.rocketmq.common.UtilAll;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.store.logfile.DefaultMappedFile;
 import org.apache.rocketmq.store.logfile.MappedFile;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
 import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.file.TieredFileAllocator;
-import org.apache.rocketmq.tieredstore.file.TieredFlatFile;
-import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
+import org.apache.rocketmq.tieredstore.file.FlatAppendFile;
+import org.apache.rocketmq.tieredstore.file.FlatFileFactory;
+import org.apache.rocketmq.tieredstore.provider.FileSegment;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class IndexStoreService extends ServiceThread implements IndexService {
 
-    private static final Logger log = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
+    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
 
     public static final String FILE_DIRECTORY_NAME = "tiered_index_file";
     public static final String FILE_COMPACTED_DIRECTORY_NAME = "compacting";
@@ -60,24 +60,35 @@ public class IndexStoreService extends ServiceThread implements IndexService {
      * File status in table example:
      * upload, upload, upload, sealed, sealed, unsealed
      */
-    private final TieredMessageStoreConfig storeConfig;
+    private final MessageStoreConfig storeConfig;
     private final ConcurrentSkipListMap<Long /* timestamp */, IndexFile> timeStoreTable;
     private final ReadWriteLock readWriteLock;
     private final AtomicLong compactTimestamp;
     private final String filePath;
-    private final TieredFileAllocator fileAllocator;
+    private final FlatFileFactory fileAllocator;
+    private final boolean autoCreateNewFile;
 
-    private IndexFile currentWriteFile;
-    private TieredFlatFile flatFile;
+    private volatile IndexFile currentWriteFile;
+    private volatile FlatAppendFile flatAppendFile;
 
-    public IndexStoreService(TieredFileAllocator fileAllocator, String filePath) {
-        this.storeConfig = fileAllocator.getStoreConfig();
+    public IndexStoreService(FlatFileFactory flatFileFactory, String filePath) {
+        this(flatFileFactory, filePath, true);
+    }
+
+    public IndexStoreService(FlatFileFactory flatFileFactory, String filePath, boolean autoCreateNewFile) {
+        this.storeConfig = flatFileFactory.getStoreConfig();
         this.filePath = filePath;
-        this.fileAllocator = fileAllocator;
+        this.fileAllocator = flatFileFactory;
         this.timeStoreTable = new ConcurrentSkipListMap<>();
         this.compactTimestamp = new AtomicLong(0L);
         this.readWriteLock = new ReentrantReadWriteLock();
+        this.autoCreateNewFile = autoCreateNewFile;
+    }
+
+    @Override
+    public void start() {
         this.recover();
+        super.start();
     }
 
     private void doConvertOldFormatFile(String filePath) {
@@ -131,30 +142,30 @@ private void recover() {
             }
         }
 
-        if (this.timeStoreTable.isEmpty()) {
+        if (this.autoCreateNewFile && this.timeStoreTable.isEmpty()) {
             this.createNewIndexFile(System.currentTimeMillis());
         }
 
-        this.currentWriteFile = this.timeStoreTable.lastEntry().getValue();
-        this.setCompactTimestamp(this.timeStoreTable.firstKey() - 1);
+        if (!this.timeStoreTable.isEmpty()) {
+            this.currentWriteFile = this.timeStoreTable.lastEntry().getValue();
+            this.setCompactTimestamp(this.timeStoreTable.firstKey() - 1);
+        }
 
         // recover remote
-        this.flatFile = fileAllocator.createFlatFileForIndexFile(filePath);
-        if (this.flatFile.getBaseOffset() == -1) {
-            this.flatFile.setBaseOffset(0);
-        }
+        this.flatAppendFile = fileAllocator.createFlatFileForIndexFile(filePath);
 
-        for (TieredFileSegment fileSegment : flatFile.getFileSegmentList()) {
+        for (FileSegment fileSegment : flatAppendFile.getFileSegmentList()) {
             IndexFile indexFile = new IndexStoreFile(storeConfig, fileSegment);
             IndexFile localFile = timeStoreTable.get(indexFile.getTimestamp());
             if (localFile != null) {
                 localFile.destroy();
             }
             timeStoreTable.put(indexFile.getTimestamp(), indexFile);
-            log.info("IndexStoreService recover load remote file, timestamp: {}", indexFile.getTimestamp());
+            log.info("IndexStoreService recover load remote file, timestamp: {}, end timestamp: {}",
+                indexFile.getTimestamp(), indexFile.getEndTimestamp());
         }
 
-        log.info("IndexStoreService recover finished, entrySize: {}, cost: {}ms, directory: {}",
+        log.info("IndexStoreService recover finished, total: {}, cost: {}ms, directory: {}",
             timeStoreTable.size(), stopwatch.elapsed(TimeUnit.MILLISECONDS), dir.getAbsolutePath());
     }
 
@@ -201,19 +212,24 @@ public AppendResult putKey(
             if (AppendResult.SUCCESS.equals(result)) {
                 return AppendResult.SUCCESS;
             } else if (AppendResult.FILE_FULL.equals(result)) {
-                this.createNewIndexFile(timestamp);
+                // use current time to ensure the order of file
+                this.createNewIndexFile(System.currentTimeMillis());
             }
         }
 
         log.error("IndexStoreService put key three times return error, topic: {}, topicId: {}, " +
             "queueId: {}, keySize: {}, timestamp: {}", topic, topicId, queueId, keySet.size(), timestamp);
-        return AppendResult.UNKNOWN_ERROR;
+        return AppendResult.SUCCESS;
     }
 
     @Override
     public CompletableFuture<List<IndexItem>> queryAsync(
         String topic, String key, int maxCount, long beginTime, long endTime) {
 
+        if (beginTime > endTime) {
+            return CompletableFuture.completedFuture(new ArrayList<>());
+        }
+
         CompletableFuture<List<IndexItem>> future = new CompletableFuture<>();
         try {
             readWriteLock.readLock().lock();
@@ -238,14 +254,16 @@ public CompletableFuture<List<IndexItem>> queryAsync(
                 .whenComplete((v, t) -> {
                     // Try to return the query results as much as possible here
                     // rather than directly throwing exceptions
-                    if (result.isEmpty() && t != null) {
-                        future.completeExceptionally(t);
-                    } else {
-                        List<IndexItem> resultList = new ArrayList<>(result.values());
-                        future.complete(resultList.subList(0, Math.min(resultList.size(), maxCount)));
+                    if (t != null) {
+                        log.error("IndexStoreService#queryAsync, topicId={}, key={}, maxCount={}, timestamp={}-{}",
+                            topic, key, maxCount, beginTime, endTime, t);
                     }
+                    List<IndexItem> resultList = new ArrayList<>(result.values());
+                    future.complete(resultList.subList(0, Math.min(resultList.size(), maxCount)));
                 });
         } catch (Exception e) {
+            log.error("IndexStoreService#queryAsync, topicId={}, key={}, maxCount={}, timestamp={}-{}",
+                topic, key, maxCount, beginTime, endTime, e);
             future.completeExceptionally(e);
         } finally {
             readWriteLock.readLock().unlock();
@@ -253,51 +271,96 @@ public CompletableFuture<List<IndexItem>> queryAsync(
         return future;
     }
 
-    public void doCompactThenUploadFile(IndexFile indexFile) {
+    @Override
+    public void forceUpload() {
+        try {
+            readWriteLock.writeLock().lock();
+            while (true) {
+                Map.Entry<Long, IndexFile> entry =
+                    this.timeStoreTable.higherEntry(this.compactTimestamp.get());
+                if (entry == null) {
+                    break;
+                }
+                if (this.doCompactThenUploadFile(entry.getValue())) {
+                    this.setCompactTimestamp(entry.getValue().getTimestamp());
+                    // The total number of files will not too much, prevent io too fast.
+                    TimeUnit.MILLISECONDS.sleep(50);
+                }
+            }
+        } catch (Exception e) {
+            log.error("IndexStoreService force upload error", e);
+            throw new RuntimeException(e);
+        } finally {
+            readWriteLock.writeLock().unlock();
+        }
+    }
+
+    public boolean doCompactThenUploadFile(IndexFile indexFile) {
         if (IndexFile.IndexStatusEnum.UPLOAD.equals(indexFile.getFileStatus())) {
             log.error("IndexStoreService file status not correct, so skip, timestamp: {}, status: {}",
                 indexFile.getTimestamp(), indexFile.getFileStatus());
             indexFile.destroy();
-            return;
+            return true;
         }
 
         Stopwatch stopwatch = Stopwatch.createStarted();
-        ByteBuffer byteBuffer = indexFile.doCompaction();
-        if (byteBuffer == null) {
-            log.error("IndexStoreService found compaction buffer is null, timestamp: {}", indexFile.getTimestamp());
-            return;
+        if (flatAppendFile.getCommitOffset() == flatAppendFile.getAppendOffset()) {
+            ByteBuffer byteBuffer = indexFile.doCompaction();
+            if (byteBuffer == null) {
+                log.error("IndexStoreService found compaction buffer is null, timestamp: {}", indexFile.getTimestamp());
+                return false;
+            }
+            flatAppendFile.rollingNewFile(Math.max(0L, flatAppendFile.getAppendOffset()));
+            flatAppendFile.append(byteBuffer, indexFile.getTimestamp());
+            flatAppendFile.getFileToWrite().setMinTimestamp(indexFile.getTimestamp());
+            flatAppendFile.getFileToWrite().setMaxTimestamp(indexFile.getEndTimestamp());
         }
-        flatFile.append(byteBuffer);
-        flatFile.commit(true);
-
-        TieredFileSegment fileSegment = flatFile.getFileByIndex(flatFile.getFileSegmentCount() - 1);
-        if (fileSegment == null || fileSegment.getMinTimestamp() != indexFile.getTimestamp()) {
-            log.warn("IndexStoreService submit compacted file to server failed, timestamp: {}", indexFile.getTimestamp());
-            return;
+        boolean result = flatAppendFile.commitAsync().join();
+
+        List<FileSegment> fileSegmentList = flatAppendFile.getFileSegmentList();
+        FileSegment fileSegment = fileSegmentList.get(fileSegmentList.size() - 1);
+        if (!result || fileSegment == null || fileSegment.getMinTimestamp() != indexFile.getTimestamp()) {
+            log.warn("IndexStoreService upload compacted file error, timestamp: {}", indexFile.getTimestamp());
+            return false;
+        } else {
+            log.info("IndexStoreService upload compacted file success, timestamp: {}", indexFile.getTimestamp());
         }
 
+        readWriteLock.writeLock().lock();
         try {
-            readWriteLock.writeLock().lock();
             IndexFile storeFile = new IndexStoreFile(storeConfig, fileSegment);
-            timeStoreTable.put(indexFile.getTimestamp(), storeFile);
+            timeStoreTable.put(storeFile.getTimestamp(), storeFile);
             indexFile.destroy();
         } catch (Exception e) {
-            log.error("IndexStoreService switch file failed, timestamp: {}, cost: {}ms",
+            log.error("IndexStoreService rolling file error, timestamp: {}, cost: {}ms",
                 indexFile.getTimestamp(), stopwatch.elapsed(TimeUnit.MILLISECONDS), e);
         } finally {
             readWriteLock.writeLock().unlock();
         }
+        return true;
     }
 
     public void destroyExpiredFile(long expireTimestamp) {
-        flatFile.cleanExpiredFile(expireTimestamp);
-        flatFile.destroyExpiredFile();
+        // delete file in time store table
+        readWriteLock.writeLock().lock();
+        try {
+            flatAppendFile.destroyExpiredFile(expireTimestamp);
+            timeStoreTable.entrySet().removeIf(entry ->
+                IndexFile.IndexStatusEnum.UPLOAD.equals(entry.getValue().getFileStatus()) &&
+                    entry.getKey() < flatAppendFile.getMinTimestamp());
+            int tableSize = (int) timeStoreTable.entrySet().stream()
+                .filter(entry -> IndexFile.IndexStatusEnum.UPLOAD.equals(entry.getValue().getFileStatus()))
+                .count();
+            log.info("IndexStoreService delete file, timestamp={}, remote={}, table={}, all={}",
+                expireTimestamp, flatAppendFile.getFileSegmentList().size(), tableSize, timeStoreTable.size());
+        } finally {
+            readWriteLock.writeLock().unlock();
+        }
     }
 
     public void destroy() {
+        readWriteLock.writeLock().lock();
         try {
-            readWriteLock.writeLock().lock();
-
             // delete local store file
             for (Map.Entry<Long, IndexFile> entry : timeStoreTable.entrySet()) {
                 IndexFile indexFile = entry.getValue();
@@ -306,10 +369,9 @@ public void destroy() {
                 }
                 indexFile.destroy();
             }
-
             // delete remote
-            if (flatFile != null) {
-                flatFile.destroy();
+            if (flatAppendFile != null) {
+                flatAppendFile.destroy();
             }
         } catch (Exception e) {
             log.error("IndexStoreService destroy all file error", e);
@@ -325,48 +387,58 @@ public String getServiceName() {
 
     public void setCompactTimestamp(long timestamp) {
         this.compactTimestamp.set(timestamp);
-        log.info("IndexStoreService compact timestamp has been set to: {}", timestamp);
+        log.debug("IndexStoreService set compact timestamp to: {}", timestamp);
     }
 
     protected IndexFile getNextSealedFile() {
-        try {
-            Map.Entry<Long, IndexFile> entry =
-                this.timeStoreTable.higherEntry(this.compactTimestamp.get());
-            if (entry != null && entry.getKey() < this.timeStoreTable.lastKey()) {
-                return entry.getValue();
-            }
-        } catch (Throwable e) {
-            log.error("Error occurred in " + getServiceName(), e);
+        Map.Entry<Long, IndexFile> entry =
+            this.timeStoreTable.higherEntry(this.compactTimestamp.get());
+        if (entry != null && entry.getKey() < this.timeStoreTable.lastKey()) {
+            return entry.getValue();
         }
         return null;
     }
 
+    @Override
+    public void shutdown() {
+        super.shutdown();
+        // Wait index service upload then clear time store table
+        while (!this.timeStoreTable.isEmpty()) {
+            try {
+                TimeUnit.MILLISECONDS.sleep(50);
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
     @Override
     public void run() {
-        log.info(this.getServiceName() + " service started");
         while (!this.isStopped()) {
             long expireTimestamp = System.currentTimeMillis()
                 - TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime());
             this.destroyExpiredFile(expireTimestamp);
-
             IndexFile indexFile = this.getNextSealedFile();
-            if (indexFile == null) {
-                this.waitForRunning(TimeUnit.SECONDS.toMillis(10));
-                continue;
+            if (indexFile != null) {
+                if (this.doCompactThenUploadFile(indexFile)) {
+                    this.setCompactTimestamp(indexFile.getTimestamp());
+                    continue;
+                }
             }
-            this.doCompactThenUploadFile(indexFile);
-            this.setCompactTimestamp(indexFile.getTimestamp());
+            this.waitForRunning(TimeUnit.SECONDS.toMillis(10));
         }
-        log.info(this.getServiceName() + " service shutdown");
-    }
-
-    @Override
-    public void shutdown() {
-        super.shutdown();
-        for (Map.Entry<Long /* timestamp */, IndexFile> entry : timeStoreTable.entrySet()) {
-            entry.getValue().shutdown();
+        readWriteLock.writeLock().lock();
+        try {
+            if (autoCreateNewFile) {
+                this.forceUpload();
+            }
+            this.timeStoreTable.forEach((timestamp, file) -> file.shutdown());
+            this.timeStoreTable.clear();
+        } catch (Exception e) {
+            log.error("IndexStoreService shutdown error", e);
+        } finally {
+            readWriteLock.writeLock().unlock();
         }
-        this.timeStoreTable.clear();
-        log.info("IndexStoreService shutdown gracefully");
+        log.info(this.getServiceName() + " service shutdown");
     }
 }
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/TieredMetadataManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/DefaultMetadataStore.java
similarity index 72%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/TieredMetadataManager.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/DefaultMetadataStore.java
index f091020241a..09500bf6da8 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/TieredMetadataManager.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/DefaultMetadataStore.java
@@ -29,27 +29,31 @@
 import java.util.function.Consumer;
 import org.apache.rocketmq.common.ConfigManager;
 import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
 import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
+import org.apache.rocketmq.tieredstore.metadata.entity.FileSegmentMetadata;
+import org.apache.rocketmq.tieredstore.metadata.entity.QueueMetadata;
+import org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata;
 
-public class TieredMetadataManager extends ConfigManager implements TieredMetadataStore {
+public class DefaultMetadataStore extends ConfigManager implements MetadataStore {
 
     private static final int DEFAULT_CAPACITY = 1024;
     private static final String DEFAULT_CONFIG_NAME = "config";
     private static final String DEFAULT_FILE_NAME = "tieredStoreMetadata.json";
 
     private final AtomicLong topicSequenceNumber;
-    private final TieredMessageStoreConfig storeConfig;
+    private final MessageStoreConfig storeConfig;
     private final ConcurrentMap<String /* topic */, TopicMetadata> topicMetadataTable;
     private final ConcurrentMap<String /* topic */, ConcurrentMap<Integer, QueueMetadata>> queueMetadataTable;
 
-    // Declare concurrent mapping tables to store file segment metadata for different types of files
+    // Declare concurrent mapping tables to store file segment metadata
     // Key: filePath -> Value: <baseOffset, metadata>
     private final ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> commitLogFileSegmentTable;
     private final ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> consumeQueueFileSegmentTable;
     private final ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> indexFileSegmentTable;
 
-    public TieredMetadataManager(TieredMessageStoreConfig storeConfig) {
+    public DefaultMetadataStore(MessageStoreConfig storeConfig) {
         this.storeConfig = storeConfig;
         this.topicSequenceNumber = new AtomicLong(-1L);
         this.topicMetadataTable = new ConcurrentHashMap<>(DEFAULT_CAPACITY);
@@ -89,6 +93,11 @@ public String configFilePath() {
         return Paths.get(storeConfig.getStorePathRootDir(), DEFAULT_CONFIG_NAME, DEFAULT_FILE_NAME).toString();
     }
 
+    @Override
+    public boolean load() {
+        return super.load();
+    }
+
     @Override
     public void decode(String jsonString) {
         if (jsonString != null) {
@@ -109,11 +118,6 @@ public void decode(String jsonString) {
         }
     }
 
-    @Override
-    public void setTopicSequenceNumber(long topicSequenceNumber) {
-        this.topicSequenceNumber.set(topicSequenceNumber);
-    }
-
     @Override
     public TopicMetadata getTopic(String topic) {
         return topicMetadataTable.get(topic);
@@ -160,7 +164,10 @@ public QueueMetadata getQueue(MessageQueue mq) {
 
     @Override
     public void iterateQueue(String topic, Consumer<QueueMetadata> callback) {
-        queueMetadataTable.get(topic).values().forEach(callback);
+        ConcurrentMap<Integer, QueueMetadata> metadataConcurrentMap = queueMetadataTable.get(topic);
+        if (metadataConcurrentMap != null) {
+            metadataConcurrentMap.values().forEach(callback);
+        }
     }
 
     @Override
@@ -274,4 +281,79 @@ public void destroy() {
         indexFileSegmentTable.clear();
         persist();
     }
+
+    static class TieredMetadataSerializeWrapper extends RemotingSerializable {
+
+        private AtomicLong topicSerialNumber = new AtomicLong(0L);
+
+        private ConcurrentMap<String /* topic */, TopicMetadata> topicMetadataTable;
+        private ConcurrentMap<String /* topic */, ConcurrentMap<Integer /* queueId */, QueueMetadata>> queueMetadataTable;
+
+        // Declare concurrent mapping tables to store file segment metadata
+        // Key: filePath -> Value: <baseOffset, metadata>
+        private ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> commitLogFileSegmentTable;
+        private ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> consumeQueueFileSegmentTable;
+        private ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> indexFileSegmentTable;
+
+        public TieredMetadataSerializeWrapper() {
+            this.topicMetadataTable = new ConcurrentHashMap<>(DEFAULT_CAPACITY);
+            this.queueMetadataTable = new ConcurrentHashMap<>(DEFAULT_CAPACITY);
+            this.commitLogFileSegmentTable = new ConcurrentHashMap<>(DEFAULT_CAPACITY);
+            this.consumeQueueFileSegmentTable = new ConcurrentHashMap<>(DEFAULT_CAPACITY);
+            this.indexFileSegmentTable = new ConcurrentHashMap<>(DEFAULT_CAPACITY);
+        }
+
+        public AtomicLong getTopicSerialNumber() {
+            return topicSerialNumber;
+        }
+
+        public void setTopicSerialNumber(AtomicLong topicSerialNumber) {
+            this.topicSerialNumber = topicSerialNumber;
+        }
+
+        public ConcurrentMap<String, TopicMetadata> getTopicMetadataTable() {
+            return topicMetadataTable;
+        }
+
+        public void setTopicMetadataTable(
+            ConcurrentMap<String, TopicMetadata> topicMetadataTable) {
+            this.topicMetadataTable = topicMetadataTable;
+        }
+
+        public ConcurrentMap<String, ConcurrentMap<Integer, QueueMetadata>> getQueueMetadataTable() {
+            return queueMetadataTable;
+        }
+
+        public void setQueueMetadataTable(
+            ConcurrentMap<String, ConcurrentMap<Integer, QueueMetadata>> queueMetadataTable) {
+            this.queueMetadataTable = queueMetadataTable;
+        }
+
+        public ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> getCommitLogFileSegmentTable() {
+            return commitLogFileSegmentTable;
+        }
+
+        public void setCommitLogFileSegmentTable(
+            ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> commitLogFileSegmentTable) {
+            this.commitLogFileSegmentTable = commitLogFileSegmentTable;
+        }
+
+        public ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> getConsumeQueueFileSegmentTable() {
+            return consumeQueueFileSegmentTable;
+        }
+
+        public void setConsumeQueueFileSegmentTable(
+            ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> consumeQueueFileSegmentTable) {
+            this.consumeQueueFileSegmentTable = consumeQueueFileSegmentTable;
+        }
+
+        public ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> getIndexFileSegmentTable() {
+            return indexFileSegmentTable;
+        }
+
+        public void setIndexFileSegmentTable(
+            ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> indexFileSegmentTable) {
+            this.indexFileSegmentTable = indexFileSegmentTable;
+        }
+    }
 }
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/TieredMetadataStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/MetadataStore.java
similarity index 60%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/TieredMetadataStore.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/MetadataStore.java
index 9d89e7582e2..0b053127d21 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/TieredMetadataStore.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/MetadataStore.java
@@ -19,18 +19,14 @@
 import java.util.function.Consumer;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.metadata.entity.FileSegmentMetadata;
+import org.apache.rocketmq.tieredstore.metadata.entity.QueueMetadata;
+import org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata;
 
 /**
  * Provides tiered metadata storage service to store metadata information of Topic, Queue, FileSegment, etc.
  */
-public interface TieredMetadataStore {
-
-    /**
-     * Set the sequence number of Topic, the start index from 0.
-     *
-     * @param topicSequenceNumber The sequence number of Topic.
-     */
-    void setTopicSequenceNumber(long topicSequenceNumber);
+public interface MetadataStore {
 
     /**
      * Get the metadata information of specified Topic.
@@ -55,11 +51,6 @@ public interface TieredMetadataStore {
 
     void deleteTopic(String topic);
 
-    /**
-     * Queue metadata operation
-     *
-     * @see QueueMetadata
-     */
     QueueMetadata getQueue(MessageQueue mq);
 
     QueueMetadata addQueue(MessageQueue mq, long baseOffset);
@@ -70,58 +61,17 @@ public interface TieredMetadataStore {
 
     void deleteQueue(MessageQueue mq);
 
-    /**
-     * Get the metadata information of specified file segment.
-     *
-     * @param basePath   The file path.
-     * @param fileType   The file type.
-     * @param baseOffset The start offset of file segment.
-     * @return The metadata information of specified file segment, or null if it does not exist.
-     */
     FileSegmentMetadata getFileSegment(String basePath, FileSegmentType fileType, long baseOffset);
 
-    /**
-     * Update the metadata information of a file segment.
-     *
-     * @param fileSegmentMetadata The metadata information of the file segment.
-     */
     void updateFileSegment(FileSegmentMetadata fileSegmentMetadata);
 
-    /**
-     * Traverse all metadata information of file segment
-     * and execute the callback function for each metadata information.
-     *
-     * @param callback The traversal callback function.
-     */
     void iterateFileSegment(Consumer<FileSegmentMetadata> callback);
 
-    /**
-     * Traverse all the metadata information of the file segments in the specified file path
-     * and execute the callback function for each metadata information.
-     *
-     * @param basePath The file path.
-     * @param callback The traversal callback function.
-     */
     void iterateFileSegment(String basePath, FileSegmentType fileType, Consumer<FileSegmentMetadata> callback);
 
-    /**
-     * Delete all the metadata information of the file segments.
-     *
-     * @param basePath The file path.
-     */
     void deleteFileSegment(String basePath, FileSegmentType fileType);
 
-    /**
-     * Delete the metadata information of a specified file segment.
-     *
-     * @param basePath   The file path.
-     * @param fileType   The file type.
-     * @param baseOffset The start offset of the file segment.
-     */
     void deleteFileSegment(String basePath, FileSegmentType fileType, long baseOffset);
 
-    /**
-     * Clean all metadata in disk
-     */
     void destroy();
 }
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/TieredMetadataSerializeWrapper.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/TieredMetadataSerializeWrapper.java
deleted file mode 100644
index fa01606dbdb..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/TieredMetadataSerializeWrapper.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.metadata;
-
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
-
-public class TieredMetadataSerializeWrapper extends RemotingSerializable {
-
-    private AtomicLong topicSerialNumber = new AtomicLong(0L);
-
-    private ConcurrentMap<String /* topic */, TopicMetadata> topicMetadataTable;
-    private ConcurrentMap<String /* topic */, ConcurrentMap<Integer /* queueId */, QueueMetadata>> queueMetadataTable;
-
-    // Declare concurrent mapping tables to store file segment metadata for different types of files
-    // Key: filePath -> Value: <baseOffset, metadata>
-    private ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> commitLogFileSegmentTable;
-    private ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> consumeQueueFileSegmentTable;
-    private ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> indexFileSegmentTable;
-
-    public TieredMetadataSerializeWrapper() {
-        this.topicMetadataTable = new ConcurrentHashMap<>(1024);
-        this.queueMetadataTable = new ConcurrentHashMap<>(1024);
-        this.commitLogFileSegmentTable = new ConcurrentHashMap<>(1024);
-        this.consumeQueueFileSegmentTable = new ConcurrentHashMap<>(1024);
-        this.indexFileSegmentTable = new ConcurrentHashMap<>(1024);
-    }
-
-    public AtomicLong getTopicSerialNumber() {
-        return topicSerialNumber;
-    }
-
-    public void setTopicSerialNumber(AtomicLong topicSerialNumber) {
-        this.topicSerialNumber = topicSerialNumber;
-    }
-
-    public ConcurrentMap<String, TopicMetadata> getTopicMetadataTable() {
-        return topicMetadataTable;
-    }
-
-    public void setTopicMetadataTable(
-        ConcurrentMap<String, TopicMetadata> topicMetadataTable) {
-        this.topicMetadataTable = topicMetadataTable;
-    }
-
-    public ConcurrentMap<String, ConcurrentMap<Integer, QueueMetadata>> getQueueMetadataTable() {
-        return queueMetadataTable;
-    }
-
-    public void setQueueMetadataTable(
-        ConcurrentMap<String, ConcurrentMap<Integer, QueueMetadata>> queueMetadataTable) {
-        this.queueMetadataTable = queueMetadataTable;
-    }
-
-    public ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> getCommitLogFileSegmentTable() {
-        return commitLogFileSegmentTable;
-    }
-
-    public void setCommitLogFileSegmentTable(
-        ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> commitLogFileSegmentTable) {
-        this.commitLogFileSegmentTable = commitLogFileSegmentTable;
-    }
-
-    public ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> getConsumeQueueFileSegmentTable() {
-        return consumeQueueFileSegmentTable;
-    }
-
-    public void setConsumeQueueFileSegmentTable(
-        ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> consumeQueueFileSegmentTable) {
-        this.consumeQueueFileSegmentTable = consumeQueueFileSegmentTable;
-    }
-
-    public ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> getIndexFileSegmentTable() {
-        return indexFileSegmentTable;
-    }
-
-    public void setIndexFileSegmentTable(
-        ConcurrentMap<String, ConcurrentMap<Long, FileSegmentMetadata>> indexFileSegmentTable) {
-        this.indexFileSegmentTable = indexFileSegmentTable;
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/entity/FileSegmentMetadata.java
similarity index 90%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/entity/FileSegmentMetadata.java
index 2f0fd71debb..4f988ca2411 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/entity/FileSegmentMetadata.java
@@ -14,8 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.tieredstore.metadata;
+package org.apache.rocketmq.tieredstore.metadata.entity;
 
+import com.alibaba.fastjson.annotation.JSONField;
 import java.util.Objects;
 
 public class FileSegmentMetadata {
@@ -24,20 +25,36 @@ public class FileSegmentMetadata {
     public static final int STATUS_SEALED = 1;
     public static final int STATUS_DELETED = 2;
 
-    private int type;
+    @JSONField(ordinal = 1)
     private String path;
+
+    @JSONField(ordinal = 2)
+    private int type;
+
+    @JSONField(ordinal = 3)
     private long baseOffset;
+
+    @JSONField(ordinal = 4)
     private int status;
+
+    @JSONField(ordinal = 5)
     private long size;
 
+    @JSONField(ordinal = 6)
     private long createTimestamp;
+
+    @JSONField(ordinal = 7)
     private long beginTimestamp;
+
+    @JSONField(ordinal = 8)
     private long endTimestamp;
+
+    @JSONField(ordinal = 9)
     private long sealTimestamp;
 
     // default constructor is used by fastjson
+    @SuppressWarnings("unused")
     public FileSegmentMetadata() {
-
     }
 
     public FileSegmentMetadata(String path, long baseOffset, int type) {
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/QueueMetadata.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/entity/QueueMetadata.java
similarity index 88%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/QueueMetadata.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/entity/QueueMetadata.java
index d479330d78f..6720f1d08ac 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/QueueMetadata.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/entity/QueueMetadata.java
@@ -14,20 +14,28 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.tieredstore.metadata;
+package org.apache.rocketmq.tieredstore.metadata.entity;
 
+import com.alibaba.fastjson.annotation.JSONField;
 import org.apache.rocketmq.common.message.MessageQueue;
 
 public class QueueMetadata {
 
+    @JSONField(ordinal = 1)
     private MessageQueue queue;
+
+    @JSONField(ordinal = 2)
     private long minOffset;
+
+    @JSONField(ordinal = 3)
     private long maxOffset;
+
+    @JSONField(ordinal = 4)
     private long updateTimestamp;
 
     // default constructor is used by fastjson
+    @SuppressWarnings("unused")
     public QueueMetadata() {
-
     }
 
     public QueueMetadata(MessageQueue queue, long minOffset, long maxOffset) {
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/TopicMetadata.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/entity/TopicMetadata.java
similarity index 88%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/TopicMetadata.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/entity/TopicMetadata.java
index 4847dafd064..80e5230e7a3 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/TopicMetadata.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/entity/TopicMetadata.java
@@ -14,26 +14,30 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.tieredstore.metadata;
+package org.apache.rocketmq.tieredstore.metadata.entity;
 
-import com.google.common.annotations.VisibleForTesting;
+import com.alibaba.fastjson.annotation.JSONField;
 
 public class TopicMetadata {
 
+    @JSONField(ordinal = 1)
     private long topicId;
+
+    @JSONField(ordinal = 2)
     private String topic;
+
+    @JSONField(ordinal = 3)
     private int status;
+
+    @JSONField(ordinal = 4)
     private long reserveTime;
+
+    @JSONField(ordinal = 5)
     private long updateTimestamp;
 
     // default constructor is used by fastjson
+    @SuppressWarnings("unused")
     public TopicMetadata() {
-
-    }
-
-    @VisibleForTesting
-    public TopicMetadata(String topic) {
-        this.topic = topic;
     }
 
     public TopicMetadata(long topicId, String topic, long reserveTime) {
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java
index 2b9fc59d821..4d083284834 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java
@@ -16,7 +16,6 @@
  */
 package org.apache.rocketmq.tieredstore.metrics;
 
-import com.github.benmanes.caffeine.cache.Policy;
 import io.opentelemetry.api.common.Attributes;
 import io.opentelemetry.api.common.AttributesBuilder;
 import io.opentelemetry.api.metrics.LongCounter;
@@ -33,25 +32,25 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import org.apache.rocketmq.common.Pair;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.metrics.NopLongCounter;
 import org.apache.rocketmq.common.metrics.NopLongHistogram;
 import org.apache.rocketmq.common.metrics.NopObservableLongGauge;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.store.MessageStore;
-import org.apache.rocketmq.tieredstore.TieredMessageFetcher;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
 import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.MessageCacheKey;
-import org.apache.rocketmq.tieredstore.common.SelectBufferResultWrapper;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.file.CompositeQueueFlatFile;
-import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
+import org.apache.rocketmq.tieredstore.core.MessageStoreFetcher;
+import org.apache.rocketmq.tieredstore.core.MessageStoreFetcherImpl;
+import org.apache.rocketmq.tieredstore.file.FlatFileStore;
+import org.apache.rocketmq.tieredstore.file.FlatMessageFile;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.GAUGE_STORAGE_SIZE;
 import static org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant.LABEL_STORAGE_MEDIUM;
@@ -77,7 +76,7 @@
 
 public class TieredStoreMetricsManager {
 
-    private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
+    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
     public static Supplier<AttributesBuilder> attributesBuilderSupplier;
     private static String storageMedium = STORAGE_MEDIUM_BLOB;
 
@@ -130,7 +129,7 @@ public static List<Pair<InstrumentSelector, ViewBuilder>> getMetricsView() {
             .build();
 
         ViewBuilder bufferSizeViewBuilder = View.builder()
-            .setAggregation(Aggregation.explicitBucketHistogram(Arrays.asList(1d * TieredStoreUtil.KB, 10d * TieredStoreUtil.KB, 100d * TieredStoreUtil.KB, 1d * TieredStoreUtil.MB, 10d * TieredStoreUtil.MB, 32d * TieredStoreUtil.MB, 50d * TieredStoreUtil.MB, 100d * TieredStoreUtil.MB)))
+            .setAggregation(Aggregation.explicitBucketHistogram(Arrays.asList(1d * MessageStoreUtil.KB, 10d * MessageStoreUtil.KB, 100d * MessageStoreUtil.KB, 1d * MessageStoreUtil.MB, 10d * MessageStoreUtil.MB, 32d * MessageStoreUtil.MB, 50d * MessageStoreUtil.MB, 100d * MessageStoreUtil.MB)))
             .setDescription("tiered_store_buffer_size_view");
 
         res.add(new Pair<>(rpcLatencySelector, rpcLatencyViewBuilder));
@@ -145,7 +144,9 @@ public static void setStorageMedium(String storageMedium) {
     }
 
     public static void init(Meter meter, Supplier<AttributesBuilder> attributesBuilderSupplier,
-        TieredMessageStoreConfig storeConfig, TieredMessageFetcher fetcher, MessageStore next) {
+        MessageStoreConfig storeConfig, MessageStoreFetcher fetcher,
+        FlatFileStore flatFileStore, MessageStore next) {
+
         TieredStoreMetricsManager.attributesBuilderSupplier = attributesBuilderSupplier;
 
         apiLatency = meter.histogramBuilder(HISTOGRAM_API_LATENCY)
@@ -176,28 +177,31 @@ public static void init(Meter meter, Supplier<AttributesBuilder> attributesBuild
             .setDescription("Tiered store dispatch behind message count")
             .ofLongs()
             .buildWithCallback(measurement -> {
-                for (CompositeQueueFlatFile flatFile :
-                    TieredFlatFileManager.getInstance(storeConfig).deepCopyFlatFileToList()) {
-
-                    MessageQueue mq = flatFile.getMessageQueue();
-                    long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId());
-                    long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1);
-                    if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > (long) storeConfig.getTieredStoreFileReservedTime() * 60 * 60 * 1000) {
-                        continue;
-                    }
+                for (FlatMessageFile flatFile : flatFileStore.deepCopyFlatFileToList()) {
+                    try {
+
+                        MessageQueue mq = flatFile.getMessageQueue();
+                        long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId());
+                        long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1);
+                        if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) {
+                            continue;
+                        }
 
-                    Attributes commitLogAttributes = newAttributesBuilder()
-                        .put(LABEL_TOPIC, mq.getTopic())
-                        .put(LABEL_QUEUE_ID, mq.getQueueId())
-                        .put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase())
-                        .build();
-                    measurement.record(Math.max(maxOffset - flatFile.getDispatchOffset(), 0), commitLogAttributes);
-                    Attributes consumeQueueAttributes = newAttributesBuilder()
-                        .put(LABEL_TOPIC, mq.getTopic())
-                        .put(LABEL_QUEUE_ID, mq.getQueueId())
-                        .put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase())
-                        .build();
-                    measurement.record(Math.max(maxOffset - flatFile.getConsumeQueueMaxOffset(), 0), consumeQueueAttributes);
+                        Attributes commitLogAttributes = newAttributesBuilder()
+                            .put(LABEL_TOPIC, mq.getTopic())
+                            .put(LABEL_QUEUE_ID, mq.getQueueId())
+                            .put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase())
+                            .build();
+
+                        Attributes consumeQueueAttributes = newAttributesBuilder()
+                            .put(LABEL_TOPIC, mq.getTopic())
+                            .put(LABEL_QUEUE_ID, mq.getQueueId())
+                            .put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase())
+                            .build();
+                        measurement.record(Math.max(maxOffset - flatFile.getConsumeQueueMaxOffset(), 0), consumeQueueAttributes);
+                    } catch (ConsumeQueueException e) {
+                        // TODO: handle exception here
+                    }
                 }
             });
 
@@ -206,39 +210,36 @@ public static void init(Meter meter, Supplier<AttributesBuilder> attributesBuild
             .setUnit("seconds")
             .ofLongs()
             .buildWithCallback(measurement -> {
-                for (CompositeQueueFlatFile flatFile :
-                    TieredFlatFileManager.getInstance(storeConfig).deepCopyFlatFileToList()) {
-
-                    MessageQueue mq = flatFile.getMessageQueue();
-                    long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId());
-                    long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1);
-                    if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > (long) storeConfig.getTieredStoreFileReservedTime() * 60 * 60 * 1000) {
-                        continue;
-                    }
+                for (FlatMessageFile flatFile : flatFileStore.deepCopyFlatFileToList()) {
+                    try {
+                        MessageQueue mq = flatFile.getMessageQueue();
 
-                    Attributes commitLogAttributes = newAttributesBuilder()
-                        .put(LABEL_TOPIC, mq.getTopic())
-                        .put(LABEL_QUEUE_ID, mq.getQueueId())
-                        .put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase())
-                        .build();
-                    long commitLogDispatchLatency = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), flatFile.getDispatchOffset());
-                    if (maxOffset <= flatFile.getDispatchOffset() || commitLogDispatchLatency < 0) {
-                        measurement.record(0, commitLogAttributes);
-                    } else {
-                        measurement.record(System.currentTimeMillis() - commitLogDispatchLatency, commitLogAttributes);
-                    }
+                        long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId());
+                        long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1);
+                        if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) {
+                            continue;
+                        }
 
-                    Attributes consumeQueueAttributes = newAttributesBuilder()
-                        .put(LABEL_TOPIC, mq.getTopic())
-                        .put(LABEL_QUEUE_ID, mq.getQueueId())
-                        .put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase())
-                        .build();
-                    long consumeQueueDispatchOffset = flatFile.getConsumeQueueMaxOffset();
-                    long consumeQueueDispatchLatency = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), consumeQueueDispatchOffset);
-                    if (maxOffset <= consumeQueueDispatchOffset || consumeQueueDispatchLatency < 0) {
-                        measurement.record(0, consumeQueueAttributes);
-                    } else {
-                        measurement.record(System.currentTimeMillis() - consumeQueueDispatchLatency, consumeQueueAttributes);
+                        Attributes commitLogAttributes = newAttributesBuilder()
+                            .put(LABEL_TOPIC, mq.getTopic())
+                            .put(LABEL_QUEUE_ID, mq.getQueueId())
+                            .put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase())
+                            .build();
+
+                        Attributes consumeQueueAttributes = newAttributesBuilder()
+                            .put(LABEL_TOPIC, mq.getTopic())
+                            .put(LABEL_QUEUE_ID, mq.getQueueId())
+                            .put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase())
+                            .build();
+                        long consumeQueueDispatchOffset = flatFile.getConsumeQueueMaxOffset();
+                        long consumeQueueDispatchLatency = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), consumeQueueDispatchOffset);
+                        if (maxOffset <= consumeQueueDispatchOffset || consumeQueueDispatchLatency < 0) {
+                            measurement.record(0, consumeQueueAttributes);
+                        } else {
+                            measurement.record(System.currentTimeMillis() - consumeQueueDispatchLatency, consumeQueueAttributes);
+                        }
+                    } catch (ConsumeQueueException e) {
+                        // TODO: handle exception
                     }
                 }
             });
@@ -258,15 +259,22 @@ public static void init(Meter meter, Supplier<AttributesBuilder> attributesBuild
         cacheCount = meter.gaugeBuilder(GAUGE_CACHE_COUNT)
             .setDescription("Tiered store cache message count")
             .ofLongs()
-            .buildWithCallback(measurement -> measurement.record(fetcher.getMessageCache().estimatedSize(), newAttributesBuilder().build()));
+            .buildWithCallback(measurement -> {
+                if (fetcher instanceof MessageStoreFetcherImpl) {
+                    long count = ((MessageStoreFetcherImpl) fetcher).getFetcherCache().stats().loadCount();
+                    measurement.record(count, newAttributesBuilder().build());
+                }
+            });
 
         cacheBytes = meter.gaugeBuilder(GAUGE_CACHE_BYTES)
             .setDescription("Tiered store cache message bytes")
             .setUnit("bytes")
             .ofLongs()
             .buildWithCallback(measurement -> {
-                Optional<Policy.Eviction<MessageCacheKey, SelectBufferResultWrapper>> eviction = fetcher.getMessageCache().policy().eviction();
-                eviction.ifPresent(resultEviction -> measurement.record(resultEviction.weightedSize().orElse(0), newAttributesBuilder().build()));
+                if (fetcher instanceof MessageStoreFetcherImpl) {
+                    long count = ((MessageStoreFetcherImpl) fetcher).getFetcherCache().estimatedSize();
+                    measurement.record(count, newAttributesBuilder().build());
+                }
             });
 
         cacheAccess = meter.counterBuilder(COUNTER_CACHE_ACCESS)
@@ -284,7 +292,7 @@ public static void init(Meter meter, Supplier<AttributesBuilder> attributesBuild
             .buildWithCallback(measurement -> {
                 Map<String, Map<FileSegmentType, Long>> topicFileSizeMap = new HashMap<>();
                 try {
-                    TieredMetadataStore metadataStore = TieredStoreUtil.getMetadataStore(storeConfig);
+                    MetadataStore metadataStore = flatFileStore.getMetadataStore();
                     metadataStore.iterateFileSegment(fileSegment -> {
                         Map<FileSegmentType, Long> subMap =
                             topicFileSizeMap.computeIfAbsent(fileSegment.getPath(), k -> new HashMap<>());
@@ -294,7 +302,7 @@ public static void init(Meter meter, Supplier<AttributesBuilder> attributesBuild
                         subMap.put(fileSegmentType, size + fileSegment.getSize());
                     });
                 } catch (Exception e) {
-                    logger.error("Failed to get storage size", e);
+                    log.error("Failed to get storage size", e);
                 }
                 topicFileSizeMap.forEach((topic, subMap) -> {
                     subMap.forEach((fileSegmentType, size) -> {
@@ -312,8 +320,8 @@ public static void init(Meter meter, Supplier<AttributesBuilder> attributesBuild
             .setUnit("milliseconds")
             .ofLongs()
             .buildWithCallback(measurement -> {
-                for (CompositeQueueFlatFile flatFile : TieredFlatFileManager.getInstance(storeConfig).deepCopyFlatFileToList()) {
-                    long timestamp = flatFile.getCommitLogBeginTimestamp();
+                for (FlatMessageFile flatFile : flatFileStore.deepCopyFlatFileToList()) {
+                    long timestamp = flatFile.getMinStoreTimestamp();
                     if (timestamp > 0) {
                         MessageQueue mq = flatFile.getMessageQueue();
                         Attributes attributes = newAttributesBuilder()
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegment.java
new file mode 100644
index 00000000000..1140add67d1
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegment.java
@@ -0,0 +1,349 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.provider;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.locks.ReentrantLock;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
+import org.apache.rocketmq.tieredstore.common.AppendResult;
+import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.exception.TieredStoreErrorCode;
+import org.apache.rocketmq.tieredstore.exception.TieredStoreException;
+import org.apache.rocketmq.tieredstore.stream.FileSegmentInputStream;
+import org.apache.rocketmq.tieredstore.stream.FileSegmentInputStreamFactory;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class FileSegment implements Comparable<FileSegment>, FileSegmentProvider {
+
+    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
+
+    protected static final Long GET_FILE_SIZE_ERROR = -1L;
+
+    protected final long baseOffset;
+    protected final String filePath;
+    protected final FileSegmentType fileType;
+    protected final MessageStoreConfig storeConfig;
+
+    protected final long maxSize;
+    protected final MessageStoreExecutor executor;
+    protected final ReentrantLock fileLock = new ReentrantLock();
+    protected final Semaphore commitLock = new Semaphore(1);
+
+    protected volatile boolean closed = false;
+    protected volatile long minTimestamp = Long.MAX_VALUE;
+    protected volatile long maxTimestamp = Long.MAX_VALUE;
+    protected volatile long commitPosition = 0L;
+    protected volatile long appendPosition = 0L;
+
+    protected volatile List<ByteBuffer> bufferList = new ArrayList<>();
+    protected volatile FileSegmentInputStream fileSegmentInputStream;
+    protected volatile CompletableFuture<Boolean> flightCommitRequest;
+
+    public FileSegment(MessageStoreConfig storeConfig, FileSegmentType fileType,
+        String filePath, long baseOffset, MessageStoreExecutor executor) {
+
+        this.storeConfig = storeConfig;
+        this.fileType = fileType;
+        this.filePath = filePath;
+        this.baseOffset = baseOffset;
+        this.executor = executor;
+        this.maxSize = this.getMaxSizeByFileType();
+    }
+
+    @Override
+    public int compareTo(FileSegment o) {
+        return Long.compare(this.baseOffset, o.baseOffset);
+    }
+
+    public long getBaseOffset() {
+        return baseOffset;
+    }
+
+    public void initPosition(long pos) {
+        fileLock.lock();
+        try {
+            this.commitPosition = pos;
+            this.appendPosition = pos;
+        } finally {
+            fileLock.unlock();
+        }
+    }
+
+    public long getCommitPosition() {
+        return commitPosition;
+    }
+
+    public long getAppendPosition() {
+        return appendPosition;
+    }
+
+    public long getCommitOffset() {
+        return baseOffset + commitPosition;
+    }
+
+    public long getAppendOffset() {
+        return baseOffset + appendPosition;
+    }
+
+    public FileSegmentType getFileType() {
+        return fileType;
+    }
+
+    public long getMaxSizeByFileType() {
+        switch (fileType) {
+            case COMMIT_LOG:
+                return storeConfig.getTieredStoreCommitLogMaxSize();
+            case CONSUME_QUEUE:
+                return storeConfig.getTieredStoreConsumeQueueMaxSize();
+            case INDEX:
+            default:
+                return Long.MAX_VALUE;
+        }
+    }
+
+    public long getMaxSize() {
+        return maxSize;
+    }
+
+    public long getMinTimestamp() {
+        return minTimestamp;
+    }
+
+    public void setMinTimestamp(long minTimestamp) {
+        this.minTimestamp = minTimestamp;
+    }
+
+    public long getMaxTimestamp() {
+        return maxTimestamp;
+    }
+
+    public void setMaxTimestamp(long maxTimestamp) {
+        this.maxTimestamp = maxTimestamp;
+    }
+
+    public boolean isClosed() {
+        return closed;
+    }
+
+    public void close() {
+        fileLock.lock();
+        try {
+            this.closed = true;
+        } finally {
+            fileLock.unlock();
+        }
+    }
+
+    protected List<ByteBuffer> borrowBuffer() {
+        List<ByteBuffer> temp;
+        fileLock.lock();
+        try {
+            temp = bufferList;
+            bufferList = new ArrayList<>();
+        } finally {
+            fileLock.unlock();
+        }
+        return temp;
+    }
+
+    @SuppressWarnings("NonAtomicOperationOnVolatileField")
+    protected void updateTimestamp(long timestamp) {
+        fileLock.lock();
+        try {
+            if (maxTimestamp == Long.MAX_VALUE && minTimestamp == Long.MAX_VALUE) {
+                maxTimestamp = timestamp;
+                minTimestamp = timestamp;
+                return;
+            }
+            maxTimestamp = Math.max(maxTimestamp, timestamp);
+            minTimestamp = Math.min(minTimestamp, timestamp);
+        } finally {
+            fileLock.unlock();
+        }
+    }
+
+    @SuppressWarnings("NonAtomicOperationOnVolatileField")
+    public AppendResult append(ByteBuffer buffer, long timestamp) {
+        fileLock.lock();
+        try {
+            if (closed) {
+                return AppendResult.FILE_CLOSED;
+            }
+            if (appendPosition + buffer.remaining() > maxSize) {
+                return AppendResult.FILE_FULL;
+            }
+            if (bufferList.size() >= storeConfig.getTieredStoreMaxGroupCommitCount()) {
+                return AppendResult.BUFFER_FULL;
+            }
+            this.appendPosition += buffer.remaining();
+            this.bufferList.add(buffer);
+            this.updateTimestamp(timestamp);
+        } finally {
+            fileLock.unlock();
+        }
+        return AppendResult.SUCCESS;
+    }
+
+    public boolean needCommit() {
+        return appendPosition > commitPosition;
+    }
+
+    @SuppressWarnings("NonAtomicOperationOnVolatileField")
+    public CompletableFuture<Boolean> commitAsync() {
+        if (closed) {
+            return CompletableFuture.completedFuture(false);
+        }
+
+        if (!needCommit()) {
+            return CompletableFuture.completedFuture(true);
+        }
+
+        // acquire lock
+        if (commitLock.drainPermits() <= 0) {
+            return CompletableFuture.completedFuture(false);
+        }
+
+        // handle last commit error
+        if (fileSegmentInputStream != null) {
+            long fileSize = this.getSize();
+            if (fileSize == GET_FILE_SIZE_ERROR) {
+                log.error("FileSegment correct position error, fileName={}, commit={}, append={}, buffer={}",
+                    this.getPath(), commitPosition, appendPosition, fileSegmentInputStream.getContentLength());
+                releaseCommitLock();
+                return CompletableFuture.completedFuture(false);
+            }
+            if (correctPosition(fileSize)) {
+                fileSegmentInputStream = null;
+            }
+        }
+
+        int bufferSize;
+        if (fileSegmentInputStream != null) {
+            fileSegmentInputStream.rewind();
+            bufferSize = fileSegmentInputStream.available();
+        } else {
+            List<ByteBuffer> bufferList = this.borrowBuffer();
+            bufferSize = bufferList.stream().mapToInt(ByteBuffer::remaining).sum();
+            if (bufferSize == 0) {
+                releaseCommitLock();
+                return CompletableFuture.completedFuture(true);
+            }
+            fileSegmentInputStream = FileSegmentInputStreamFactory.build(
+                fileType, this.getCommitOffset(), bufferList, null, bufferSize);
+        }
+
+        boolean append = fileType != FileSegmentType.INDEX;
+        return flightCommitRequest =
+            this.commit0(fileSegmentInputStream, commitPosition, bufferSize, append)
+                .thenApply(result -> {
+                    if (result) {
+                        commitPosition += bufferSize;
+                        fileSegmentInputStream = null;
+                        return true;
+                    } else {
+                        fileSegmentInputStream.rewind();
+                        return false;
+                    }
+                })
+                .exceptionally(this::handleCommitException)
+                .whenComplete((result, e) -> releaseCommitLock());
+    }
+
+    private boolean handleCommitException(Throwable e) {
+
+        log.warn("FileSegment commit exception, filePath={}", this.filePath, e);
+
+        // Get root cause here
+        Throwable rootCause = e.getCause() != null ? e.getCause() : e;
+
+        long fileSize = rootCause instanceof TieredStoreException ?
+            ((TieredStoreException) rootCause).getPosition() : this.getSize();
+
+        long expectPosition = commitPosition + fileSegmentInputStream.getContentLength();
+        if (fileSize == GET_FILE_SIZE_ERROR) {
+            log.error("Get file size error after commit, FileName: {}, Commit: {}, Content: {}, Expect: {}, Append: {}",
+                this.getPath(), commitPosition, fileSegmentInputStream.getContentLength(), expectPosition, appendPosition);
+            return false;
+        }
+
+        if (correctPosition(fileSize)) {
+            fileSegmentInputStream = null;
+            return true;
+        } else {
+            fileSegmentInputStream.rewind();
+            return false;
+        }
+    }
+
+    private void releaseCommitLock() {
+        if (commitLock.availablePermits() == 0) {
+            commitLock.release();
+        }
+    }
+
+    /**
+     * return true to clear buffer
+     */
+    private boolean correctPosition(long fileSize) {
+
+        // Current we have three offsets here: commit offset, expect offset, file size.
+        // We guarantee that the commit offset is less than or equal to the expect offset.
+        // Max offset will increase because we can continuously put in new buffers
+
+        // We are believing that the file size returned by the server is correct,
+        // can reset the commit offset to the file size reported by the storage system.
+
+        long expectPosition = commitPosition + fileSegmentInputStream.getContentLength();
+        commitPosition = fileSize;
+        return expectPosition == fileSize;
+    }
+
+    public ByteBuffer read(long position, int length) {
+        return readAsync(position, length).join();
+    }
+
+    public CompletableFuture<ByteBuffer> readAsync(long position, int length) {
+        CompletableFuture<ByteBuffer> future = new CompletableFuture<>();
+        if (position < 0 || position >= commitPosition) {
+            future.completeExceptionally(new TieredStoreException(
+                TieredStoreErrorCode.ILLEGAL_PARAM, "FileSegment read position is illegal position"));
+            return future;
+        }
+
+        if (length <= 0) {
+            future.completeExceptionally(new TieredStoreException(
+                TieredStoreErrorCode.ILLEGAL_PARAM, "FileSegment read length illegal"));
+            return future;
+        }
+
+        int readableBytes = (int) (commitPosition - position);
+        if (readableBytes < length) {
+            length = readableBytes;
+            log.debug("FileSegment#readAsync, expect request position is greater than commit position, " +
+                    "file: {}, request position: {}, commit position: {}, change length from {} to {}",
+                getPath(), position, commitPosition, length, readableBytes);
+        }
+        return this.read0(position, length);
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentAllocator.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentAllocator.java
deleted file mode 100644
index c4b1e67afe2..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentAllocator.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.tieredstore.provider;
-
-import java.lang.reflect.Constructor;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-
-public class FileSegmentAllocator {
-
-    private static final Logger log = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
-
-    private final TieredMessageStoreConfig storeConfig;
-
-    private final Constructor<? extends TieredFileSegment> fileSegmentConstructor;
-
-    public FileSegmentAllocator(
-        TieredMessageStoreConfig storeConfig) throws ClassNotFoundException, NoSuchMethodException {
-        this.storeConfig = storeConfig;
-        Class<? extends TieredFileSegment> clazz =
-            Class.forName(storeConfig.getTieredBackendServiceProvider()).asSubclass(TieredFileSegment.class);
-        fileSegmentConstructor = clazz.getConstructor(
-            TieredMessageStoreConfig.class, FileSegmentType.class, String.class, Long.TYPE);
-    }
-
-    public TieredMessageStoreConfig getStoreConfig() {
-        return storeConfig;
-    }
-
-    public TieredMetadataStore getMetadataStore() {
-        return TieredStoreUtil.getMetadataStore(storeConfig);
-    }
-
-    public TieredFileSegment createSegment(
-        FileSegmentType fileType, String filePath, long baseOffset) {
-
-        switch (fileType) {
-            case COMMIT_LOG:
-                return this.createCommitLogFileSegment(filePath, baseOffset);
-            case CONSUME_QUEUE:
-                return this.createConsumeQueueFileSegment(filePath, baseOffset);
-            case INDEX:
-                return this.createIndexFileSegment(filePath, baseOffset);
-        }
-        return null;
-    }
-
-    public TieredFileSegment createCommitLogFileSegment(String filePath, long baseOffset) {
-        TieredFileSegment segment = null;
-        try {
-            segment = fileSegmentConstructor.newInstance(
-                this.storeConfig, FileSegmentType.COMMIT_LOG, filePath, baseOffset);
-        } catch (Exception e) {
-            log.error("create file segment of commitlog failed, filePath: {}, baseOffset: {}",
-                filePath, baseOffset, e);
-        }
-        return segment;
-    }
-
-    public TieredFileSegment createConsumeQueueFileSegment(String filePath, long baseOffset) {
-        TieredFileSegment segment = null;
-        try {
-            segment = fileSegmentConstructor.newInstance(
-                this.storeConfig, FileSegmentType.CONSUME_QUEUE, filePath, baseOffset);
-        } catch (Exception e) {
-            log.error("create file segment of commitlog failed, filePath: {}, baseOffset: {}",
-                filePath, baseOffset, e);
-        }
-        return segment;
-    }
-
-    public TieredFileSegment createIndexFileSegment(String filePath, long baseOffset) {
-        TieredFileSegment segment = null;
-        try {
-            segment = fileSegmentConstructor.newInstance(
-                this.storeConfig, FileSegmentType.INDEX, filePath, baseOffset);
-        } catch (Exception e) {
-            log.error("create file segment of commitlog failed, filePath: {}, baseOffset: {}",
-                filePath, baseOffset, e);
-        }
-        return segment;
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactory.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactory.java
new file mode 100644
index 00000000000..ace6d8f08fd
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactory.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.tieredstore.provider;
+
+import java.lang.reflect.Constructor;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
+import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+
+public class FileSegmentFactory {
+
+    private final MetadataStore metadataStore;
+    private final MessageStoreConfig storeConfig;
+    private final MessageStoreExecutor executor;
+    private final Constructor<? extends FileSegment> fileSegmentConstructor;
+
+    public FileSegmentFactory(MetadataStore metadataStore,
+        MessageStoreConfig storeConfig, MessageStoreExecutor executor) {
+
+        try {
+            this.storeConfig = storeConfig;
+            this.metadataStore = metadataStore;
+            this.executor = executor;
+            Class<? extends FileSegment> clazz =
+                Class.forName(storeConfig.getTieredBackendServiceProvider()).asSubclass(FileSegment.class);
+            fileSegmentConstructor = clazz.getConstructor(
+                MessageStoreConfig.class, FileSegmentType.class, String.class, Long.TYPE, MessageStoreExecutor.class);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public MetadataStore getMetadataStore() {
+        return metadataStore;
+    }
+
+    public MessageStoreConfig getStoreConfig() {
+        return storeConfig;
+    }
+
+    public FileSegment createSegment(FileSegmentType fileType, String filePath, long baseOffset) {
+        try {
+            return fileSegmentConstructor.newInstance(this.storeConfig, fileType, filePath, baseOffset, executor);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public FileSegment createCommitLogFileSegment(String filePath, long baseOffset) {
+        return this.createSegment(FileSegmentType.COMMIT_LOG, filePath, baseOffset);
+    }
+
+    public FileSegment createConsumeQueueFileSegment(String filePath, long baseOffset) {
+        return this.createSegment(FileSegmentType.CONSUME_QUEUE, filePath, baseOffset);
+    }
+
+    public FileSegment createIndexServiceFileSegment(String filePath, long baseOffset) {
+        return this.createSegment(FileSegmentType.INDEX, filePath, baseOffset);
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentProvider.java
similarity index 95%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentProvider.java
index b9938b7a8a0..1ce643e0e8c 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentProvider.java
@@ -18,9 +18,9 @@
 
 import java.nio.ByteBuffer;
 import java.util.concurrent.CompletableFuture;
-import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
+import org.apache.rocketmq.tieredstore.stream.FileSegmentInputStream;
 
-public interface TieredStoreProvider {
+public interface FileSegmentProvider {
 
     /**
      * Get file path in backend file system
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegment.java
similarity index 59%
rename from tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegment.java
index 80ad41f6859..93ad74541b6 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegment.java
@@ -14,62 +14,52 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.tieredstore.provider.memory;
+package org.apache.rocketmq.tieredstore.provider;
 
 import java.nio.ByteBuffer;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
 import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
-import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-import org.junit.Assert;
+import org.apache.rocketmq.tieredstore.stream.FileSegmentInputStream;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-public class MemoryFileSegment extends TieredFileSegment {
+public class MemoryFileSegment extends FileSegment {
 
-    protected final ByteBuffer memStore;
-
-    public CompletableFuture<Boolean> blocker;
+    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
 
+    protected final ByteBuffer memStore;
+    protected CompletableFuture<Boolean> blocker;
     protected int size = 0;
-
     protected boolean checkSize = true;
 
-    public MemoryFileSegment(FileSegmentType fileType, MessageQueue messageQueue, long baseOffset,
-        TieredMessageStoreConfig storeConfig) {
-        this(storeConfig, fileType, TieredStoreUtil.toPath(messageQueue), baseOffset);
-    }
+    public MemoryFileSegment(MessageStoreConfig storeConfig,
+        FileSegmentType fileType, String filePath, long baseOffset, MessageStoreExecutor executor) {
 
-    public MemoryFileSegment(TieredMessageStoreConfig storeConfig,
-        FileSegmentType fileType, String filePath, long baseOffset) {
-        super(storeConfig, fileType, filePath, baseOffset);
-        switch (fileType) {
-            case COMMIT_LOG:
-            case INDEX:
-            case CONSUME_QUEUE:
-                memStore = ByteBuffer.allocate(10000);
-                break;
-            default:
-                memStore = null;
-                break;
-        }
+        super(storeConfig, fileType, filePath, baseOffset, executor);
+        memStore = ByteBuffer.allocate(10000);
         memStore.position((int) getSize());
     }
 
-    public boolean isCheckSize() {
-        return checkSize;
+    @Override
+    public boolean exists() {
+        return false;
     }
 
-    public void setCheckSize(boolean checkSize) {
-        this.checkSize = checkSize;
+    @Override
+    public void createFile() {
     }
 
     public ByteBuffer getMemStore() {
         return memStore;
     }
 
+    public void setCheckSize(boolean checkSize) {
+        this.checkSize = checkSize;
+    }
+
     @Override
     public String getPath() {
         return filePath;
@@ -87,11 +77,6 @@ public void setSize(int size) {
         this.size = size;
     }
 
-    @Override
-    public void createFile() {
-
-    }
-
     @Override
     public CompletableFuture<ByteBuffer> read0(long position, int length) {
         ByteBuffer buffer = memStore.duplicate();
@@ -107,36 +92,22 @@ public CompletableFuture<Boolean> commit0(
 
         try {
             if (blocker != null && !blocker.get()) {
-                throw new IllegalStateException("Commit Exception for Memory Test");
+                log.info("Commit Blocker Exception for Memory Test");
+                return CompletableFuture.completedFuture(false);
             }
-        } catch (InterruptedException | ExecutionException e) {
-            Assert.fail(e.getMessage());
-        }
-
-        Assert.assertTrue(!checkSize || position >= getSize());
 
-        byte[] buffer = new byte[1024];
-        int startPos = memStore.position();
-        try {
             int len;
+            byte[] buffer = new byte[1024];
             while ((len = inputStream.read(buffer)) > 0) {
                 memStore.put(buffer, 0, len);
             }
-            Assert.assertEquals(length, memStore.position() - startPos);
         } catch (Exception e) {
-            Assert.fail(e.getMessage());
             return CompletableFuture.completedFuture(false);
         }
         return CompletableFuture.completedFuture(true);
     }
 
-    @Override
-    public boolean exists() {
-        return false;
-    }
-
     @Override
     public void destroyFile() {
-
     }
 }
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java
new file mode 100644
index 00000000000..3ab5914161d
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.provider;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Supplier;
+import com.google.common.io.ByteStreams;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.common.AttributesBuilder;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Paths;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
+import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager;
+import org.apache.rocketmq.tieredstore.stream.FileSegmentInputStream;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_FILE_TYPE;
+import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_OPERATION;
+import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_PATH;
+import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_SUCCESS;
+
+/**
+ * this class is experimental and may change without notice.
+ */
+public class PosixFileSegment extends FileSegment {
+
+    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
+
+    private static final String OPERATION_POSIX_READ = "read";
+    private static final String OPERATION_POSIX_WRITE = "write";
+
+    private final String fullPath;
+    private volatile File file;
+    private volatile FileChannel readFileChannel;
+    private volatile FileChannel writeFileChannel;
+
+    public PosixFileSegment(MessageStoreConfig storeConfig,
+        FileSegmentType fileType, String filePath, long baseOffset, MessageStoreExecutor executor) {
+
+        super(storeConfig, fileType, filePath, baseOffset, executor);
+
+        // basePath
+        String basePath = StringUtils.defaultString(storeConfig.getTieredStoreFilePath(),
+            StringUtils.appendIfMissing(storeConfig.getTieredStoreFilePath(), File.separator));
+
+        // fullPath: basePath/hash_cluster/broker/topic/queueId/fileType/baseOffset
+        String clusterName = storeConfig.getBrokerClusterName();
+        String clusterBasePath = String.format("%s_%s", MessageStoreUtil.getHash(clusterName), clusterName);
+        fullPath = Paths.get(basePath, clusterBasePath, filePath,
+            fileType.toString(), MessageStoreUtil.offset2FileName(baseOffset)).toString();
+        log.info("Constructing Posix FileSegment, filePath: {}", fullPath);
+
+        this.createFile();
+    }
+
+    protected AttributesBuilder newAttributesBuilder() {
+        return TieredStoreMetricsManager.newAttributesBuilder()
+            .put(LABEL_PATH, filePath)
+            .put(LABEL_FILE_TYPE, fileType.name().toLowerCase());
+    }
+
+    @Override
+    public String getPath() {
+        return filePath;
+    }
+
+    @Override
+    public long getSize() {
+        if (exists()) {
+            return file.length();
+        }
+        return 0L;
+    }
+
+    @Override
+    public boolean exists() {
+        return file != null && file.exists();
+    }
+
+    @Override
+    public void createFile() {
+        if (this.file == null) {
+            synchronized (this) {
+                if (this.file == null) {
+                    this.createFile0();
+                }
+            }
+        }
+    }
+
+    @SuppressWarnings({"resource", "ResultOfMethodCallIgnored"})
+    private void createFile0() {
+        try {
+            File file = new File(fullPath);
+            File dir = file.getParentFile();
+            if (!dir.exists()) {
+                dir.mkdirs();
+            }
+            if (!file.exists()) {
+                if (file.createNewFile()) {
+                    log.debug("Create Posix FileSegment, filePath: {}", fullPath);
+                }
+            }
+            this.readFileChannel = new RandomAccessFile(file, "r").getChannel();
+            this.writeFileChannel = new RandomAccessFile(file, "rwd").getChannel();
+            this.file = file;
+        } catch (Exception e) {
+            log.error("PosixFileSegment#createFile: create file {} failed: ", filePath, e);
+        }
+    }
+
+    @Override
+
+    public void destroyFile() {
+        this.close();
+        if (file != null && file.exists()) {
+            if (file.delete()) {
+                log.info("Destroy Posix FileSegment, filePath: {}", fullPath);
+            } else {
+                log.warn("Destroy Posix FileSegment error, filePath: {}", fullPath);
+            }
+        }
+    }
+
+    @Override
+    public void close() {
+        super.close();
+        try {
+            if (readFileChannel != null && readFileChannel.isOpen()) {
+                readFileChannel.close();
+                readFileChannel = null;
+            }
+            if (writeFileChannel != null && writeFileChannel.isOpen()) {
+                writeFileChannel.close();
+                writeFileChannel = null;
+            }
+        } catch (IOException e) {
+            log.error("Destroy Posix FileSegment failed, filePath: {}", fullPath, e);
+        }
+    }
+
+    @Override
+    public CompletableFuture<ByteBuffer> read0(long position, int length) {
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        AttributesBuilder attributesBuilder = newAttributesBuilder()
+            .put(LABEL_OPERATION, OPERATION_POSIX_READ);
+
+        return CompletableFuture.supplyAsync((Supplier<ByteBuffer>) () -> {
+            ByteBuffer byteBuffer = ByteBuffer.allocate(length);
+            try {
+                readFileChannel.position(position);
+                readFileChannel.read(byteBuffer);
+                byteBuffer.flip();
+                byteBuffer.limit(length);
+
+                attributesBuilder.put(LABEL_SUCCESS, true);
+                long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
+                TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build());
+
+                Attributes metricsAttributes = newAttributesBuilder()
+                    .put(LABEL_OPERATION, OPERATION_POSIX_READ)
+                    .build();
+                int downloadedBytes = byteBuffer.remaining();
+                TieredStoreMetricsManager.downloadBytes.record(downloadedBytes, metricsAttributes);
+            } catch (IOException e) {
+                long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
+                attributesBuilder.put(LABEL_SUCCESS, false);
+                TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build());
+            }
+            return byteBuffer;
+        }, executor.bufferFetchExecutor);
+    }
+
+    @Override
+    @SuppressWarnings("ResultOfMethodCallIgnored")
+    public CompletableFuture<Boolean> commit0(
+        FileSegmentInputStream inputStream, long position, int length, boolean append) {
+
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        AttributesBuilder attributesBuilder = newAttributesBuilder()
+            .put(LABEL_OPERATION, OPERATION_POSIX_WRITE);
+
+        return CompletableFuture.supplyAsync(() -> {
+            try {
+                byte[] byteArray = ByteStreams.toByteArray(inputStream);
+                writeFileChannel.position(position);
+                ByteBuffer buffer = ByteBuffer.wrap(byteArray);
+                while (buffer.hasRemaining()) {
+                    writeFileChannel.write(buffer);
+                }
+                writeFileChannel.force(true);
+                attributesBuilder.put(LABEL_SUCCESS, true);
+                long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
+                TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build());
+
+                Attributes metricsAttributes = newAttributesBuilder()
+                    .put(LABEL_OPERATION, OPERATION_POSIX_WRITE)
+                    .build();
+                TieredStoreMetricsManager.uploadBytes.record(length, metricsAttributes);
+            } catch (Exception e) {
+                long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
+                attributesBuilder.put(LABEL_SUCCESS, false);
+                TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build());
+                return false;
+            }
+            return true;
+        }, executor.bufferCommitExecutor);
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java
deleted file mode 100644
index 6703de9403f..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java
+++ /dev/null
@@ -1,485 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.provider;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.locks.ReentrantLock;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.exception.TieredStoreErrorCode;
-import org.apache.rocketmq.tieredstore.exception.TieredStoreException;
-import org.apache.rocketmq.tieredstore.file.TieredCommitLog;
-import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue;
-import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
-import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStreamFactory;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-
-import static org.apache.rocketmq.tieredstore.index.IndexStoreFile.INDEX_BEGIN_TIME_STAMP;
-import static org.apache.rocketmq.tieredstore.index.IndexStoreFile.INDEX_END_TIME_STAMP;
-
-public abstract class TieredFileSegment implements Comparable<TieredFileSegment>, TieredStoreProvider {
-
-    private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
-
-    protected final String filePath;
-    protected final long baseOffset;
-    protected final FileSegmentType fileType;
-    protected final TieredMessageStoreConfig storeConfig;
-
-    private final long maxSize;
-    private final ReentrantLock bufferLock = new ReentrantLock();
-    private final Semaphore commitLock = new Semaphore(1);
-
-    private volatile boolean full = false;
-    private volatile boolean closed = false;
-
-    private volatile long minTimestamp = Long.MAX_VALUE;
-    private volatile long maxTimestamp = Long.MAX_VALUE;
-    private volatile long commitPosition = 0L;
-    private volatile long appendPosition = 0L;
-
-    // only used in commitLog
-    private volatile long dispatchCommitOffset = 0L;
-
-    private ByteBuffer codaBuffer;
-    private List<ByteBuffer> bufferList = new ArrayList<>();
-    private FileSegmentInputStream fileSegmentInputStream;
-    private CompletableFuture<Boolean> flightCommitRequest = CompletableFuture.completedFuture(false);
-
-    public TieredFileSegment(TieredMessageStoreConfig storeConfig,
-        FileSegmentType fileType, String filePath, long baseOffset) {
-
-        this.storeConfig = storeConfig;
-        this.fileType = fileType;
-        this.filePath = filePath;
-        this.baseOffset = baseOffset;
-        this.maxSize = getMaxSizeByFileType();
-    }
-
-    /**
-     * The max segment size of a file is determined by the file type
-     */
-    protected long getMaxSizeByFileType() {
-        switch (fileType) {
-            case COMMIT_LOG:
-                return storeConfig.getTieredStoreCommitLogMaxSize();
-            case CONSUME_QUEUE:
-                return storeConfig.getTieredStoreConsumeQueueMaxSize();
-            case INDEX:
-                return Long.MAX_VALUE;
-            default:
-                throw new IllegalArgumentException("Unsupported file type: " + fileType);
-        }
-    }
-
-    @Override
-    public int compareTo(TieredFileSegment o) {
-        return Long.compare(this.baseOffset, o.baseOffset);
-    }
-
-    public long getBaseOffset() {
-        return baseOffset;
-    }
-
-    public long getCommitOffset() {
-        return baseOffset + commitPosition;
-    }
-
-    public long getCommitPosition() {
-        return commitPosition;
-    }
-
-    public long getDispatchCommitOffset() {
-        return dispatchCommitOffset;
-    }
-
-    public long getMaxOffset() {
-        return baseOffset + appendPosition;
-    }
-
-    public long getMaxSize() {
-        return maxSize;
-    }
-
-    public long getMinTimestamp() {
-        return minTimestamp;
-    }
-
-    public void setMinTimestamp(long minTimestamp) {
-        this.minTimestamp = minTimestamp;
-    }
-
-    public long getMaxTimestamp() {
-        return maxTimestamp;
-    }
-
-    public void setMaxTimestamp(long maxTimestamp) {
-        this.maxTimestamp = maxTimestamp;
-    }
-
-    public boolean isFull() {
-        return full;
-    }
-
-    public void setFull() {
-        setFull(true);
-    }
-
-    public void setFull(boolean appendCoda) {
-        bufferLock.lock();
-        try {
-            full = true;
-            if (fileType == FileSegmentType.COMMIT_LOG && appendCoda) {
-                appendCoda();
-            }
-        } finally {
-            bufferLock.unlock();
-        }
-    }
-
-    public boolean isClosed() {
-        return closed;
-    }
-
-    public void close() {
-        closed = true;
-    }
-
-    public FileSegmentType getFileType() {
-        return fileType;
-    }
-
-    public void initPosition(long pos) {
-        this.commitPosition = pos;
-        this.appendPosition = pos;
-    }
-
-    private List<ByteBuffer> borrowBuffer() {
-        bufferLock.lock();
-        try {
-            List<ByteBuffer> tmp = bufferList;
-            bufferList = new ArrayList<>();
-            return tmp;
-        } finally {
-            bufferLock.unlock();
-        }
-    }
-
-    @SuppressWarnings("NonAtomicOperationOnVolatileField")
-    public AppendResult append(ByteBuffer byteBuf, long timestamp) {
-        if (closed) {
-            return AppendResult.FILE_CLOSED;
-        }
-
-        bufferLock.lock();
-        try {
-            if (full || codaBuffer != null) {
-                return AppendResult.FILE_FULL;
-            }
-
-            if (fileType == FileSegmentType.INDEX) {
-                minTimestamp = byteBuf.getLong(INDEX_BEGIN_TIME_STAMP);
-                maxTimestamp = byteBuf.getLong(INDEX_END_TIME_STAMP);
-
-                appendPosition += byteBuf.remaining();
-                // IndexFile is large and not change after compaction, no need deep copy
-                bufferList.add(byteBuf);
-                setFull();
-                return AppendResult.SUCCESS;
-            }
-
-            if (appendPosition + byteBuf.remaining() > maxSize) {
-                setFull();
-                return AppendResult.FILE_FULL;
-            }
-
-            if (bufferList.size() > storeConfig.getTieredStoreGroupCommitCount()
-                || appendPosition - commitPosition > storeConfig.getTieredStoreGroupCommitSize()) {
-                commitAsync();
-            }
-
-            if (bufferList.size() > storeConfig.getTieredStoreMaxGroupCommitCount()) {
-                logger.debug("File segment append buffer full, file: {}, buffer size: {}, pending bytes: {}",
-                    getPath(), bufferList.size(), appendPosition - commitPosition);
-                return AppendResult.BUFFER_FULL;
-            }
-
-            if (timestamp != Long.MAX_VALUE) {
-                maxTimestamp = timestamp;
-                if (minTimestamp == Long.MAX_VALUE) {
-                    minTimestamp = timestamp;
-                }
-            }
-
-            appendPosition += byteBuf.remaining();
-
-            // deep copy buffer
-            ByteBuffer byteBuffer = ByteBuffer.allocateDirect(byteBuf.remaining());
-            byteBuffer.put(byteBuf);
-            byteBuffer.flip();
-            byteBuf.rewind();
-
-            bufferList.add(byteBuffer);
-            return AppendResult.SUCCESS;
-        } finally {
-            bufferLock.unlock();
-        }
-    }
-
-    public void setCommitPosition(long commitPosition) {
-        this.commitPosition = commitPosition;
-    }
-
-    public long getAppendPosition() {
-        return appendPosition;
-    }
-
-    public void setAppendPosition(long appendPosition) {
-        this.appendPosition = appendPosition;
-    }
-
-    @SuppressWarnings("NonAtomicOperationOnVolatileField")
-    private void appendCoda() {
-        if (codaBuffer != null) {
-            return;
-        }
-        codaBuffer = ByteBuffer.allocate(TieredCommitLog.CODA_SIZE);
-        codaBuffer.putInt(TieredCommitLog.CODA_SIZE);
-        codaBuffer.putInt(TieredCommitLog.BLANK_MAGIC_CODE);
-        codaBuffer.putLong(maxTimestamp);
-        codaBuffer.flip();
-        appendPosition += TieredCommitLog.CODA_SIZE;
-    }
-
-    public ByteBuffer read(long position, int length) {
-        return readAsync(position, length).join();
-    }
-
-    public CompletableFuture<ByteBuffer> readAsync(long position, int length) {
-        CompletableFuture<ByteBuffer> future = new CompletableFuture<>();
-        if (position < 0 || length < 0) {
-            future.completeExceptionally(
-                new TieredStoreException(TieredStoreErrorCode.ILLEGAL_PARAM, "position or length is negative"));
-            return future;
-        }
-        if (length == 0) {
-            future.completeExceptionally(
-                new TieredStoreException(TieredStoreErrorCode.ILLEGAL_PARAM, "length is zero"));
-            return future;
-        }
-        if (position >= commitPosition) {
-            future.completeExceptionally(
-                new TieredStoreException(TieredStoreErrorCode.ILLEGAL_PARAM, "position is illegal"));
-            return future;
-        }
-        if (position + length > commitPosition) {
-            logger.debug("TieredFileSegment#readAsync request position + length is greater than commit position," +
-                    " correct length using commit position, file: {}, request position: {}, commit position:{}, change length from {} to {}",
-                getPath(), position, commitPosition, length, commitPosition - position);
-            length = (int) (commitPosition - position);
-            if (length == 0) {
-                future.completeExceptionally(
-                    new TieredStoreException(TieredStoreErrorCode.NO_NEW_DATA, "request position is equal to commit position"));
-                return future;
-            }
-            if (fileType == FileSegmentType.CONSUME_QUEUE && length % TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE != 0) {
-                future.completeExceptionally(
-                    new TieredStoreException(TieredStoreErrorCode.ILLEGAL_PARAM, "position and length is illegal"));
-                return future;
-            }
-        }
-        return read0(position, length);
-    }
-
-    public boolean needCommit() {
-        return appendPosition > commitPosition;
-    }
-
-    public boolean commit() {
-        if (closed) {
-            return false;
-        }
-        // result is false when we send real commit request
-        // use join for wait flight request done
-        Boolean result = commitAsync().join();
-        if (!result) {
-            result = flightCommitRequest.join();
-        }
-        return result;
-    }
-
-    private void releaseCommitLock() {
-        if (commitLock.availablePermits() == 0) {
-            commitLock.release();
-        } else {
-            logger.error("[Bug] FileSegmentCommitAsync, lock is already released: available permits: {}",
-                commitLock.availablePermits());
-        }
-    }
-
-    private void updateDispatchCommitOffset(List<ByteBuffer> bufferList) {
-        if (fileType == FileSegmentType.COMMIT_LOG && bufferList.size() > 0) {
-            dispatchCommitOffset =
-                MessageBufferUtil.getQueueOffset(bufferList.get(bufferList.size() - 1));
-        }
-    }
-
-    /**
-     * @return false: commit, true: no commit operation
-     */
-    @SuppressWarnings("NonAtomicOperationOnVolatileField")
-    public CompletableFuture<Boolean> commitAsync() {
-        if (closed) {
-            return CompletableFuture.completedFuture(false);
-        }
-
-        if (!needCommit()) {
-            return CompletableFuture.completedFuture(true);
-        }
-
-        if (commitLock.drainPermits() <= 0) {
-            return CompletableFuture.completedFuture(false);
-        }
-
-        try {
-            if (fileSegmentInputStream != null) {
-                long fileSize = this.getSize();
-                if (fileSize == -1L) {
-                    logger.error("Get commit position error before commit, Commit: {}, Expect: {}, Current Max: {}, FileName: {}",
-                        commitPosition, commitPosition + fileSegmentInputStream.getContentLength(), appendPosition, getPath());
-                    releaseCommitLock();
-                    return CompletableFuture.completedFuture(false);
-                } else {
-                    if (correctPosition(fileSize, null)) {
-                        updateDispatchCommitOffset(fileSegmentInputStream.getBufferList());
-                        fileSegmentInputStream = null;
-                    }
-                }
-            }
-
-            int bufferSize;
-            if (fileSegmentInputStream != null) {
-                bufferSize = fileSegmentInputStream.available();
-            } else {
-                List<ByteBuffer> bufferList = borrowBuffer();
-                bufferSize = bufferList.stream().mapToInt(ByteBuffer::remaining).sum()
-                    + (codaBuffer != null ? codaBuffer.remaining() : 0);
-                if (bufferSize == 0) {
-                    releaseCommitLock();
-                    return CompletableFuture.completedFuture(true);
-                }
-                fileSegmentInputStream = FileSegmentInputStreamFactory.build(
-                    fileType, baseOffset + commitPosition, bufferList, codaBuffer, bufferSize);
-            }
-
-            return flightCommitRequest = this
-                .commit0(fileSegmentInputStream, commitPosition, bufferSize, fileType != FileSegmentType.INDEX)
-                .thenApply(result -> {
-                    if (result) {
-                        updateDispatchCommitOffset(fileSegmentInputStream.getBufferList());
-                        commitPosition += bufferSize;
-                        fileSegmentInputStream = null;
-                        return true;
-                    } else {
-                        fileSegmentInputStream.rewind();
-                        return false;
-                    }
-                })
-                .exceptionally(this::handleCommitException)
-                .whenComplete((result, e) -> releaseCommitLock());
-
-        } catch (Exception e) {
-            handleCommitException(e);
-            releaseCommitLock();
-        }
-        return CompletableFuture.completedFuture(false);
-    }
-
-    private long getCorrectFileSize(Throwable throwable) {
-        if (throwable instanceof TieredStoreException) {
-            long fileSize = ((TieredStoreException) throwable).getPosition();
-            if (fileSize > 0) {
-                return fileSize;
-            }
-        }
-        return getSize();
-    }
-
-    private boolean handleCommitException(Throwable e) {
-        // Get root cause here
-        Throwable cause = e.getCause() != null ? e.getCause() : e;
-        long fileSize = this.getCorrectFileSize(cause);
-
-        if (fileSize == -1L) {
-            logger.error("Get commit position error, Commit: %d, Expect: %d, Current Max: %d, FileName: %s",
-                commitPosition, commitPosition + fileSegmentInputStream.getContentLength(), appendPosition, getPath());
-            fileSegmentInputStream.rewind();
-            return false;
-        }
-
-        if (correctPosition(fileSize, cause)) {
-            updateDispatchCommitOffset(fileSegmentInputStream.getBufferList());
-            fileSegmentInputStream = null;
-            return true;
-        } else {
-            fileSegmentInputStream.rewind();
-            return false;
-        }
-    }
-
-    /**
-     * return true to clear buffer
-     */
-    private boolean correctPosition(long fileSize, Throwable throwable) {
-
-        // Current we have three offsets here: commit offset, expect offset, file size.
-        // We guarantee that the commit offset is less than or equal to the expect offset.
-        // Max offset will increase because we can continuously put in new buffers
-        String handleInfo = throwable == null ? "before commit" : "after commit";
-        long expectPosition = commitPosition + fileSegmentInputStream.getContentLength();
-
-        String offsetInfo = String.format("Correct Commit Position, %s, result=[{}], " +
-                "Commit: %d, Expect: %d, Current Max: %d, FileSize: %d, FileName: %s",
-            handleInfo, commitPosition, expectPosition, appendPosition, fileSize, this.getPath());
-
-        // We are believing that the file size returned by the server is correct,
-        // can reset the commit offset to the file size reported by the storage system.
-        if (fileSize == expectPosition) {
-            logger.info(offsetInfo, "Success", throwable);
-            commitPosition = fileSize;
-            return true;
-        }
-
-        if (fileSize < commitPosition) {
-            logger.error(offsetInfo, "FileSizeIncorrect", throwable);
-        } else if (fileSize == commitPosition) {
-            logger.warn(offsetInfo, "CommitFailed", throwable);
-        } else if (fileSize > commitPosition) {
-            logger.warn(offsetInfo, "PartialSuccess", throwable);
-        }
-        commitPosition = fileSize;
-        return false;
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java
deleted file mode 100644
index ee56b1e68bd..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.provider.posix;
-
-import com.google.common.base.Stopwatch;
-import com.google.common.io.ByteStreams;
-import io.opentelemetry.api.common.Attributes;
-import io.opentelemetry.api.common.AttributesBuilder;
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Paths;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager;
-import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
-import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-
-import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_FILE_TYPE;
-import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_OPERATION;
-import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_PATH;
-import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_SUCCESS;
-
-/**
- * this class is experimental and may change without notice.
- */
-public class PosixFileSegment extends TieredFileSegment {
-
-    private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
-
-    private static final String UNDERLINE = "_";
-    private static final String OPERATION_POSIX_READ = "read";
-    private static final String OPERATION_POSIX_WRITE = "write";
-
-    private final String fullPath;
-    private volatile File file;
-    private volatile FileChannel readFileChannel;
-    private volatile FileChannel writeFileChannel;
-
-    public PosixFileSegment(TieredMessageStoreConfig storeConfig,
-        FileSegmentType fileType, String filePath, long baseOffset) {
-
-        super(storeConfig, fileType, filePath, baseOffset);
-
-        // basePath
-        String basePath = StringUtils.defaultString(storeConfig.getTieredStoreFilePath(),
-            StringUtils.appendIfMissing(storeConfig.getTieredStoreFilePath(), File.separator));
-
-        // fullPath: basePath/hash_cluster/broker/topic/queueId/fileType/baseOffset
-        String brokerClusterName = storeConfig.getBrokerClusterName();
-        String clusterBasePath = TieredStoreUtil.getHash(brokerClusterName) + UNDERLINE + brokerClusterName;
-        this.fullPath = Paths.get(basePath, clusterBasePath, filePath,
-            fileType.toString(), TieredStoreUtil.offset2FileName(baseOffset)).toString();
-        logger.info("Constructing Posix FileSegment, filePath: {}", fullPath);
-
-        createFile();
-    }
-
-    protected AttributesBuilder newAttributesBuilder() {
-        return TieredStoreMetricsManager.newAttributesBuilder()
-            .put(LABEL_PATH, filePath)
-            .put(LABEL_FILE_TYPE, fileType.name().toLowerCase());
-    }
-
-    @Override
-    public String getPath() {
-        return fullPath;
-    }
-
-    @Override
-    public long getSize() {
-        if (exists()) {
-            return file.length();
-        }
-        return -1;
-    }
-
-    @Override
-    public boolean exists() {
-        return file != null && file.exists();
-    }
-
-    @Override
-    public void createFile() {
-        if (file == null) {
-            synchronized (this) {
-                if (file == null) {
-                    File file = new File(fullPath);
-                    try {
-                        File dir = file.getParentFile();
-                        if (!dir.exists()) {
-                            dir.mkdirs();
-                        }
-
-                        // TODO use direct IO to avoid polluting the page cache
-                        file.createNewFile();
-                        this.readFileChannel = new RandomAccessFile(file, "r").getChannel();
-                        this.writeFileChannel = new RandomAccessFile(file, "rwd").getChannel();
-                        this.file = file;
-                    } catch (Exception e) {
-                        logger.error("PosixFileSegment#createFile: create file {} failed: ", filePath, e);
-                    }
-                }
-            }
-        }
-    }
-
-    @Override
-    public void destroyFile() {
-        try {
-            if (readFileChannel != null && readFileChannel.isOpen()) {
-                readFileChannel.close();
-            }
-            if (writeFileChannel != null && writeFileChannel.isOpen()) {
-                writeFileChannel.close();
-            }
-            logger.info("Destroy Posix FileSegment, filePath: {}", fullPath);
-        } catch (IOException e) {
-            logger.error("Destroy Posix FileSegment failed, filePath: {}", fullPath, e);
-        }
-
-        if (file.exists()) {
-            file.delete();
-        }
-    }
-
-    @Override
-    public CompletableFuture<ByteBuffer> read0(long position, int length) {
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        AttributesBuilder attributesBuilder = newAttributesBuilder()
-            .put(LABEL_OPERATION, OPERATION_POSIX_READ);
-
-        CompletableFuture<ByteBuffer> future = new CompletableFuture<>();
-        ByteBuffer byteBuffer = ByteBuffer.allocate(length);
-        try {
-            readFileChannel.position(position);
-            readFileChannel.read(byteBuffer);
-            byteBuffer.flip();
-            byteBuffer.limit(length);
-
-            attributesBuilder.put(LABEL_SUCCESS, true);
-            long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
-            TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build());
-
-            Attributes metricsAttributes = newAttributesBuilder()
-                .put(LABEL_OPERATION, OPERATION_POSIX_READ)
-                .build();
-            int downloadedBytes = byteBuffer.remaining();
-            TieredStoreMetricsManager.downloadBytes.record(downloadedBytes, metricsAttributes);
-
-            future.complete(byteBuffer);
-        } catch (IOException e) {
-            long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
-            attributesBuilder.put(LABEL_SUCCESS, false);
-            TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build());
-            logger.error("PosixFileSegment#read0: read file {} failed: position: {}, length: {}",
-                filePath, position, length, e);
-            future.completeExceptionally(e);
-        }
-        return future;
-    }
-
-    @Override
-    public CompletableFuture<Boolean> commit0(
-        FileSegmentInputStream inputStream, long position, int length, boolean append) {
-
-        Stopwatch stopwatch = Stopwatch.createStarted();
-        AttributesBuilder attributesBuilder = newAttributesBuilder()
-            .put(LABEL_OPERATION, OPERATION_POSIX_WRITE);
-
-        CompletableFuture<Boolean> future = new CompletableFuture<>();
-        try {
-            TieredStoreExecutor.commitExecutor.execute(() -> {
-                try {
-                    byte[] byteArray = ByteStreams.toByteArray(inputStream);
-                    if (byteArray.length != length) {
-                        logger.error("PosixFileSegment#commit0: append file {} failed: real data size: {}, is not equal to length: {}",
-                            filePath, byteArray.length, length);
-                        future.complete(false);
-                        return;
-                    }
-                    writeFileChannel.position(position);
-                    ByteBuffer buffer = ByteBuffer.wrap(byteArray);
-                    while (buffer.hasRemaining()) {
-                        writeFileChannel.write(buffer);
-                    }
-
-                    attributesBuilder.put(LABEL_SUCCESS, true);
-                    long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
-                    TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build());
-
-                    Attributes metricsAttributes = newAttributesBuilder()
-                        .put(LABEL_OPERATION, OPERATION_POSIX_WRITE)
-                        .build();
-                    TieredStoreMetricsManager.uploadBytes.record(length, metricsAttributes);
-
-                    future.complete(true);
-                } catch (Exception e) {
-                    long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
-                    attributesBuilder.put(LABEL_SUCCESS, false);
-                    TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build());
-
-                    logger.error("PosixFileSegment#commit0: append file {} failed: position: {}, length: {}",
-                        filePath, position, length, e);
-                    future.completeExceptionally(e);
-                }
-            });
-        } catch (Exception e) {
-            // commit task cannot be executed
-            long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
-            attributesBuilder.put(LABEL_SUCCESS, false);
-            TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build());
-
-            future.completeExceptionally(e);
-        }
-        return future;
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/stream/CommitLogInputStream.java
similarity index 91%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/stream/CommitLogInputStream.java
index 13b6e0ef9c9..e2d7354755d 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/stream/CommitLogInputStream.java
@@ -15,13 +15,13 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.tieredstore.provider.stream;
+package org.apache.rocketmq.tieredstore.stream;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.List;
 import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtil;
 
 public class CommitLogInputStream extends FileSegmentInputStream {
 
@@ -90,9 +90,9 @@ public int read() {
             commitLogOffset += readPosInCurBuffer;
             readPosInCurBuffer = 0;
         }
-        if (readPosInCurBuffer >= MessageBufferUtil.PHYSICAL_OFFSET_POSITION
-            && readPosInCurBuffer < MessageBufferUtil.SYS_FLAG_OFFSET_POSITION) {
-            res = (int) ((commitLogOffset >> (8 * (MessageBufferUtil.SYS_FLAG_OFFSET_POSITION - readPosInCurBuffer - 1))) & 0xff);
+        if (readPosInCurBuffer >= MessageFormatUtil.PHYSICAL_OFFSET_POSITION
+            && readPosInCurBuffer < MessageFormatUtil.SYS_FLAG_OFFSET_POSITION) {
+            res = (int) ((commitLogOffset >> (8 * (MessageFormatUtil.SYS_FLAG_OFFSET_POSITION - readPosInCurBuffer - 1))) & 0xff);
             readPosInCurBuffer++;
         } else {
             res = curBuffer.get(readPosInCurBuffer++) & 0xff;
@@ -150,18 +150,18 @@ public int read(byte[] b, int off, int len) {
             remaining = curBuf.remaining() - posInCurBuffer;
             readLen = Math.min(remaining, needRead);
             curBuf = bufferList.get(bufIndex);
-            if (posInCurBuffer < MessageBufferUtil.PHYSICAL_OFFSET_POSITION) {
-                realReadLen = Math.min(MessageBufferUtil.PHYSICAL_OFFSET_POSITION - posInCurBuffer, readLen);
+            if (posInCurBuffer < MessageFormatUtil.PHYSICAL_OFFSET_POSITION) {
+                realReadLen = Math.min(MessageFormatUtil.PHYSICAL_OFFSET_POSITION - posInCurBuffer, readLen);
                 // read from commitLog buffer
                 curBuf.position(posInCurBuffer);
                 curBuf.get(b, off, realReadLen);
                 curBuf.position(0);
-            } else if (posInCurBuffer < MessageBufferUtil.SYS_FLAG_OFFSET_POSITION) {
-                realReadLen = Math.min(MessageBufferUtil.SYS_FLAG_OFFSET_POSITION - posInCurBuffer, readLen);
+            } else if (posInCurBuffer < MessageFormatUtil.SYS_FLAG_OFFSET_POSITION) {
+                realReadLen = Math.min(MessageFormatUtil.SYS_FLAG_OFFSET_POSITION - posInCurBuffer, readLen);
                 // read from converted PHYSICAL_OFFSET_POSITION
                 byte[] physicalOffsetBytes = new byte[realReadLen];
                 for (int i = 0; i < realReadLen; i++) {
-                    physicalOffsetBytes[i] = (byte) ((curCommitLogOffset >> (8 * (MessageBufferUtil.SYS_FLAG_OFFSET_POSITION - posInCurBuffer - i - 1))) & 0xff);
+                    physicalOffsetBytes[i] = (byte) ((curCommitLogOffset >> (8 * (MessageFormatUtil.SYS_FLAG_OFFSET_POSITION - posInCurBuffer - i - 1))) & 0xff);
                 }
                 System.arraycopy(physicalOffsetBytes, 0, b, off, realReadLen);
             } else {
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/stream/FileSegmentInputStream.java
similarity index 99%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/stream/FileSegmentInputStream.java
index 9e9d5135cd7..5020ff199d0 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/stream/FileSegmentInputStream.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.tieredstore.provider.stream;
+package org.apache.rocketmq.tieredstore.stream;
 
 import java.io.IOException;
 import java.io.InputStream;
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/stream/FileSegmentInputStreamFactory.java
similarity index 87%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/stream/FileSegmentInputStreamFactory.java
index a90baff3ae6..6872296bbc0 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/stream/FileSegmentInputStreamFactory.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.tieredstore.provider.stream;
+package org.apache.rocketmq.tieredstore.stream;
 
 import java.nio.ByteBuffer;
 import java.util.List;
@@ -32,8 +32,7 @@ public static FileSegmentInputStream build(
 
         switch (fileType) {
             case COMMIT_LOG:
-                return new CommitLogInputStream(
-                    fileType, offset, bufferList, byteBuffer, length);
+                return new CommitLogInputStream(fileType, offset, bufferList, byteBuffer, length);
             case CONSUME_QUEUE:
                 return new FileSegmentInputStream(fileType, bufferList, length);
             case INDEX:
@@ -42,7 +41,7 @@ public static FileSegmentInputStream build(
                 }
                 return new FileSegmentInputStream(fileType, bufferList, length);
             default:
-                throw new IllegalArgumentException("file type is not supported");
+                throw new IllegalArgumentException("file type not supported");
         }
     }
 }
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/MessageBufferUtil.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/MessageBufferUtil.java
deleted file mode 100644
index 2c4a6e5784b..00000000000
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/MessageBufferUtil.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.util;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import org.apache.rocketmq.common.UtilAll;
-import org.apache.rocketmq.common.message.MessageDecoder;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.tieredstore.common.SelectBufferResult;
-import org.apache.rocketmq.tieredstore.file.TieredCommitLog;
-import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue;
-
-public class MessageBufferUtil {
-    private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
-
-    public static final int QUEUE_OFFSET_POSITION = 4 /* total size */
-        + 4 /* magic code */
-        + 4 /* body CRC */
-        + 4 /* queue id */
-        + 4; /* flag */
-
-    public static final int PHYSICAL_OFFSET_POSITION = 4 /* total size */
-        + 4 /* magic code */
-        + 4 /* body CRC */
-        + 4 /* queue id */
-        + 4 /* flag */
-        + 8; /* queue offset */
-
-    public static final int SYS_FLAG_OFFSET_POSITION = 4 /* total size */
-        + 4 /* magic code */
-        + 4 /* body CRC */
-        + 4 /* queue id */
-        + 4 /* flag */
-        + 8 /* queue offset */
-        + 8; /* physical offset */
-
-    public static final int STORE_TIMESTAMP_POSITION = 4 /* total size */
-        + 4 /* magic code */
-        + 4 /* body CRC */
-        + 4 /* queue id */
-        + 4 /* flag */
-        + 8 /* queue offset */
-        + 8 /* physical offset */
-        + 4 /* sys flag */
-        + 8 /* born timestamp */
-        + 8; /* born host */
-
-    public static final int STORE_HOST_POSITION = 4 /* total size */
-        + 4 /* magic code */
-        + 4 /* body CRC */
-        + 4 /* queue id */
-        + 4 /* flag */
-        + 8 /* queue offset */
-        + 8 /* physical offset */
-        + 4 /* sys flag */
-        + 8 /* born timestamp */
-        + 8 /* born host */
-        + 8; /* store timestamp */
-
-    public static int getTotalSize(ByteBuffer message) {
-        return message.getInt(message.position());
-    }
-
-    public static int getMagicCode(ByteBuffer message) {
-        return message.getInt(message.position() + 4);
-    }
-
-    public static long getQueueOffset(ByteBuffer message) {
-        return message.getLong(message.position() + QUEUE_OFFSET_POSITION);
-    }
-
-    public static long getCommitLogOffset(ByteBuffer message) {
-        return message.getLong(message.position() + PHYSICAL_OFFSET_POSITION);
-    }
-
-    public static long getStoreTimeStamp(ByteBuffer message) {
-        return message.getLong(message.position() + STORE_TIMESTAMP_POSITION);
-    }
-
-    public static ByteBuffer getOffsetIdBuffer(ByteBuffer message) {
-        ByteBuffer idBuffer = ByteBuffer.allocate(TieredStoreUtil.MSG_ID_LENGTH);
-        idBuffer.limit(TieredStoreUtil.MSG_ID_LENGTH);
-        idBuffer.putLong(message.getLong(message.position() + STORE_HOST_POSITION));
-        idBuffer.putLong(getCommitLogOffset(message));
-        idBuffer.flip();
-        return idBuffer;
-    }
-
-    public static String getOffsetId(ByteBuffer message) {
-        return UtilAll.bytes2string(getOffsetIdBuffer(message).array());
-    }
-
-    public static Map<String, String> getProperties(ByteBuffer message) {
-        ByteBuffer slice = message.slice();
-        return MessageDecoder.decodeProperties(slice);
-    }
-
-    public static List<SelectBufferResult> splitMessageBuffer(ByteBuffer cqBuffer, ByteBuffer msgBuffer) {
-
-        cqBuffer.rewind();
-        msgBuffer.rewind();
-
-        List<SelectBufferResult> bufferResultList = new ArrayList<>(
-            cqBuffer.remaining() / TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-
-        if (msgBuffer.remaining() == 0) {
-            logger.error("MessageBufferUtil#splitMessage, msg buffer length is zero");
-            return bufferResultList;
-        }
-
-        if (cqBuffer.remaining() % TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE != 0) {
-            logger.error("MessageBufferUtil#splitMessage, consume queue buffer size incorrect, {}", cqBuffer.remaining());
-            return bufferResultList;
-        }
-
-        try {
-            long firstCommitLogOffset = CQItemBufferUtil.getCommitLogOffset(cqBuffer);
-
-            for (int position = cqBuffer.position(); position < cqBuffer.limit();
-                position += TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE) {
-
-                cqBuffer.position(position);
-                long logOffset = CQItemBufferUtil.getCommitLogOffset(cqBuffer);
-                int bufferSize = CQItemBufferUtil.getSize(cqBuffer);
-                long tagCode = CQItemBufferUtil.getTagCode(cqBuffer);
-
-                int offset = (int) (logOffset - firstCommitLogOffset);
-                if (offset + bufferSize > msgBuffer.limit()) {
-                    logger.error("MessageBufferUtil#splitMessage, message buffer size incorrect. " +
-                        "Expect length in consume queue: {}, actual length: {}", offset + bufferSize, msgBuffer.limit());
-                    break;
-                }
-
-                msgBuffer.position(offset);
-                int magicCode = getMagicCode(msgBuffer);
-                if (magicCode == TieredCommitLog.BLANK_MAGIC_CODE) {
-                    offset += TieredCommitLog.CODA_SIZE;
-                    msgBuffer.position(offset);
-                    magicCode = getMagicCode(msgBuffer);
-                }
-                if (magicCode != MessageDecoder.MESSAGE_MAGIC_CODE &&
-                    magicCode != MessageDecoder.MESSAGE_MAGIC_CODE_V2) {
-                    logger.warn("MessageBufferUtil#splitMessage, found unknown magic code. " +
-                        "Message offset: {}, wrong magic code: {}", offset, magicCode);
-                    continue;
-                }
-
-                if (bufferSize != getTotalSize(msgBuffer)) {
-                    logger.warn("MessageBufferUtil#splitMessage, message length in commitlog incorrect. " +
-                        "Except length in commitlog: {}, actual: {}", getTotalSize(msgBuffer), bufferSize);
-                    continue;
-                }
-
-                ByteBuffer sliceBuffer = msgBuffer.slice();
-                sliceBuffer.limit(bufferSize);
-                bufferResultList.add(new SelectBufferResult(sliceBuffer, offset, bufferSize, tagCode));
-            }
-        } catch (Exception e) {
-            logger.error("MessageBufferUtil#splitMessage, split message buffer error", e);
-        } finally {
-            cqBuffer.rewind();
-            msgBuffer.rewind();
-        }
-        return bufferResultList;
-    }
-}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/MessageFormatUtil.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/MessageFormatUtil.java
new file mode 100644
index 00000000000..560234d050a
--- /dev/null
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/MessageFormatUtil.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.util;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.tieredstore.common.SelectBufferResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MessageFormatUtil {
+
+    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
+
+    public static final int MSG_ID_LENGTH = 8 + 8;
+    public static final int MAGIC_CODE_POSITION = 4;
+    public static final int QUEUE_OFFSET_POSITION = 20;
+    public static final int PHYSICAL_OFFSET_POSITION = 28;
+    public static final int SYS_FLAG_OFFSET_POSITION = 36;
+    public static final int STORE_TIMESTAMP_POSITION = 56;
+    public static final int STORE_HOST_POSITION = 64;
+
+    /**
+     * item size:           int, 4 bytes
+     * magic code:          int, 4 bytes
+     * max store timestamp: long, 8 bytes
+     */
+    public static final int COMMIT_LOG_CODA_SIZE = 4 + 8 + 4;
+    public static final int BLANK_MAGIC_CODE = 0xBBCCDDEE ^ 1880681586 + 8;
+
+    /**
+     * commit log offset: long, 8 bytes
+     * message size: int, 4 bytes
+     * tag hash code: long, 8 bytes
+     */
+    public static final int CONSUME_QUEUE_UNIT_SIZE = 8 + 4 + 8;
+
+    public static int getTotalSize(ByteBuffer message) {
+        return message.getInt(message.position());
+    }
+
+    public static int getMagicCode(ByteBuffer message) {
+        return message.getInt(message.position() + MAGIC_CODE_POSITION);
+    }
+
+    public static long getQueueOffset(ByteBuffer message) {
+        return message.getLong(message.position() + QUEUE_OFFSET_POSITION);
+    }
+
+    public static long getCommitLogOffset(ByteBuffer message) {
+        return message.getLong(message.position() + PHYSICAL_OFFSET_POSITION);
+    }
+
+    public static long getStoreTimeStamp(ByteBuffer message) {
+        return message.getLong(message.position() + STORE_TIMESTAMP_POSITION);
+    }
+
+    public static ByteBuffer getOffsetIdBuffer(ByteBuffer message) {
+        ByteBuffer buffer = ByteBuffer.allocate(MSG_ID_LENGTH);
+        buffer.putLong(message.getLong(message.position() + STORE_HOST_POSITION));
+        buffer.putLong(getCommitLogOffset(message));
+        buffer.flip();
+        return buffer;
+    }
+
+    public static String getOffsetId(ByteBuffer message) {
+        return UtilAll.bytes2string(getOffsetIdBuffer(message).array());
+    }
+
+    public static Map<String, String> getProperties(ByteBuffer message) {
+        return MessageDecoder.decodeProperties(message.slice());
+    }
+
+    public static long getCommitLogOffsetFromItem(ByteBuffer cqItem) {
+        return cqItem.getLong(cqItem.position());
+    }
+
+    public static int getSizeFromItem(ByteBuffer cqItem) {
+        return cqItem.getInt(cqItem.position() + 8);
+    }
+
+    public static long getTagCodeFromItem(ByteBuffer cqItem) {
+        return cqItem.getLong(cqItem.position() + 12);
+    }
+
+    public static List<SelectBufferResult> splitMessageBuffer(ByteBuffer cqBuffer, ByteBuffer msgBuffer) {
+
+        if (cqBuffer == null || msgBuffer == null) {
+            log.error("MessageFormatUtil split buffer error, cq buffer or msg buffer is null");
+            return new ArrayList<>();
+        }
+
+        cqBuffer.rewind();
+        msgBuffer.rewind();
+
+        List<SelectBufferResult> bufferResultList = new ArrayList<>(
+            cqBuffer.remaining() / CONSUME_QUEUE_UNIT_SIZE);
+
+        if (msgBuffer.remaining() == 0) {
+            log.error("MessageFormatUtil split buffer error, msg buffer length is 0");
+            return bufferResultList;
+        }
+
+        if (cqBuffer.remaining() == 0 || cqBuffer.remaining() % CONSUME_QUEUE_UNIT_SIZE != 0) {
+            log.error("MessageFormatUtil split buffer error, cq buffer size is {}", cqBuffer.remaining());
+            return bufferResultList;
+        }
+
+        try {
+            long firstCommitLogOffset = MessageFormatUtil.getCommitLogOffsetFromItem(cqBuffer);
+
+            for (int position = cqBuffer.position(); position < cqBuffer.limit();
+                position += CONSUME_QUEUE_UNIT_SIZE) {
+
+                cqBuffer.position(position);
+                long logOffset = MessageFormatUtil.getCommitLogOffsetFromItem(cqBuffer);
+                int bufferSize = MessageFormatUtil.getSizeFromItem(cqBuffer);
+                long tagCode = MessageFormatUtil.getTagCodeFromItem(cqBuffer);
+
+                int offset = (int) (logOffset - firstCommitLogOffset);
+                if (offset + bufferSize > msgBuffer.limit()) {
+                    log.error("MessageFormatUtil split buffer error, message buffer offset exceeded limit. " +
+                        "Expect length: {}, Actual length: {}", offset + bufferSize, msgBuffer.limit());
+                    break;
+                }
+
+                msgBuffer.position(offset);
+                int magicCode = getMagicCode(msgBuffer);
+                if (magicCode == BLANK_MAGIC_CODE) {
+                    offset += COMMIT_LOG_CODA_SIZE;
+                    msgBuffer.position(offset);
+                    magicCode = getMagicCode(msgBuffer);
+                }
+                if (magicCode != MessageDecoder.MESSAGE_MAGIC_CODE &&
+                    magicCode != MessageDecoder.MESSAGE_MAGIC_CODE_V2) {
+                    log.error("MessageFormatUtil split buffer error, found unknown magic code. " +
+                        "Message offset: {}, wrong magic code: {}", offset, magicCode);
+                    continue;
+                }
+
+                if (bufferSize != getTotalSize(msgBuffer)) {
+                    log.error("MessageFormatUtil split buffer error, message length not match. " +
+                        "CommitLog length: {}, buffer length: {}", getTotalSize(msgBuffer), bufferSize);
+                    continue;
+                }
+
+                ByteBuffer sliceBuffer = msgBuffer.slice();
+                sliceBuffer.limit(bufferSize);
+                bufferResultList.add(new SelectBufferResult(sliceBuffer, offset, bufferSize, tagCode));
+            }
+        } finally {
+            cqBuffer.rewind();
+            msgBuffer.rewind();
+        }
+        return bufferResultList;
+    }
+}
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/TieredStoreUtil.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/MessageStoreUtil.java
similarity index 54%
rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/TieredStoreUtil.java
rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/MessageStoreUtil.java
index d15765fcd03..eccde8cad76 100644
--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/TieredStoreUtil.java
+++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/util/MessageStoreUtil.java
@@ -16,27 +16,18 @@
  */
 package org.apache.rocketmq.tieredstore.util;
 
-import com.google.common.annotations.VisibleForTesting;
-import java.io.File;
-import java.lang.reflect.Constructor;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.text.DecimalFormat;
 import java.text.NumberFormat;
-import java.util.LinkedList;
-import java.util.List;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.common.topic.TopicValidator;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
 
-public class TieredStoreUtil {
+public class MessageStoreUtil {
 
-    private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
+    public static final String TIERED_STORE_LOGGER_NAME = "RocketmqTieredStore";
+    public static final String RMQ_SYS_TIERED_STORE_INDEX_TOPIC = "rmq_sys_INDEX";
 
     public static final long BYTE = 1L;
     public static final long KB = BYTE << 10;
@@ -46,23 +37,8 @@ public class TieredStoreUtil {
     public static final long PB = TB << 10;
     public static final long EB = PB << 10;
 
-    public static final String TIERED_STORE_LOGGER_NAME = "RocketmqTieredStore";
-    public static final String RMQ_SYS_TIERED_STORE_INDEX_TOPIC = "rmq_sys_INDEX";
-    public final static int MSG_ID_LENGTH = 8 + 8;
-
     private static final DecimalFormat DEC_FORMAT = new DecimalFormat("#.##");
 
-    private final static List<String> SYSTEM_TOPIC_LIST = new LinkedList<String>() {
-        {
-            add(RMQ_SYS_TIERED_STORE_INDEX_TOPIC);
-        }
-    };
-
-    private final static List<String> SYSTEM_TOPIC_WHITE_LIST = new LinkedList<>();
-
-    @VisibleForTesting
-    public volatile static TieredMetadataStore metadataStoreInstance;
-
     private static String formatSize(long size, long divider, String unitName) {
         return DEC_FORMAT.format((double) size / divider) + unitName;
     }
@@ -82,7 +58,7 @@ public static String toHumanReadable(long size) {
             return formatSize(size, MB, "MB");
         if (size >= KB)
             return formatSize(size, KB, "KB");
-        return formatSize(size, BYTE, "Bytes");
+        return formatSize(size, BYTE, "B");
     }
 
     public static String getHash(String str) {
@@ -91,23 +67,27 @@ public static String getHash(String str) {
             md.update(str.getBytes(StandardCharsets.UTF_8));
             byte[] digest = md.digest();
             return String.format("%032x", new BigInteger(1, digest)).substring(0, 8);
-        } catch (Exception ignore) {
-            return "";
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
         }
     }
 
+    public static String toFilePath(MessageQueue mq) {
+        return String.format("%s/%s/%s", mq.getBrokerName(), mq.getTopic(), mq.getQueueId());
+    }
+
+    public static String getIndexFilePath(String brokerName) {
+        return toFilePath(new MessageQueue(RMQ_SYS_TIERED_STORE_INDEX_TOPIC, brokerName, 0));
+    }
+
     public static String offset2FileName(final long offset) {
         final NumberFormat numberFormat = NumberFormat.getInstance();
-
         numberFormat.setMinimumIntegerDigits(20);
         numberFormat.setMaximumFractionDigits(0);
         numberFormat.setGroupingUsed(false);
-
         try {
             MessageDigest md = MessageDigest.getInstance("MD5");
-
             md.update(Long.toString(offset).getBytes(StandardCharsets.UTF_8));
-
             byte[] digest = md.digest();
             String hash = String.format("%032x", new BigInteger(1, digest)).substring(0, 8);
             return hash + numberFormat.format(offset);
@@ -119,52 +99,4 @@ public static String offset2FileName(final long offset) {
     public static long fileName2Offset(final String fileName) {
         return Long.parseLong(fileName.substring(fileName.length() - 20));
     }
-
-    public static void addSystemTopic(final String topic) {
-        SYSTEM_TOPIC_LIST.add(topic);
-    }
-
-    public static boolean isSystemTopic(final String topic) {
-        if (StringUtils.isBlank(topic)) {
-            return false;
-        }
-
-        if (SYSTEM_TOPIC_WHITE_LIST.contains(topic)) {
-            return false;
-        }
-
-        if (SYSTEM_TOPIC_LIST.contains(topic)) {
-            return true;
-        }
-        return TopicValidator.isSystemTopic(topic);
-    }
-
-    public static TieredMetadataStore getMetadataStore(TieredMessageStoreConfig storeConfig) {
-        if (storeConfig == null) {
-            return metadataStoreInstance;
-        }
-
-        if (metadataStoreInstance == null) {
-            synchronized (TieredMetadataStore.class) {
-                if (metadataStoreInstance == null) {
-                    try {
-                        Class<? extends TieredMetadataStore> clazz = Class.forName(
-                            storeConfig.getTieredMetadataServiceProvider()).asSubclass(TieredMetadataStore.class);
-                        Constructor<? extends TieredMetadataStore> constructor =
-                            clazz.getConstructor(TieredMessageStoreConfig.class);
-                        metadataStoreInstance = constructor.newInstance(storeConfig);
-                    } catch (Exception e) {
-                        logger.error("TieredMetadataStore#getInstance: " +
-                            "build metadata store failed, provider class: {}",
-                            storeConfig.getTieredMetadataServiceProvider(), e);
-                    }
-                }
-            }
-        }
-        return metadataStoreInstance;
-    }
-
-    public static String toPath(MessageQueue mq) {
-        return mq.getBrokerName() + File.separator + mq.getTopic() + File.separator + mq.getQueueId();
-    }
 }
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java
deleted file mode 100644
index 5791dc9a4e2..00000000000
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Objects;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.store.ConsumeQueue;
-import org.apache.rocketmq.store.DefaultMessageStore;
-import org.apache.rocketmq.store.DispatchRequest;
-import org.apache.rocketmq.store.SelectMappedBufferResult;
-import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.file.CompositeQueueFlatFile;
-import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue;
-import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager;
-import org.apache.rocketmq.tieredstore.metadata.FileSegmentMetadata;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
-import org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-public class TieredDispatcherTest {
-
-    private final String storePath = TieredStoreTestUtil.getRandomStorePath();
-    private TieredMessageStoreConfig storeConfig;
-    private MessageQueue mq;
-    private TieredMetadataStore metadataStore;
-
-    @Before
-    public void setUp() {
-        storeConfig = new TieredMessageStoreConfig();
-        storeConfig.setStorePathRootDir(storePath);
-        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegmentWithoutCheck");
-        storeConfig.setBrokerName(storeConfig.getBrokerName());
-        mq = new MessageQueue("CompositeQueueFlatFileTest", storeConfig.getBrokerName(), 0);
-        metadataStore = TieredStoreUtil.getMetadataStore(storeConfig);
-        TieredStoreExecutor.init();
-    }
-
-    @After
-    public void tearDown() throws IOException {
-        TieredStoreTestUtil.destroyCompositeFlatFileManager();
-        TieredStoreTestUtil.destroyMetadataStore();
-        TieredStoreTestUtil.destroyTempDir(storePath);
-        TieredStoreExecutor.shutdown();
-    }
-
-    @Test
-    public void testDispatch() {
-        metadataStore.addQueue(mq, 6);
-        MemoryFileSegment segment = new MemoryFileSegment(FileSegmentType.COMMIT_LOG, mq, 1000, storeConfig);
-        segment.initPosition(segment.getSize());
-
-        String filePath1 = TieredStoreUtil.toPath(mq);
-        FileSegmentMetadata segmentMetadata = new FileSegmentMetadata(
-            filePath1, 1000, FileSegmentType.COMMIT_LOG.getType());
-        metadataStore.updateFileSegment(segmentMetadata);
-        metadataStore.updateFileSegment(segmentMetadata);
-
-        segment = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, mq,
-            6 * TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE, storeConfig);
-        FileSegmentMetadata segmentMetadata2 = new FileSegmentMetadata(
-            filePath1, segment.getBaseOffset(), FileSegmentType.CONSUME_QUEUE.getType());
-        metadataStore.updateFileSegment(segmentMetadata2);
-
-        TieredFlatFileManager flatFileManager = TieredFlatFileManager.getInstance(storeConfig);
-        DefaultMessageStore defaultMessageStore = Mockito.mock(DefaultMessageStore.class);
-        TieredDispatcher dispatcher = new TieredDispatcher(defaultMessageStore, storeConfig);
-
-        SelectMappedBufferResult mockResult = new SelectMappedBufferResult(0, MessageBufferUtilTest.buildMockedMessageBuffer(), MessageBufferUtilTest.MSG_LEN, null);
-        Mockito.when(defaultMessageStore.selectOneMessageByOffset(7, MessageBufferUtilTest.MSG_LEN)).thenReturn(mockResult);
-        DispatchRequest request = new DispatchRequest(mq.getTopic(), mq.getQueueId(), 6, 7, MessageBufferUtilTest.MSG_LEN, 1);
-        dispatcher.dispatch(request);
-        Assert.assertNotNull(flatFileManager.getFlatFile(mq));
-        Assert.assertEquals(7, Objects.requireNonNull(flatFileManager.getFlatFile(mq)).getDispatchOffset());
-
-        CompositeQueueFlatFile flatFile = flatFileManager.getOrCreateFlatFileIfAbsent(mq);
-        Assert.assertNotNull(flatFile);
-        flatFile.commit(true);
-        Assert.assertEquals(6, flatFile.getConsumeQueueMaxOffset());
-
-        dispatcher.buildConsumeQueueAndIndexFile();
-        Assert.assertEquals(7, flatFile.getConsumeQueueMaxOffset());
-
-        ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer();
-        buffer1.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 7);
-        flatFile.appendCommitLog(buffer1);
-        ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer();
-        buffer2.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 8);
-        flatFile.appendCommitLog(buffer2);
-        ByteBuffer buffer3 = MessageBufferUtilTest.buildMockedMessageBuffer();
-        buffer3.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 9);
-        flatFile.appendCommitLog(buffer3);
-        flatFile.commitCommitLog();
-        Assert.assertEquals(9 + 1, flatFile.getDispatchOffset());
-        Assert.assertEquals(9, flatFile.getCommitLogDispatchCommitOffset());
-
-        dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 8, 8, 0, 0, buffer1);
-        dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 9, 9, 0, 0, buffer2);
-        dispatcher.buildConsumeQueueAndIndexFile();
-        Assert.assertEquals(7, flatFile.getConsumeQueueMaxOffset());
-
-        dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 7, 7, 0, 0, buffer1);
-        dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 8, 8, 0, 0, buffer2);
-        dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 9, 9, 0, 0, buffer3);
-        dispatcher.buildConsumeQueueAndIndexFile();
-        Assert.assertEquals(6, flatFile.getConsumeQueueMinOffset());
-        Assert.assertEquals(9 + 1, flatFile.getConsumeQueueMaxOffset());
-    }
-
-    @Test
-    public void testDispatchByFlatFile() {
-        metadataStore.addQueue(mq, 6);
-        TieredFlatFileManager flatFileManager = TieredFlatFileManager.getInstance(storeConfig);
-        DefaultMessageStore defaultStore = Mockito.mock(DefaultMessageStore.class);
-        Mockito.when(defaultStore.getConsumeQueue(mq.getTopic(), mq.getQueueId())).thenReturn(Mockito.mock(ConsumeQueue.class));
-        TieredDispatcher dispatcher = new TieredDispatcher(defaultStore, storeConfig);
-
-        Mockito.when(defaultStore.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId())).thenReturn(0L);
-        Mockito.when(defaultStore.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId())).thenReturn(9L);
-
-        // mock cq item, position = 7
-        ByteBuffer cqItem = ByteBuffer.allocate(ConsumeQueue.CQ_STORE_UNIT_SIZE);
-        cqItem.putLong(7);
-        cqItem.putInt(MessageBufferUtilTest.MSG_LEN);
-        cqItem.putLong(1);
-        cqItem.flip();
-        SelectMappedBufferResult mockResult = new SelectMappedBufferResult(0, cqItem, ConsumeQueue.CQ_STORE_UNIT_SIZE, null);
-        Mockito.when(((ConsumeQueue) defaultStore.getConsumeQueue(mq.getTopic(), mq.getQueueId())).getIndexBuffer(6)).thenReturn(mockResult);
-
-        // mock cq item, position = 8
-        cqItem = ByteBuffer.allocate(ConsumeQueue.CQ_STORE_UNIT_SIZE);
-        cqItem.putLong(8);
-        cqItem.putInt(MessageBufferUtilTest.MSG_LEN);
-        cqItem.putLong(1);
-        cqItem.flip();
-        mockResult = new SelectMappedBufferResult(0, cqItem, ConsumeQueue.CQ_STORE_UNIT_SIZE, null);
-        Mockito.when(((ConsumeQueue) defaultStore.getConsumeQueue(mq.getTopic(), mq.getQueueId())).getIndexBuffer(7)).thenReturn(mockResult);
-
-        mockResult = new SelectMappedBufferResult(0, MessageBufferUtilTest.buildMockedMessageBuffer(), MessageBufferUtilTest.MSG_LEN, null);
-        Mockito.when(defaultStore.selectOneMessageByOffset(7, MessageBufferUtilTest.MSG_LEN)).thenReturn(mockResult);
-
-        ByteBuffer msg = MessageBufferUtilTest.buildMockedMessageBuffer();
-        msg.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 7);
-        mockResult = new SelectMappedBufferResult(0, msg, MessageBufferUtilTest.MSG_LEN, null);
-        Mockito.when(defaultStore.selectOneMessageByOffset(8, MessageBufferUtilTest.MSG_LEN)).thenReturn(mockResult);
-
-        CompositeQueueFlatFile flatFile = flatFileManager.getOrCreateFlatFileIfAbsent(mq);
-        Assert.assertNotNull(flatFile);
-        flatFile.initOffset(7);
-        dispatcher.dispatchFlatFile(flatFile);
-        Assert.assertEquals(8, flatFileManager.getFlatFile(mq).getDispatchOffset());
-    }
-}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java
deleted file mode 100644
index 4e8287533f5..00000000000
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Objects;
-import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang3.SystemUtils;
-import org.apache.commons.lang3.tuple.Triple;
-import org.apache.rocketmq.common.BoundaryType;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.store.DispatchRequest;
-import org.apache.rocketmq.store.GetMessageResult;
-import org.apache.rocketmq.store.GetMessageStatus;
-import org.apache.rocketmq.store.QueryMessageResult;
-import org.apache.rocketmq.store.SelectMappedBufferResult;
-import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.SelectBufferResultWrapper;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.file.CompositeFlatFile;
-import org.apache.rocketmq.tieredstore.file.CompositeQueueFlatFile;
-import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-import org.awaitility.Awaitility;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TieredMessageFetcherTest {
-
-    private final String storePath = TieredStoreTestUtil.getRandomStorePath();
-    private TieredMessageStoreConfig storeConfig;
-    private MessageQueue mq;
-
-    @Before
-    public void setUp() {
-        storeConfig = new TieredMessageStoreConfig();
-        storeConfig.setStorePathRootDir(storePath);
-        storeConfig.setBrokerName(storeConfig.getBrokerName());
-        storeConfig.setReadAheadCacheExpireDuration(Long.MAX_VALUE);
-        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegmentWithoutCheck");
-        storeConfig.setTieredStoreIndexFileMaxHashSlotNum(2);
-        storeConfig.setTieredStoreIndexFileMaxIndexNum(3);
-        mq = new MessageQueue("TieredMessageFetcherTest", storeConfig.getBrokerName(), 0);
-        TieredStoreUtil.getMetadataStore(storeConfig);
-        TieredStoreExecutor.init();
-    }
-
-    @After
-    public void tearDown() throws IOException {
-        TieredStoreTestUtil.destroyCompositeFlatFileManager();
-        TieredStoreTestUtil.destroyMetadataStore();
-        TieredStoreTestUtil.destroyTempDir(storePath);
-        TieredStoreExecutor.shutdown();
-    }
-
-    public Triple<TieredMessageFetcher, ByteBuffer, ByteBuffer> buildFetcher() {
-        TieredFlatFileManager flatFileManager = TieredFlatFileManager.getInstance(storeConfig);
-        TieredMessageFetcher fetcher = new TieredMessageFetcher(storeConfig);
-        GetMessageResult getMessageResult = fetcher.getMessageAsync("group", mq.getTopic(), mq.getQueueId(), 0, 32, null).join();
-        Assert.assertEquals(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE, getMessageResult.getStatus());
-
-        CompositeFlatFile flatFile = flatFileManager.getOrCreateFlatFileIfAbsent(mq);
-        Assert.assertNotNull(flatFile);
-        flatFile.initOffset(0);
-
-        getMessageResult = fetcher.getMessageAsync("group", mq.getTopic(), mq.getQueueId(), 0, 32, null).join();
-        Assert.assertEquals(GetMessageStatus.NO_MESSAGE_IN_QUEUE, getMessageResult.getStatus());
-
-        ByteBuffer msg1 = MessageBufferUtilTest.buildMockedMessageBuffer();
-        msg1.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 0);
-        msg1.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, 0);
-        AppendResult result = flatFile.appendCommitLog(msg1);
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-
-        ByteBuffer msg2 = MessageBufferUtilTest.buildMockedMessageBuffer();
-        msg2.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 1);
-        msg2.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN);
-        flatFile.appendCommitLog(msg2);
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-
-        result = flatFile.appendConsumeQueue(new DispatchRequest(mq.getTopic(), mq.getQueueId(), 0, 0, MessageBufferUtilTest.MSG_LEN, 0));
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-        result = flatFile.appendConsumeQueue(new DispatchRequest(mq.getTopic(), mq.getQueueId(), 1, MessageBufferUtilTest.MSG_LEN, MessageBufferUtilTest.MSG_LEN, 0));
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-
-        flatFile.commit(true);
-        return Triple.of(fetcher, msg1, msg2);
-    }
-
-    @Test
-    public void testGetMessageFromTieredStoreAsync() {
-        Triple<TieredMessageFetcher, ByteBuffer, ByteBuffer> triple = buildFetcher();
-        TieredMessageFetcher fetcher = triple.getLeft();
-        ByteBuffer msg1 = triple.getMiddle();
-        ByteBuffer msg2 = triple.getRight();
-        CompositeQueueFlatFile flatFile = TieredFlatFileManager.getInstance(storeConfig).getFlatFile(mq);
-        Assert.assertNotNull(flatFile);
-
-        GetMessageResult getMessageResult = fetcher.getMessageFromTieredStoreAsync(flatFile, 0, 32).join();
-        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
-        Assert.assertEquals(2, getMessageResult.getMessageBufferList().size());
-        Assert.assertEquals(msg1, getMessageResult.getMessageBufferList().get(0));
-        Assert.assertEquals(msg2, getMessageResult.getMessageBufferList().get(1));
-
-        AppendResult result = flatFile.appendConsumeQueue(new DispatchRequest(mq.getTopic(), mq.getQueueId(), 2, storeConfig.getReadAheadMessageSizeThreshold(), MessageBufferUtilTest.MSG_LEN, 0));
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-        flatFile.commit(true);
-        getMessageResult = fetcher.getMessageFromTieredStoreAsync(flatFile, 0, 32).join();
-        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
-        Assert.assertEquals(2, getMessageResult.getMessageBufferList().size());
-    }
-
-    @Test
-    public void testGetMessageFromCacheAsync() {
-        Triple<TieredMessageFetcher, ByteBuffer, ByteBuffer> triple = buildFetcher();
-        TieredMessageFetcher fetcher = triple.getLeft();
-        ByteBuffer msg1 = triple.getMiddle();
-        ByteBuffer msg2 = triple.getRight();
-        CompositeQueueFlatFile flatFile = TieredFlatFileManager.getInstance(storeConfig).getFlatFile(mq);
-        Assert.assertNotNull(flatFile);
-
-        fetcher.recordCacheAccess(flatFile, "prevent-invalid-cache", 0, new ArrayList<>());
-        Assert.assertEquals(0, fetcher.getMessageCache().estimatedSize());
-        SelectMappedBufferResult bufferResult = new SelectMappedBufferResult(0, msg1, msg1.remaining(), null);
-        fetcher.putMessageToCache(flatFile, new SelectBufferResultWrapper(bufferResult, 0, 0, false));
-        Assert.assertEquals(1, fetcher.getMessageCache().estimatedSize());
-
-        GetMessageResult getMessageResult = fetcher.getMessageFromCacheAsync(flatFile, "group", 0, 32, true).join();
-        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
-        Assert.assertEquals(1, getMessageResult.getMessageBufferList().size());
-        Assert.assertEquals(msg1, getMessageResult.getMessageBufferList().get(0));
-
-        Awaitility.waitAtMost(3, TimeUnit.SECONDS)
-            .until(() -> fetcher.getMessageCache().estimatedSize() == 2);
-        ArrayList<SelectBufferResultWrapper> wrapperList = new ArrayList<>();
-        wrapperList.add(fetcher.getMessageFromCache(flatFile, 0));
-        fetcher.recordCacheAccess(flatFile, "prevent-invalid-cache", 0, wrapperList);
-        Assert.assertEquals(1, fetcher.getMessageCache().estimatedSize());
-        wrapperList.clear();
-        wrapperList.add(fetcher.getMessageFromCache(flatFile, 1));
-        fetcher.recordCacheAccess(flatFile, "prevent-invalid-cache", 0, wrapperList);
-        Assert.assertEquals(1, fetcher.getMessageCache().estimatedSize());
-
-        SelectMappedBufferResult messageFromCache =
-            Objects.requireNonNull(fetcher.getMessageFromCache(flatFile, 1)).getDuplicateResult();
-        fetcher.recordCacheAccess(flatFile, "group", 0, wrapperList);
-        Assert.assertNotNull(messageFromCache);
-        Assert.assertEquals(msg2, messageFromCache.getByteBuffer());
-        Assert.assertEquals(0, fetcher.getMessageCache().estimatedSize());
-    }
-
-    @Test
-    public void testGetMessageAsync() {
-        Triple<TieredMessageFetcher, ByteBuffer, ByteBuffer> triple = buildFetcher();
-        TieredMessageFetcher fetcher = triple.getLeft();
-        ByteBuffer msg1 = triple.getMiddle();
-        ByteBuffer msg2 = triple.getRight();
-
-        GetMessageResult getMessageResult = fetcher.getMessageAsync("group", mq.getTopic(), mq.getQueueId(), -1, 32, null).join();
-        Assert.assertEquals(GetMessageStatus.OFFSET_TOO_SMALL, getMessageResult.getStatus());
-
-        getMessageResult = fetcher.getMessageAsync("group", mq.getTopic(), mq.getQueueId(), 2, 32, null).join();
-        Assert.assertEquals(GetMessageStatus.OFFSET_OVERFLOW_ONE, getMessageResult.getStatus());
-
-        getMessageResult = fetcher.getMessageAsync("group", mq.getTopic(), mq.getQueueId(), 3, 32, null).join();
-        Assert.assertEquals(GetMessageStatus.OFFSET_OVERFLOW_BADLY, getMessageResult.getStatus());
-
-        getMessageResult = fetcher.getMessageAsync("group", mq.getTopic(), mq.getQueueId(), 0, 32, null).join();
-        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
-        Assert.assertEquals(2, getMessageResult.getMessageBufferList().size());
-        Assert.assertEquals(msg1, getMessageResult.getMessageBufferList().get(0));
-        Assert.assertEquals(msg2, getMessageResult.getMessageBufferList().get(1));
-    }
-
-    @Test
-    public void testGetMessageStoreTimeStampAsync() {
-        TieredMessageFetcher fetcher = new TieredMessageFetcher(storeConfig);
-        CompositeFlatFile flatFile = TieredFlatFileManager.getInstance(storeConfig).getOrCreateFlatFileIfAbsent(mq);
-        Assert.assertNotNull(flatFile);
-        flatFile.initOffset(0);
-
-        ByteBuffer msg1 = MessageBufferUtilTest.buildMockedMessageBuffer();
-        msg1.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 0);
-        msg1.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, 0);
-        long currentTimeMillis1 = System.currentTimeMillis();
-        msg1.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, currentTimeMillis1);
-        AppendResult result = flatFile.appendCommitLog(msg1);
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-
-        ByteBuffer msg2 = MessageBufferUtilTest.buildMockedMessageBuffer();
-        msg2.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 1);
-        msg2.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN);
-        long currentTimeMillis2 = System.currentTimeMillis();
-        msg2.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, currentTimeMillis2);
-        flatFile.appendCommitLog(msg2);
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-
-        result = flatFile.appendConsumeQueue(new DispatchRequest(mq.getTopic(), mq.getQueueId(), 0, 0, MessageBufferUtilTest.MSG_LEN, 0));
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-        result = flatFile.appendConsumeQueue(new DispatchRequest(mq.getTopic(), mq.getQueueId(), 1, MessageBufferUtilTest.MSG_LEN, MessageBufferUtilTest.MSG_LEN, 0));
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-
-        flatFile.commit(true);
-
-        long result1 = fetcher.getEarliestMessageTimeAsync(mq.getTopic(), mq.getQueueId()).join();
-        long result2 = fetcher.getMessageStoreTimeStampAsync(mq.getTopic(), mq.getQueueId(), 0).join();
-        Assert.assertEquals(result1, result2);
-        Assert.assertEquals(currentTimeMillis1, result1);
-
-        long result3 = fetcher.getMessageStoreTimeStampAsync(mq.getTopic(), mq.getQueueId(), 1).join();
-        Assert.assertEquals(currentTimeMillis2, result3);
-    }
-
-    @Test
-    public void testGetOffsetInQueueByTime() {
-        TieredMessageFetcher fetcher = new TieredMessageFetcher(storeConfig);
-        Assert.assertEquals(-1, fetcher.getOffsetInQueueByTime(mq.getTopic(), mq.getQueueId(), 0, BoundaryType.LOWER));
-
-        CompositeQueueFlatFile flatFile = TieredFlatFileManager.getInstance(storeConfig).getOrCreateFlatFileIfAbsent(mq);
-        Assert.assertEquals(-1, fetcher.getOffsetInQueueByTime(mq.getTopic(), mq.getQueueId(), 0, BoundaryType.LOWER));
-        Assert.assertNotNull(flatFile);
-
-        // offset has not been initialized, so put message would be failed
-        AppendResult appendResult = flatFile.appendConsumeQueue(new DispatchRequest(mq.getTopic(), mq.getQueueId(), 50, 0, MessageBufferUtilTest.MSG_LEN, 0), true);
-        Assert.assertEquals(AppendResult.OFFSET_INCORRECT, appendResult);
-        flatFile.commit(true);
-        Assert.assertEquals(-1, fetcher.getOffsetInQueueByTime(mq.getTopic(), mq.getQueueId(), 0, BoundaryType.LOWER));
-
-        long timestamp = System.currentTimeMillis();
-        ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-        buffer.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 50);
-        buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, timestamp);
-        flatFile.initOffset(50);
-        flatFile.appendCommitLog(buffer, true);
-        appendResult = flatFile.appendConsumeQueue(new DispatchRequest(mq.getTopic(), mq.getQueueId(), 0, MessageBufferUtilTest.MSG_LEN, 0, timestamp, 50, "", "", 0, 0, null), true);
-        Assert.assertEquals(AppendResult.SUCCESS, appendResult);
-        flatFile.commit(true);
-        Assert.assertEquals(50, fetcher.getOffsetInQueueByTime(mq.getTopic(), mq.getQueueId(), 0, BoundaryType.LOWER));
-    }
-
-    @Test
-    public void testQueryMessageAsync() {
-        // skip this test on windows
-        Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS);
-        Assume.assumeFalse(SystemUtils.IS_OS_LINUX);
-
-        TieredMessageFetcher fetcher = new TieredMessageFetcher(storeConfig);
-        Assert.assertEquals(0, fetcher.queryMessageAsync(mq.getTopic(), "key", 32, 0, Long.MAX_VALUE).join().getMessageMapedList().size());
-
-        CompositeQueueFlatFile flatFile = TieredFlatFileManager.getInstance(storeConfig).getOrCreateFlatFileIfAbsent(mq);
-        Assert.assertEquals(0, fetcher.queryMessageAsync(mq.getTopic(), "key", 32, 0, Long.MAX_VALUE).join().getMessageMapedList().size());
-
-        Assert.assertNotNull(flatFile);
-        flatFile.initOffset(0);
-        ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-        buffer.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 0);
-        flatFile.appendCommitLog(buffer);
-        buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-        buffer.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 1);
-        flatFile.appendCommitLog(buffer);
-        buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-        buffer.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 2);
-        flatFile.appendCommitLog(buffer);
-
-        long timestamp = System.currentTimeMillis();
-        DispatchRequest request = new DispatchRequest(mq.getTopic(), mq.getQueueId(), 0, MessageBufferUtilTest.MSG_LEN, 0, timestamp, 0, "", "key", 0, 0, null);
-        flatFile.appendIndexFile(request);
-        request = new DispatchRequest(mq.getTopic(), mq.getQueueId(), MessageBufferUtilTest.MSG_LEN, MessageBufferUtilTest.MSG_LEN, 0, timestamp + 1, 0, "", "key", 0, 0, null);
-        flatFile.appendIndexFile(request);
-        request = new DispatchRequest(mq.getTopic(), mq.getQueueId(), MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtilTest.MSG_LEN, 0, timestamp + 2, 0, "", "another-key", 0, 0, null);
-        flatFile.appendIndexFile(request);
-        flatFile.commit(true);
-        Assert.assertEquals(1, fetcher.queryMessageAsync(mq.getTopic(), "key", 1, 0, Long.MAX_VALUE).join().getMessageMapedList().size());
-
-        QueryMessageResult result = fetcher.queryMessageAsync(mq.getTopic(), "key", 32, 0, Long.MAX_VALUE).join();
-        Assert.assertEquals(2, result.getMessageMapedList().size());
-        Assert.assertEquals(0, result.getMessageMapedList().get(0).getByteBuffer().getLong(MessageBufferUtil.QUEUE_OFFSET_POSITION));
-        Assert.assertEquals(1, result.getMessageMapedList().get(1).getByteBuffer().getLong(MessageBufferUtil.QUEUE_OFFSET_POSITION));
-    }
-}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java
index 07af1fc8b1f..bb259ae811a 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java
@@ -18,30 +18,40 @@
 
 import io.opentelemetry.api.common.Attributes;
 import io.opentelemetry.sdk.OpenTelemetrySdk;
+import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Locale;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.common.BoundaryType;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.Configuration;
-import org.apache.rocketmq.store.CommitLog;
 import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.DispatchRequest;
 import org.apache.rocketmq.store.GetMessageResult;
 import org.apache.rocketmq.store.GetMessageStatus;
-import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.QueryMessageResult;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.plugin.MessageStorePluginContext;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.file.CompositeQueueFlatFile;
-import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-import org.apache.rocketmq.common.BoundaryType;
+import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
+import org.apache.rocketmq.store.queue.CqUnit;
+import org.apache.rocketmq.tieredstore.core.MessageStoreFetcher;
+import org.apache.rocketmq.tieredstore.file.FlatFileStore;
+import org.apache.rocketmq.tieredstore.file.FlatMessageFile;
+import org.apache.rocketmq.tieredstore.provider.PosixFileSegment;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtil;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtilTest;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -53,186 +63,231 @@
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class TieredMessageStoreTest {
 
-    private final String storePath = TieredStoreTestUtil.getRandomStorePath();
+    private final String brokerName = "brokerName";
+    private final String storePath = MessageStoreUtilTest.getRandomStorePath();
+    private final MessageQueue mq = new MessageQueue("MessageStoreTest", brokerName, 0);
 
-    private MessageStoreConfig storeConfig;
-    private MessageQueue mq;
-    private MessageStore nextStore;
-    private TieredMessageStore store;
-    private TieredMessageFetcher fetcher;
     private Configuration configuration;
-    private TieredFlatFileManager flatFileManager;
+    private DefaultMessageStore defaultStore;
+    private TieredMessageStore currentStore;
+    private FlatFileStore flatFileStore;
+    private MessageStoreFetcher fetcher;
 
     @Before
-    public void setUp() {
-        storeConfig = new MessageStoreConfig();
-        storeConfig.setStorePathRootDir(storePath);
-        mq = new MessageQueue("TieredMessageStoreTest", "broker", 0);
-
-        nextStore = Mockito.mock(DefaultMessageStore.class);
-        CommitLog commitLog = mock(CommitLog.class);
-        when(commitLog.getMinOffset()).thenReturn(100L);
-        when(nextStore.getCommitLog()).thenReturn(commitLog);
-
+    public void init() throws Exception {
         BrokerConfig brokerConfig = new BrokerConfig();
-        brokerConfig.setBrokerName("broker");
-        configuration = new Configuration(LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME), "/tmp/rmqut/config", storeConfig, brokerConfig);
+        brokerConfig.setBrokerName(brokerName);
+
         Properties properties = new Properties();
-        properties.setProperty("tieredBackendServiceProvider", "org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment");
+        properties.setProperty("recordGetMessageResult", Boolean.TRUE.toString().toLowerCase(Locale.ROOT));
+        properties.setProperty("tieredBackendServiceProvider", PosixFileSegment.class.getName());
+
+        configuration = new Configuration(LoggerFactory.getLogger(
+            MessageStoreUtil.TIERED_STORE_LOGGER_NAME), storePath + File.separator + "conf",
+            new org.apache.rocketmq.tieredstore.MessageStoreConfig(), brokerConfig);
         configuration.registerConfig(properties);
-        MessageStorePluginContext context = new MessageStorePluginContext(new MessageStoreConfig(), null, null, brokerConfig, configuration);
 
-        store = new TieredMessageStore(context, nextStore);
+        MessageStorePluginContext context = new MessageStorePluginContext(
+            new MessageStoreConfig(), null, null, brokerConfig, configuration);
+
+        defaultStore = Mockito.mock(DefaultMessageStore.class);
+        Mockito.when(defaultStore.load()).thenReturn(true);
+
+        currentStore = new TieredMessageStore(context, defaultStore);
+        Assert.assertNotNull(currentStore.getStoreConfig());
+        Assert.assertNotNull(currentStore.getBrokerName());
+        Assert.assertEquals(defaultStore, currentStore.getDefaultStore());
+        Assert.assertNotNull(currentStore.getMetadataStore());
+        Assert.assertNotNull(currentStore.getTopicFilter());
+        Assert.assertNotNull(currentStore.getStoreExecutor());
+        Assert.assertNotNull(currentStore.getFlatFileStore());
+        Assert.assertNotNull(currentStore.getIndexService());
 
-        fetcher = Mockito.mock(TieredMessageFetcher.class);
+        fetcher = Mockito.spy(currentStore.fetcher);
         try {
-            Field field = store.getClass().getDeclaredField("fetcher");
+            Field field = currentStore.getClass().getDeclaredField("fetcher");
             field.setAccessible(true);
-            field.set(store, fetcher);
+            field.set(currentStore, fetcher);
         } catch (NoSuchFieldException | IllegalAccessException e) {
             Assert.fail(e.getClass().getCanonicalName() + ": " + e.getMessage());
         }
 
-        TieredFlatFileManager.getInstance(store.getStoreConfig()).getOrCreateFlatFileIfAbsent(mq);
+        flatFileStore = currentStore.getFlatFileStore();
+
+        Mockito.when(defaultStore.getMinOffsetInQueue(anyString(), anyInt())).thenReturn(100L);
+        Mockito.when(defaultStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(200L);
+        ConsumeQueueInterface cq = Mockito.mock(ConsumeQueueInterface.class);
+        Mockito.when(defaultStore.getConsumeQueue(anyString(), anyInt())).thenReturn(cq);
+
+        ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+        Mockito.when(cq.get(anyLong())).thenReturn(
+            new CqUnit(100, 1000, buffer.remaining(), 0L));
+        Mockito.when(defaultStore.selectOneMessageByOffset(anyLong(), anyInt())).thenReturn(
+            new SelectMappedBufferResult(0L, buffer.asReadOnlyBuffer(), buffer.remaining(), null));
+        currentStore.load();
+
+        FlatMessageFile flatFile = currentStore.getFlatFileStore().computeIfAbsent(mq);
+        Assert.assertNotNull(flatFile);
+        currentStore.dispatcher.doScheduleDispatch(flatFile, true).join();
+
+        for (int i = 100; i < 200; i++) {
+            SelectMappedBufferResult bufferResult = new SelectMappedBufferResult(
+                0L, buffer, buffer.remaining(), null);
+            DispatchRequest request = new DispatchRequest(mq.getTopic(), mq.getQueueId(),
+                MessageFormatUtil.getCommitLogOffset(buffer), buffer.remaining(), 0L,
+                MessageFormatUtil.getStoreTimeStamp(buffer), 0L,
+                "", "", 0, 0L, new HashMap<>());
+            flatFile.appendCommitLog(bufferResult);
+            flatFile.appendConsumeQueue(request);
+        }
+        currentStore.dispatcher.doScheduleDispatch(flatFile, true).join();
     }
 
     @After
-    public void tearDown() throws IOException {
-        TieredStoreExecutor.shutdown();
-        TieredStoreTestUtil.destroyCompositeFlatFileManager();
-        TieredStoreTestUtil.destroyMetadataStore();
-        TieredStoreTestUtil.destroyTempDir(storePath);
-    }
-
-    private void mockCompositeFlatFile() {
-        flatFileManager = Mockito.mock(TieredFlatFileManager.class);
-        CompositeQueueFlatFile flatFile = Mockito.mock(CompositeQueueFlatFile.class);
-        when(flatFile.getConsumeQueueCommitOffset()).thenReturn(Long.MAX_VALUE);
-        when(flatFileManager.getFlatFile(mq)).thenReturn(flatFile);
-        try {
-            Field field = store.getClass().getDeclaredField("flatFileManager");
-            field.setAccessible(true);
-            field.set(store, flatFileManager);
-        } catch (NoSuchFieldException | IllegalAccessException e) {
-            Assert.fail(e.getClass().getCanonicalName() + ": " + e.getMessage());
-        }
+    public void shutdown() throws IOException {
+        currentStore.shutdown();
+        currentStore.destroy();
+        MessageStoreUtilTest.deleteStoreDirectory(storePath);
     }
 
     @Test
     public void testViaTieredStorage() {
-        mockCompositeFlatFile();
         Properties properties = new Properties();
+
         // TieredStorageLevel.DISABLE
         properties.setProperty("tieredStorageLevel", "0");
         configuration.update(properties);
-        Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
+        Assert.assertFalse(currentStore.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
 
         // TieredStorageLevel.NOT_IN_DISK
         properties.setProperty("tieredStorageLevel", "1");
         configuration.update(properties);
-        when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false);
-        Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
+        when(defaultStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false);
+        Assert.assertTrue(currentStore.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
 
-        when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true);
-        Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
+        when(defaultStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true);
+        Assert.assertFalse(currentStore.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
 
         // TieredStorageLevel.NOT_IN_MEM
         properties.setProperty("tieredStorageLevel", "2");
         configuration.update(properties);
-        Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false);
-        Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true);
-        Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
+        Mockito.when(defaultStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false);
+        Mockito.when(defaultStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true);
+        Assert.assertTrue(currentStore.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
 
-        Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true);
-        Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(false);
-        Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
+        Mockito.when(defaultStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true);
+        Mockito.when(defaultStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(false);
+        Assert.assertTrue(currentStore.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
 
-        Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true);
-        Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true);
-        Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
+        Mockito.when(defaultStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true);
+        Mockito.when(defaultStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true);
+        Assert.assertFalse(currentStore.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
 
         // TieredStorageLevel.FORCE
         properties.setProperty("tieredStorageLevel", "3");
         configuration.update(properties);
-        Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
+        Assert.assertTrue(currentStore.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0));
     }
 
     @Test
     public void testGetMessageAsync() {
-        mockCompositeFlatFile();
-        GetMessageResult result1 = new GetMessageResult();
-        result1.setStatus(GetMessageStatus.FOUND);
-        GetMessageResult result2 = new GetMessageResult();
-        result2.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY);
-
-        when(fetcher.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(CompletableFuture.completedFuture(result1));
-        when(nextStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(result2);
-        Assert.assertSame(result1, store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null));
-
-        result1.setStatus(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE);
-        Assert.assertSame(result1, store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null));
-
-        result1.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE);
-        Assert.assertSame(result1, store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null));
+        GetMessageResult expect = new GetMessageResult();
+        expect.setStatus(GetMessageStatus.FOUND);
+        expect.setMinOffset(100L);
+        expect.setMaxOffset(200L);
+
+        // topic filter
+        Mockito.when(defaultStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any()))
+            .thenReturn(CompletableFuture.completedFuture(expect));
+        String groupName = "groupName";
+        GetMessageResult result = currentStore.getMessage(
+            groupName, TopicValidator.SYSTEM_TOPIC_PREFIX, mq.getQueueId(), 100, 0, null);
+        Assert.assertSame(expect, result);
+
+        // fetch from default
+        Mockito.when(fetcher.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any()))
+            .thenReturn(CompletableFuture.completedFuture(expect));
+
+        result = currentStore.getMessage(
+            groupName, mq.getTopic(), mq.getQueueId(), 100, 0, null);
+        Assert.assertSame(expect, result);
+
+        expect.setStatus(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE);
+        Assert.assertSame(expect, currentStore.getMessage(
+            groupName, mq.getTopic(), mq.getQueueId(), 0, 0, null));
+
+        expect.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE);
+        Assert.assertSame(expect, currentStore.getMessage(
+            groupName, mq.getTopic(), mq.getQueueId(), 0, 0, null));
+
+        expect.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY);
+        Assert.assertSame(expect, currentStore.getMessage(
+            groupName, mq.getTopic(), mq.getQueueId(), 0, 0, null));
+
+        expect.setStatus(GetMessageStatus.OFFSET_RESET);
+        Assert.assertSame(expect, currentStore.getMessage(
+            groupName, mq.getTopic(), mq.getQueueId(), 0, 0, null));
+    }
 
-        result1.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY);
-        Assert.assertSame(result1, store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null));
+    @Test
+    public void testGetMinOffsetInQueue() {
+        FlatMessageFile flatFile = flatFileStore.getFlatFile(mq);
+        Mockito.when(defaultStore.getMinOffsetInQueue(anyString(), anyInt())).thenReturn(100L);
+        Assert.assertEquals(100L, currentStore.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId()));
 
-        // TieredStorageLevel.FORCE
-        Properties properties = new Properties();
-        properties.setProperty("tieredStorageLevel", "3");
-        configuration.update(properties);
-        when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true);
-        Assert.assertEquals(result2.getStatus(),
-            store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null).getStatus());
+        Mockito.when(flatFile.getConsumeQueueMinOffset()).thenReturn(10L);
+        Assert.assertEquals(10L, currentStore.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId()));
     }
 
     @Test
     public void testGetEarliestMessageTimeAsync() {
         when(fetcher.getEarliestMessageTimeAsync(anyString(), anyInt())).thenReturn(CompletableFuture.completedFuture(1L));
-        Assert.assertEquals(1, (long) store.getEarliestMessageTimeAsync(mq.getTopic(), mq.getQueueId()).join());
+        Assert.assertEquals(0, (long) currentStore.getEarliestMessageTimeAsync(mq.getTopic(), mq.getQueueId()).join());
 
         when(fetcher.getEarliestMessageTimeAsync(anyString(), anyInt())).thenReturn(CompletableFuture.completedFuture(-1L));
-        when(nextStore.getEarliestMessageTime(anyString(), anyInt())).thenReturn(2L);
-        Assert.assertEquals(2, (long) store.getEarliestMessageTimeAsync(mq.getTopic(), mq.getQueueId()).join());
+        when(defaultStore.getEarliestMessageTime(anyString(), anyInt())).thenReturn(2L);
+        Assert.assertEquals(2, (long) currentStore.getEarliestMessageTimeAsync(mq.getTopic(), mq.getQueueId()).join());
     }
 
     @Test
     public void testGetMessageStoreTimeStampAsync() {
-        mockCompositeFlatFile();
         // TieredStorageLevel.DISABLE
         Properties properties = new Properties();
         properties.setProperty("tieredStorageLevel", "DISABLE");
         configuration.update(properties);
         when(fetcher.getMessageStoreTimeStampAsync(anyString(), anyInt(), anyLong())).thenReturn(CompletableFuture.completedFuture(1L));
-        when(nextStore.getMessageStoreTimeStampAsync(anyString(), anyInt(), anyLong())).thenReturn(CompletableFuture.completedFuture(2L));
-        when(nextStore.getMessageStoreTimeStamp(anyString(), anyInt(), anyLong())).thenReturn(3L);
-        Assert.assertEquals(2, (long) store.getMessageStoreTimeStampAsync(mq.getTopic(), mq.getQueueId(), 0).join());
+        when(defaultStore.getMessageStoreTimeStampAsync(anyString(), anyInt(), anyLong())).thenReturn(CompletableFuture.completedFuture(2L));
+        when(defaultStore.getMessageStoreTimeStamp(anyString(), anyInt(), anyLong())).thenReturn(3L);
+        Assert.assertEquals(2, (long) currentStore.getMessageStoreTimeStampAsync(mq.getTopic(), mq.getQueueId(), 0).join());
 
         // TieredStorageLevel.FORCE
         properties.setProperty("tieredStorageLevel", "FORCE");
         configuration.update(properties);
-        Assert.assertEquals(1, (long) store.getMessageStoreTimeStampAsync(mq.getTopic(), mq.getQueueId(), 0).join());
+        Assert.assertEquals(1, (long) currentStore.getMessageStoreTimeStampAsync(mq.getTopic(), mq.getQueueId(), 0).join());
 
         Mockito.when(fetcher.getMessageStoreTimeStampAsync(anyString(), anyInt(), anyLong())).thenReturn(CompletableFuture.completedFuture(-1L));
-        Assert.assertEquals(3, (long) store.getMessageStoreTimeStampAsync(mq.getTopic(), mq.getQueueId(), 0).join());
+        Assert.assertEquals(3, (long) currentStore.getMessageStoreTimeStampAsync(mq.getTopic(), mq.getQueueId(), 0).join());
     }
 
     @Test
     public void testGetOffsetInQueueByTime() {
+        Properties properties = new Properties();
+        properties.setProperty("tieredStorageLevel", "FORCE");
+        configuration.update(properties);
+
         Mockito.when(fetcher.getOffsetInQueueByTime(anyString(), anyInt(), anyLong(), eq(BoundaryType.LOWER))).thenReturn(1L);
-        Mockito.when(nextStore.getOffsetInQueueByTime(anyString(), anyInt(), anyLong())).thenReturn(2L);
-        Mockito.when(nextStore.getEarliestMessageTime()).thenReturn(100L);
-        Assert.assertEquals(1, store.getOffsetInQueueByTime(mq.getTopic(), mq.getQueueId(), 0, BoundaryType.LOWER));
-        Assert.assertEquals(2, store.getOffsetInQueueByTime(mq.getTopic(), mq.getQueueId(), 1000, BoundaryType.LOWER));
+        Mockito.when(defaultStore.getOffsetInQueueByTime(anyString(), anyInt(), anyLong())).thenReturn(2L);
+        Mockito.when(defaultStore.getEarliestMessageTime()).thenReturn(100L);
+        Assert.assertEquals(1L, currentStore.getOffsetInQueueByTime(mq.getTopic(), mq.getQueueId(), 1000, BoundaryType.LOWER));
+        Assert.assertEquals(1L, currentStore.getOffsetInQueueByTime(mq.getTopic(), mq.getQueueId(), 0, BoundaryType.LOWER));
 
         Mockito.when(fetcher.getOffsetInQueueByTime(anyString(), anyInt(), anyLong(), eq(BoundaryType.LOWER))).thenReturn(-1L);
-        Assert.assertEquals(2, store.getOffsetInQueueByTime(mq.getTopic(), mq.getQueueId(), 0, BoundaryType.LOWER));
+        Assert.assertEquals(-1L, currentStore.getOffsetInQueueByTime(mq.getTopic(), mq.getQueueId(), 0));
+        Assert.assertEquals(-1L, currentStore.getOffsetInQueueByTime(mq.getTopic(), mq.getQueueId(), 0, BoundaryType.LOWER));
     }
 
     @Test
@@ -243,55 +298,36 @@ public void testQueryMessage() {
         when(fetcher.queryMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyLong())).thenReturn(CompletableFuture.completedFuture(result1));
         QueryMessageResult result2 = new QueryMessageResult();
         result2.addMessage(new SelectMappedBufferResult(0, null, 0, null));
-        when(nextStore.queryMessage(anyString(), anyString(), anyInt(), anyLong(), anyLong())).thenReturn(result2);
-        when(nextStore.getEarliestMessageTime()).thenReturn(100L);
-        Assert.assertEquals(2, store.queryMessage(mq.getTopic(), "key", 32, 0, 99).getMessageMapedList().size());
-        Assert.assertEquals(1, store.queryMessage(mq.getTopic(), "key", 32, 100, 200).getMessageMapedList().size());
-        Assert.assertEquals(3, store.queryMessage(mq.getTopic(), "key", 32, 0, 200).getMessageMapedList().size());
-    }
-
-    @Test
-    public void testGetMinOffsetInQueue() {
-        mockCompositeFlatFile();
-        CompositeQueueFlatFile flatFile = flatFileManager.getFlatFile(mq);
-        when(nextStore.getMinOffsetInQueue(anyString(), anyInt())).thenReturn(100L);
-        when(flatFileManager.getFlatFile(mq)).thenReturn(null);
-        Assert.assertEquals(100L, store.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId()));
-
-        when(flatFileManager.getFlatFile(mq)).thenReturn(flatFile);
-        when(flatFile.getConsumeQueueMinOffset()).thenReturn(10L);
-        Assert.assertEquals(10L, store.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId()));
+        when(defaultStore.queryMessage(anyString(), anyString(), anyInt(), anyLong(), anyLong())).thenReturn(result2);
+        when(defaultStore.getEarliestMessageTime()).thenReturn(100L);
+        Assert.assertEquals(2, currentStore.queryMessage(mq.getTopic(), "key", 32, 0, 99).getMessageMapedList().size());
+        Assert.assertEquals(1, currentStore.queryMessage(mq.getTopic(), "key", 32, 100, 200).getMessageMapedList().size());
+        Assert.assertEquals(3, currentStore.queryMessage(mq.getTopic(), "key", 32, 0, 200).getMessageMapedList().size());
     }
 
     @Test
     public void testCleanUnusedTopics() {
         Set<String> topicSet = new HashSet<>();
-        store.cleanUnusedTopic(topicSet);
-        Assert.assertNull(TieredFlatFileManager.getInstance(store.getStoreConfig()).getFlatFile(mq));
-        Assert.assertNull(TieredStoreUtil.getMetadataStore(store.getStoreConfig()).getTopic(mq.getTopic()));
-        Assert.assertNull(TieredStoreUtil.getMetadataStore(store.getStoreConfig()).getQueue(mq));
+        currentStore.cleanUnusedTopic(topicSet);
+        Assert.assertNull(flatFileStore.getFlatFile(mq));
+        Assert.assertNull(flatFileStore.getMetadataStore().getTopic(mq.getTopic()));
+        Assert.assertNull(flatFileStore.getMetadataStore().getQueue(mq));
     }
 
     @Test
     public void testDeleteTopics() {
         Set<String> topicSet = new HashSet<>();
         topicSet.add(mq.getTopic());
-        store.deleteTopics(topicSet);
-        Assert.assertNull(TieredFlatFileManager.getInstance(store.getStoreConfig()).getFlatFile(mq));
-        Assert.assertNull(TieredStoreUtil.getMetadataStore(store.getStoreConfig()).getTopic(mq.getTopic()));
-        Assert.assertNull(TieredStoreUtil.getMetadataStore(store.getStoreConfig()).getQueue(mq));
+        currentStore.deleteTopics(topicSet);
+        Assert.assertNull(flatFileStore.getFlatFile(mq));
+        Assert.assertNull(flatFileStore.getMetadataStore().getTopic(mq.getTopic()));
+        Assert.assertNull(flatFileStore.getMetadataStore().getQueue(mq));
     }
 
     @Test
     public void testMetrics() {
-        store.getMetricsView();
-        store.initMetrics(OpenTelemetrySdk.builder().build().getMeter(""),
-            Attributes::builder);
-    }
-
-    @Test
-    public void testShutdownAndDestroy() {
-        store.shutdown();
-        store.destroy();
+        currentStore.getMetricsView();
+        currentStore.initMetrics(
+            OpenTelemetrySdk.builder().build().getMeter(""), Attributes::builder);
     }
 }
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredStoreTestUtil.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredStoreTestUtil.java
deleted file mode 100644
index fb11b60f05a..00000000000
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredStoreTestUtil.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore;
-
-import java.io.File;
-import java.lang.reflect.Field;
-import java.util.UUID;
-import org.apache.commons.io.FileUtils;
-import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-import org.junit.Assert;
-
-public class TieredStoreTestUtil {
-
-    public static String getRandomStorePath() {
-        return FileUtils.getTempDirectory() + File.separator + "unit_test_tiered_store" + UUID.randomUUID();
-    }
-
-    public static void destroyMetadataStore() {
-        TieredMetadataStore metadataStore = TieredStoreUtil.getMetadataStore(null);
-        if (metadataStore != null) {
-            metadataStore.destroy();
-        }
-        try {
-            Field field = TieredStoreUtil.class.getDeclaredField("metadataStoreInstance");
-            field.setAccessible(true);
-            field.set(null, null);
-        } catch (NoSuchFieldException | IllegalAccessException e) {
-            Assert.fail(e.getClass().getCanonicalName() + ": " + e.getMessage());
-        }
-    }
-
-    public static void destroyCompositeFlatFileManager() {
-        TieredFlatFileManager flatFileManagerManager = TieredFlatFileManager.getInstance(null);
-        if (flatFileManagerManager != null) {
-            flatFileManagerManager.destroy();
-        }
-        try {
-            Field field = TieredFlatFileManager.class.getDeclaredField("instance");
-            field.setAccessible(true);
-            field.set(null, null);
-        } catch (NoSuchFieldException | IllegalAccessException e) {
-            Assert.fail(e.getClass().getCanonicalName() + ": " + e.getMessage());
-        }
-    }
-
-    public static void destroyTempDir(String storePath) {
-        try {
-            FileUtils.deleteDirectory(new File(storePath));
-        } catch (Exception ignore) {
-        }
-    }
-}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/FileSegmentTypeTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/FileSegmentTypeTest.java
new file mode 100644
index 00000000000..28439e06e6f
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/FileSegmentTypeTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.common;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class FileSegmentTypeTest {
+
+    @Test
+    public void getTypeCodeTest() {
+        assertEquals(0, FileSegmentType.COMMIT_LOG.getCode());
+        assertEquals(1, FileSegmentType.CONSUME_QUEUE.getCode());
+        assertEquals(2, FileSegmentType.INDEX.getCode());
+    }
+
+    @Test
+    public void getTypeFromValueTest() {
+        assertEquals(FileSegmentType.COMMIT_LOG, FileSegmentType.valueOf(0));
+        assertEquals(FileSegmentType.CONSUME_QUEUE, FileSegmentType.valueOf(1));
+        assertEquals(FileSegmentType.INDEX, FileSegmentType.valueOf(2));
+    }
+}
\ No newline at end of file
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/GetMessageResultExtTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/GetMessageResultExtTest.java
index deb8770d281..69240a420a7 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/GetMessageResultExtTest.java
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/GetMessageResultExtTest.java
@@ -23,30 +23,32 @@
 import org.apache.rocketmq.store.GetMessageStatus;
 import org.apache.rocketmq.store.MessageFilter;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtilTest;
 import org.junit.Assert;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-
 public class GetMessageResultExtTest {
 
     @Test
     public void doFilterTest() {
         GetMessageResultExt resultExt = new GetMessageResultExt();
+        Assert.assertNull(resultExt.getStatus());
         Assert.assertEquals(0, resultExt.doFilterMessage(null).getMessageCount());
+
         resultExt.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE);
         Assert.assertEquals(0, resultExt.doFilterMessage(null).getMessageCount());
+
         resultExt.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY);
         Assert.assertEquals(0, resultExt.doFilterMessage(null).getMessageCount());
 
-        resultExt.addMessageExt(new SelectMappedBufferResult(
-                1000L, MessageBufferUtilTest.buildMockedMessageBuffer(), 100, null),
-            0, "TagA".hashCode());
-        resultExt.addMessageExt(new SelectMappedBufferResult(
-                2000L, MessageBufferUtilTest.buildMockedMessageBuffer(), 100, null),
-            0, "TagB".hashCode());
-        assertEquals(2, resultExt.getMessageCount());
+        int total = 3;
+        for (int i = 0; i < total; i++) {
+            resultExt.addMessageExt(new SelectMappedBufferResult(i * 1000L,
+                    MessageFormatUtilTest.buildMockedMessageBuffer(), 1000, null),
+                0, ("Tag" + i).hashCode());
+        }
+        Assert.assertEquals(total, resultExt.getMessageCount());
+        Assert.assertEquals(total, resultExt.getTagCodeList().size());
 
         resultExt.setStatus(GetMessageStatus.FOUND);
         GetMessageResult getMessageResult = resultExt.doFilterMessage(new MessageFilter() {
@@ -61,5 +63,31 @@ public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> pr
             }
         });
         Assert.assertEquals(0, getMessageResult.getMessageCount());
+
+        getMessageResult = resultExt.doFilterMessage(new MessageFilter() {
+            @Override
+            public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
+                return "Tag1".hashCode() == tagsCode;
+            }
+
+            @Override
+            public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {
+                return false;
+            }
+        });
+        Assert.assertEquals(0, getMessageResult.getMessageCount());
+
+        getMessageResult = resultExt.doFilterMessage(new MessageFilter() {
+            @Override
+            public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
+                return "Tag1".hashCode() == tagsCode;
+            }
+
+            @Override
+            public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {
+                return true;
+            }
+        });
+        Assert.assertEquals(1, getMessageResult.getMessageCount());
     }
 }
\ No newline at end of file
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/GroupCommitContextTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/GroupCommitContextTest.java
new file mode 100644
index 00000000000..e692360761d
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/GroupCommitContextTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.common;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.rocketmq.store.DispatchRequest;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class GroupCommitContextTest {
+
+    @Test
+    public void groupCommitContextTest() {
+        GroupCommitContext releaseGroupCommitContext = new GroupCommitContext();
+        releaseGroupCommitContext.release();
+
+        long endOffset = 1000;
+        List<DispatchRequest> dispatchRequestList = new ArrayList<>();
+        dispatchRequestList.add(new DispatchRequest(1000));
+        List<SelectMappedBufferResult> selectMappedBufferResultList = new ArrayList<>();
+        selectMappedBufferResultList.add(new SelectMappedBufferResult(100, ByteBuffer.allocate(10), 1000, null));
+        GroupCommitContext groupCommitContext = new GroupCommitContext();
+        groupCommitContext.setEndOffset(endOffset);
+        groupCommitContext.setBufferList(selectMappedBufferResultList);
+        groupCommitContext.setDispatchRequests(dispatchRequestList);
+
+        Assert.assertTrue(groupCommitContext.getEndOffset() == endOffset);
+        Assert.assertTrue(groupCommitContext.getBufferList().equals(selectMappedBufferResultList));
+        Assert.assertTrue(groupCommitContext.getDispatchRequests().equals(dispatchRequestList));
+        groupCommitContext.release();
+        Assert.assertTrue(groupCommitContext.getDispatchRequests() == null);
+        Assert.assertTrue(groupCommitContext.getBufferList() == null);
+        Assert.assertTrue(dispatchRequestList.isEmpty());
+        Assert.assertTrue(selectMappedBufferResultList.isEmpty());
+    }
+
+}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/InFlightRequestFutureTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/InFlightRequestFutureTest.java
deleted file mode 100644
index 54b88f38d49..00000000000
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/InFlightRequestFutureTest.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.common;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import org.apache.commons.lang3.tuple.Pair;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class InFlightRequestFutureTest {
-
-    @Test
-    public void testInFlightRequestFuture() {
-        List<Pair<Integer, CompletableFuture<Long>>> futureList = new ArrayList<>();
-        futureList.add(Pair.of(32, CompletableFuture.completedFuture(1031L)));
-        futureList.add(Pair.of(256, CompletableFuture.completedFuture(1287L)));
-        InFlightRequestFuture future = new InFlightRequestFuture(1000, futureList);
-
-        Assert.assertEquals(1000, future.getStartOffset());
-        Assert.assertTrue(future.isFirstDone());
-        Assert.assertTrue(future.isAllDone());
-        Assert.assertEquals(1031, future.getFirstFuture().join().longValue());
-        Assert.assertEquals(-1L, future.getFuture(0).join().longValue());
-        Assert.assertEquals(1031L, future.getFuture(1024).join().longValue());
-        Assert.assertEquals(1287L, future.getFuture(1200).join().longValue());
-        Assert.assertEquals(-1L, future.getFuture(2000).join().longValue());
-        Assert.assertEquals(1287L, future.getLastFuture().join().longValue());
-        Assert.assertArrayEquals(futureList.stream().map(Pair::getRight).toArray(), future.getAllFuture().toArray());
-    }
-
-    @Test
-    public void testInFlightRequestKey() {
-        InFlightRequestKey requestKey1 = new InFlightRequestKey("group", 0, 0);
-        InFlightRequestKey requestKey2 = new InFlightRequestKey("group", 1, 1);
-        Assert.assertEquals(requestKey1, requestKey2);
-        Assert.assertEquals(requestKey1.hashCode(), requestKey2.hashCode());
-        Assert.assertEquals(requestKey1.getGroup(), requestKey2.getGroup());
-        Assert.assertNotEquals(requestKey1.getOffset(), requestKey2.getOffset());
-        Assert.assertNotEquals(requestKey1.getBatchSize(), requestKey2.getBatchSize());
-    }
-
-    @Test
-    public void testGetStartOffset() {
-        List<Pair<Integer, CompletableFuture<Long>>> futureList = new ArrayList<>();
-        CompletableFuture<Long> completableFuture = new CompletableFuture<>();
-        futureList.add(Pair.of(1, completableFuture));
-        InFlightRequestFuture inFlightRequestFuture = new InFlightRequestFuture(10, futureList);
-        long startOffset = inFlightRequestFuture.getStartOffset();
-        Assert.assertEquals(10, startOffset);
-    }
-
-    @Test
-    public void testGetFirstFuture() throws ExecutionException, InterruptedException {
-        List<Pair<Integer, CompletableFuture<Long>>> futureList = new ArrayList<>();
-        CompletableFuture<Long> completableFuture = new CompletableFuture<>();
-        completableFuture.complete(20L);
-        futureList.add(Pair.of(1, completableFuture));
-        InFlightRequestFuture inFlightRequestFuture = new InFlightRequestFuture(10, futureList);
-        CompletableFuture<Long> firstFuture = inFlightRequestFuture.getFirstFuture();
-        Assert.assertEquals(new Long(20), firstFuture.get());
-    }
-
-    @Test
-    public void testGetFuture() {
-        List<Pair<Integer, CompletableFuture<Long>>> futureList = new ArrayList<>();
-        CompletableFuture<Long> completableFuture = new CompletableFuture<>();
-        completableFuture.complete(20L);
-        futureList.add(Pair.of(1, completableFuture));
-        InFlightRequestFuture inFlightRequestFuture = new InFlightRequestFuture(10, futureList);
-        CompletableFuture<Long> future = inFlightRequestFuture.getFuture(11);
-        Assert.assertEquals(new Long(-1L), future.join());
-    }
-
-    @Test
-    public void testGetLastFuture() throws ExecutionException, InterruptedException {
-        List<Pair<Integer, CompletableFuture<Long>>> futureList = new ArrayList<>();
-        CompletableFuture<Long> completableFuture = new CompletableFuture<>();
-        completableFuture.complete(20L);
-        futureList.add(Pair.of(1, completableFuture));
-        InFlightRequestFuture inFlightRequestFuture = new InFlightRequestFuture(10, futureList);
-        CompletableFuture<Long> lastFuture = inFlightRequestFuture.getLastFuture();
-        Assert.assertEquals(new Long(20), lastFuture.get());
-    }
-
-    @Test
-    public void testIsFirstDone() {
-        List<Pair<Integer, CompletableFuture<Long>>> futureList = new ArrayList<>();
-        CompletableFuture<Long> completableFuture = new CompletableFuture<>();
-        completableFuture.complete(20L);
-        futureList.add(Pair.of(1, completableFuture));
-        InFlightRequestFuture inFlightRequestFuture = new InFlightRequestFuture(10, futureList);
-        Assert.assertTrue(inFlightRequestFuture.isFirstDone());
-    }
-
-    @Test
-    public void testIsAllDone() {
-        List<Pair<Integer, CompletableFuture<Long>>> futureList = new ArrayList<>();
-        CompletableFuture<Long> completableFuture1 = new CompletableFuture<>();
-        CompletableFuture<Long> completableFuture2 = new CompletableFuture<>();
-        CompletableFuture<Long> completableFuture3 = new CompletableFuture<>();
-        CompletableFuture<Long> completableFuture4 = new CompletableFuture<>();
-        completableFuture1.complete(20L);
-        completableFuture2.complete(30L);
-        completableFuture3.complete(40L);
-        futureList.add(Pair.of(1, completableFuture1));
-        futureList.add(Pair.of(2, completableFuture2));
-        futureList.add(Pair.of(3, completableFuture3));
-        futureList.add(Pair.of(4, completableFuture4));
-        InFlightRequestFuture inFlightRequestFuture = new InFlightRequestFuture(10, futureList);
-        Assert.assertFalse(inFlightRequestFuture.isAllDone());
-    }
-
-    @Test
-    public void testGetAllFuture() {
-        List<Pair<Integer, CompletableFuture<Long>>> futureList = new ArrayList<>();
-        CompletableFuture<Long> completableFuture1 = new CompletableFuture<>();
-        CompletableFuture<Long> completableFuture2 = new CompletableFuture<>();
-        CompletableFuture<Long> completableFuture3 = new CompletableFuture<>();
-        CompletableFuture<Long> completableFuture4 = new CompletableFuture<>();
-        futureList.add(Pair.of(1, completableFuture1));
-        futureList.add(Pair.of(2, completableFuture2));
-        futureList.add(Pair.of(3, completableFuture3));
-        futureList.add(Pair.of(4, completableFuture4));
-        InFlightRequestFuture inFlightRequestFuture = new InFlightRequestFuture(10, futureList);
-        List<CompletableFuture<Long>> allFuture = inFlightRequestFuture.getAllFuture();
-        Assert.assertEquals(4, allFuture.size());
-    }
-}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/SelectBufferResultTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/SelectBufferResultTest.java
index b7e6e639f0c..f9dfce9447c 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/SelectBufferResultTest.java
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/common/SelectBufferResultTest.java
@@ -21,8 +21,9 @@
 import org.junit.Test;
 
 public class SelectBufferResultTest {
+
     @Test
-    public void testSelectBufferResult() {
+    public void selectBufferResultTest() {
         ByteBuffer buffer = ByteBuffer.allocate(10);
         long startOffset = 5L;
         int size = 10;
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImplTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImplTest.java
new file mode 100644
index 00000000000..7a43e1ede83
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImplTest.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.core;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.store.DispatchRequest;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
+import org.apache.rocketmq.store.queue.CqUnit;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
+import org.apache.rocketmq.tieredstore.TieredMessageStore;
+import org.apache.rocketmq.tieredstore.common.GroupCommitContext;
+import org.apache.rocketmq.tieredstore.file.FlatFileFactory;
+import org.apache.rocketmq.tieredstore.file.FlatFileStore;
+import org.apache.rocketmq.tieredstore.file.FlatMessageFile;
+import org.apache.rocketmq.tieredstore.index.IndexItem;
+import org.apache.rocketmq.tieredstore.index.IndexService;
+import org.apache.rocketmq.tieredstore.index.IndexStoreService;
+import org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.provider.PosixFileSegment;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtil;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtilTest;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
+import org.awaitility.Awaitility;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+
+public class MessageStoreDispatcherImplTest {
+
+    protected final String storePath = MessageStoreUtilTest.getRandomStorePath();
+    protected MessageQueue mq;
+    protected MetadataStore metadataStore;
+    protected MessageStoreConfig storeConfig;
+    protected MessageStoreExecutor executor;
+    protected FlatFileStore fileStore;
+    protected TieredMessageStore messageStore;
+
+    @Before
+    public void init() {
+        storeConfig = new MessageStoreConfig();
+        storeConfig.setBrokerName("brokerName");
+        storeConfig.setStorePathRootDir(storePath);
+        storeConfig.setTieredStoreFilePath(storePath);
+        storeConfig.setTieredBackendServiceProvider(PosixFileSegment.class.getName());
+        mq = new MessageQueue("StoreTest", storeConfig.getBrokerName(), 1);
+        metadataStore = new DefaultMetadataStore(storeConfig);
+        executor = new MessageStoreExecutor();
+        fileStore = new FlatFileStore(storeConfig, metadataStore, executor);
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        if (messageStore != null) {
+            messageStore.destroy();
+        }
+        MessageStoreUtilTest.deleteStoreDirectory(storePath);
+    }
+
+    @Test
+    public void dispatchFromCommitLogTest() throws Exception {
+        MessageStore defaultStore = Mockito.mock(MessageStore.class);
+        Mockito.when(defaultStore.getMinOffsetInQueue(anyString(), anyInt())).thenReturn(100L);
+        Mockito.when(defaultStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(200L);
+
+        messageStore = Mockito.mock(TieredMessageStore.class);
+        IndexService indexService =
+            new IndexStoreService(new FlatFileFactory(metadataStore, storeConfig), storePath);
+        indexService.start();
+        Mockito.when(messageStore.getDefaultStore()).thenReturn(defaultStore);
+        Mockito.when(messageStore.getStoreConfig()).thenReturn(storeConfig);
+        Mockito.when(messageStore.getStoreExecutor()).thenReturn(executor);
+        Mockito.when(messageStore.getFlatFileStore()).thenReturn(fileStore);
+        Mockito.when(messageStore.getIndexService()).thenReturn(indexService);
+        Mockito.when(messageStore.getMessageStoreConfig()).thenReturn(new org.apache.rocketmq.store.config.MessageStoreConfig());
+
+        // mock message
+        ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+        MessageExt messageExt = MessageDecoder.decode(buffer);
+        messageExt.setKeys("Key");
+        MessageAccessor.putProperty(
+            messageExt, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "uk");
+        messageExt.setBody(new byte[10]);
+        messageExt.setStoreSize(0);
+        buffer = ByteBuffer.wrap(MessageDecoder.encode(messageExt, false));
+        buffer.putInt(0, buffer.remaining());
+
+        DispatchRequest request = new DispatchRequest(mq.getTopic(), mq.getQueueId(),
+            MessageFormatUtil.getCommitLogOffset(buffer), buffer.remaining(), 0L,
+            MessageFormatUtil.getStoreTimeStamp(buffer), 0L,
+            "", "", 0, 0L, new HashMap<>());
+
+        // construct flat file
+        MessageStoreDispatcher dispatcher = new MessageStoreDispatcherImpl(messageStore);
+        dispatcher.dispatch(request);
+        FlatMessageFile flatFile = fileStore.getFlatFile(mq);
+        Assert.assertNotNull(flatFile);
+
+        // init offset
+        dispatcher.doScheduleDispatch(flatFile, true).join();
+        Assert.assertEquals(100L, flatFile.getConsumeQueueMinOffset());
+        Assert.assertEquals(100L, flatFile.getConsumeQueueMaxOffset());
+        Assert.assertEquals(100L, flatFile.getConsumeQueueCommitOffset());
+
+        ConsumeQueueInterface cq = Mockito.mock(ConsumeQueueInterface.class);
+        Mockito.when(defaultStore.getConsumeQueue(anyString(), anyInt())).thenReturn(cq);
+        Mockito.when(cq.get(anyLong())).thenReturn(
+            new CqUnit(100, 1000, buffer.remaining(), 0L));
+        Mockito.when(defaultStore.selectOneMessageByOffset(anyLong(), anyInt())).thenReturn(
+            new SelectMappedBufferResult(0L, buffer.asReadOnlyBuffer(), buffer.remaining(), null));
+        dispatcher.doScheduleDispatch(flatFile, true).join();
+
+        Awaitility.await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(30)).until(() -> {
+            List<IndexItem> resultList1 = indexService.queryAsync(
+                mq.getTopic(), "uk", 32, 0L, System.currentTimeMillis()).join();
+            List<IndexItem> resultList2 = indexService.queryAsync(
+                mq.getTopic(), "uk", 120, 0L, System.currentTimeMillis()).join();
+            Assert.assertEquals(32, resultList1.size());
+            Assert.assertEquals(100, resultList2.size());
+            return true;
+        });
+
+        Assert.assertEquals(100L, flatFile.getConsumeQueueMinOffset());
+        Assert.assertEquals(200L, flatFile.getConsumeQueueMaxOffset());
+        Assert.assertEquals(200L, flatFile.getConsumeQueueCommitOffset());
+    }
+
+    @Test
+    public void dispatchCommitFailedTest() throws Exception {
+        MessageStore defaultStore = Mockito.mock(MessageStore.class);
+        Mockito.when(defaultStore.getMinOffsetInQueue(anyString(), anyInt())).thenReturn(100L);
+        Mockito.when(defaultStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(200L);
+
+        messageStore = Mockito.mock(TieredMessageStore.class);
+        IndexService indexService =
+            new IndexStoreService(new FlatFileFactory(metadataStore, storeConfig), storePath);
+        indexService.start();
+        Mockito.when(messageStore.getDefaultStore()).thenReturn(defaultStore);
+        Mockito.when(messageStore.getStoreConfig()).thenReturn(storeConfig);
+        Mockito.when(messageStore.getStoreExecutor()).thenReturn(executor);
+        Mockito.when(messageStore.getFlatFileStore()).thenReturn(fileStore);
+        Mockito.when(messageStore.getIndexService()).thenReturn(indexService);
+
+        // mock message
+        ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+        MessageExt messageExt = MessageDecoder.decode(buffer);
+        messageExt.setKeys("Key");
+        MessageAccessor.putProperty(
+            messageExt, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "uk");
+        messageExt.setBody(new byte[10]);
+        messageExt.setStoreSize(0);
+        buffer = ByteBuffer.wrap(MessageDecoder.encode(messageExt, false));
+        buffer.putInt(0, buffer.remaining());
+
+        DispatchRequest request = new DispatchRequest(mq.getTopic(), mq.getQueueId(),
+            MessageFormatUtil.getCommitLogOffset(buffer), buffer.remaining(), 0L,
+            MessageFormatUtil.getStoreTimeStamp(buffer), 0L,
+            "", "", 0, 0L, new HashMap<>());
+
+        // construct flat file
+        MessageStoreDispatcher dispatcher = new MessageStoreDispatcherImpl(messageStore);
+        dispatcher.dispatch(request);
+        FlatMessageFile flatFile = fileStore.getFlatFile(mq);
+        Assert.assertNotNull(flatFile);
+
+        // init offset
+        dispatcher.doScheduleDispatch(flatFile, true).join();
+        Assert.assertEquals(100L, flatFile.getConsumeQueueMinOffset());
+        Assert.assertEquals(100L, flatFile.getConsumeQueueMaxOffset());
+        Assert.assertEquals(100L, flatFile.getConsumeQueueCommitOffset());
+
+        ConsumeQueueInterface cq = Mockito.mock(ConsumeQueueInterface.class);
+        Mockito.when(defaultStore.getConsumeQueue(anyString(), anyInt())).thenReturn(cq);
+        Mockito.when(cq.get(anyLong())).thenReturn(
+            new CqUnit(100, 1000, buffer.remaining(), 0L));
+        Mockito.when(defaultStore.selectOneMessageByOffset(anyLong(), anyInt())).thenReturn(
+            new SelectMappedBufferResult(0L, buffer.asReadOnlyBuffer(), buffer.remaining(), null));
+        flatFile.getCommitLock().drainPermits();
+        dispatcher.doScheduleDispatch(flatFile, true).join();
+        GroupCommitContext groupCommitContext = ((MessageStoreDispatcherImpl)dispatcher).getFailedGroupCommitMap().get(flatFile);
+        Assert.assertTrue(groupCommitContext != null);
+        Assert.assertTrue(groupCommitContext.getEndOffset() == 200);
+        flatFile.getCommitLock().release();
+        flatFile.commitAsync().join();
+        dispatcher.doScheduleDispatch(flatFile, true).join();
+        Assert.assertTrue(((MessageStoreDispatcherImpl)dispatcher).getFailedGroupCommitMap().get(flatFile) == null);
+        ((MessageStoreDispatcherImpl)dispatcher).flatFileStore.destroyFile(mq);
+        ((MessageStoreDispatcherImpl)dispatcher).releaseClosedPendingGroupCommit();
+
+    }
+
+    @Test
+    public void dispatchFailedGroupCommitMapReleaseTest() throws Exception {
+        MessageStore defaultStore = Mockito.mock(MessageStore.class);
+        Mockito.when(defaultStore.getMinOffsetInQueue(anyString(), anyInt())).thenReturn(100L);
+        Mockito.when(defaultStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(200L);
+
+        messageStore = Mockito.mock(TieredMessageStore.class);
+        IndexService indexService =
+            new IndexStoreService(new FlatFileFactory(metadataStore, storeConfig), storePath);
+        indexService.start();
+        Mockito.when(messageStore.getDefaultStore()).thenReturn(defaultStore);
+        Mockito.when(messageStore.getStoreConfig()).thenReturn(storeConfig);
+        Mockito.when(messageStore.getStoreExecutor()).thenReturn(executor);
+        Mockito.when(messageStore.getFlatFileStore()).thenReturn(fileStore);
+        Mockito.when(messageStore.getIndexService()).thenReturn(indexService);
+
+        // mock message
+        ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+        MessageExt messageExt = MessageDecoder.decode(buffer);
+        messageExt.setKeys("Key");
+        MessageAccessor.putProperty(
+            messageExt, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "uk");
+        messageExt.setBody(new byte[10]);
+        messageExt.setStoreSize(0);
+        buffer = ByteBuffer.wrap(MessageDecoder.encode(messageExt, false));
+        buffer.putInt(0, buffer.remaining());
+
+        DispatchRequest request = new DispatchRequest(mq.getTopic(), mq.getQueueId(),
+            MessageFormatUtil.getCommitLogOffset(buffer), buffer.remaining(), 0L,
+            MessageFormatUtil.getStoreTimeStamp(buffer), 0L,
+            "", "", 0, 0L, new HashMap<>());
+
+        // construct flat file
+        MessageStoreDispatcher dispatcher = new MessageStoreDispatcherImpl(messageStore);
+        dispatcher.dispatch(request);
+        FlatMessageFile flatFile = fileStore.getFlatFile(mq);
+        Assert.assertNotNull(flatFile);
+
+        // init offset
+        dispatcher.doScheduleDispatch(flatFile, true).join();
+        Assert.assertEquals(100L, flatFile.getConsumeQueueMinOffset());
+        Assert.assertEquals(100L, flatFile.getConsumeQueueMaxOffset());
+        Assert.assertEquals(100L, flatFile.getConsumeQueueCommitOffset());
+
+        ConsumeQueueInterface cq = Mockito.mock(ConsumeQueueInterface.class);
+        Mockito.when(defaultStore.getConsumeQueue(anyString(), anyInt())).thenReturn(cq);
+        Mockito.when(cq.get(anyLong())).thenReturn(
+            new CqUnit(100, 1000, buffer.remaining(), 0L));
+        Mockito.when(defaultStore.selectOneMessageByOffset(anyLong(), anyInt())).thenReturn(
+            new SelectMappedBufferResult(0L, buffer.asReadOnlyBuffer(), buffer.remaining(), null));
+        flatFile.getCommitLock().drainPermits();
+        dispatcher.doScheduleDispatch(flatFile, true).join();
+        GroupCommitContext groupCommitContext = ((MessageStoreDispatcherImpl)dispatcher).getFailedGroupCommitMap().get(flatFile);
+        Assert.assertTrue(groupCommitContext != null);
+        ((MessageStoreDispatcherImpl)dispatcher).flatFileStore.destroyFile(mq);
+        ((MessageStoreDispatcherImpl)dispatcher).releaseClosedPendingGroupCommit();
+        Assert.assertTrue(((MessageStoreDispatcherImpl)dispatcher).getFailedGroupCommitMap().get(flatFile) == null);
+
+    }
+
+    @Test
+    public void dispatchServiceTest() {
+        MessageStore defaultStore = Mockito.mock(MessageStore.class);
+        messageStore = Mockito.mock(TieredMessageStore.class);
+        IndexService indexService =
+            new IndexStoreService(new FlatFileFactory(metadataStore, storeConfig), storePath);
+        Mockito.when(messageStore.getDefaultStore()).thenReturn(defaultStore);
+        Mockito.when(messageStore.getStoreConfig()).thenReturn(storeConfig);
+        Mockito.when(messageStore.getStoreExecutor()).thenReturn(executor);
+        Mockito.when(messageStore.getFlatFileStore()).thenReturn(fileStore);
+        Mockito.when(messageStore.getIndexService()).thenReturn(indexService);
+
+        // construct flat file
+        ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+        DispatchRequest request = new DispatchRequest(mq.getTopic(), mq.getQueueId(),
+            MessageFormatUtil.getCommitLogOffset(buffer), buffer.remaining(), 0L,
+            MessageFormatUtil.getStoreTimeStamp(buffer), 0L,
+            "", "", 0, 0L, new HashMap<>());
+        MessageStoreDispatcherImpl dispatcher = new MessageStoreDispatcherImpl(messageStore);
+        dispatcher.dispatch(request);
+        FlatMessageFile flatFile = fileStore.getFlatFile(mq);
+        Assert.assertNotNull(flatFile);
+
+        AtomicBoolean result = new AtomicBoolean(false);
+        MessageStoreDispatcherImpl dispatcherSpy = Mockito.spy(dispatcher);
+        Mockito.doAnswer(mock -> {
+            result.set(true);
+            return true;
+        }).when(dispatcherSpy).dispatchWithSemaphore(any());
+        dispatcherSpy.start();
+        Awaitility.await().atMost(Duration.ofSeconds(10)).until(result::get);
+        dispatcherSpy.shutdown();
+    }
+}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImplTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImplTest.java
new file mode 100644
index 00000000000..fdcdec066fe
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImplTest.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.core;
+
+import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.rocketmq.common.BoundaryType;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.store.DefaultMessageFilter;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+import org.apache.rocketmq.store.MessageFilter;
+import org.apache.rocketmq.store.QueryMessageResult;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.TieredMessageStore;
+import org.apache.rocketmq.tieredstore.common.SelectBufferResult;
+import org.apache.rocketmq.tieredstore.file.FlatMessageFile;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtilTest;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
+import org.awaitility.Awaitility;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.apache.rocketmq.tieredstore.core.MessageStoreFetcherImpl.CACHE_KEY_FORMAT;
+
+public class MessageStoreFetcherImplTest {
+
+    private String groupName;
+    private MessageQueue mq;
+    private MessageStoreConfig storeConfig;
+    private TieredMessageStore messageStore;
+    private MessageStoreDispatcherImplTest dispatcherTest;
+    private MessageStoreFetcherImpl fetcher;
+
+    @Before
+    public void init() throws Exception {
+        groupName = "GID-fetcherTest";
+        dispatcherTest = new MessageStoreDispatcherImplTest();
+        dispatcherTest.init();
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        if (messageStore != null) {
+            messageStore.destroy();
+        }
+        MessageStoreUtilTest.deleteStoreDirectory(dispatcherTest.storePath);
+    }
+
+    @Test
+    public void getMessageFromTieredStoreTest() throws Exception {
+        dispatcherTest.dispatchFromCommitLogTest();
+        mq = dispatcherTest.mq;
+        messageStore = dispatcherTest.messageStore;
+        storeConfig = dispatcherTest.storeConfig;
+
+        storeConfig.setReadAheadCacheEnable(true);
+        fetcher = new MessageStoreFetcherImpl(messageStore);
+        GetMessageResult getMessageResult = fetcher.getMessageAsync(
+            groupName, mq.getTopic(), 0, 0, 32, null).join();
+        Assert.assertEquals(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE, getMessageResult.getStatus());
+
+        getMessageResult = fetcher.getMessageAsync(
+            groupName, mq.getTopic(), mq.getQueueId(), 0, 32, null).join();
+        Assert.assertEquals(GetMessageStatus.OFFSET_TOO_SMALL, getMessageResult.getStatus());
+        Assert.assertEquals(100L, getMessageResult.getMinOffset());
+        Assert.assertEquals(200L, getMessageResult.getMaxOffset());
+        Assert.assertEquals(100L, getMessageResult.getNextBeginOffset());
+
+        getMessageResult = fetcher.getMessageAsync(
+            groupName, mq.getTopic(), mq.getQueueId(), 200, 32, null).join();
+        Assert.assertEquals(GetMessageStatus.OFFSET_OVERFLOW_ONE, getMessageResult.getStatus());
+        Assert.assertEquals(100L, getMessageResult.getMinOffset());
+        Assert.assertEquals(200L, getMessageResult.getMaxOffset());
+        Assert.assertEquals(200L, getMessageResult.getNextBeginOffset());
+
+        getMessageResult = fetcher.getMessageAsync(
+            groupName, mq.getTopic(), mq.getQueueId(), 300, 32, null).join();
+        Assert.assertEquals(GetMessageStatus.OFFSET_OVERFLOW_BADLY, getMessageResult.getStatus());
+        Assert.assertEquals(100L, getMessageResult.getMinOffset());
+        Assert.assertEquals(200L, getMessageResult.getMaxOffset());
+        Assert.assertEquals(200L, getMessageResult.getNextBeginOffset());
+
+        FlatMessageFile flatFile = dispatcherTest.fileStore.getFlatFile(mq);
+
+        // direct
+        getMessageResult = fetcher.getMessageFromTieredStoreAsync(flatFile, 0, 32).join();
+        Assert.assertEquals(GetMessageStatus.OFFSET_TOO_SMALL, getMessageResult.getStatus());
+        Assert.assertEquals(100L, getMessageResult.getMinOffset());
+        Assert.assertEquals(200L, getMessageResult.getMaxOffset());
+        Assert.assertEquals(100L, getMessageResult.getNextBeginOffset());
+
+        getMessageResult = fetcher.getMessageFromTieredStoreAsync(flatFile, 200, 32).join();
+        Assert.assertEquals(GetMessageStatus.OFFSET_OVERFLOW_ONE, getMessageResult.getStatus());
+        Assert.assertEquals(100L, getMessageResult.getMinOffset());
+        Assert.assertEquals(200L, getMessageResult.getMaxOffset());
+        Assert.assertEquals(200L, getMessageResult.getNextBeginOffset());
+
+        getMessageResult = fetcher.getMessageFromTieredStoreAsync(flatFile, 300, 32).join();
+        Assert.assertEquals(GetMessageStatus.OFFSET_OVERFLOW_BADLY, getMessageResult.getStatus());
+        Assert.assertEquals(100L, getMessageResult.getMinOffset());
+        Assert.assertEquals(200L, getMessageResult.getMaxOffset());
+        Assert.assertEquals(200L, getMessageResult.getNextBeginOffset());
+
+        getMessageResult = fetcher.getMessageFromTieredStoreAsync(flatFile, 100, 32).join();
+        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
+        Assert.assertEquals(100L, getMessageResult.getMinOffset());
+        Assert.assertEquals(200L, getMessageResult.getMaxOffset());
+        Assert.assertEquals(100L + 32L, getMessageResult.getNextBeginOffset());
+
+        getMessageResult = fetcher.getMessageFromTieredStoreAsync(flatFile, 180, 32).join();
+        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
+        Assert.assertEquals(20, getMessageResult.getMessageCount());
+        Assert.assertEquals(100L, getMessageResult.getMinOffset());
+        Assert.assertEquals(200L, getMessageResult.getMaxOffset());
+        Assert.assertEquals(200L, getMessageResult.getNextBeginOffset());
+
+        // limit count or size
+        int expect = 8;
+        int size = getMessageResult.getMessageBufferList().get(0).remaining();
+        storeConfig.setReadAheadMessageSizeThreshold(expect * size + 10);
+        getMessageResult = fetcher.getMessageFromTieredStoreAsync(flatFile, 180, 32).join();
+        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
+        Assert.assertEquals(expect, getMessageResult.getMessageCount());
+        Assert.assertEquals(100L, getMessageResult.getMinOffset());
+        Assert.assertEquals(200L, getMessageResult.getMaxOffset());
+        Assert.assertEquals(180L + expect, getMessageResult.getNextBeginOffset());
+
+        storeConfig.setReadAheadMessageCountThreshold(expect);
+        storeConfig.setReadAheadMessageSizeThreshold(expect * size + expect * 2);
+        getMessageResult = fetcher.getMessageFromTieredStoreAsync(flatFile, 180, 32).join();
+        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
+        Assert.assertEquals(expect, getMessageResult.getMessageCount());
+        Assert.assertEquals(100L, getMessageResult.getMinOffset());
+        Assert.assertEquals(200L, getMessageResult.getMaxOffset());
+        Assert.assertEquals(180L + expect, getMessageResult.getNextBeginOffset());
+    }
+
+    @Test
+    public void getMessageFromCacheTest() throws Exception {
+        this.getMessageFromTieredStoreTest();
+        mq = dispatcherTest.mq;
+        messageStore = dispatcherTest.messageStore;
+        storeConfig = dispatcherTest.storeConfig;
+
+        storeConfig.setReadAheadCacheEnable(true);
+        storeConfig.setReadAheadMessageCountThreshold(32);
+        storeConfig.setReadAheadMessageSizeThreshold(Integer.MAX_VALUE);
+
+        int batchSize = 4;
+        AtomicLong times = new AtomicLong(0L);
+        AtomicLong offset = new AtomicLong(100L);
+        FlatMessageFile flatFile = dispatcherTest.fileStore.getFlatFile(mq);
+        Awaitility.await().atMost(Duration.ofSeconds(10)).until(() -> {
+            GetMessageResult getMessageResult =
+                fetcher.getMessageFromCacheAsync(flatFile, groupName, offset.get(), batchSize, null).join();
+            offset.set(getMessageResult.getNextBeginOffset());
+            times.incrementAndGet();
+            return offset.get() == 200L;
+        });
+        Assert.assertEquals(100 / times.get(), batchSize);
+    }
+
+    @Test
+    public void getMessageFromCacheTagFilterTest() throws Exception {
+        dispatcherTest.dispatchFromCommitLogTest();
+        mq = dispatcherTest.mq;
+        messageStore = dispatcherTest.messageStore;
+        storeConfig = dispatcherTest.storeConfig;
+
+        storeConfig.setReadAheadCacheEnable(true);
+        fetcher = new MessageStoreFetcherImpl(messageStore);
+
+        FlatMessageFile flatFile = Mockito.mock(FlatMessageFile.class);
+        Mockito.when(flatFile.getMessageQueue()).thenReturn(mq);
+        Mockito.when(flatFile.getConsumeQueueMinOffset()).thenReturn(100L);
+        Mockito.when(flatFile.getConsumeQueueMaxOffset()).thenReturn(200L);
+
+        for (int i = 100; i < 200; i++) {
+            ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+            SelectBufferResult bufferResult = new SelectBufferResult(buffer, i, buffer.remaining(), i % 2);
+            fetcher.getFetcherCache().put(
+                String.format(CACHE_KEY_FORMAT, mq.getTopic(), mq.getQueueId(), i), bufferResult);
+        }
+
+        SubscriptionData subscriptionData = new SubscriptionData();
+        subscriptionData.setSubString("1 || 2");
+        subscriptionData.setCodeSet(Sets.newHashSet(1, 2));
+        MessageFilter filter = new DefaultMessageFilter(subscriptionData);
+
+        GetMessageResult getMessageResult =
+            fetcher.getMessageFromCacheAsync(flatFile, groupName, 100L, 32, filter).join();
+        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
+        Assert.assertEquals(32, getMessageResult.getMessageCount());
+        Assert.assertEquals(164L, getMessageResult.getNextBeginOffset());
+
+        getMessageResult =
+            fetcher.getMessageFromCacheAsync(flatFile, groupName, 164L, 32, filter).join();
+        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
+        Assert.assertEquals(18, getMessageResult.getMessageCount());
+        Assert.assertEquals(200L, getMessageResult.getNextBeginOffset());
+
+        getMessageResult =
+            fetcher.getMessageFromCacheAsync(flatFile, groupName, 200L, 32, filter).join();
+        Assert.assertEquals(GetMessageStatus.NO_MATCHED_MESSAGE, getMessageResult.getStatus());
+        Assert.assertEquals(200L, getMessageResult.getNextBeginOffset());
+
+        subscriptionData.setCodeSet(Sets.newHashSet(0));
+        filter = new DefaultMessageFilter(subscriptionData);
+        getMessageResult =
+            fetcher.getMessageFromCacheAsync(flatFile, groupName, 100L, 32, filter).join();
+        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
+        Assert.assertEquals(32, getMessageResult.getMessageCount());
+        Assert.assertEquals(164L - 1L, getMessageResult.getNextBeginOffset());
+    }
+
+    @Test
+    public void getMessageFromCacheTagFilter2Test() throws Exception {
+        dispatcherTest.dispatchFromCommitLogTest();
+        mq = dispatcherTest.mq;
+        messageStore = dispatcherTest.messageStore;
+        storeConfig = dispatcherTest.storeConfig;
+
+        storeConfig.setReadAheadCacheEnable(true);
+        fetcher = new MessageStoreFetcherImpl(messageStore);
+
+        FlatMessageFile flatFile = Mockito.mock(FlatMessageFile.class);
+        Mockito.when(flatFile.getMessageQueue()).thenReturn(mq);
+        Mockito.when(flatFile.getConsumeQueueMinOffset()).thenReturn(100L);
+        Mockito.when(flatFile.getConsumeQueueMaxOffset()).thenReturn(200L);
+
+        for (int i = 100; i < 200; i++) {
+            ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+            SelectBufferResult bufferResult = new SelectBufferResult(buffer, i, buffer.remaining(), i - 100L);
+            fetcher.getFetcherCache().put(
+                String.format(CACHE_KEY_FORMAT, mq.getTopic(), mq.getQueueId(), i), bufferResult);
+        }
+
+        SubscriptionData subscriptionData = new SubscriptionData();
+        subscriptionData.setSubString("1 || 2");
+        subscriptionData.setCodeSet(Sets.newHashSet(10, 20));
+        MessageFilter filter = new DefaultMessageFilter(subscriptionData);
+
+        GetMessageResult getMessageResult =
+            fetcher.getMessageFromCacheAsync(flatFile, groupName, 100L, 2, filter).join();
+        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
+        Assert.assertEquals(2, getMessageResult.getMessageCount());
+        Assert.assertEquals(121L, getMessageResult.getNextBeginOffset());
+    }
+
+    @Test
+    public void testGetMessageStoreTimeStampAsync() throws Exception {
+        this.getMessageFromTieredStoreTest();
+        mq = dispatcherTest.mq;
+        messageStore = dispatcherTest.messageStore;
+        storeConfig = dispatcherTest.storeConfig;
+
+        long result1 = fetcher.getEarliestMessageTimeAsync(mq.getTopic(), 0).join();
+        Assert.assertEquals(-1L, result1);
+
+        long result2 = fetcher.getEarliestMessageTimeAsync(mq.getTopic(), mq.getQueueId()).join();
+        Assert.assertEquals(11L, result2);
+
+        long result3 = fetcher.getMessageStoreTimeStampAsync(mq.getTopic(), 0, 100).join();
+        Assert.assertEquals(-1L, result3);
+
+        long result4 = fetcher.getMessageStoreTimeStampAsync(mq.getTopic(), mq.getQueueId(), 100).join();
+        Assert.assertEquals(11L, result4);
+
+        long result5 = fetcher.getMessageStoreTimeStampAsync(mq.getTopic(), mq.getQueueId(), 120).join();
+        Assert.assertEquals(11L, result5);
+    }
+
+    @Test
+    public void testGetOffsetInQueueByTime() throws Exception {
+        this.getMessageFromTieredStoreTest();
+        mq = dispatcherTest.mq;
+        messageStore = dispatcherTest.messageStore;
+        storeConfig = dispatcherTest.storeConfig;
+
+        // message time is all 11
+        Assert.assertEquals(-1L, fetcher.getOffsetInQueueByTime(mq.getTopic(), 0, 10, BoundaryType.LOWER));
+
+        Assert.assertEquals(100L, fetcher.getOffsetInQueueByTime(mq.getTopic(), 1, 10, BoundaryType.LOWER));
+        Assert.assertEquals(100L, fetcher.getOffsetInQueueByTime(mq.getTopic(), 1, 11, BoundaryType.LOWER));
+        Assert.assertEquals(200L, fetcher.getOffsetInQueueByTime(mq.getTopic(), 1, 12, BoundaryType.LOWER));
+
+        Assert.assertEquals(100L, fetcher.getOffsetInQueueByTime(mq.getTopic(), 1, 10, BoundaryType.UPPER));
+        Assert.assertEquals(199L, fetcher.getOffsetInQueueByTime(mq.getTopic(), 1, 11, BoundaryType.UPPER));
+        Assert.assertEquals(200L, fetcher.getOffsetInQueueByTime(mq.getTopic(), 1, 12, BoundaryType.UPPER));
+    }
+
+    @Test
+    public void testQueryMessageAsync() throws Exception {
+        this.getMessageFromTieredStoreTest();
+        mq = dispatcherTest.mq;
+        messageStore = dispatcherTest.messageStore;
+        storeConfig = dispatcherTest.storeConfig;
+
+        QueryMessageResult queryMessageResult = fetcher.queryMessageAsync(
+            mq.getTopic(), "uk", 32, 0L, System.currentTimeMillis()).join();
+        Assert.assertEquals(32, queryMessageResult.getMessageBufferList().size());
+
+        queryMessageResult = fetcher.queryMessageAsync(
+            mq.getTopic(), "uk", 120, 0L, System.currentTimeMillis()).join();
+        Assert.assertEquals(100, queryMessageResult.getMessageBufferList().size());
+    }
+}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreTopicFilterTest.java
similarity index 84%
rename from tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java
rename to tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreTopicFilterTest.java
index fbaafa1b4cf..d5c3703152c 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreTopicFilterTest.java
@@ -14,17 +14,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.tieredstore.provider;
+package org.apache.rocketmq.tieredstore.core;
 
 import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
 import org.junit.Assert;
 import org.junit.Test;
 
-public class TieredStoreTopicBlackListFilterTest {
+public class MessageStoreTopicFilterTest {
 
     @Test
     public void filterTopicTest() {
-        TieredStoreTopicFilter topicFilter = new TieredStoreTopicBlackListFilter();
+        MessageStoreFilter topicFilter = new MessageStoreTopicFilter(new MessageStoreConfig());
         Assert.assertTrue(topicFilter.filterTopic(""));
         Assert.assertTrue(topicFilter.filterTopic(TopicValidator.SYSTEM_TOPIC_PREFIX + "_Topic"));
 
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/exception/TieredStoreExceptionTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/exception/TieredStoreExceptionTest.java
new file mode 100644
index 00000000000..1de891a8acc
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/exception/TieredStoreExceptionTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.exception;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TieredStoreExceptionTest {
+
+    @Test
+    public void testMessageStoreException() {
+        long position = 100L;
+        String requestId = "requestId";
+        String error = "ErrorMessage";
+
+        TieredStoreException tieredStoreException = new TieredStoreException(TieredStoreErrorCode.IO_ERROR, error);
+        Assert.assertEquals(TieredStoreErrorCode.IO_ERROR, tieredStoreException.getErrorCode());
+        Assert.assertEquals(error, tieredStoreException.getMessage());
+
+        tieredStoreException.setRequestId(requestId);
+        Assert.assertEquals(requestId, tieredStoreException.getRequestId());
+
+        tieredStoreException.setPosition(position);
+        Assert.assertEquals(position, tieredStoreException.getPosition());
+        Assert.assertNotNull(tieredStoreException.toString());
+    }
+}
\ No newline at end of file
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java
deleted file mode 100644
index 58842430483..00000000000
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.file;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.store.ConsumeQueue;
-import org.apache.rocketmq.store.DispatchRequest;
-import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
-import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.metadata.QueueMetadata;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
-import org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-import org.apache.rocketmq.common.BoundaryType;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CompositeQueueFlatFileTest {
-
-    private final String storePath = TieredStoreTestUtil.getRandomStorePath();
-    private TieredMessageStoreConfig storeConfig;
-    private TieredMetadataStore metadataStore;
-    private TieredFileAllocator tieredFileAllocator;
-    private MessageQueue mq;
-
-    @Before
-    public void setUp() throws ClassNotFoundException, NoSuchMethodException {
-        storeConfig = new TieredMessageStoreConfig();
-        storeConfig.setBrokerName("brokerName");
-        storeConfig.setStorePathRootDir(storePath);
-        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment");
-        storeConfig.setCommitLogRollingInterval(0);
-        storeConfig.setCommitLogRollingMinimumSize(999);
-        mq = new MessageQueue("CompositeQueueFlatFileTest", storeConfig.getBrokerName(), 0);
-        metadataStore = TieredStoreUtil.getMetadataStore(storeConfig);
-        tieredFileAllocator = new TieredFileAllocator(storeConfig);
-        TieredStoreExecutor.init();
-    }
-
-    @After
-    public void tearDown() throws IOException {
-        TieredStoreTestUtil.destroyCompositeFlatFileManager();
-        TieredStoreTestUtil.destroyMetadataStore();
-        TieredStoreTestUtil.destroyTempDir(storePath);
-        TieredStoreExecutor.shutdown();
-    }
-
-    @Test
-    public void testAppendCommitLog() {
-        CompositeQueueFlatFile flatFile = new CompositeQueueFlatFile(tieredFileAllocator, mq);
-        ByteBuffer message = MessageBufferUtilTest.buildMockedMessageBuffer();
-        AppendResult result = flatFile.appendCommitLog(message);
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-        Assert.assertEquals(123L, flatFile.commitLog.getFlatFile().getFileToWrite().getAppendPosition());
-        Assert.assertEquals(0L, flatFile.commitLog.getFlatFile().getFileToWrite().getCommitPosition());
-
-        flatFile = new CompositeQueueFlatFile(tieredFileAllocator, mq);
-        flatFile.initOffset(6);
-        result = flatFile.appendCommitLog(message);
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-
-        message.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 7);
-        result = flatFile.appendCommitLog(message);
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-
-        flatFile.commit(true);
-        Assert.assertEquals(7, flatFile.getCommitLogDispatchCommitOffset());
-
-        flatFile.cleanExpiredFile(0);
-        flatFile.destroyExpiredFile();
-    }
-
-    @Test
-    public void testAppendConsumeQueue() {
-        CompositeQueueFlatFile file = new CompositeQueueFlatFile(tieredFileAllocator, mq);
-        DispatchRequest request = new DispatchRequest(
-            mq.getTopic(), mq.getQueueId(), 51, 2, 3, 4);
-        AppendResult result = file.appendConsumeQueue(request);
-        Assert.assertEquals(AppendResult.OFFSET_INCORRECT, result);
-
-        // Create new segment in file queue
-        MemoryFileSegment segment = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, mq, 20, storeConfig);
-        segment.initPosition(segment.getSize());
-        file.consumeQueue.getFlatFile().setBaseOffset(20L);
-        file.consumeQueue.getFlatFile().getFileToWrite();
-
-        // Recreate will load metadata and build consume queue
-        file = new CompositeQueueFlatFile(tieredFileAllocator, mq);
-        segment.initPosition(ConsumeQueue.CQ_STORE_UNIT_SIZE);
-        result = file.appendConsumeQueue(request);
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-
-        request = new DispatchRequest(
-            mq.getTopic(), mq.getQueueId(), 52, 2, 3, 4);
-        result = file.appendConsumeQueue(request);
-        Assert.assertEquals(AppendResult.SUCCESS, result);
-
-        file.commit(true);
-        file.flushMetadata();
-
-        QueueMetadata queueMetadata = metadataStore.getQueue(mq);
-        Assert.assertEquals(53, queueMetadata.getMaxOffset());
-    }
-
-    @Test
-    public void testBinarySearchInQueueByTime() throws ClassNotFoundException, NoSuchMethodException {
-
-        // replace provider, need new factory again
-        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegmentWithoutCheck");
-        tieredFileAllocator = new TieredFileAllocator(storeConfig);
-
-        // inject store time: 0, +100, +100, +100, +200
-        CompositeQueueFlatFile flatFile = new CompositeQueueFlatFile(tieredFileAllocator, mq);
-        flatFile.initOffset(50);
-        long timestamp1 = System.currentTimeMillis();
-        ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-        buffer.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 50);
-        buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, timestamp1);
-        flatFile.appendCommitLog(buffer, true);
-
-        long timestamp2 = timestamp1 + 100;
-        buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-        buffer.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 51);
-        buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, timestamp2);
-        flatFile.appendCommitLog(buffer, true);
-        buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-        buffer.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 52);
-        buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, timestamp2);
-        flatFile.appendCommitLog(buffer, true);
-        buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-        buffer.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 53);
-        buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, timestamp2);
-        flatFile.appendCommitLog(buffer, true);
-
-        long timestamp3 = timestamp2 + 100;
-        buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-        buffer.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 54);
-        buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, timestamp3);
-        flatFile.appendCommitLog(buffer, true);
-
-        // append message to consume queue
-        flatFile.consumeQueue.getFlatFile().setBaseOffset(50 * ConsumeQueue.CQ_STORE_UNIT_SIZE);
-
-        for (int i = 0; i < 5; i++) {
-            AppendResult appendResult = flatFile.appendConsumeQueue(new DispatchRequest(
-                mq.getTopic(), mq.getQueueId(), MessageBufferUtilTest.MSG_LEN * i,
-                MessageBufferUtilTest.MSG_LEN, 0, timestamp1, 50 + i,
-                "", "", 0, 0, null), true);
-            Assert.assertEquals(AppendResult.SUCCESS, appendResult);
-        }
-
-        // commit message will increase max consume queue offset
-        flatFile.commit(true);
-
-        Assert.assertEquals(54, flatFile.getOffsetInConsumeQueueByTime(timestamp3 + 1, BoundaryType.UPPER));
-        Assert.assertEquals(54, flatFile.getOffsetInConsumeQueueByTime(timestamp3, BoundaryType.UPPER));
-
-        Assert.assertEquals(50, flatFile.getOffsetInConsumeQueueByTime(timestamp1 - 1, BoundaryType.LOWER));
-        Assert.assertEquals(50, flatFile.getOffsetInConsumeQueueByTime(timestamp1, BoundaryType.LOWER));
-
-        Assert.assertEquals(51, flatFile.getOffsetInConsumeQueueByTime(timestamp1 + 1, BoundaryType.LOWER));
-        Assert.assertEquals(51, flatFile.getOffsetInConsumeQueueByTime(timestamp2, BoundaryType.LOWER));
-        Assert.assertEquals(54, flatFile.getOffsetInConsumeQueueByTime(timestamp2 + 1, BoundaryType.LOWER));
-        Assert.assertEquals(54, flatFile.getOffsetInConsumeQueueByTime(timestamp3, BoundaryType.LOWER));
-
-        Assert.assertEquals(50, flatFile.getOffsetInConsumeQueueByTime(timestamp1, BoundaryType.UPPER));
-        Assert.assertEquals(50, flatFile.getOffsetInConsumeQueueByTime(timestamp1 + 1, BoundaryType.UPPER));
-        Assert.assertEquals(53, flatFile.getOffsetInConsumeQueueByTime(timestamp2, BoundaryType.UPPER));
-        Assert.assertEquals(53, flatFile.getOffsetInConsumeQueueByTime(timestamp2 + 1, BoundaryType.UPPER));
-
-        Assert.assertEquals(0, flatFile.getOffsetInConsumeQueueByTime(timestamp1 - 1, BoundaryType.UPPER));
-        Assert.assertEquals(55, flatFile.getOffsetInConsumeQueueByTime(timestamp3 + 1, BoundaryType.LOWER));
-    }
-}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatAppendFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatAppendFileTest.java
new file mode 100644
index 00000000000..2e6943728e2
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatAppendFileTest.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.file;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.concurrent.CompletionException;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.exception.TieredStoreErrorCode;
+import org.apache.rocketmq.tieredstore.exception.TieredStoreException;
+import org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.entity.FileSegmentMetadata;
+import org.apache.rocketmq.tieredstore.provider.FileSegment;
+import org.apache.rocketmq.tieredstore.provider.PosixFileSegment;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FlatAppendFileTest {
+
+    private final String storePath = MessageStoreUtilTest.getRandomStorePath();
+    private MessageQueue queue;
+    private MetadataStore metadataStore;
+    private MessageStoreConfig storeConfig;
+    private FlatFileFactory flatFileFactory;
+
+    @Before
+    public void init() throws ClassNotFoundException, NoSuchMethodException {
+        storeConfig = new MessageStoreConfig();
+        storeConfig.setBrokerName("brokerName");
+        storeConfig.setStorePathRootDir(storePath);
+        storeConfig.setTieredStoreFilePath(storePath);
+        storeConfig.setTieredBackendServiceProvider(PosixFileSegment.class.getName());
+        storeConfig.setTieredStoreCommitLogMaxSize(2000L);
+        storeConfig.setTieredStoreConsumeQueueMaxSize(2000L);
+        queue = new MessageQueue("TieredFlatFileTest", storeConfig.getBrokerName(), 0);
+        metadataStore = new DefaultMetadataStore(storeConfig);
+        flatFileFactory = new FlatFileFactory(metadataStore, storeConfig);
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        MessageStoreUtilTest.deleteStoreDirectory(storePath);
+    }
+
+    public ByteBuffer allocateBuffer(int size) {
+        byte[] byteArray = new byte[size];
+        ByteBuffer buffer = ByteBuffer.wrap(byteArray);
+        Arrays.fill(byteArray, (byte) 0);
+        return buffer;
+    }
+
+    @Test
+    public void recoverFileSizeTest() {
+        String filePath = MessageStoreUtil.toFilePath(queue);
+        FlatAppendFile flatFile = flatFileFactory.createFlatFileForConsumeQueue(filePath);
+        flatFile.rollingNewFile(500L);
+
+        FileSegment fileSegment = flatFile.getFileToWrite();
+        flatFile.append(allocateBuffer(1000), 1L);
+        flatFile.commitAsync().join();
+        flatFile.flushFileSegmentMeta(fileSegment);
+    }
+
+    @Test
+    public void testRecoverFile() {
+        String filePath = MessageStoreUtil.toFilePath(queue);
+        FlatAppendFile flatFile = flatFileFactory.createFlatFileForConsumeQueue(filePath);
+        flatFile.rollingNewFile(500L);
+
+        FileSegment fileSegment = flatFile.getFileToWrite();
+        flatFile.append(allocateBuffer(1000), 1L);
+        flatFile.commitAsync().join();
+        flatFile.flushFileSegmentMeta(fileSegment);
+
+        FileSegmentMetadata metadata =
+            metadataStore.getFileSegment(filePath, FileSegmentType.CONSUME_QUEUE, 500L);
+        Assert.assertEquals(fileSegment.getPath(), metadata.getPath());
+        Assert.assertEquals(FileSegmentType.CONSUME_QUEUE, FileSegmentType.valueOf(metadata.getType()));
+        Assert.assertEquals(500L, metadata.getBaseOffset());
+        Assert.assertEquals(1000L, metadata.getSize());
+        Assert.assertEquals(0L, metadata.getSealTimestamp());
+
+        fileSegment.close();
+        flatFile.rollingNewFile(flatFile.getAppendOffset());
+        flatFile.append(allocateBuffer(200), 1L);
+        flatFile.commitAsync().join();
+        flatFile.flushFileSegmentMeta(fileSegment);
+        Assert.assertEquals(2, flatFile.getFileSegmentList().size());
+        flatFile.getFileToWrite().close();
+
+        metadata = metadataStore.getFileSegment(filePath, FileSegmentType.CONSUME_QUEUE, 1500L);
+        Assert.assertEquals(fileSegment.getPath(), metadata.getPath());
+        Assert.assertEquals(FileSegmentType.CONSUME_QUEUE, FileSegmentType.valueOf(metadata.getType()));
+        Assert.assertEquals(1500L, metadata.getBaseOffset());
+        Assert.assertEquals(200L, metadata.getSize());
+        Assert.assertEquals(0L, metadata.getSealTimestamp());
+
+        // reference same file
+        flatFile = flatFileFactory.createFlatFileForConsumeQueue(filePath);
+        Assert.assertEquals(2, flatFile.fileSegmentTable.size());
+        metadata = metadataStore.getFileSegment(filePath, FileSegmentType.CONSUME_QUEUE, 1500L);
+        Assert.assertEquals(fileSegment.getPath(), metadata.getPath());
+        Assert.assertEquals(FileSegmentType.CONSUME_QUEUE, FileSegmentType.valueOf(metadata.getType()));
+        Assert.assertEquals(1500L, metadata.getBaseOffset());
+        Assert.assertEquals(200L, metadata.getSize());
+        Assert.assertEquals(0L, metadata.getSealTimestamp());
+        flatFile.destroy();
+    }
+
+    @Test
+    public void testFileSegment() {
+        String filePath = MessageStoreUtil.toFilePath(queue);
+        FlatAppendFile flatFile = flatFileFactory.createFlatFileForConsumeQueue(filePath);
+        Assert.assertThrows(IllegalStateException.class, flatFile::getFileToWrite);
+
+        flatFile.commitAsync().join();
+        flatFile.rollingNewFile(0L);
+        Assert.assertEquals(0L, flatFile.getMinOffset());
+        Assert.assertEquals(0L, flatFile.getCommitOffset());
+        Assert.assertEquals(0L, flatFile.getAppendOffset());
+
+        flatFile.append(allocateBuffer(1000), 1L);
+        Assert.assertEquals(0L, flatFile.getMinOffset());
+        Assert.assertEquals(0L, flatFile.getCommitOffset());
+        Assert.assertEquals(1000L, flatFile.getAppendOffset());
+        Assert.assertEquals(1L, flatFile.getMinTimestamp());
+        Assert.assertEquals(1L, flatFile.getMaxTimestamp());
+
+        flatFile.commitAsync().join();
+        Assert.assertEquals(filePath, flatFile.getFilePath());
+        Assert.assertEquals(FileSegmentType.CONSUME_QUEUE, flatFile.getFileType());
+        Assert.assertEquals(0L, flatFile.getMinOffset());
+        Assert.assertEquals(1000L, flatFile.getCommitOffset());
+        Assert.assertEquals(1000L, flatFile.getAppendOffset());
+        Assert.assertEquals(1L, flatFile.getMinTimestamp());
+        Assert.assertEquals(1L, flatFile.getMaxTimestamp());
+
+        // file full
+        flatFile.append(allocateBuffer(1000), 1L);
+        flatFile.append(allocateBuffer(1000), 1L);
+        flatFile.commitAsync().join();
+        Assert.assertEquals(2, flatFile.fileSegmentTable.size());
+        flatFile.destroy();
+    }
+
+    @Test
+    public void testAppendAndRead() {
+        FlatAppendFile flatFile = flatFileFactory.createFlatFileForConsumeQueue(MessageStoreUtil.toFilePath(queue));
+        flatFile.rollingNewFile(500L);
+        Assert.assertEquals(500L, flatFile.getCommitOffset());
+        Assert.assertEquals(500L, flatFile.getAppendOffset());
+
+        flatFile.append(allocateBuffer(1000), 1L);
+
+        // no commit
+        CompletionException exception = Assert.assertThrows(
+            CompletionException.class, () -> flatFile.readAsync(500, 200).join());
+        Assert.assertTrue(exception.getCause() instanceof TieredStoreException);
+        Assert.assertEquals(TieredStoreErrorCode.ILLEGAL_PARAM,
+            ((TieredStoreException) exception.getCause()).getErrorCode());
+        flatFile.commitAsync().join();
+        Assert.assertEquals(200, flatFile.readAsync(500, 200).join().remaining());
+
+        // 500-1500, 1500-3000
+        flatFile.append(allocateBuffer(1500), 1L);
+        flatFile.commitAsync().join();
+        Assert.assertEquals(2, flatFile.fileSegmentTable.size());
+        Assert.assertEquals(1000, flatFile.readAsync(1000, 1000).join().remaining());
+        flatFile.destroy();
+    }
+
+    @Test
+    public void testCleanExpiredFile() {
+        FlatAppendFile flatFile = flatFileFactory.createFlatFileForConsumeQueue(MessageStoreUtil.toFilePath(queue));
+        flatFile.destroyExpiredFile(1);
+
+        flatFile.rollingNewFile(500L);
+        flatFile.append(allocateBuffer(1000), 2L);
+        flatFile.commitAsync().join();
+        Assert.assertEquals(1, flatFile.fileSegmentTable.size());
+        flatFile.destroyExpiredFile(1);
+        Assert.assertEquals(1, flatFile.fileSegmentTable.size());
+        flatFile.destroyExpiredFile(3);
+        Assert.assertEquals(0, flatFile.fileSegmentTable.size());
+
+        flatFile.rollingNewFile(1500L);
+        flatFile.append(allocateBuffer(1000), 2L);
+        flatFile.append(allocateBuffer(1000), 2L);
+        flatFile.commitAsync().join();
+        flatFile.destroy();
+        Assert.assertEquals(0, flatFile.fileSegmentTable.size());
+    }
+}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFileTest.java
new file mode 100644
index 00000000000..0fbf5a6a843
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFileTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.file;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.common.AppendResult;
+import org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.provider.PosixFileSegment;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtil;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtilTest;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FlatCommitLogFileTest {
+
+    private final String storePath = MessageStoreUtilTest.getRandomStorePath();
+    private MessageQueue queue;
+    private MetadataStore metadataStore;
+    private MessageStoreConfig storeConfig;
+    private FlatFileFactory flatFileFactory;
+
+    @Before
+    public void init() throws ClassNotFoundException, NoSuchMethodException {
+        storeConfig = new MessageStoreConfig();
+        storeConfig.setBrokerName("brokerName");
+        storeConfig.setStorePathRootDir(storePath);
+        storeConfig.setTieredStoreFilePath(storePath);
+        storeConfig.setTieredBackendServiceProvider(PosixFileSegment.class.getName());
+        storeConfig.setTieredStoreCommitLogMaxSize(2000L);
+        storeConfig.setTieredStoreConsumeQueueMaxSize(2000L);
+        queue = new MessageQueue("TieredFlatFileTest", storeConfig.getBrokerName(), 0);
+        metadataStore = new DefaultMetadataStore(storeConfig);
+        flatFileFactory = new FlatFileFactory(metadataStore, storeConfig);
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        MessageStoreUtilTest.deleteStoreDirectory(storePath);
+    }
+
+    @Test
+    public void constructTest() {
+        String filePath = MessageStoreUtil.toFilePath(queue);
+        FlatAppendFile flatFile = flatFileFactory.createFlatFileForCommitLog(filePath);
+        Assert.assertEquals(1L, flatFile.fileSegmentTable.size());
+    }
+
+    @Test
+    public void tryRollingFileTest() throws InterruptedException {
+        String filePath = MessageStoreUtil.toFilePath(queue);
+        FlatCommitLogFile flatFile = flatFileFactory.createFlatFileForCommitLog(filePath);
+        for (int i = 0; i < 3; i++) {
+            ByteBuffer byteBuffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+            byteBuffer.putLong(MessageFormatUtil.QUEUE_OFFSET_POSITION, i);
+            Assert.assertEquals(AppendResult.SUCCESS, flatFile.append(byteBuffer, i));
+            TimeUnit.MILLISECONDS.sleep(2);
+            storeConfig.setCommitLogRollingMinimumSize(byteBuffer.remaining());
+            Assert.assertTrue(flatFile.tryRollingFile(1));
+        }
+        Assert.assertEquals(4, flatFile.fileSegmentTable.size());
+        Assert.assertFalse(flatFile.tryRollingFile(1000));
+        flatFile.destroy();
+    }
+
+    @Test
+    public void getMinOffsetFromFileAsyncTest() {
+        String filePath = MessageStoreUtil.toFilePath(queue);
+        FlatCommitLogFile flatFile = flatFileFactory.createFlatFileForCommitLog(filePath);
+
+        // append some messages
+        for (int i = 6; i < 9; i++) {
+            ByteBuffer byteBuffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+            byteBuffer.putLong(MessageFormatUtil.QUEUE_OFFSET_POSITION, i);
+            Assert.assertEquals(AppendResult.SUCCESS, flatFile.append(byteBuffer, i));
+        }
+        Assert.assertEquals(-1L, flatFile.getMinOffsetFromFileAsync().join().longValue());
+
+        // append some messages
+        for (int i = 9; i < 30; i++) {
+            if (i == 20) {
+                flatFile.commitAsync().join();
+                flatFile.rollingNewFile(flatFile.getAppendOffset());
+            }
+            ByteBuffer byteBuffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+            byteBuffer.putLong(MessageFormatUtil.QUEUE_OFFSET_POSITION, i);
+            Assert.assertEquals(AppendResult.SUCCESS, flatFile.append(byteBuffer, i));
+        }
+
+        flatFile.commitAsync().join();
+        Assert.assertEquals(6L, flatFile.getMinOffsetFromFile());
+        Assert.assertEquals(6L, flatFile.getMinOffsetFromFileAsync().join().longValue());
+
+        // recalculate min offset here
+        flatFile.destroyExpiredFile(20L);
+        Assert.assertEquals(20L, flatFile.getMinOffsetFromFile());
+        Assert.assertEquals(20L, flatFile.getMinOffsetFromFileAsync().join().longValue());
+
+        // clean expired file again
+        flatFile.destroyExpiredFile(20L);
+        Assert.assertEquals(20L, flatFile.getMinOffsetFromFile());
+        Assert.assertEquals(20L, flatFile.getMinOffsetFromFileAsync().join().longValue());
+    }
+}
\ No newline at end of file
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatFileFactoryTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatFileFactoryTest.java
new file mode 100644
index 00000000000..bc8ebaf1cb6
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatFileFactoryTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.file;
+
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class FlatFileFactoryTest {
+
+    @Test
+    public void factoryTest() {
+        MessageStoreConfig storeConfig = new MessageStoreConfig();
+        storeConfig.setTieredStoreFilePath(MessageStoreUtilTest.getRandomStorePath());
+        MetadataStore metadataStore = new DefaultMetadataStore(storeConfig);
+        FlatFileFactory factory = new FlatFileFactory(metadataStore, storeConfig);
+        Assert.assertEquals(storeConfig, factory.getStoreConfig());
+        Assert.assertEquals(metadataStore, factory.getMetadataStore());
+
+        FlatAppendFile flatFile1 = factory.createFlatFileForCommitLog("CommitLog");
+        FlatAppendFile flatFile2 = factory.createFlatFileForConsumeQueue("ConsumeQueue");
+        FlatAppendFile flatFile3 = factory.createFlatFileForIndexFile("IndexFile");
+
+        Assert.assertNotNull(flatFile1);
+        Assert.assertNotNull(flatFile2);
+        Assert.assertNotNull(flatFile3);
+
+        flatFile1.destroy();
+        flatFile2.destroy();
+        flatFile3.destroy();
+    }
+}
\ No newline at end of file
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatFileStoreTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatFileStoreTest.java
new file mode 100644
index 00000000000..2a007af4e9d
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatFileStoreTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.file;
+
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
+import org.apache.rocketmq.tieredstore.exception.TieredStoreErrorCode;
+import org.apache.rocketmq.tieredstore.exception.TieredStoreException;
+import org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.provider.PosixFileSegment;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.mockito.ArgumentMatchers.any;
+
+public class FlatFileStoreTest {
+
+    private final String storePath = MessageStoreUtilTest.getRandomStorePath();
+    private MessageStoreConfig storeConfig;
+    private MetadataStore metadataStore;
+
+    @Before
+    public void init() {
+        storeConfig = new MessageStoreConfig();
+        storeConfig.setStorePathRootDir(storePath);
+        storeConfig.setTieredBackendServiceProvider(PosixFileSegment.class.getName());
+        storeConfig.setBrokerName("brokerName");
+        metadataStore = new DefaultMetadataStore(storeConfig);
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        MessageStoreUtilTest.deleteStoreDirectory(storePath);
+    }
+
+    @Test
+    public void flatFileStoreTest() {
+        // Empty recover
+        MessageStoreExecutor executor = new MessageStoreExecutor();
+        FlatFileStore fileStore = new FlatFileStore(storeConfig, metadataStore, executor);
+        Assert.assertTrue(fileStore.load());
+
+        Assert.assertEquals(storeConfig, fileStore.getStoreConfig());
+        Assert.assertEquals(metadataStore, fileStore.getMetadataStore());
+        Assert.assertNotNull(fileStore.getFlatFileFactory());
+
+        for (int i = 0; i < 4; i++) {
+            MessageQueue mq = new MessageQueue("flatFileStoreTest", storeConfig.getBrokerName(), i);
+            FlatMessageFile flatFile = fileStore.computeIfAbsent(mq);
+            FlatMessageFile flatFileGet = fileStore.getFlatFile(mq);
+            Assert.assertEquals(flatFile, flatFileGet);
+        }
+        Assert.assertEquals(4, fileStore.deepCopyFlatFileToList().size());
+        fileStore.shutdown();
+
+        fileStore = new FlatFileStore(storeConfig, metadataStore, executor);
+        Assert.assertTrue(fileStore.load());
+        Assert.assertEquals(4, fileStore.deepCopyFlatFileToList().size());
+
+        for (int i = 1; i < 3; i++) {
+            MessageQueue mq = new MessageQueue("flatFileStoreTest", storeConfig.getBrokerName(), i);
+            fileStore.destroyFile(mq);
+        }
+        Assert.assertEquals(2, fileStore.deepCopyFlatFileToList().size());
+        fileStore.shutdown();
+
+        FlatFileStore fileStoreSpy = Mockito.spy(fileStore);
+        Mockito.when(fileStoreSpy.recoverAsync(any())).thenReturn(CompletableFuture.supplyAsync(() -> {
+            throw new TieredStoreException(TieredStoreErrorCode.ILLEGAL_PARAM, "Test");
+        }));
+        Assert.assertFalse(fileStoreSpy.load());
+
+        Mockito.reset(fileStoreSpy);
+        fileStore.load();
+        Assert.assertEquals(2, fileStore.deepCopyFlatFileToList().size());
+        fileStore.destroy();
+        Assert.assertEquals(0, fileStore.deepCopyFlatFileToList().size());
+    }
+}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatMessageFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatMessageFileTest.java
new file mode 100644
index 00000000000..97768d0658a
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatMessageFileTest.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.file;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.rocketmq.common.BoundaryType;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.store.ConsumeQueue;
+import org.apache.rocketmq.store.DispatchRequest;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.common.AppendResult;
+import org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.provider.PosixFileSegment;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtil;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtilTest;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FlatMessageFileTest {
+
+    private final String storePath = MessageStoreUtilTest.getRandomStorePath();
+    private MessageStoreConfig storeConfig;
+    private MetadataStore metadataStore;
+    private FlatFileFactory flatFileFactory;
+
+    @Before
+    public void init() throws ClassNotFoundException, NoSuchMethodException {
+        storeConfig = new MessageStoreConfig();
+        storeConfig.setBrokerName("brokerName");
+        storeConfig.setStorePathRootDir(storePath);
+        storeConfig.setTieredBackendServiceProvider(PosixFileSegment.class.getName());
+        storeConfig.setCommitLogRollingInterval(0);
+        storeConfig.setCommitLogRollingMinimumSize(999);
+        metadataStore = new DefaultMetadataStore(storeConfig);
+        flatFileFactory = new FlatFileFactory(metadataStore, storeConfig);
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        MessageStoreUtilTest.deleteStoreDirectory(storePath);
+    }
+
+    @Test
+    public void testAppendCommitLog() {
+        String topic = "CommitLogTest";
+        FlatMessageFile flatFile = new FlatMessageFile(flatFileFactory, topic, 0);
+        Assert.assertTrue(flatFile.getTopicId() >= 0);
+        Assert.assertEquals(topic, flatFile.getMessageQueue().getTopic());
+        Assert.assertEquals(0, flatFile.getMessageQueue().getQueueId());
+        Assert.assertFalse(flatFile.isFlatFileInit());
+
+        flatFile.flushMetadata();
+        Assert.assertNotNull(metadataStore.getQueue(flatFile.getMessageQueue()));
+
+        long offset = 100;
+        flatFile.initOffset(offset);
+        for (int i = 0; i < 5; i++) {
+            ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+            DispatchRequest request = new DispatchRequest(
+                topic, 0, i, (long) buffer.remaining() * i, buffer.remaining(), 0L);
+            flatFile.appendCommitLog(buffer);
+            flatFile.appendConsumeQueue(request);
+        }
+
+        Assert.assertNotNull(flatFile.getFileLock());
+
+        long time = MessageFormatUtil.getStoreTimeStamp(MessageFormatUtilTest.buildMockedMessageBuffer());
+        Assert.assertEquals(time, flatFile.getMinStoreTimestamp());
+        Assert.assertEquals(time, flatFile.getMaxStoreTimestamp());
+
+        long size = MessageFormatUtilTest.buildMockedMessageBuffer().remaining();
+        Assert.assertEquals(-1L, flatFile.getFirstMessageOffset());
+        Assert.assertEquals(0L, flatFile.getCommitLogMinOffset());
+        Assert.assertEquals(0L, flatFile.getCommitLogCommitOffset());
+        Assert.assertEquals(5 * size, flatFile.getCommitLogMaxOffset());
+
+        Assert.assertEquals(offset, flatFile.getConsumeQueueMinOffset());
+        Assert.assertEquals(offset, flatFile.getConsumeQueueCommitOffset());
+        Assert.assertEquals(offset + 5L, flatFile.getConsumeQueueMaxOffset());
+
+        Assert.assertTrue(flatFile.commitAsync().join());
+        Assert.assertEquals(6L, flatFile.getFirstMessageOffset());
+        Assert.assertEquals(0L, flatFile.getCommitLogMinOffset());
+        Assert.assertEquals(5 * size, flatFile.getCommitLogCommitOffset());
+        Assert.assertEquals(5 * size, flatFile.getCommitLogMaxOffset());
+
+        Assert.assertEquals(offset, flatFile.getConsumeQueueMinOffset());
+        Assert.assertEquals(offset + 5L, flatFile.getConsumeQueueCommitOffset());
+        Assert.assertEquals(offset + 5L, flatFile.getConsumeQueueMaxOffset());
+
+        // test read
+        ByteBuffer buffer = flatFile.getMessageAsync(offset).join();
+        Assert.assertNotNull(buffer);
+        Assert.assertEquals(size, buffer.remaining());
+        Assert.assertEquals(6L, MessageFormatUtil.getQueueOffset(buffer));
+
+        flatFile.destroyExpiredFile(0);
+        flatFile.destroy();
+    }
+
+    @Test
+    public void testEquals() {
+        String topic = "EqualsTest";
+        FlatMessageFile flatFile1 = new FlatMessageFile(flatFileFactory, topic, 0);
+        FlatMessageFile flatFile2 = new FlatMessageFile(flatFileFactory, topic, 0);
+        FlatMessageFile flatFile3 = new FlatMessageFile(flatFileFactory, topic, 1);
+        Assert.assertEquals(flatFile1, flatFile2);
+        Assert.assertEquals(flatFile1.hashCode(), flatFile2.hashCode());
+        Assert.assertNotEquals(flatFile1, flatFile3);
+
+        flatFile1.shutdown();
+        flatFile2.shutdown();
+        flatFile3.shutdown();
+
+        flatFile1.destroy();
+        flatFile2.destroy();
+        flatFile3.destroy();
+    }
+
+    @Test
+    public void testBinarySearchInQueueByTime() {
+
+        // replace provider, need new factory again
+        storeConfig.setTieredBackendServiceProvider(PosixFileSegment.class.getName());
+        flatFileFactory = new FlatFileFactory(metadataStore, storeConfig);
+
+        // inject store time: 0, +100, +100, +100, +200
+        MessageQueue mq = new MessageQueue("TopicTest", "BrokerName", 1);
+        FlatMessageFile flatFile = new FlatMessageFile(flatFileFactory, MessageStoreUtil.toFilePath(mq));
+        flatFile.initOffset(50);
+        long timestamp1 = 1000;
+        ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+        buffer.putLong(MessageFormatUtil.QUEUE_OFFSET_POSITION, 50);
+        buffer.putLong(MessageFormatUtil.STORE_TIMESTAMP_POSITION, timestamp1);
+        flatFile.appendCommitLog(buffer);
+
+        long timestamp2 = timestamp1 + 100;
+        buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+        buffer.putLong(MessageFormatUtil.QUEUE_OFFSET_POSITION, 51);
+        buffer.putLong(MessageFormatUtil.STORE_TIMESTAMP_POSITION, timestamp2);
+        flatFile.appendCommitLog(buffer);
+        buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+        buffer.putLong(MessageFormatUtil.QUEUE_OFFSET_POSITION, 52);
+        buffer.putLong(MessageFormatUtil.STORE_TIMESTAMP_POSITION, timestamp2);
+        flatFile.appendCommitLog(buffer);
+        buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+        buffer.putLong(MessageFormatUtil.QUEUE_OFFSET_POSITION, 53);
+        buffer.putLong(MessageFormatUtil.STORE_TIMESTAMP_POSITION, timestamp2);
+        flatFile.appendCommitLog(buffer);
+
+        long timestamp3 = timestamp2 + 100;
+        buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+        buffer.putLong(MessageFormatUtil.QUEUE_OFFSET_POSITION, 54);
+        buffer.putLong(MessageFormatUtil.STORE_TIMESTAMP_POSITION, timestamp3);
+        flatFile.appendCommitLog(buffer);
+
+        // append message to consume queue
+        flatFile.consumeQueue.initOffset(50 * ConsumeQueue.CQ_STORE_UNIT_SIZE);
+
+        AppendResult appendResult = flatFile.appendConsumeQueue(new DispatchRequest(
+                mq.getTopic(), mq.getQueueId(), 0,
+                MessageFormatUtilTest.MSG_LEN, 0, timestamp1, 50,
+                "", "", 0, 0, null));
+        Assert.assertEquals(AppendResult.SUCCESS, appendResult);
+
+        appendResult = flatFile.appendConsumeQueue(new DispatchRequest(
+                mq.getTopic(), mq.getQueueId(), MessageFormatUtilTest.MSG_LEN,
+                MessageFormatUtilTest.MSG_LEN, 0, timestamp2, 51,
+                "", "", 0, 0, null));
+        Assert.assertEquals(AppendResult.SUCCESS, appendResult);
+
+        appendResult = flatFile.appendConsumeQueue(new DispatchRequest(
+                mq.getTopic(), mq.getQueueId(), MessageFormatUtilTest.MSG_LEN * 2,
+                MessageFormatUtilTest.MSG_LEN, 0, timestamp2, 52,
+                "", "", 0, 0, null));
+        Assert.assertEquals(AppendResult.SUCCESS, appendResult);
+
+        appendResult = flatFile.appendConsumeQueue(new DispatchRequest(
+                mq.getTopic(), mq.getQueueId(), MessageFormatUtilTest.MSG_LEN * 3,
+                MessageFormatUtilTest.MSG_LEN, 0, timestamp2, 53,
+                "", "", 0, 0, null));
+        Assert.assertEquals(AppendResult.SUCCESS, appendResult);
+
+        appendResult = flatFile.appendConsumeQueue(new DispatchRequest(
+                mq.getTopic(), mq.getQueueId(), MessageFormatUtilTest.MSG_LEN * 4,
+                MessageFormatUtilTest.MSG_LEN, 0, timestamp3, 54,
+                "", "", 0, 0, null));
+        Assert.assertEquals(AppendResult.SUCCESS, appendResult);
+
+        // commit message will increase max consume queue offset
+        Assert.assertTrue(flatFile.commitAsync().join());
+
+        // offset:            50,  51,   52,   53,   54
+        // inject store time: 0, +100, +100, +100, +200
+        Assert.assertEquals(50, flatFile.getQueueOffsetByTimeAsync(0, BoundaryType.LOWER).join().longValue());
+        Assert.assertEquals(50, flatFile.getQueueOffsetByTimeAsync(0, BoundaryType.UPPER).join().longValue());
+
+        Assert.assertEquals(50, flatFile.getQueueOffsetByTimeAsync(timestamp1 - 1, BoundaryType.LOWER).join().longValue());
+        Assert.assertEquals(50, flatFile.getQueueOffsetByTimeAsync(timestamp1 - 1, BoundaryType.UPPER).join().longValue());
+
+        Assert.assertEquals(50, flatFile.getQueueOffsetByTimeAsync(timestamp1, BoundaryType.LOWER).join().longValue());
+        Assert.assertEquals(50, flatFile.getQueueOffsetByTimeAsync(timestamp1, BoundaryType.UPPER).join().longValue());
+
+        Assert.assertEquals(51, flatFile.getQueueOffsetByTimeAsync(timestamp1 + 1, BoundaryType.LOWER).join().longValue());
+        Assert.assertEquals(51, flatFile.getQueueOffsetByTimeAsync(timestamp1 + 1, BoundaryType.UPPER).join().longValue());
+
+        Assert.assertEquals(51, flatFile.getQueueOffsetByTimeAsync(timestamp2, BoundaryType.LOWER).join().longValue());
+        Assert.assertEquals(53, flatFile.getQueueOffsetByTimeAsync(timestamp2, BoundaryType.UPPER).join().longValue());
+
+        Assert.assertEquals(54, flatFile.getQueueOffsetByTimeAsync(timestamp2 + 1, BoundaryType.UPPER).join().longValue());
+        Assert.assertEquals(54, flatFile.getQueueOffsetByTimeAsync(timestamp2 + 1, BoundaryType.LOWER).join().longValue());
+
+        Assert.assertEquals(54, flatFile.getQueueOffsetByTimeAsync(timestamp3, BoundaryType.LOWER).join().longValue());
+        Assert.assertEquals(54, flatFile.getQueueOffsetByTimeAsync(timestamp3, BoundaryType.UPPER).join().longValue());
+
+        Assert.assertEquals(55, flatFile.getQueueOffsetByTimeAsync(timestamp3 + 1, BoundaryType.LOWER).join().longValue());
+        Assert.assertEquals(55, flatFile.getQueueOffsetByTimeAsync(timestamp3 + 1, BoundaryType.UPPER).join().longValue());
+
+        flatFile.destroy();
+    }
+
+    @Test
+    public void testCommitLock() {
+        String topic = "CommitLogTest";
+        FlatMessageFile flatFile = new FlatMessageFile(flatFileFactory, topic, 0);
+        flatFile.getCommitLock().drainPermits();
+        Assert.assertFalse(flatFile.commitAsync().join());
+    }
+}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredCommitLogTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredCommitLogTest.java
deleted file mode 100644
index 6693d3cb790..00000000000
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredCommitLogTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.file;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
-import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.metadata.FileSegmentMetadata;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
-import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TieredCommitLogTest {
-
-    private final String storePath = TieredStoreTestUtil.getRandomStorePath();
-    private MessageQueue mq;
-    private TieredFileAllocator fileAllocator;
-    private TieredMetadataStore metadataStore;
-
-    @Before
-    public void setUp() throws ClassNotFoundException, NoSuchMethodException {
-        TieredMessageStoreConfig storeConfig = new TieredMessageStoreConfig();
-        storeConfig.setBrokerName("brokerName");
-        storeConfig.setStorePathRootDir(storePath);
-        storeConfig.setTieredStoreFilePath(storePath + File.separator);
-        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment");
-        storeConfig.setCommitLogRollingInterval(0);
-        storeConfig.setTieredStoreCommitLogMaxSize(1000);
-
-        metadataStore = TieredStoreUtil.getMetadataStore(storeConfig);
-        fileAllocator = new TieredFileAllocator(storeConfig);
-        mq = new MessageQueue("CommitLogTest", storeConfig.getBrokerName(), 0);
-        TieredStoreExecutor.init();
-    }
-
-    @After
-    public void tearDown() throws IOException {
-        TieredStoreTestUtil.destroyCompositeFlatFileManager();
-        TieredStoreTestUtil.destroyMetadataStore();
-        TieredStoreTestUtil.destroyTempDir(storePath);
-        TieredStoreExecutor.shutdown();
-    }
-
-    @Test
-    public void correctMinOffsetTest() {
-        String filePath = TieredStoreUtil.toPath(mq);
-        TieredCommitLog tieredCommitLog = new TieredCommitLog(fileAllocator, filePath);
-        Assert.assertEquals(0L, tieredCommitLog.getMinOffset());
-        Assert.assertEquals(0L, tieredCommitLog.getCommitOffset());
-        Assert.assertEquals(0L, tieredCommitLog.getDispatchCommitOffset());
-
-        // append some messages
-        for (int i = 6; i < 50; i++) {
-            ByteBuffer byteBuffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-            byteBuffer.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, i);
-            Assert.assertEquals(AppendResult.SUCCESS, tieredCommitLog.append(byteBuffer));
-        }
-
-        tieredCommitLog.commit(true);
-        tieredCommitLog.correctMinOffset();
-
-        // single file store: 1000 / 122 = 8, file count: 44 / 8 = 5
-        Assert.assertEquals(6, tieredCommitLog.getFlatFile().getFileSegmentCount());
-
-        metadataStore.iterateFileSegment(filePath, FileSegmentType.COMMIT_LOG, metadata -> {
-            if (metadata.getBaseOffset() < 1000) {
-                metadata.setStatus(FileSegmentMetadata.STATUS_DELETED);
-                metadataStore.updateFileSegment(metadata);
-            }
-        });
-
-        // manually delete file
-        List<TieredFileSegment> segmentList = tieredCommitLog.getFlatFile().getFileSegmentList();
-        segmentList.remove(0).destroyFile();
-        segmentList.remove(0).destroyFile();
-
-        tieredCommitLog.correctMinOffset();
-        Assert.assertEquals(4, tieredCommitLog.getFlatFile().getFileSegmentCount());
-        Assert.assertEquals(6 + 8 + 8, tieredCommitLog.getMinConsumeQueueOffset());
-    }
-}
\ No newline at end of file
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java
deleted file mode 100644
index 20fe4dd7022..00000000000
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.file;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-import org.awaitility.Awaitility;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TieredFlatFileManagerTest {
-
-    private final String storePath = TieredStoreTestUtil.getRandomStorePath();
-    private TieredMessageStoreConfig storeConfig;
-    private MessageQueue mq;
-    private TieredMetadataStore metadataStore;
-
-    @Before
-    public void setUp() {
-        storeConfig = new TieredMessageStoreConfig();
-        storeConfig.setStorePathRootDir(storePath);
-        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment");
-        storeConfig.setBrokerName(storeConfig.getBrokerName());
-        mq = new MessageQueue("TieredFlatFileManagerTest", storeConfig.getBrokerName(), 0);
-        metadataStore = TieredStoreUtil.getMetadataStore(storeConfig);
-        TieredStoreExecutor.init();
-    }
-
-    @After
-    public void tearDown() throws IOException {
-        TieredStoreTestUtil.destroyCompositeFlatFileManager();
-        TieredStoreTestUtil.destroyMetadataStore();
-        TieredStoreTestUtil.destroyTempDir(storePath);
-        TieredStoreExecutor.shutdown();
-    }
-
-    @Test
-    public void testLoadAndDestroy() {
-        metadataStore.addTopic(mq.getTopic(), 0);
-        metadataStore.addQueue(mq, 100);
-        MessageQueue mq1 = new MessageQueue(mq.getTopic(), mq.getBrokerName(), 1);
-        metadataStore.addQueue(mq1, 200);
-        TieredFlatFileManager flatFileManager = TieredFlatFileManager.getInstance(storeConfig);
-        boolean load = flatFileManager.load();
-        Assert.assertTrue(load);
-
-        Awaitility.await()
-            .atMost(3, TimeUnit.SECONDS)
-            .until(() -> flatFileManager.deepCopyFlatFileToList().size() == 2);
-
-        CompositeFlatFile flatFile = flatFileManager.getFlatFile(mq);
-        Assert.assertNotNull(flatFile);
-        Assert.assertEquals(-1L, flatFile.getDispatchOffset());
-        flatFile.initOffset(100L);
-        Assert.assertEquals(100L, flatFile.getDispatchOffset());
-        flatFile.initOffset(200L);
-        Assert.assertEquals(100L, flatFile.getDispatchOffset());
-
-        CompositeFlatFile flatFile1 = flatFileManager.getFlatFile(mq1);
-        Assert.assertNotNull(flatFile1);
-        flatFile1.initOffset(200L);
-        Assert.assertEquals(200, flatFile1.getDispatchOffset());
-
-        flatFileManager.destroyCompositeFile(mq);
-        Assert.assertTrue(flatFile.isClosed());
-        Assert.assertNull(flatFileManager.getFlatFile(mq));
-        Assert.assertNull(metadataStore.getQueue(mq));
-
-        flatFileManager.destroy();
-        Assert.assertTrue(flatFile1.isClosed());
-        Assert.assertNull(flatFileManager.getFlatFile(mq1));
-        Assert.assertNull(metadataStore.getQueue(mq1));
-    }
-}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java
deleted file mode 100644
index 7e2fbf20136..00000000000
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.file;
-
-import org.apache.rocketmq.common.BoundaryType;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.metadata.FileSegmentMetadata;
-import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore;
-import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
-import org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-public class TieredFlatFileTest {
-
-    private final String storePath = TieredStoreTestUtil.getRandomStorePath();
-    private MessageQueue queue;
-    private TieredMessageStoreConfig storeConfig;
-    private TieredFileAllocator fileQueueFactory;
-
-    @Before
-    public void setUp() throws ClassNotFoundException, NoSuchMethodException {
-        storeConfig = new TieredMessageStoreConfig();
-        storeConfig.setBrokerName("brokerName");
-        storeConfig.setStorePathRootDir(storePath);
-        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment");
-        queue = new MessageQueue("TieredFlatFileTest", storeConfig.getBrokerName(), 0);
-        fileQueueFactory = new TieredFileAllocator(storeConfig);
-    }
-
-    @After
-    public void tearDown() throws IOException {
-        TieredStoreTestUtil.destroyMetadataStore();
-        TieredStoreTestUtil.destroyTempDir(storePath);
-        TieredStoreExecutor.shutdown();
-    }
-
-    private List<FileSegmentMetadata> getSegmentMetadataList(TieredMetadataStore metadataStore) {
-        List<FileSegmentMetadata> result = new ArrayList<>();
-        metadataStore.iterateFileSegment(result::add);
-        return result;
-    }
-
-    @Test
-    public void testFileSegment() {
-        MemoryFileSegment fileSegment = new MemoryFileSegment(
-            FileSegmentType.COMMIT_LOG, queue, 100, storeConfig);
-        fileSegment.initPosition(fileSegment.getSize());
-
-        String filePath = TieredStoreUtil.toPath(queue);
-        TieredFlatFile fileQueue = fileQueueFactory.createFlatFileForCommitLog(filePath);
-        fileQueue.updateFileSegment(fileSegment);
-
-        TieredMetadataStore metadataStore = TieredStoreUtil.getMetadataStore(storeConfig);
-        FileSegmentMetadata metadata =
-            metadataStore.getFileSegment(filePath, FileSegmentType.COMMIT_LOG, 100);
-        Assert.assertEquals(fileSegment.getPath(), metadata.getPath());
-        Assert.assertEquals(FileSegmentType.COMMIT_LOG, FileSegmentType.valueOf(metadata.getType()));
-        Assert.assertEquals(100, metadata.getBaseOffset());
-        Assert.assertEquals(0, metadata.getSealTimestamp());
-
-        fileSegment.setFull();
-        fileQueue.updateFileSegment(fileSegment);
-        metadata = metadataStore.getFileSegment(fileSegment.getPath(), FileSegmentType.COMMIT_LOG, 100);
-        Assert.assertEquals(1000, metadata.getSize());
-        Assert.assertEquals(0, metadata.getSealTimestamp());
-
-        fileSegment.commit();
-        fileQueue.updateFileSegment(fileSegment);
-        metadata = metadataStore.getFileSegment(fileSegment.getPath(), FileSegmentType.COMMIT_LOG, 100);
-        Assert.assertEquals(1000 + TieredCommitLog.CODA_SIZE, metadata.getSize());
-        Assert.assertTrue(metadata.getSealTimestamp() > 0);
-
-        MemoryFileSegment fileSegment2 = new MemoryFileSegment(FileSegmentType.COMMIT_LOG,
-            queue, 1100, storeConfig);
-        fileQueue.updateFileSegment(fileSegment2);
-        List<FileSegmentMetadata> list = getSegmentMetadataList(metadataStore);
-        Assert.assertEquals(2, list.size());
-        Assert.assertEquals(100, list.get(0).getBaseOffset());
-        Assert.assertEquals(1100, list.get(1).getBaseOffset());
-
-        Assert.assertNotNull(metadataStore.getFileSegment(
-            fileSegment.getPath(), fileSegment.getFileType(), fileSegment.getBaseOffset()));
-        metadataStore.deleteFileSegment(fileSegment.getPath(), fileSegment.getFileType());
-        Assert.assertEquals(0L, getSegmentMetadataList(metadataStore).size());
-    }
-
-    /**
-     * Test whether the file is continuous after switching to write.
-     */
-    @Test
-    public void testGetFileSegment() {
-        TieredFlatFile fileQueue = fileQueueFactory.createFlatFileForCommitLog(TieredStoreUtil.toPath(queue));
-        fileQueue.setBaseOffset(0);
-        TieredFileSegment segment1 = fileQueue.getFileToWrite();
-        segment1.initPosition(1000);
-        segment1.append(ByteBuffer.allocate(100), 0);
-        segment1.setFull();
-        segment1.commit();
-
-        TieredFileSegment segment2 = fileQueue.getFileToWrite();
-        Assert.assertNotSame(segment1, segment2);
-        Assert.assertEquals(1000 + 100 + TieredCommitLog.CODA_SIZE, segment1.getMaxOffset());
-        Assert.assertEquals(1000 + 100 + TieredCommitLog.CODA_SIZE, segment2.getBaseOffset());
-
-        Assert.assertSame(fileQueue.getSegmentIndexByOffset(1000), 0);
-        Assert.assertSame(fileQueue.getSegmentIndexByOffset(1050), 0);
-        Assert.assertSame(fileQueue.getSegmentIndexByOffset(1100 + TieredCommitLog.CODA_SIZE), 1);
-        Assert.assertSame(fileQueue.getSegmentIndexByOffset(1150), -1);
-    }
-
-    @Test
-    public void testAppendAndRead() {
-        TieredFlatFile fileQueue = fileQueueFactory.createFlatFileForConsumeQueue(TieredStoreUtil.toPath(queue));
-        fileQueue.setBaseOffset(0);
-        Assert.assertEquals(0, fileQueue.getMinOffset());
-        Assert.assertEquals(0, fileQueue.getDispatchCommitOffset());
-
-        TieredFileSegment segment1 = fileQueue.getFileToWrite();
-        segment1.initPosition(segment1.getSize());
-        Assert.assertEquals(0, segment1.getBaseOffset());
-        Assert.assertEquals(1000, fileQueue.getCommitOffset());
-        Assert.assertEquals(1000, fileQueue.getMaxOffset());
-
-        ByteBuffer buffer = ByteBuffer.allocate(100);
-        long currentTimeMillis = System.currentTimeMillis();
-        buffer.putLong(currentTimeMillis);
-        buffer.rewind();
-        fileQueue.append(buffer);
-        Assert.assertEquals(1100, segment1.getMaxOffset());
-
-        segment1.setFull();
-        fileQueue.commit(true);
-        Assert.assertEquals(1100, segment1.getCommitOffset());
-
-        ByteBuffer readBuffer = fileQueue.readAsync(1000, 8).join();
-        Assert.assertEquals(currentTimeMillis, readBuffer.getLong());
-
-        TieredFileSegment segment2 = fileQueue.getFileToWrite();
-        Assert.assertNotEquals(segment1, segment2);
-        segment2.initPosition(segment2.getSize());
-        buffer.rewind();
-        fileQueue.append(buffer);
-        fileQueue.commit(true);
-        readBuffer = fileQueue.readAsync(1000, 1200).join();
-        Assert.assertEquals(currentTimeMillis, readBuffer.getLong(1100));
-    }
-
-    @Test
-    public void testLoadFromMetadata() {
-        String filePath = TieredStoreUtil.toPath(queue);
-        TieredFlatFile fileQueue = fileQueueFactory.createFlatFileForCommitLog(filePath);
-
-        MemoryFileSegment fileSegment1 =
-            new MemoryFileSegment(FileSegmentType.COMMIT_LOG, queue, 100, storeConfig);
-        fileSegment1.initPosition(fileSegment1.getSize());
-        fileSegment1.setFull();
-
-        fileQueue.updateFileSegment(fileSegment1);
-        fileQueue.updateFileSegment(fileSegment1);
-
-        MemoryFileSegment fileSegment2 =
-            new MemoryFileSegment(FileSegmentType.COMMIT_LOG, queue, 1100, storeConfig);
-        fileQueue.updateFileSegment(fileSegment2);
-
-        // Set instance to null and reload from disk
-        TieredStoreUtil.metadataStoreInstance = null;
-        fileQueue = fileQueueFactory.createFlatFileForCommitLog(filePath);
-        Assert.assertEquals(2, fileQueue.getNeedCommitFileSegmentList().size());
-        TieredFileSegment file1 = fileQueue.getFileByIndex(0);
-        Assert.assertNotNull(file1);
-        Assert.assertEquals(100, file1.getBaseOffset());
-        Assert.assertFalse(file1.isFull());
-
-        TieredFileSegment file2 = fileQueue.getFileByIndex(1);
-        Assert.assertNotNull(file2);
-        Assert.assertEquals(1100, file2.getBaseOffset());
-        Assert.assertFalse(file2.isFull());
-
-        TieredFileSegment file3 = fileQueue.getFileByIndex(2);
-        Assert.assertNull(file3);
-    }
-
-    @Test
-    public void testCheckFileSize() {
-        String filePath = TieredStoreUtil.toPath(queue);
-        TieredFlatFile tieredFlatFile = fileQueueFactory.createFlatFileForCommitLog(filePath);
-
-        TieredFileSegment fileSegment1 = new MemoryFileSegment(
-            FileSegmentType.CONSUME_QUEUE, queue, 100, storeConfig);
-        fileSegment1.initPosition(fileSegment1.getSize() - 100);
-        fileSegment1.setFull();
-        tieredFlatFile.updateFileSegment(fileSegment1);
-        tieredFlatFile.updateFileSegment(fileSegment1);
-
-        TieredFileSegment fileSegment2 = new MemoryFileSegment(
-            FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig);
-        fileSegment2.initPosition(fileSegment2.getSize() - 100);
-        tieredFlatFile.updateFileSegment(fileSegment2);
-        tieredFlatFile.updateFileSegment(fileSegment2);
-
-        TieredFlatFile fileQueue = fileQueueFactory.createFlatFileForConsumeQueue(filePath);
-        Assert.assertEquals(1, fileQueue.getNeedCommitFileSegmentList().size());
-
-        fileSegment1 = fileQueue.getFileByIndex(0);
-        Assert.assertTrue(fileSegment1.isFull());
-        Assert.assertEquals(fileSegment1.getSize() + 100, fileSegment1.getCommitOffset());
-
-        fileSegment2 = fileQueue.getFileByIndex(1);
-        Assert.assertEquals(1000, fileSegment2.getCommitPosition());
-
-        fileSegment2.setFull();
-        fileQueue.commit(true);
-        Assert.assertEquals(0, fileQueue.getNeedCommitFileSegmentList().size());
-
-        fileQueue.getFileToWrite();
-        Assert.assertEquals(1, fileQueue.getNeedCommitFileSegmentList().size());
-    }
-
-    @Test
-    public void testCleanExpiredFile() {
-        String filePath = TieredStoreUtil.toPath(queue);
-        TieredFlatFile tieredFlatFile = fileQueueFactory.createFlatFileForCommitLog(filePath);
-
-        TieredFileSegment fileSegment1 = new MemoryFileSegment(
-            FileSegmentType.CONSUME_QUEUE, queue, 100, storeConfig);
-        fileSegment1.initPosition(fileSegment1.getSize() - 100);
-        fileSegment1.setFull(false);
-        fileSegment1.setMaxTimestamp(System.currentTimeMillis() - 1);
-        tieredFlatFile.updateFileSegment(fileSegment1);
-        tieredFlatFile.updateFileSegment(fileSegment1);
-
-        long file1CreateTimeStamp = System.currentTimeMillis();
-
-        TieredFileSegment fileSegment2 = new MemoryFileSegment(
-            FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig);
-        fileSegment2.initPosition(fileSegment2.getSize());
-        fileSegment2.setMaxTimestamp(System.currentTimeMillis() + 1);
-        tieredFlatFile.updateFileSegment(fileSegment2);
-        tieredFlatFile.updateFileSegment(fileSegment2);
-
-        TieredFlatFile fileQueue = fileQueueFactory.createFlatFileForConsumeQueue(filePath);
-        Assert.assertEquals(2, fileQueue.getFileSegmentCount());
-
-        TieredMetadataStore metadataStore = TieredStoreUtil.getMetadataStore(storeConfig);
-        fileQueue.cleanExpiredFile(file1CreateTimeStamp);
-        fileQueue.destroyExpiredFile();
-        Assert.assertEquals(1, fileQueue.getFileSegmentCount());
-        Assert.assertNull(getMetadata(metadataStore, fileSegment1));
-        Assert.assertNotNull(getMetadata(metadataStore, fileSegment2));
-
-        fileQueue.cleanExpiredFile(Long.MAX_VALUE);
-        fileQueue.destroyExpiredFile();
-        Assert.assertEquals(0, fileQueue.getFileSegmentCount());
-        Assert.assertNull(getMetadata(metadataStore, fileSegment1));
-        Assert.assertNull(getMetadata(metadataStore, fileSegment2));
-    }
-
-    private FileSegmentMetadata getMetadata(TieredMetadataStore metadataStore, TieredFileSegment fileSegment) {
-        return metadataStore.getFileSegment(
-            fileSegment.getPath(), fileSegment.getFileType(), fileSegment.getBaseOffset());
-    }
-
-    @Test
-    public void testRollingNewFile() {
-        String filePath = TieredStoreUtil.toPath(queue);
-        TieredFlatFile tieredFlatFile = fileQueueFactory.createFlatFileForCommitLog(filePath);
-
-        TieredFileSegment fileSegment1 = new MemoryFileSegment(
-            FileSegmentType.CONSUME_QUEUE, queue, 100, storeConfig);
-        fileSegment1.initPosition(fileSegment1.getSize() - 100);
-        tieredFlatFile.updateFileSegment(fileSegment1);
-
-        TieredFlatFile fileQueue = fileQueueFactory.createFlatFileForConsumeQueue(filePath);
-        Assert.assertEquals(1, fileQueue.getFileSegmentCount());
-
-        fileQueue.rollingNewFile();
-        Assert.assertEquals(2, fileQueue.getFileSegmentCount());
-    }
-
-    @Test
-    public void testGetFileByTime() {
-        String filePath = TieredStoreUtil.toPath(queue);
-        TieredFlatFile tieredFlatFile = fileQueueFactory.createFlatFileForCommitLog(filePath);
-        TieredFileSegment fileSegment1 = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig);
-        fileSegment1.setMinTimestamp(100);
-        fileSegment1.setMaxTimestamp(200);
-
-        TieredFileSegment fileSegment2 = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig);
-        fileSegment2.setMinTimestamp(200);
-        fileSegment2.setMaxTimestamp(300);
-
-        tieredFlatFile.getFileSegmentList().add(fileSegment1);
-        tieredFlatFile.getFileSegmentList().add(fileSegment2);
-
-        TieredFileSegment segmentUpper = tieredFlatFile.getFileByTime(400, BoundaryType.UPPER);
-        Assert.assertEquals(fileSegment2, segmentUpper);
-
-        TieredFileSegment segmentLower = tieredFlatFile.getFileByTime(400, BoundaryType.LOWER);
-        Assert.assertEquals(fileSegment2, segmentLower);
-
-
-        TieredFileSegment segmentUpper2 = tieredFlatFile.getFileByTime(0, BoundaryType.UPPER);
-        Assert.assertEquals(fileSegment1, segmentUpper2);
-
-        TieredFileSegment segmentLower2 = tieredFlatFile.getFileByTime(0, BoundaryType.LOWER);
-        Assert.assertEquals(fileSegment1, segmentLower2);
-
-
-        TieredFileSegment segmentUpper3 = tieredFlatFile.getFileByTime(200, BoundaryType.UPPER);
-        Assert.assertEquals(fileSegment1, segmentUpper3);
-
-        TieredFileSegment segmentLower3 = tieredFlatFile.getFileByTime(200, BoundaryType.LOWER);
-        Assert.assertEquals(fileSegment2, segmentLower3);
-    }
-}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreFileTest.java
index b408a7c3cf6..d19b562463d 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreFileTest.java
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreFileTest.java
@@ -28,13 +28,13 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
-import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
 import org.apache.rocketmq.tieredstore.common.AppendResult;
 import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.provider.TieredFileSegment;
-import org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment;
+import org.apache.rocketmq.tieredstore.provider.FileSegment;
+import org.apache.rocketmq.tieredstore.provider.PosixFileSegment;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -51,20 +51,19 @@ public class IndexStoreFileTest {
     private static final Set<String> KEY_SET = Collections.singleton(KEY);
 
     private String filePath;
-    private TieredMessageStoreConfig storeConfig;
+    private MessageStoreConfig storeConfig;
     private IndexStoreFile indexStoreFile;
 
     @Before
     public void init() throws IOException {
-        TieredStoreExecutor.init();
         filePath = UUID.randomUUID().toString().replace("-", "").substring(0, 8);
         String directory = Paths.get(System.getProperty("user.home"), "store_test", filePath).toString();
-        storeConfig = new TieredMessageStoreConfig();
+        storeConfig = new MessageStoreConfig();
         storeConfig.setStorePathRootDir(directory);
         storeConfig.setTieredStoreFilePath(directory);
         storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5);
         storeConfig.setTieredStoreIndexFileMaxIndexNum(20);
-        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment");
+        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.PosixFileSegment");
         indexStoreFile = new IndexStoreFile(storeConfig, System.currentTimeMillis());
     }
 
@@ -74,10 +73,7 @@ public void shutdown() {
             this.indexStoreFile.shutdown();
             this.indexStoreFile.destroy();
         }
-        TieredStoreTestUtil.destroyMetadataStore();
-        TieredStoreTestUtil.destroyTempDir(storeConfig.getStorePathRootDir());
-        TieredStoreTestUtil.destroyTempDir(storeConfig.getTieredStoreFilePath());
-        TieredStoreExecutor.shutdown();
+        MessageStoreUtilTest.deleteStoreDirectory(storeConfig.getTieredStoreFilePath());
     }
 
     @Test
@@ -215,7 +211,7 @@ public void recoverFileTest() throws IOException {
     }
 
     @Test
-    public void doCompactionTest() throws Exception {
+    public void doCompactionTest() {
         long timestamp = indexStoreFile.getTimestamp();
         for (int i = 0; i < 10; i++) {
             Assert.assertEquals(AppendResult.SUCCESS, indexStoreFile.putKey(
@@ -223,10 +219,10 @@ public void doCompactionTest() throws Exception {
         }
 
         ByteBuffer byteBuffer = indexStoreFile.doCompaction();
-        TieredFileSegment fileSegment = new PosixFileSegment(
-            storeConfig, FileSegmentType.INDEX, filePath, 0L);
+        FileSegment fileSegment = new PosixFileSegment(
+            storeConfig, FileSegmentType.INDEX, filePath, 0L, new MessageStoreExecutor());
         fileSegment.append(byteBuffer, timestamp);
-        fileSegment.commit();
+        fileSegment.commitAsync().join();
         Assert.assertEquals(byteBuffer.limit(), fileSegment.getSize());
         fileSegment.destroyFile();
     }
@@ -256,10 +252,10 @@ public void queryAsyncFromSegmentFileTest() throws ExecutionException, Interrupt
         }
 
         ByteBuffer byteBuffer = indexStoreFile.doCompaction();
-        TieredFileSegment fileSegment = new PosixFileSegment(
-            storeConfig, FileSegmentType.INDEX, filePath, 0L);
+        FileSegment fileSegment = new PosixFileSegment(
+            storeConfig, FileSegmentType.INDEX, filePath, 0L, new MessageStoreExecutor());
         fileSegment.append(byteBuffer, timestamp);
-        fileSegment.commit();
+        fileSegment.commitAsync().join();
         Assert.assertEquals(byteBuffer.limit(), fileSegment.getSize());
         indexStoreFile.destroy();
 
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceBenchTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceBenchTest.java
index 57d00eefe15..fcb28402ea9 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceBenchTest.java
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceBenchTest.java
@@ -27,13 +27,12 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.LongAdder;
 import org.apache.rocketmq.common.UtilAll;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
 import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.file.TieredFileAllocator;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
+import org.apache.rocketmq.tieredstore.file.FlatFileFactory;
+import org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
 import org.junit.Assert;
 import org.junit.Ignore;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -51,15 +50,17 @@
 import org.openjdk.jmh.runner.Runner;
 import org.openjdk.jmh.runner.options.Options;
 import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Ignore
 @State(Scope.Benchmark)
 @Fork(value = 1, jvmArgs = {"-Djava.net.preferIPv4Stack=true", "-Djmh.rmi.port=1099"})
 public class IndexStoreServiceBenchTest {
 
-    private static final Logger log = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
+    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
     private static final String TOPIC_NAME = "TopicTest";
-    private TieredMessageStoreConfig storeConfig;
+    private MessageStoreConfig storeConfig;
     private IndexStoreService indexStoreService;
     private final LongAdder failureCount = new LongAdder();
 
@@ -68,25 +69,23 @@ public void init() throws ClassNotFoundException, NoSuchMethodException {
         String storePath = Paths.get(System.getProperty("user.home"), "store_test", "index").toString();
         UtilAll.deleteFile(new File(storePath));
         UtilAll.deleteFile(new File("./e96d41b2_IndexService"));
-        storeConfig = new TieredMessageStoreConfig();
+        storeConfig = new MessageStoreConfig();
         storeConfig.setBrokerClusterName("IndexService");
         storeConfig.setBrokerName("IndexServiceBroker");
         storeConfig.setStorePathRootDir(storePath);
-        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment");
+        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.PosixFileSegment");
         storeConfig.setTieredStoreIndexFileMaxHashSlotNum(500 * 1000);
         storeConfig.setTieredStoreIndexFileMaxIndexNum(2000 * 1000);
-        TieredStoreUtil.getMetadataStore(storeConfig);
-        TieredStoreExecutor.init();
-        TieredFileAllocator tieredFileAllocator = new TieredFileAllocator(storeConfig);
-        indexStoreService = new IndexStoreService(tieredFileAllocator, storePath);
+        MetadataStore metadataStore = new DefaultMetadataStore(storeConfig);
+        FlatFileFactory flatFileFactory = new FlatFileFactory(metadataStore, storeConfig);
+        indexStoreService = new IndexStoreService(flatFileFactory, storePath);
         indexStoreService.start();
     }
 
-    @TearDown
+    @TearDown()
     public void shutdown() throws IOException {
         indexStoreService.shutdown();
         indexStoreService.destroy();
-        TieredStoreExecutor.shutdown();
     }
 
     //@Benchmark
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java
index 20b4acbfa11..7b881ddd447 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java
@@ -33,26 +33,30 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.store.logfile.DefaultMappedFile;
-import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
 import org.apache.rocketmq.tieredstore.common.AppendResult;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.file.TieredFileAllocator;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
+import org.apache.rocketmq.tieredstore.file.FlatAppendFile;
+import org.apache.rocketmq.tieredstore.file.FlatFileFactory;
+import org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
+import org.awaitility.Awaitility;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.awaitility.Awaitility.await;
 
 public class IndexStoreServiceTest {
 
-    private static final Logger log = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME);
+    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
 
     private static final String TOPIC_NAME = "TopicTest";
     private static final int TOPIC_ID = 123;
@@ -62,22 +66,22 @@ public class IndexStoreServiceTest {
     private static final Set<String> KEY_SET = Collections.singleton("MessageKey");
 
     private String filePath;
-    private TieredMessageStoreConfig storeConfig;
-    private TieredFileAllocator fileAllocator;
+    private MessageStoreConfig storeConfig;
+    private FlatFileFactory fileAllocator;
     private IndexStoreService indexService;
 
     @Before
     public void init() throws IOException, ClassNotFoundException, NoSuchMethodException {
-        TieredStoreExecutor.init();
         filePath = UUID.randomUUID().toString().replace("-", "").substring(0, 8);
         String directory = Paths.get(System.getProperty("user.home"), "store_test", filePath).toString();
-        storeConfig = new TieredMessageStoreConfig();
+        storeConfig = new MessageStoreConfig();
         storeConfig.setStorePathRootDir(directory);
         storeConfig.setTieredStoreFilePath(directory);
         storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5);
         storeConfig.setTieredStoreIndexFileMaxIndexNum(20);
-        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment");
-        fileAllocator = new TieredFileAllocator(storeConfig);
+        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.PosixFileSegment");
+        MetadataStore metadataStore = new DefaultMetadataStore(storeConfig);
+        fileAllocator = new FlatFileFactory(metadataStore, storeConfig);
     }
 
     @After
@@ -86,15 +90,13 @@ public void shutdown() {
             indexService.shutdown();
             indexService.destroy();
         }
-        TieredStoreTestUtil.destroyMetadataStore();
-        TieredStoreTestUtil.destroyTempDir(storeConfig.getStorePathRootDir());
-        TieredStoreTestUtil.destroyTempDir(storeConfig.getTieredStoreFilePath());
-        TieredStoreExecutor.shutdown();
+        MessageStoreUtilTest.deleteStoreDirectory(storeConfig.getTieredStoreFilePath());
     }
 
     @Test
     public void basicServiceTest() throws InterruptedException {
         indexService = new IndexStoreService(fileAllocator, filePath);
+        indexService.start();
         for (int i = 0; i < 50; i++) {
             Assert.assertEquals(AppendResult.SUCCESS, indexService.putKey(
                 TOPIC_NAME, TOPIC_ID, QUEUE_ID, KEY_SET, i * 100, MESSAGE_SIZE, System.currentTimeMillis()));
@@ -107,6 +109,7 @@ public void basicServiceTest() throws InterruptedException {
     @Test
     public void doConvertOldFormatTest() throws IOException {
         indexService = new IndexStoreService(fileAllocator, filePath);
+        indexService.start();
         long timestamp = indexService.getTimeStoreTable().firstKey();
         Assert.assertEquals(AppendResult.SUCCESS, indexService.putKey(
             TOPIC_NAME, TOPIC_ID, QUEUE_ID, KEY_SET, MESSAGE_OFFSET, MESSAGE_SIZE, timestamp));
@@ -118,8 +121,9 @@ public void doConvertOldFormatTest() throws IOException {
         mappedFile.shutdown(10 * 1000);
 
         indexService = new IndexStoreService(fileAllocator, filePath);
+        indexService.start();
         ConcurrentSkipListMap<Long, IndexFile> timeStoreTable = indexService.getTimeStoreTable();
-        Assert.assertEquals(1, timeStoreTable.size());
+        Assert.assertEquals(2, timeStoreTable.size());
         Assert.assertEquals(Long.valueOf(timestamp), timeStoreTable.firstKey());
         mappedFile.destroy(10 * 1000);
     }
@@ -131,6 +135,7 @@ public void concurrentPutTest() throws InterruptedException {
         storeConfig.setTieredStoreIndexFileMaxHashSlotNum(500);
         storeConfig.setTieredStoreIndexFileMaxIndexNum(2000);
         indexService = new IndexStoreService(fileAllocator, filePath);
+        indexService.start();
         long timestamp = System.currentTimeMillis();
 
         // first item is invalid
@@ -204,9 +209,43 @@ public void runServiceTest() throws InterruptedException {
         });
     }
 
+    @Test
+    public void deleteFileTest() throws InterruptedException, IllegalAccessException {
+        indexService = new IndexStoreService(fileAllocator, filePath);
+        indexService.start();
+
+        for (int i = 0; i < 2 * 20; i++) {
+            AppendResult result = indexService.putKey(
+                TOPIC_NAME, TOPIC_ID, QUEUE_ID, Collections.singleton(String.valueOf(i)),
+                i * 100L, MESSAGE_SIZE, System.currentTimeMillis());
+            Assert.assertEquals(AppendResult.SUCCESS, result);
+            TimeUnit.MILLISECONDS.sleep(1);
+        }
+
+        indexService.wakeup();
+        Awaitility.await().until(() -> {
+            int tableSize = (int) indexService.getTimeStoreTable().entrySet().stream()
+                .filter(entry -> IndexFile.IndexStatusEnum.UPLOAD.equals(entry.getValue().getFileStatus()))
+                .count();
+            return tableSize == 2;
+        });
+
+        long timestamp = indexService.getTimeStoreTable().firstEntry().getValue().getEndTimestamp();
+        FlatAppendFile flatAppendFile = (FlatAppendFile)
+            FieldUtils.readField(indexService, "flatAppendFile", true);
+
+        indexService.destroyExpiredFile(timestamp);
+        Assert.assertEquals(2, flatAppendFile.getFileSegmentList().size());
+        Assert.assertEquals(3, indexService.getTimeStoreTable().size());
+        indexService.destroyExpiredFile(timestamp + 1);
+        Assert.assertEquals(1, flatAppendFile.getFileSegmentList().size());
+        Assert.assertEquals(2, indexService.getTimeStoreTable().size());
+    }
+
     @Test
     public void restartServiceTest() throws InterruptedException {
         indexService = new IndexStoreService(fileAllocator, filePath);
+        indexService.start();
         for (int i = 0; i < 20; i++) {
             AppendResult result = indexService.putKey(
                 TOPIC_NAME, TOPIC_ID, QUEUE_ID, Collections.singleton(String.valueOf(i)),
@@ -216,10 +255,10 @@ public void restartServiceTest() throws InterruptedException {
         }
         long timestamp = indexService.getTimeStoreTable().firstKey();
         indexService.shutdown();
-        indexService = new IndexStoreService(fileAllocator, filePath);
-        Assert.assertEquals(timestamp, indexService.getTimeStoreTable().firstKey().longValue());
 
+        indexService = new IndexStoreService(fileAllocator, filePath);
         indexService.start();
+        Assert.assertEquals(timestamp, indexService.getTimeStoreTable().firstKey().longValue());
         await().atMost(Duration.ofMinutes(1)).pollInterval(Duration.ofSeconds(1)).until(() -> {
             ArrayList<IndexFile> files = new ArrayList<>(indexService.getTimeStoreTable().values());
             return IndexFile.IndexStatusEnum.UPLOAD.equals(files.get(0).getFileStatus());
@@ -227,8 +266,9 @@ public void restartServiceTest() throws InterruptedException {
         indexService.shutdown();
 
         indexService = new IndexStoreService(fileAllocator, filePath);
+        indexService.start();
         Assert.assertEquals(timestamp, indexService.getTimeStoreTable().firstKey().longValue());
-        Assert.assertEquals(2, indexService.getTimeStoreTable().size());
+        Assert.assertEquals(4, indexService.getTimeStoreTable().size());
         Assert.assertEquals(IndexFile.IndexStatusEnum.UPLOAD,
             indexService.getTimeStoreTable().firstEntry().getValue().getFileStatus());
     }
@@ -237,6 +277,7 @@ public void restartServiceTest() throws InterruptedException {
     public void queryFromFileTest() throws InterruptedException, ExecutionException {
         long timestamp = System.currentTimeMillis();
         indexService = new IndexStoreService(fileAllocator, filePath);
+        indexService.start();
 
         // three files, echo contains 19 items
         for (int i = 0; i < 3; i++) {
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/metadata/TieredMetadataManagerTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/metadata/DefaultMetadataStoreTest.java
similarity index 77%
rename from tieredstore/src/test/java/org/apache/rocketmq/tieredstore/metadata/TieredMetadataManagerTest.java
rename to tieredstore/src/test/java/org/apache/rocketmq/tieredstore/metadata/DefaultMetadataStoreTest.java
index f7d2c352a2b..7a33903d84f 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/metadata/TieredMetadataManagerTest.java
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/metadata/DefaultMetadataStoreTest.java
@@ -24,39 +24,42 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
 import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
+import org.apache.rocketmq.tieredstore.metadata.entity.FileSegmentMetadata;
+import org.apache.rocketmq.tieredstore.metadata.entity.QueueMetadata;
+import org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-public class TieredMetadataManagerTest {
+public class DefaultMetadataStoreTest {
 
-    private final String storePath = TieredStoreTestUtil.getRandomStorePath();
+    private final String storePath = MessageStoreUtilTest.getRandomStorePath();
     private MessageQueue mq0;
     private MessageQueue mq1;
     private MessageQueue mq2;
-    private TieredMessageStoreConfig storeConfig;
-    private TieredMetadataStore metadataStore;
+    private MessageStoreConfig storeConfig;
+    private MetadataStore metadataStore;
 
     @Before
-    public void setUp() {
-        storeConfig = new TieredMessageStoreConfig();
+    public void init() {
+        storeConfig = new MessageStoreConfig();
         storeConfig.setBrokerName("brokerName");
         storeConfig.setStorePathRootDir(storePath);
         mq0 = new MessageQueue("MetadataStoreTest0", storeConfig.getBrokerName(), 0);
         mq1 = new MessageQueue("MetadataStoreTest1", storeConfig.getBrokerName(), 0);
         mq2 = new MessageQueue("MetadataStoreTest1", storeConfig.getBrokerName(), 1);
-        metadataStore = new TieredMetadataManager(storeConfig);
+        metadataStore = new DefaultMetadataStore(storeConfig);
     }
 
     @After
-    public void tearDown() throws IOException {
-        TieredStoreTestUtil.destroyMetadataStore();
-        TieredStoreTestUtil.destroyTempDir(storePath);
+    public void shutdown() throws IOException {
+        metadataStore.destroy();
+        MessageStoreUtilTest.deleteStoreDirectory(storePath);
     }
 
     @Test
@@ -142,13 +145,13 @@ public void testTopic() {
         Assert.assertNotNull(metadataStore.getTopic(topic1));
     }
 
-    private long countFileSegment(TieredMetadataStore metadataStore) {
+    private long countFileSegment(MetadataStore metadataStore) {
         AtomicLong count = new AtomicLong();
         metadataStore.iterateFileSegment(segmentMetadata -> count.incrementAndGet());
         return count.get();
     }
 
-    private long countFileSegment(TieredMetadataStore metadataStore, String filePath) {
+    private long countFileSegment(MetadataStore metadataStore, String filePath) {
         AtomicLong count = new AtomicLong();
         metadataStore.iterateFileSegment(
             filePath, FileSegmentType.COMMIT_LOG, segmentMetadata -> count.incrementAndGet());
@@ -157,14 +160,14 @@ private long countFileSegment(TieredMetadataStore metadataStore, String filePath
 
     @Test
     public void testFileSegment() {
-        String filePath = TieredStoreUtil.toPath(mq0);
+        String filePath = MessageStoreUtil.toFilePath(mq0);
         FileSegmentMetadata segmentMetadata1 = new FileSegmentMetadata(
-            filePath, 0L, FileSegmentType.COMMIT_LOG.getType());
+            filePath, 0L, FileSegmentType.COMMIT_LOG.getCode());
         metadataStore.updateFileSegment(segmentMetadata1);
         Assert.assertEquals(1L, countFileSegment(metadataStore));
 
         FileSegmentMetadata segmentMetadata2 = new FileSegmentMetadata(
-            filePath, 100, FileSegmentType.COMMIT_LOG.getType());
+            filePath, 100, FileSegmentType.COMMIT_LOG.getCode());
         metadataStore.updateFileSegment(segmentMetadata2);
         Assert.assertEquals(2L, countFileSegment(metadataStore));
 
@@ -186,15 +189,15 @@ public void testFileSegment() {
 
     @Test
     public void testFileSegmentDelete() {
-        String filePath0 = TieredStoreUtil.toPath(mq0);
-        String filePath1 = TieredStoreUtil.toPath(mq1);
+        String filePath0 = MessageStoreUtil.toFilePath(mq0);
+        String filePath1 = MessageStoreUtil.toFilePath(mq1);
         for (int i = 0; i < 10; i++) {
             FileSegmentMetadata segmentMetadata = new FileSegmentMetadata(
-                filePath0, i * 1000L * 1000L, FileSegmentType.COMMIT_LOG.getType());
+                filePath0, i * 1000L * 1000L, FileSegmentType.COMMIT_LOG.getCode());
             metadataStore.updateFileSegment(segmentMetadata);
 
             segmentMetadata = new FileSegmentMetadata(
-                filePath1, i * 1000L * 1000L, FileSegmentType.COMMIT_LOG.getType());
+                filePath1, i * 1000L * 1000L, FileSegmentType.COMMIT_LOG.getCode());
             metadataStore.updateFileSegment(segmentMetadata);
         }
         Assert.assertEquals(20, countFileSegment(metadataStore));
@@ -213,52 +216,52 @@ public void testFileSegmentDelete() {
 
     @Test
     public void testReload() {
-        TieredMetadataManager metadataManager = (TieredMetadataManager) metadataStore;
-        metadataManager.addTopic(mq0.getTopic(), 1);
-        metadataManager.addTopic(mq1.getTopic(), 2);
+        DefaultMetadataStore defaultMetadataStore = (DefaultMetadataStore) metadataStore;
+        defaultMetadataStore.addTopic(mq0.getTopic(), 1);
+        defaultMetadataStore.addTopic(mq1.getTopic(), 2);
 
-        metadataManager.addQueue(mq0, 2);
-        metadataManager.addQueue(mq1, 4);
-        metadataManager.addQueue(mq2, 8);
+        defaultMetadataStore.addQueue(mq0, 2);
+        defaultMetadataStore.addQueue(mq1, 4);
+        defaultMetadataStore.addQueue(mq2, 8);
 
-        String filePath0 = TieredStoreUtil.toPath(mq0);
+        String filePath0 = MessageStoreUtil.toFilePath(mq0);
         FileSegmentMetadata segmentMetadata =
-            new FileSegmentMetadata(filePath0, 100, FileSegmentType.COMMIT_LOG.getType());
+            new FileSegmentMetadata(filePath0, 100, FileSegmentType.COMMIT_LOG.getCode());
         metadataStore.updateFileSegment(segmentMetadata);
         segmentMetadata =
-            new FileSegmentMetadata(filePath0, 200, FileSegmentType.COMMIT_LOG.getType());
+            new FileSegmentMetadata(filePath0, 200, FileSegmentType.COMMIT_LOG.getCode());
         metadataStore.updateFileSegment(segmentMetadata);
 
-        Assert.assertTrue(new File(metadataManager.configFilePath()).exists());
+        Assert.assertTrue(new File(defaultMetadataStore.configFilePath()).exists());
 
         // Reload from disk
-        metadataManager = new TieredMetadataManager(storeConfig);
-        metadataManager.load();
-        TopicMetadata topicMetadata = metadataManager.getTopic(mq0.getTopic());
+        defaultMetadataStore = new DefaultMetadataStore(storeConfig);
+        defaultMetadataStore.load();
+        TopicMetadata topicMetadata = defaultMetadataStore.getTopic(mq0.getTopic());
         Assert.assertNotNull(topicMetadata);
         Assert.assertEquals(topicMetadata.getReserveTime(), 1);
 
-        topicMetadata = metadataManager.getTopic(mq1.getTopic());
+        topicMetadata = defaultMetadataStore.getTopic(mq1.getTopic());
         Assert.assertNotNull(topicMetadata);
         Assert.assertEquals(topicMetadata.getReserveTime(), 2);
 
-        QueueMetadata queueMetadata = metadataManager.getQueue(mq0);
+        QueueMetadata queueMetadata = defaultMetadataStore.getQueue(mq0);
         Assert.assertNotNull(queueMetadata);
         Assert.assertEquals(mq0, queueMetadata.getQueue());
         Assert.assertEquals(queueMetadata.getMinOffset(), 2);
 
-        queueMetadata = metadataManager.getQueue(mq1);
+        queueMetadata = defaultMetadataStore.getQueue(mq1);
         Assert.assertNotNull(queueMetadata);
         Assert.assertEquals(mq1, queueMetadata.getQueue());
         Assert.assertEquals(queueMetadata.getMinOffset(), 4);
 
-        queueMetadata = metadataManager.getQueue(mq2);
+        queueMetadata = defaultMetadataStore.getQueue(mq2);
         Assert.assertNotNull(queueMetadata);
         Assert.assertEquals(mq2, queueMetadata.getQueue());
         Assert.assertEquals(queueMetadata.getMinOffset(), 8);
 
         Map<Long, FileSegmentMetadata> map = new HashMap<>();
-        metadataManager.iterateFileSegment(metadata -> map.put(metadata.getBaseOffset(), metadata));
+        defaultMetadataStore.iterateFileSegment(metadata -> map.put(metadata.getBaseOffset(), metadata));
         FileSegmentMetadata fileSegmentMetadata = map.get(100L);
         Assert.assertNotNull(fileSegmentMetadata);
         Assert.assertEquals(filePath0, fileSegmentMetadata.getPath());
@@ -267,4 +270,15 @@ public void testReload() {
         Assert.assertNotNull(fileSegmentMetadata);
         Assert.assertEquals(filePath0, fileSegmentMetadata.getPath());
     }
+
+    @Test
+    public void basicTest() {
+        this.testTopic();
+        this.testQueue();
+        this.testFileSegment();
+
+        ((DefaultMetadataStore) metadataStore).encode();
+        ((DefaultMetadataStore) metadataStore).encode(false);
+        ((DefaultMetadataStore) metadataStore).encode(true);
+    }
 }
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManagerTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManagerTest.java
index 26b38b9706e..cc4d9e2c68b 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManagerTest.java
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManagerTest.java
@@ -17,23 +17,17 @@
 package org.apache.rocketmq.tieredstore.metrics;
 
 import io.opentelemetry.sdk.OpenTelemetrySdk;
-import java.io.IOException;
-import org.apache.rocketmq.tieredstore.TieredMessageFetcher;
-import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.junit.After;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.TieredMessageStore;
+import org.apache.rocketmq.tieredstore.core.MessageStoreFetcherImpl;
+import org.apache.rocketmq.tieredstore.file.FlatFileStore;
+import org.apache.rocketmq.tieredstore.provider.PosixFileSegment;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 public class TieredStoreMetricsManagerTest {
 
-    @After
-    public void tearDown() throws IOException {
-        TieredStoreTestUtil.destroyCompositeFlatFileManager();
-        TieredStoreTestUtil.destroyMetadataStore();
-        TieredStoreExecutor.shutdown();
-    }
-
     @Test
     public void getMetricsView() {
         TieredStoreMetricsManager.getMetricsView();
@@ -41,11 +35,17 @@ public void getMetricsView() {
 
     @Test
     public void init() {
-        TieredStoreExecutor.init();
-        TieredMessageStoreConfig storeConfig = new TieredMessageStoreConfig();
-        storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment");
-        TieredStoreMetricsManager.init(OpenTelemetrySdk.builder().build().getMeter(""),
-            null, storeConfig, new TieredMessageFetcher(storeConfig), null);
+        MessageStoreConfig storeConfig = new MessageStoreConfig();
+        storeConfig.setTieredBackendServiceProvider(PosixFileSegment.class.getName());
+        TieredMessageStore messageStore = Mockito.mock(TieredMessageStore.class);
+        Mockito.when(messageStore.getStoreConfig()).thenReturn(storeConfig);
+        Mockito.when(messageStore.getFlatFileStore()).thenReturn(Mockito.mock(FlatFileStore.class));
+        MessageStoreFetcherImpl fetcher = Mockito.spy(new MessageStoreFetcherImpl(messageStore));
+
+        TieredStoreMetricsManager.init(
+            OpenTelemetrySdk.builder().build().getMeter(""),
+            null, storeConfig, fetcher,
+            Mockito.mock(FlatFileStore.class), Mockito.mock(DefaultMessageStore.class));
     }
 
     @Test
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactoryTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactoryTest.java
new file mode 100644
index 00000000000..3b44b10f47b
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactoryTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.provider;
+
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
+import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class FileSegmentFactoryTest {
+
+    @Test
+    public void fileSegmentInstanceTest() throws ClassNotFoundException, NoSuchMethodException {
+        int baseOffset = 1000;
+        String filePath = "FileSegmentFactoryPath";
+        String storePath = MessageStoreUtilTest.getRandomStorePath();
+        MessageStoreConfig storeConfig = new MessageStoreConfig();
+        storeConfig.setTieredStoreCommitLogMaxSize(1024);
+        storeConfig.setTieredStoreFilePath(storePath);
+        MessageStoreExecutor executor = new MessageStoreExecutor();
+
+        MetadataStore metadataStore = new DefaultMetadataStore(storeConfig);
+        FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig, executor);
+
+        Assert.assertEquals(metadataStore, factory.getMetadataStore());
+        Assert.assertEquals(storeConfig, factory.getStoreConfig());
+
+        FileSegment fileSegment = factory.createCommitLogFileSegment(filePath, baseOffset);
+        Assert.assertEquals(1000, fileSegment.getBaseOffset());
+        Assert.assertEquals(FileSegmentType.COMMIT_LOG, fileSegment.getFileType());
+        fileSegment.destroyFile();
+
+        fileSegment = factory.createConsumeQueueFileSegment(filePath, baseOffset);
+        Assert.assertEquals(1000, fileSegment.getBaseOffset());
+        Assert.assertEquals(FileSegmentType.CONSUME_QUEUE, fileSegment.getFileType());
+        fileSegment.destroyFile();
+
+        fileSegment = factory.createIndexServiceFileSegment(filePath, baseOffset);
+        Assert.assertEquals(1000, fileSegment.getBaseOffset());
+        Assert.assertEquals(FileSegmentType.INDEX, fileSegment.getFileType());
+        fileSegment.destroyFile();
+
+        Assert.assertThrows(RuntimeException.class,
+            () -> factory.createSegment(null, null, 0L));
+        storeConfig.setTieredBackendServiceProvider(null);
+        Assert.assertThrows(RuntimeException.class,
+            () -> new FileSegmentFactory(metadataStore, storeConfig, executor));
+
+        executor.shutdown();
+        MessageStoreUtilTest.deleteStoreDirectory(storePath);
+    }
+}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentTest.java
new file mode 100644
index 00000000000..26844113cd0
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentTest.java
@@ -0,0 +1,469 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.provider;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
+import org.apache.rocketmq.tieredstore.common.AppendResult;
+import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.exception.TieredStoreErrorCode;
+import org.apache.rocketmq.tieredstore.exception.TieredStoreException;
+import org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtil;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtilTest;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+
+public class FileSegmentTest {
+
+    public int baseOffset = 1000;
+    private final String storePath = MessageStoreUtilTest.getRandomStorePath();
+    private MessageStoreConfig storeConfig;
+    private MessageQueue mq;
+    private MessageStoreExecutor storeExecutor;
+
+    @Before
+    public void init() {
+        storeConfig = new MessageStoreConfig();
+        storeConfig.setTieredStoreCommitLogMaxSize(2000);
+        storeConfig.setTieredStoreFilePath(storePath);
+        storeConfig.setTieredBackendServiceProvider(PosixFileSegment.class.getName());
+        mq = new MessageQueue("FileSegmentTest", "brokerName", 0);
+        storeExecutor = new MessageStoreExecutor();
+    }
+
+    @After
+    public void shutdown() {
+        MessageStoreUtilTest.deleteStoreDirectory(storePath);
+        storeExecutor.shutdown();
+    }
+
+    @Test
+    public void fileAttributesTest() {
+        int baseOffset = 1000;
+        FileSegment fileSegment = new PosixFileSegment(
+            storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), baseOffset, storeExecutor);
+
+        // for default value check
+        Assert.assertEquals(baseOffset, fileSegment.getBaseOffset());
+        Assert.assertEquals(0L, fileSegment.getCommitPosition());
+        Assert.assertEquals(0L, fileSegment.getAppendPosition());
+        Assert.assertEquals(baseOffset, fileSegment.getCommitOffset());
+        Assert.assertEquals(baseOffset, fileSegment.getAppendOffset());
+        Assert.assertEquals(FileSegmentType.COMMIT_LOG, fileSegment.getFileType());
+        Assert.assertEquals(Long.MAX_VALUE, fileSegment.getMinTimestamp());
+        Assert.assertEquals(Long.MAX_VALUE, fileSegment.getMaxTimestamp());
+
+        // for recover
+        long timestamp = System.currentTimeMillis();
+        fileSegment.setMinTimestamp(timestamp);
+        fileSegment.setMaxTimestamp(timestamp);
+        Assert.assertEquals(timestamp, fileSegment.getMinTimestamp());
+        Assert.assertEquals(timestamp, fileSegment.getMaxTimestamp());
+
+        // for file status change
+        Assert.assertFalse(fileSegment.isClosed());
+        fileSegment.close();
+        Assert.assertTrue(fileSegment.isClosed());
+
+        fileSegment.destroyFile();
+    }
+
+    @Test
+    public void fileSortByOffsetTest() {
+        FileSegment fileSegment1 = new PosixFileSegment(
+            storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), 200L, storeExecutor);
+        FileSegment fileSegment2 = new PosixFileSegment(
+            storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), 100L, storeExecutor);
+        FileSegment[] fileSegments = new FileSegment[] {fileSegment1, fileSegment2};
+        Arrays.sort(fileSegments);
+        Assert.assertEquals(fileSegments[0], fileSegment2);
+        Assert.assertEquals(fileSegments[1], fileSegment1);
+    }
+
+    @Test
+    public void fileMaxSizeTest() {
+        FileSegment fileSegment = new PosixFileSegment(
+            storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), 100L, storeExecutor);
+        Assert.assertEquals(storeConfig.getTieredStoreCommitLogMaxSize(), fileSegment.getMaxSize());
+        fileSegment.destroyFile();
+
+        fileSegment = new PosixFileSegment(
+            storeConfig, FileSegmentType.CONSUME_QUEUE, MessageStoreUtil.toFilePath(mq), 100L, storeExecutor);
+        Assert.assertEquals(storeConfig.getTieredStoreConsumeQueueMaxSize(), fileSegment.getMaxSize());
+        fileSegment.destroyFile();
+
+        fileSegment = new PosixFileSegment(
+            storeConfig, FileSegmentType.INDEX, MessageStoreUtil.toFilePath(mq), 100L, storeExecutor);
+        Assert.assertEquals(Long.MAX_VALUE, fileSegment.getMaxSize());
+        fileSegment.destroyFile();
+    }
+
+    @Test
+    public void unexpectedCaseTest() {
+        MetadataStore metadataStore = new DefaultMetadataStore(storeConfig);
+        FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig, new MessageStoreExecutor());
+        FileSegment fileSegment = factory.createCommitLogFileSegment(MessageStoreUtil.toFilePath(mq), baseOffset);
+
+        fileSegment.initPosition(fileSegment.getSize());
+        Assert.assertFalse(fileSegment.needCommit());
+        Assert.assertTrue(fileSegment.commitAsync().join());
+
+        fileSegment.append(ByteBuffer.allocate(0), 0L);
+        Assert.assertTrue(fileSegment.commitAsync().join());
+
+        ByteBuffer byteBuffer = ByteBuffer.allocate(8);
+        byteBuffer.putLong(0L);
+        byteBuffer.flip();
+        fileSegment.append(byteBuffer, 0L);
+
+        byteBuffer.getLong();
+        Assert.assertTrue(fileSegment.commitAsync().join());
+        fileSegment.destroyFile();
+    }
+
+    @Test
+    public void commitLogTest() throws InterruptedException {
+        MetadataStore metadataStore = new DefaultMetadataStore(storeConfig);
+        FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig, new MessageStoreExecutor());
+        FileSegment fileSegment = factory.createCommitLogFileSegment(MessageStoreUtil.toFilePath(mq), baseOffset);
+        long lastSize = fileSegment.getSize();
+        fileSegment.initPosition(fileSegment.getSize());
+        Assert.assertFalse(fileSegment.needCommit());
+        Assert.assertTrue(fileSegment.commitAsync().join());
+
+        fileSegment.append(MessageFormatUtilTest.buildMockedMessageBuffer(), 0L);
+        fileSegment.append(MessageFormatUtilTest.buildMockedMessageBuffer(), 0L);
+        Assert.assertTrue(fileSegment.needCommit());
+
+        fileSegment.commitLock.acquire();
+        Assert.assertFalse(fileSegment.commitAsync().join());
+        fileSegment.commitLock.release();
+
+        long storeTimestamp = System.currentTimeMillis();
+        ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+        buffer.putLong(MessageFormatUtil.STORE_TIMESTAMP_POSITION, storeTimestamp);
+        buffer.putLong(MessageFormatUtil.QUEUE_OFFSET_POSITION, 100L);
+        fileSegment.append(buffer, storeTimestamp);
+
+        Assert.assertTrue(fileSegment.needCommit());
+        Assert.assertEquals(baseOffset, fileSegment.getBaseOffset());
+        Assert.assertEquals(baseOffset + lastSize + MessageFormatUtilTest.MSG_LEN * 3, fileSegment.getAppendOffset());
+        Assert.assertEquals(0L, fileSegment.getMinTimestamp());
+        Assert.assertEquals(storeTimestamp, fileSegment.getMaxTimestamp());
+
+        List<ByteBuffer> buffers = fileSegment.borrowBuffer();
+        Assert.assertEquals(3, buffers.size());
+        fileSegment.bufferList.addAll(buffers);
+
+        fileSegment.commitAsync().join();
+        Assert.assertFalse(fileSegment.needCommit());
+        Assert.assertEquals(fileSegment.getCommitOffset(), fileSegment.getAppendOffset());
+
+        // offset will change when type is commitLog
+        ByteBuffer msg1 = fileSegment.read(lastSize, MessageFormatUtilTest.MSG_LEN);
+        Assert.assertEquals(baseOffset + lastSize, MessageFormatUtil.getCommitLogOffset(msg1));
+
+        ByteBuffer msg2 = fileSegment.read(lastSize + MessageFormatUtilTest.MSG_LEN, MessageFormatUtilTest.MSG_LEN);
+        Assert.assertEquals(baseOffset + lastSize + MessageFormatUtilTest.MSG_LEN, MessageFormatUtil.getCommitLogOffset(msg2));
+
+        ByteBuffer msg3 = fileSegment.read(lastSize + MessageFormatUtilTest.MSG_LEN * 2, MessageFormatUtilTest.MSG_LEN);
+        Assert.assertEquals(baseOffset + lastSize + MessageFormatUtilTest.MSG_LEN * 2, MessageFormatUtil.getCommitLogOffset(msg3));
+
+        // buffer full
+        fileSegment.bufferList.addAll(buffers);
+        storeConfig.setTieredStoreMaxGroupCommitCount(3);
+        Assert.assertEquals(AppendResult.BUFFER_FULL,
+            fileSegment.append(MessageFormatUtilTest.buildMockedMessageBuffer(), 0L));
+
+        // file full
+        fileSegment.initPosition(storeConfig.getTieredStoreCommitLogMaxSize() - MessageFormatUtilTest.MSG_LEN + 1);
+        Assert.assertEquals(AppendResult.FILE_FULL,
+            fileSegment.append(MessageFormatUtilTest.buildMockedMessageBuffer(), 0L));
+
+        // file close
+        fileSegment.close();
+        Assert.assertEquals(AppendResult.FILE_CLOSED,
+            fileSegment.append(MessageFormatUtilTest.buildMockedMessageBuffer(), 0L));
+        Assert.assertFalse(fileSegment.commitAsync().join());
+
+        fileSegment.destroyFile();
+    }
+
+    @Test
+    public void consumeQueueTest() throws ClassNotFoundException, NoSuchMethodException {
+        MetadataStore metadataStore = new DefaultMetadataStore(storeConfig);
+        FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig, new MessageStoreExecutor());
+        FileSegment fileSegment = factory.createConsumeQueueFileSegment(MessageStoreUtil.toFilePath(mq), baseOffset);
+
+        long storeTimestamp = System.currentTimeMillis();
+        int messageSize = MessageFormatUtilTest.MSG_LEN;
+        int unitSize = MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE;
+        long initPosition = 5 * unitSize;
+
+        fileSegment.initPosition(initPosition);
+        fileSegment.append(MessageFormatUtilTest.buildMockedConsumeQueueBuffer().putLong(0, baseOffset), 0);
+        fileSegment.append(MessageFormatUtilTest.buildMockedConsumeQueueBuffer().putLong(0, baseOffset + messageSize), 0);
+        fileSegment.append(MessageFormatUtilTest.buildMockedConsumeQueueBuffer().putLong(0, baseOffset + messageSize * 2), storeTimestamp);
+
+        Assert.assertEquals(initPosition + unitSize * 3, fileSegment.getAppendPosition());
+        Assert.assertEquals(0, fileSegment.getMinTimestamp());
+        Assert.assertEquals(storeTimestamp, fileSegment.getMaxTimestamp());
+
+        fileSegment.commitAsync().join();
+        Assert.assertEquals(fileSegment.getAppendOffset(), fileSegment.getCommitOffset());
+
+        ByteBuffer cqItem1 = fileSegment.read(initPosition, unitSize);
+        Assert.assertEquals(baseOffset, cqItem1.getLong());
+
+        ByteBuffer cqItem2 = fileSegment.read(initPosition + unitSize, unitSize);
+        Assert.assertEquals(baseOffset + messageSize, cqItem2.getLong());
+
+        ByteBuffer cqItem3 = fileSegment.read(initPosition + unitSize * 2, unitSize);
+        Assert.assertEquals(baseOffset + messageSize * 2, cqItem3.getLong());
+    }
+
+    @Test
+    public void fileSegmentReadTest() throws ClassNotFoundException, NoSuchMethodException {
+        MetadataStore metadataStore = new DefaultMetadataStore(storeConfig);
+        FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig, new MessageStoreExecutor());
+        FileSegment fileSegment = factory.createConsumeQueueFileSegment(MessageStoreUtil.toFilePath(mq), baseOffset);
+
+        long storeTimestamp = System.currentTimeMillis();
+        int messageSize = MessageFormatUtilTest.MSG_LEN;
+        int unitSize = MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE;
+        long initPosition = 5 * unitSize;
+
+        fileSegment.initPosition(initPosition);
+        fileSegment.append(MessageFormatUtilTest.buildMockedConsumeQueueBuffer().putLong(0, baseOffset), 0);
+        fileSegment.append(MessageFormatUtilTest.buildMockedConsumeQueueBuffer().putLong(0, baseOffset + messageSize), 0);
+        fileSegment.append(MessageFormatUtilTest.buildMockedConsumeQueueBuffer().putLong(0, baseOffset + messageSize * 2), storeTimestamp);
+        fileSegment.commitAsync().join();
+
+        CompletionException exception = Assert.assertThrows(
+            CompletionException.class, () -> fileSegment.read(-1, -1));
+        Assert.assertTrue(exception.getCause() instanceof TieredStoreException);
+        Assert.assertEquals(TieredStoreErrorCode.ILLEGAL_PARAM, ((TieredStoreException) exception.getCause()).getErrorCode());
+
+        exception = Assert.assertThrows(
+            CompletionException.class, () -> fileSegment.read(100, 0));
+        Assert.assertTrue(exception.getCause() instanceof TieredStoreException);
+        Assert.assertEquals(TieredStoreErrorCode.ILLEGAL_PARAM, ((TieredStoreException) exception.getCause()).getErrorCode());
+
+        // at most three messages
+        Assert.assertEquals(unitSize * 3,
+            fileSegment.read(100, messageSize * 3).remaining());
+        Assert.assertEquals(unitSize * 3,
+            fileSegment.read(100, messageSize * 5).remaining());
+    }
+
+    @Test
+    public void commitFailedThenSuccessTest() {
+        MemoryFileSegment segment = new MemoryFileSegment(
+            storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), baseOffset, storeExecutor);
+
+        long lastSize = segment.getSize();
+        segment.setCheckSize(false);
+        segment.initPosition(lastSize);
+        segment.setSize((int) lastSize);
+
+        int messageSize = MessageFormatUtilTest.MSG_LEN;
+        ByteBuffer buffer1 = MessageFormatUtilTest.buildMockedMessageBuffer().putLong(
+            MessageFormatUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize);
+        ByteBuffer buffer2 = MessageFormatUtilTest.buildMockedMessageBuffer().putLong(
+            MessageFormatUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + messageSize);
+        Assert.assertEquals(AppendResult.SUCCESS, segment.append(buffer1, 0));
+        Assert.assertEquals(AppendResult.SUCCESS, segment.append(buffer2, 0));
+
+        // Mock new message arrive
+        long timestamp = System.currentTimeMillis();
+        segment.blocker = new CompletableFuture<>();
+        new Thread(() -> {
+            try {
+                TimeUnit.SECONDS.sleep(1);
+            } catch (InterruptedException e) {
+                Assert.fail(e.getMessage());
+            }
+            ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+            buffer.putLong(MessageFormatUtil.PHYSICAL_OFFSET_POSITION, messageSize * 2);
+            buffer.putLong(MessageFormatUtil.STORE_TIMESTAMP_POSITION, timestamp);
+            segment.append(buffer, 0);
+            segment.blocker.complete(false);
+        }).start();
+
+        // Commit failed
+        segment.commitAsync().join();
+        segment.blocker.join();
+        segment.blocker = null;
+
+        // Copy data and assume commit success
+        segment.getMemStore().put(buffer1);
+        segment.getMemStore().put(buffer2);
+        segment.setSize((int) (lastSize + messageSize * 2));
+
+        segment.commitAsync().join();
+        Assert.assertEquals(lastSize + messageSize * 3, segment.getCommitPosition());
+        Assert.assertEquals(baseOffset + lastSize + messageSize * 3, segment.getCommitOffset());
+        Assert.assertEquals(baseOffset + lastSize + messageSize * 3, segment.getAppendOffset());
+
+        ByteBuffer msg1 = segment.read(lastSize, messageSize);
+        Assert.assertEquals(baseOffset + lastSize, MessageFormatUtil.getCommitLogOffset(msg1));
+
+        ByteBuffer msg2 = segment.read(lastSize + messageSize, messageSize);
+        Assert.assertEquals(baseOffset + lastSize + messageSize, MessageFormatUtil.getCommitLogOffset(msg2));
+
+        ByteBuffer msg3 = segment.read(lastSize + messageSize * 2, messageSize);
+        Assert.assertEquals(baseOffset + lastSize + messageSize * 2, MessageFormatUtil.getCommitLogOffset(msg3));
+    }
+
+    @Test
+    public void commitFailedMoreTimes() {
+        long startTime = System.currentTimeMillis();
+        MemoryFileSegment segment = new MemoryFileSegment(
+            storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), baseOffset, storeExecutor);
+
+        long lastSize = segment.getSize();
+        segment.setCheckSize(false);
+        segment.initPosition(lastSize);
+        segment.setSize((int) lastSize);
+
+        ByteBuffer buffer1 = MessageFormatUtilTest.buildMockedMessageBuffer().putLong(
+            MessageFormatUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize);
+        ByteBuffer buffer2 = MessageFormatUtilTest.buildMockedMessageBuffer().putLong(
+            MessageFormatUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageFormatUtilTest.MSG_LEN);
+        segment.append(buffer1, 0);
+        segment.append(buffer2, 0);
+
+        // Mock new message arrive
+        segment.blocker = new CompletableFuture<>();
+        new Thread(() -> {
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException e) {
+                Assert.fail(e.getMessage());
+            }
+            ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer();
+            buffer.putLong(MessageFormatUtil.PHYSICAL_OFFSET_POSITION, MessageFormatUtilTest.MSG_LEN * 2);
+            buffer.putLong(MessageFormatUtil.STORE_TIMESTAMP_POSITION, startTime);
+            segment.append(buffer, 0);
+            segment.blocker.complete(false);
+        }).start();
+
+        for (int i = 0; i < 3; i++) {
+            Assert.assertFalse(segment.commitAsync().join());
+        }
+
+        FileSegment fileSpySegment = Mockito.spy(segment);
+        Mockito.when(fileSpySegment.getSize()).thenReturn(-1L);
+        Assert.assertFalse(fileSpySegment.commitAsync().join());
+
+        Assert.assertEquals(lastSize, segment.getCommitPosition());
+        Assert.assertEquals(baseOffset + lastSize, segment.getCommitOffset());
+        Assert.assertEquals(baseOffset + lastSize + MessageFormatUtilTest.MSG_LEN * 3, segment.getAppendOffset());
+
+        segment.blocker.join();
+        segment.blocker = null;
+
+        segment.commitAsync().join();
+        Assert.assertEquals(lastSize + MessageFormatUtilTest.MSG_LEN * 2, segment.getCommitPosition());
+        Assert.assertEquals(baseOffset + lastSize + MessageFormatUtilTest.MSG_LEN * 2, segment.getCommitOffset());
+        Assert.assertEquals(baseOffset + lastSize + MessageFormatUtilTest.MSG_LEN * 3, segment.getAppendOffset());
+
+        segment.commitAsync().join();
+        Assert.assertEquals(lastSize + MessageFormatUtilTest.MSG_LEN * 3, segment.getCommitPosition());
+        Assert.assertEquals(baseOffset + lastSize + MessageFormatUtilTest.MSG_LEN * 3, segment.getCommitOffset());
+        Assert.assertEquals(baseOffset + lastSize + MessageFormatUtilTest.MSG_LEN * 3, segment.getAppendOffset());
+
+        ByteBuffer msg1 = segment.read(lastSize, MessageFormatUtilTest.MSG_LEN);
+        Assert.assertEquals(baseOffset + lastSize, MessageFormatUtil.getCommitLogOffset(msg1));
+
+        ByteBuffer msg2 = segment.read(lastSize + MessageFormatUtilTest.MSG_LEN, MessageFormatUtilTest.MSG_LEN);
+        Assert.assertEquals(baseOffset + lastSize + MessageFormatUtilTest.MSG_LEN, MessageFormatUtil.getCommitLogOffset(msg2));
+
+        ByteBuffer msg3 = segment.read(lastSize + MessageFormatUtilTest.MSG_LEN * 2, MessageFormatUtilTest.MSG_LEN);
+        Assert.assertEquals(baseOffset + lastSize + MessageFormatUtilTest.MSG_LEN * 2, MessageFormatUtil.getCommitLogOffset(msg3));
+    }
+
+    @Test
+    public void handleCommitExceptionTest() {
+        MetadataStore metadataStore = new DefaultMetadataStore(storeConfig);
+        FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig, storeExecutor);
+
+        {
+            FileSegment fileSegment = factory.createCommitLogFileSegment(MessageStoreUtil.toFilePath(mq), baseOffset);
+            FileSegment fileSpySegment = Mockito.spy(fileSegment);
+            fileSpySegment.append(MessageFormatUtilTest.buildMockedMessageBuffer(), 0L);
+            fileSpySegment.append(MessageFormatUtilTest.buildMockedMessageBuffer(), 0L);
+
+            Mockito.when(fileSpySegment.commit0(any(), anyLong(), anyInt(), anyBoolean()))
+                .thenReturn(CompletableFuture.supplyAsync(() -> {
+                    throw new TieredStoreException(TieredStoreErrorCode.IO_ERROR, "Test");
+                }));
+            Assert.assertFalse(fileSpySegment.commitAsync().join());
+            fileSegment.destroyFile();
+        }
+
+        {
+            FileSegment fileSegment = factory.createCommitLogFileSegment(MessageStoreUtil.toFilePath(mq), baseOffset);
+            FileSegment fileSpySegment = Mockito.spy(fileSegment);
+            fileSpySegment.append(MessageFormatUtilTest.buildMockedMessageBuffer(), 0L);
+            fileSpySegment.append(MessageFormatUtilTest.buildMockedMessageBuffer(), 0L);
+
+            Mockito.when(fileSpySegment.commit0(any(), anyLong(), anyInt(), anyBoolean()))
+                .thenReturn(CompletableFuture.supplyAsync(() -> {
+                    long size = MessageFormatUtilTest.buildMockedMessageBuffer().remaining();
+                    TieredStoreException exception = new TieredStoreException(TieredStoreErrorCode.IO_ERROR, "Test");
+                    exception.setPosition(size * 2L);
+                    throw exception;
+                }));
+            Assert.assertTrue(fileSpySegment.commitAsync().join());
+            fileSegment.destroyFile();
+        }
+
+        {
+            FileSegment fileSegment = factory.createCommitLogFileSegment(MessageStoreUtil.toFilePath(mq), baseOffset);
+            FileSegment fileSpySegment = Mockito.spy(fileSegment);
+            fileSpySegment.append(MessageFormatUtilTest.buildMockedMessageBuffer(), 0L);
+            fileSpySegment.append(MessageFormatUtilTest.buildMockedMessageBuffer(), 0L);
+
+            Mockito.when(fileSpySegment.commit0(any(), anyLong(), anyInt(), anyBoolean()))
+                .thenReturn(CompletableFuture.supplyAsync(() -> {
+                    throw new RuntimeException("Runtime Error for Test");
+                }));
+            Mockito.when(fileSpySegment.getSize()).thenReturn(0L);
+            Assert.assertFalse(fileSpySegment.commitAsync().join());
+        }
+    }
+}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegmentTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegmentTest.java
new file mode 100644
index 00000000000..72396506b10
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegmentTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.provider;
+
+import java.io.IOException;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.tieredstore.MessageStoreConfig;
+import org.apache.rocketmq.tieredstore.MessageStoreExecutor;
+import org.apache.rocketmq.tieredstore.common.FileSegmentType;
+import org.apache.rocketmq.tieredstore.stream.FileSegmentInputStream;
+import org.apache.rocketmq.tieredstore.util.MessageStoreUtil;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.mockito.ArgumentMatchers.any;
+
+public class MemoryFileSegmentTest {
+
+    @Test
+    public void memoryTest() throws IOException {
+        MemoryFileSegment fileSegment = new MemoryFileSegment(
+            new MessageStoreConfig(), FileSegmentType.COMMIT_LOG,
+            MessageStoreUtil.toFilePath(new MessageQueue()), 0L, new MessageStoreExecutor());
+        Assert.assertFalse(fileSegment.exists());
+        fileSegment.createFile();
+        MemoryFileSegment fileSpySegment = Mockito.spy(fileSegment);
+        FileSegmentInputStream inputStream = Mockito.mock(FileSegmentInputStream.class);
+        Mockito.when(inputStream.read(any())).thenThrow(new RuntimeException());
+        Assert.assertFalse(fileSpySegment.commit0(inputStream, 0L, 0, false).join());
+        fileSegment.destroyFile();
+    }
+}
\ No newline at end of file
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java
deleted file mode 100644
index a655710a500..00000000000
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.provider;
-
-import java.nio.ByteBuffer;
-import java.util.concurrent.CompletableFuture;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.file.TieredCommitLog;
-import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue;
-import org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TieredFileSegmentTest {
-
-    public int baseOffset = 1000;
-
-    public TieredFileSegment createFileSegment(FileSegmentType fileType) {
-        String brokerName = new TieredMessageStoreConfig().getBrokerName();
-        return new MemoryFileSegment(fileType, new MessageQueue("TieredFileSegmentTest", brokerName, 0),
-            baseOffset, new TieredMessageStoreConfig());
-    }
-
-    @Test
-    public void testCommitLog() {
-        TieredFileSegment segment = createFileSegment(FileSegmentType.COMMIT_LOG);
-        segment.initPosition(segment.getSize());
-        long lastSize = segment.getSize();
-        segment.append(MessageBufferUtilTest.buildMockedMessageBuffer(), 0);
-        segment.append(MessageBufferUtilTest.buildMockedMessageBuffer(), 0);
-        Assert.assertTrue(segment.needCommit());
-
-        ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-        long msg3StoreTime = System.currentTimeMillis();
-        buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, msg3StoreTime);
-        long queueOffset = baseOffset * 1000L;
-        buffer.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, queueOffset);
-        segment.append(buffer, msg3StoreTime);
-
-        Assert.assertEquals(baseOffset, segment.getBaseOffset());
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset());
-        Assert.assertEquals(0, segment.getMinTimestamp());
-        Assert.assertEquals(msg3StoreTime, segment.getMaxTimestamp());
-
-        segment.setFull();
-        segment.commit();
-        Assert.assertFalse(segment.needCommit());
-        Assert.assertEquals(segment.getMaxOffset(), segment.getCommitOffset());
-        Assert.assertEquals(queueOffset, segment.getDispatchCommitOffset());
-
-        ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN);
-        Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1));
-
-        ByteBuffer msg2 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtilTest.MSG_LEN);
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtil.getCommitLogOffset(msg2));
-
-        ByteBuffer msg3 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtilTest.MSG_LEN);
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtil.getCommitLogOffset(msg3));
-
-        ByteBuffer coda = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN * 3, TieredCommitLog.CODA_SIZE);
-        Assert.assertEquals(msg3StoreTime, coda.getLong(4 + 4));
-    }
-
-    private ByteBuffer buildConsumeQueue(long commitLogOffset) {
-        ByteBuffer cqItem = ByteBuffer.allocate(TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-        cqItem.putLong(commitLogOffset);
-        cqItem.putInt(2);
-        cqItem.putLong(3);
-        cqItem.flip();
-        return cqItem;
-    }
-
-    @Test
-    public void testConsumeQueue() {
-        TieredFileSegment segment = createFileSegment(FileSegmentType.CONSUME_QUEUE);
-        segment.initPosition(segment.getSize());
-        long lastSize = segment.getSize();
-        segment.append(buildConsumeQueue(baseOffset), 0);
-        segment.append(buildConsumeQueue(baseOffset + MessageBufferUtilTest.MSG_LEN), 0);
-        long cqItem3Timestamp = System.currentTimeMillis();
-        segment.append(buildConsumeQueue(baseOffset + MessageBufferUtilTest.MSG_LEN * 2), cqItem3Timestamp);
-
-        Assert.assertEquals(baseOffset + lastSize + TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE * 3, segment.getMaxOffset());
-        Assert.assertEquals(0, segment.getMinTimestamp());
-        Assert.assertEquals(cqItem3Timestamp, segment.getMaxTimestamp());
-
-        segment.commit();
-        Assert.assertEquals(segment.getMaxOffset(), segment.getCommitOffset());
-
-        ByteBuffer cqItem1 = segment.read(lastSize, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-        Assert.assertEquals(baseOffset, cqItem1.getLong());
-
-        ByteBuffer cqItem2 = segment.read(lastSize + TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-        Assert.assertEquals(baseOffset + MessageBufferUtilTest.MSG_LEN, cqItem2.getLong());
-
-        ByteBuffer cqItem3 = segment.read(lastSize + TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE * 2, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-        Assert.assertEquals(baseOffset + MessageBufferUtilTest.MSG_LEN * 2, cqItem3.getLong());
-    }
-
-    @Test
-    public void testCommitFailedThenSuccess() {
-        long startTime = System.currentTimeMillis();
-        MemoryFileSegment segment = (MemoryFileSegment) createFileSegment(FileSegmentType.COMMIT_LOG);
-        long lastSize = segment.getSize();
-        segment.setCheckSize(false);
-        segment.initPosition(lastSize);
-        segment.setSize((int) lastSize);
-
-        ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong(
-            MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize);
-        ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong(
-            MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN);
-        segment.append(buffer1, 0);
-        segment.append(buffer2, 0);
-
-        // Mock new message arrive
-        segment.blocker = new CompletableFuture<>();
-        new Thread(() -> {
-            try {
-                Thread.sleep(1000);
-            } catch (InterruptedException e) {
-                Assert.fail(e.getMessage());
-            }
-            ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-            buffer.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN * 2);
-            buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, startTime);
-            segment.append(buffer, 0);
-            segment.blocker.complete(false);
-        }).start();
-
-        // Commit failed
-        segment.commit();
-        segment.blocker.join();
-        segment.blocker = null;
-
-        // Copy data and assume commit success
-        segment.getMemStore().put(buffer1);
-        segment.getMemStore().put(buffer2);
-        segment.setSize((int) (lastSize + MessageBufferUtilTest.MSG_LEN * 2));
-
-        segment.commit();
-        Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitPosition());
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitOffset());
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset());
-
-        ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN);
-        Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1));
-
-        ByteBuffer msg2 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtilTest.MSG_LEN);
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtil.getCommitLogOffset(msg2));
-
-        ByteBuffer msg3 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtilTest.MSG_LEN);
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtil.getCommitLogOffset(msg3));
-    }
-
-    @Test
-    public void testCommitFailed3Times() {
-        long startTime = System.currentTimeMillis();
-        MemoryFileSegment segment = (MemoryFileSegment) createFileSegment(FileSegmentType.COMMIT_LOG);
-        long lastSize = segment.getSize();
-        segment.setCheckSize(false);
-        segment.initPosition(lastSize);
-        segment.setSize((int) lastSize);
-
-        ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong(
-            MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize);
-        ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong(
-            MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN);
-        segment.append(buffer1, 0);
-        segment.append(buffer2, 0);
-
-        // Mock new message arrive
-        segment.blocker = new CompletableFuture<>();
-        new Thread(() -> {
-            try {
-                Thread.sleep(3000);
-            } catch (InterruptedException e) {
-                Assert.fail(e.getMessage());
-            }
-            ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer();
-            buffer.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN * 2);
-            buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, startTime);
-            segment.append(buffer, 0);
-            segment.blocker.complete(false);
-        }).start();
-
-        for (int i = 0; i < 3; i++) {
-            segment.commit();
-        }
-
-        Assert.assertEquals(lastSize, segment.getCommitPosition());
-        Assert.assertEquals(baseOffset + lastSize, segment.getCommitOffset());
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset());
-
-        segment.blocker.join();
-        segment.blocker = null;
-
-        segment.commit();
-        Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 2, segment.getCommitPosition());
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, segment.getCommitOffset());
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset());
-
-        segment.commit();
-        Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitPosition());
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitOffset());
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset());
-
-        ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN);
-        Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1));
-
-        ByteBuffer msg2 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtilTest.MSG_LEN);
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtil.getCommitLogOffset(msg2));
-
-        ByteBuffer msg3 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtilTest.MSG_LEN);
-        Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtil.getCommitLogOffset(msg3));
-    }
-}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java
deleted file mode 100644
index 630fd22236c..00000000000
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.provider.memory;
-
-import java.io.File;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-import org.junit.Assert;
-
-public class MemoryFileSegmentWithoutCheck extends MemoryFileSegment {
-
-    public MemoryFileSegmentWithoutCheck(FileSegmentType fileType,
-        MessageQueue messageQueue, long baseOffset, TieredMessageStoreConfig storeConfig) {
-        super(storeConfig, fileType,
-            storeConfig.getStorePathRootDir() + File.separator + TieredStoreUtil.toPath(messageQueue),
-            baseOffset);
-    }
-
-    public MemoryFileSegmentWithoutCheck(TieredMessageStoreConfig storeConfig,
-        FileSegmentType fileType, String filePath, long baseOffset) {
-        super(storeConfig, fileType, filePath, baseOffset);
-    }
-
-    @Override
-    public long getSize() {
-        return 0;
-    }
-
-    @Override
-    public CompletableFuture<Boolean> commit0(FileSegmentInputStream inputStream, long position, int length,
-        boolean append) {
-        try {
-            if (blocker != null && !blocker.get()) {
-                throw new IllegalStateException();
-            }
-        } catch (InterruptedException | ExecutionException e) {
-            Assert.fail(e.getMessage());
-        }
-
-        byte[] buffer = new byte[1024];
-
-        int startPos = memStore.position();
-        try {
-            int len;
-            while ((len = inputStream.read(buffer)) > 0) {
-                memStore.put(buffer, 0, len);
-            }
-            Assert.assertEquals(length, memStore.position() - startPos);
-        } catch (Exception e) {
-            Assert.fail(e.getMessage());
-            return CompletableFuture.completedFuture(false);
-        }
-        return CompletableFuture.completedFuture(true);
-    }
-}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegmentTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegmentTest.java
deleted file mode 100644
index db33ae847f2..00000000000
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegmentTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.provider.posix;
-
-import com.google.common.io.ByteStreams;
-import com.google.common.io.Files;
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Random;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
-import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
-import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
-import org.apache.rocketmq.tieredstore.util.TieredStoreUtil;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class PosixFileSegmentTest {
-
-    private final String storePath = TieredStoreTestUtil.getRandomStorePath();
-    private TieredMessageStoreConfig storeConfig;
-    private MessageQueue mq;
-
-    @Before
-    public void setUp() {
-        storeConfig = new TieredMessageStoreConfig();
-        storeConfig.setTieredStoreFilePath(storePath);
-        mq = new MessageQueue("OSSFileSegmentTest", "broker", 0);
-        TieredStoreExecutor.init();
-    }
-
-    @After
-    public void tearDown() throws IOException {
-        TieredStoreTestUtil.destroyCompositeFlatFileManager();
-        TieredStoreTestUtil.destroyMetadataStore();
-        TieredStoreTestUtil.destroyTempDir(storePath);
-        TieredStoreExecutor.shutdown();
-    }
-
-    @Test
-    public void testCommitAndRead() throws IOException {
-        PosixFileSegment fileSegment = new PosixFileSegment(
-            storeConfig, FileSegmentType.CONSUME_QUEUE, TieredStoreUtil.toPath(mq), 0);
-        byte[] source = new byte[4096];
-        new Random().nextBytes(source);
-        ByteBuffer buffer = ByteBuffer.wrap(source);
-        fileSegment.append(buffer, 0);
-        fileSegment.commit();
-
-        File file = new File(fileSegment.getPath());
-        Assert.assertTrue(file.exists());
-        byte[] result = new byte[4096];
-        ByteStreams.read(Files.asByteSource(file).openStream(), result, 0, 4096);
-        Assert.assertArrayEquals(source, result);
-
-        ByteBuffer read = fileSegment.read(0, 4096);
-        Assert.assertArrayEquals(source, read.array());
-    }
-}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/stream/FileSegmentInputStreamTest.java
similarity index 87%
rename from tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java
rename to tieredstore/src/test/java/org/apache/rocketmq/tieredstore/stream/FileSegmentInputStreamTest.java
index 743d9182ce3..3d0dd57a8b9 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/stream/FileSegmentInputStreamTest.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.tieredstore.provider;
+package org.apache.rocketmq.tieredstore.stream;
 
 import com.google.common.base.Supplier;
 import java.io.IOException;
@@ -26,20 +26,16 @@
 import java.util.List;
 import java.util.Random;
 import org.apache.rocketmq.tieredstore.common.FileSegmentType;
-import org.apache.rocketmq.tieredstore.file.TieredCommitLog;
-import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue;
-import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream;
-import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStreamFactory;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtil;
-import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtil;
+import org.apache.rocketmq.tieredstore.util.MessageFormatUtilTest;
 import org.junit.Assert;
 import org.junit.Test;
 
-public class TieredFileSegmentInputStreamTest {
+public class FileSegmentInputStreamTest {
 
     private final static long COMMIT_LOG_START_OFFSET = 13131313;
 
-    private final static int MSG_LEN = MessageBufferUtilTest.MSG_LEN;
+    private final static int MSG_LEN = MessageFormatUtilTest.MSG_LEN;
 
     private final static int MSG_NUM = 10;
 
@@ -52,7 +48,7 @@ public void testCommitLogTypeInputStream() {
         List<ByteBuffer> uploadBufferList = new ArrayList<>();
         int bufferSize = 0;
         for (int i = 0; i < MSG_NUM; i++) {
-            ByteBuffer byteBuffer = MessageBufferUtilTest.buildMockedMessageBuffer();
+            ByteBuffer byteBuffer = MessageFormatUtilTest.buildMockedMessageBuffer();
             uploadBufferList.add(byteBuffer);
             bufferSize += byteBuffer.remaining();
         }
@@ -66,13 +62,13 @@ public void testCommitLogTypeInputStream() {
         // set real physical offset
         for (int i = 0; i < MSG_NUM; i++) {
             long physicalOffset = COMMIT_LOG_START_OFFSET + i * MSG_LEN;
-            int position = i * MSG_LEN + MessageBufferUtil.PHYSICAL_OFFSET_POSITION;
+            int position = i * MSG_LEN + MessageFormatUtil.PHYSICAL_OFFSET_POSITION;
             expectedByteBuffer.putLong(position, physicalOffset);
         }
 
         int finalBufferSize = bufferSize;
         int[] batchReadSizeTestSet = {
-            MessageBufferUtil.PHYSICAL_OFFSET_POSITION - 1, MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtil.PHYSICAL_OFFSET_POSITION + 1, MSG_LEN - 1, MSG_LEN, MSG_LEN + 1
+            MessageFormatUtil.PHYSICAL_OFFSET_POSITION - 1, MessageFormatUtil.PHYSICAL_OFFSET_POSITION, MessageFormatUtil.PHYSICAL_OFFSET_POSITION + 1, MSG_LEN - 1, MSG_LEN, MSG_LEN + 1
         };
         verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build(
             FileSegmentType.COMMIT_LOG, COMMIT_LOG_START_OFFSET, uploadBufferList, null, finalBufferSize), finalBufferSize, batchReadSizeTestSet);
@@ -84,14 +80,14 @@ public void testCommitLogTypeInputStreamWithCoda() {
         List<ByteBuffer> uploadBufferList = new ArrayList<>();
         int bufferSize = 0;
         for (int i = 0; i < MSG_NUM; i++) {
-            ByteBuffer byteBuffer = MessageBufferUtilTest.buildMockedMessageBuffer();
+            ByteBuffer byteBuffer = MessageFormatUtilTest.buildMockedMessageBuffer();
             uploadBufferList.add(byteBuffer);
             bufferSize += byteBuffer.remaining();
         }
 
-        ByteBuffer codaBuffer = ByteBuffer.allocate(TieredCommitLog.CODA_SIZE);
-        codaBuffer.putInt(TieredCommitLog.CODA_SIZE);
-        codaBuffer.putInt(TieredCommitLog.BLANK_MAGIC_CODE);
+        ByteBuffer codaBuffer = ByteBuffer.allocate(MessageFormatUtil.COMMIT_LOG_CODA_SIZE);
+        codaBuffer.putInt(MessageFormatUtil.COMMIT_LOG_CODA_SIZE);
+        codaBuffer.putInt(MessageFormatUtil.BLANK_MAGIC_CODE);
         long timeMillis = System.currentTimeMillis();
         codaBuffer.putLong(timeMillis);
         codaBuffer.flip();
@@ -109,13 +105,13 @@ public void testCommitLogTypeInputStreamWithCoda() {
         // set real physical offset
         for (int i = 0; i < MSG_NUM; i++) {
             long physicalOffset = COMMIT_LOG_START_OFFSET + i * MSG_LEN;
-            int position = i * MSG_LEN + MessageBufferUtil.PHYSICAL_OFFSET_POSITION;
+            int position = i * MSG_LEN + MessageFormatUtil.PHYSICAL_OFFSET_POSITION;
             expectedByteBuffer.putLong(position, physicalOffset);
         }
 
         int finalBufferSize = bufferSize;
         int[] batchReadSizeTestSet = {
-            MessageBufferUtil.PHYSICAL_OFFSET_POSITION - 1, MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtil.PHYSICAL_OFFSET_POSITION + 1,
+            MessageFormatUtil.PHYSICAL_OFFSET_POSITION - 1, MessageFormatUtil.PHYSICAL_OFFSET_POSITION, MessageFormatUtil.PHYSICAL_OFFSET_POSITION + 1,
             MSG_LEN - 1, MSG_LEN, MSG_LEN + 1,
             bufferSize - 1, bufferSize, bufferSize + 1
         };
@@ -129,7 +125,7 @@ public void testConsumeQueueTypeInputStream() {
         List<ByteBuffer> uploadBufferList = new ArrayList<>();
         int bufferSize = 0;
         for (int i = 0; i < MSG_NUM; i++) {
-            ByteBuffer byteBuffer = MessageBufferUtilTest.buildMockedConsumeQueueBuffer();
+            ByteBuffer byteBuffer = MessageFormatUtilTest.buildMockedConsumeQueueBuffer();
             uploadBufferList.add(byteBuffer);
             bufferSize += byteBuffer.remaining();
         }
@@ -142,7 +138,7 @@ public void testConsumeQueueTypeInputStream() {
         }
 
         int finalBufferSize = bufferSize;
-        int[] batchReadSizeTestSet = {TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE - 1, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE + 1};
+        int[] batchReadSizeTestSet = {MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE - 1, MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE, MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE + 1};
         verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build(
             FileSegmentType.CONSUME_QUEUE, COMMIT_LOG_START_OFFSET, uploadBufferList, null, finalBufferSize), bufferSize, batchReadSizeTestSet);
     }
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/MessageBufferUtilTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/MessageFormatUtilTest.java
similarity index 54%
rename from tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/MessageBufferUtilTest.java
rename to tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/MessageFormatUtilTest.java
index a0b43894817..be4805be833 100644
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/MessageBufferUtilTest.java
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/MessageFormatUtilTest.java
@@ -25,70 +25,37 @@
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.tieredstore.common.SelectBufferResult;
-import org.apache.rocketmq.tieredstore.file.TieredCommitLog;
-import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue;
 import org.junit.Assert;
 import org.junit.Test;
 
-public class MessageBufferUtilTest {
-    public static final int MSG_LEN = 4 //TOTALSIZE
-        + 4 //MAGICCODE
-        + 4 //BODYCRC
-        + 4 //QUEUEID
-        + 4 //FLAG
-        + 8 //QUEUEOFFSET
-        + 8 //PHYSICALOFFSET
-        + 4 //SYSFLAG
-        + 8 //BORNTIMESTAMP
-        + 8 //BORNHOST
-        + 8 //STORETIMESTAMP
-        + 8 //STOREHOSTADDRESS
-        + 4 //RECONSUMETIMES
-        + 8 //Prepared Transaction Offset
-        + 4 + 0 //BODY
-        + 2 + 0 //TOPIC
-        + 2 + 31 //properties
-        + 0;
+import static org.apache.rocketmq.tieredstore.util.MessageFormatUtil.COMMIT_LOG_CODA_SIZE;
+
+public class MessageFormatUtilTest {
+
+    public static final int MSG_LEN = 123;
 
     public static ByteBuffer buildMockedMessageBuffer() {
-        // Initialization of storage space
         ByteBuffer buffer = ByteBuffer.allocate(MSG_LEN);
-        // 1 TOTALSIZE
         buffer.putInt(MSG_LEN);
-        // 2 MAGICCODE
         buffer.putInt(MessageDecoder.MESSAGE_MAGIC_CODE_V2);
-        // 3 BODYCRC
         buffer.putInt(3);
-        // 4 QUEUEID
         buffer.putInt(4);
-        // 5 FLAG
         buffer.putInt(5);
-        // 6 QUEUEOFFSET
         buffer.putLong(6);
-        // 7 PHYSICALOFFSET
         buffer.putLong(7);
-        // 8 SYSFLAG
         buffer.putInt(8);
-        // 9 BORNTIMESTAMP
         buffer.putLong(9);
-        // 10 BORNHOST
         buffer.putLong(10);
-        // 11 STORETIMESTAMP
         buffer.putLong(11);
-        // 12 STOREHOSTADDRESS
         buffer.putLong(10);
-        // 13 RECONSUMETIMES
         buffer.putInt(13);
-        // 14 Prepared Transaction Offset
         buffer.putLong(14);
-        // 15 BODY
         buffer.putInt(0);
-        // 16 TOPIC
         buffer.putShort((short) 0);
-        // 17 PROPERTIES
+
         Map<String, String> map = new HashMap<>();
         map.put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "uk");
-        map.put("userkey", "uservalue0");
+        map.put("UserKey", "UserValue0");
         String properties = MessageDecoder.messageProperties2String(map);
         byte[] propertiesBytes = properties.getBytes(StandardCharsets.UTF_8);
         buffer.putShort((short) propertiesBytes.length);
@@ -99,19 +66,9 @@ public static ByteBuffer buildMockedMessageBuffer() {
         return buffer;
     }
 
-    public static ByteBuffer buildMockedConsumeQueueBuffer() {
-        ByteBuffer byteBuffer = ByteBuffer.allocate(TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-        // 1 COMMIT_LOG_OFFSET
-        byteBuffer.putLong(1);
-        // 2 MESSAGE_SIZE
-        byteBuffer.putInt(2);
-        // 3 TAG_HASH_CODE
-        byteBuffer.putLong(3);
-        byteBuffer.flip();
-        return byteBuffer;
-    }
-
-    public static void verifyMockedMessageBuffer(ByteBuffer buffer, int phyOffset) {
+    @Test
+    public void verifyMockedMessageBuffer() {
+        ByteBuffer buffer = buildMockedMessageBuffer();
         Assert.assertEquals(MSG_LEN, buffer.remaining());
         Assert.assertEquals(MSG_LEN, buffer.getInt());
         Assert.assertEquals(MessageDecoder.MESSAGE_MAGIC_CODE_V2, buffer.getInt());
@@ -119,7 +76,7 @@ public static void verifyMockedMessageBuffer(ByteBuffer buffer, int phyOffset) {
         Assert.assertEquals(4, buffer.getInt());
         Assert.assertEquals(5, buffer.getInt());
         Assert.assertEquals(6, buffer.getLong());
-        Assert.assertEquals(phyOffset, buffer.getLong());
+        Assert.assertEquals(7, buffer.getLong());
         Assert.assertEquals(8, buffer.getInt());
         Assert.assertEquals(9, buffer.getLong());
         Assert.assertEquals(10, buffer.getLong());
@@ -130,38 +87,79 @@ public static void verifyMockedMessageBuffer(ByteBuffer buffer, int phyOffset) {
         Assert.assertEquals(0, buffer.getInt());
         Assert.assertEquals(0, buffer.getShort());
         buffer.rewind();
-        Map<String, String> properties = MessageBufferUtil.getProperties(buffer);
+        Map<String, String> properties = MessageFormatUtil.getProperties(buffer);
         Assert.assertEquals("uk", properties.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
-        Assert.assertEquals("uservalue0", properties.get("userkey"));
+        Assert.assertEquals("UserValue0", properties.get("UserKey"));
+    }
+
+    public static ByteBuffer buildMockedConsumeQueueBuffer() {
+        ByteBuffer buffer = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);
+        buffer.putLong(1L);
+        buffer.putInt(2);
+        buffer.putLong(3L);
+        buffer.flip();
+        return buffer;
     }
 
     @Test
-    public void testGetTotalSize() {
+    public void verifyMockedConsumeQueueBuffer() {
+        ByteBuffer buffer = buildMockedConsumeQueueBuffer();
+        Assert.assertEquals(1L, MessageFormatUtil.getCommitLogOffsetFromItem(buffer));
+        Assert.assertEquals(2, MessageFormatUtil.getSizeFromItem(buffer));
+        Assert.assertEquals(3L, MessageFormatUtil.getTagCodeFromItem(buffer));
+    }
+
+    @Test
+    public void messageFormatBasicTest() {
         ByteBuffer buffer = buildMockedMessageBuffer();
-        int totalSize = MessageBufferUtil.getTotalSize(buffer);
-        Assert.assertEquals(MSG_LEN, totalSize);
+        Assert.assertEquals(MSG_LEN, MessageFormatUtil.getTotalSize(buffer));
+        Assert.assertEquals(MessageDecoder.MESSAGE_MAGIC_CODE_V2, MessageFormatUtil.getMagicCode(buffer));
+        Assert.assertEquals(6L, MessageFormatUtil.getQueueOffset(buffer));
+        Assert.assertEquals(7L, MessageFormatUtil.getCommitLogOffset(buffer));
+        Assert.assertEquals(11L, MessageFormatUtil.getStoreTimeStamp(buffer));
     }
 
     @Test
-    public void testGetMagicCode() {
+    public void getOffsetIdTest() {
         ByteBuffer buffer = buildMockedMessageBuffer();
-        int magicCode = MessageBufferUtil.getMagicCode(buffer);
-        Assert.assertEquals(MessageDecoder.MESSAGE_MAGIC_CODE_V2, magicCode);
+        InetSocketAddress inetSocketAddress = new InetSocketAddress("127.0.0.1", 65535);
+        ByteBuffer address = ByteBuffer.allocate(Long.BYTES);
+        address.put(inetSocketAddress.getAddress().getAddress(), 0, 4);
+        address.putInt(inetSocketAddress.getPort());
+        address.flip();
+        for (int i = 0; i < address.remaining(); i++) {
+            buffer.put(MessageFormatUtil.STORE_HOST_POSITION + i, address.get(i));
+        }
+        String excepted = MessageDecoder.createMessageId(
+            ByteBuffer.allocate(MessageFormatUtil.MSG_ID_LENGTH), address, 7);
+        String offsetId = MessageFormatUtil.getOffsetId(buffer);
+        Assert.assertEquals(excepted, offsetId);
+    }
+
+    @Test
+    public void getPropertiesTest() {
+        ByteBuffer buffer = buildMockedMessageBuffer();
+        Map<String, String> properties = MessageFormatUtil.getProperties(buffer);
+        Assert.assertEquals(2, properties.size());
+        Assert.assertTrue(properties.containsKey(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
+        Assert.assertEquals("uk", properties.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
+        Assert.assertTrue(properties.containsKey("UserKey"));
+        Assert.assertEquals("UserValue0", properties.get("UserKey"));
     }
 
     @Test
     public void testSplitMessages() {
         ByteBuffer msgBuffer1 = buildMockedMessageBuffer();
-        msgBuffer1.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 10);
+        msgBuffer1.putLong(MessageFormatUtil.QUEUE_OFFSET_POSITION, 10);
 
-        ByteBuffer msgBuffer2 = ByteBuffer.allocate(TieredCommitLog.CODA_SIZE);
-        msgBuffer2.putInt(TieredCommitLog.CODA_SIZE);
-        msgBuffer2.putInt(TieredCommitLog.BLANK_MAGIC_CODE);
+        ByteBuffer msgBuffer2 = ByteBuffer.allocate(COMMIT_LOG_CODA_SIZE);
+        msgBuffer2.putInt(MessageFormatUtil.COMMIT_LOG_CODA_SIZE);
+        msgBuffer2.putInt(MessageFormatUtil.BLANK_MAGIC_CODE);
         msgBuffer2.putLong(System.currentTimeMillis());
         msgBuffer2.flip();
 
         ByteBuffer msgBuffer3 = buildMockedMessageBuffer();
-        msgBuffer3.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 11);
+        msgBuffer3.putLong(MessageFormatUtil.QUEUE_OFFSET_POSITION, 11);
 
         ByteBuffer msgBuffer = ByteBuffer.allocate(
             msgBuffer1.remaining() + msgBuffer2.remaining() + msgBuffer3.remaining());
@@ -170,116 +168,99 @@ public void testSplitMessages() {
         msgBuffer.put(msgBuffer3);
         msgBuffer.flip();
 
-        ByteBuffer cqBuffer1 = ByteBuffer.allocate(TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
+        ByteBuffer cqBuffer1 = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);
         cqBuffer1.putLong(1000);
         cqBuffer1.putInt(MSG_LEN);
         cqBuffer1.putLong(0);
         cqBuffer1.flip();
 
-        ByteBuffer cqBuffer2 = ByteBuffer.allocate(TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-        cqBuffer2.putLong(1000 + TieredCommitLog.CODA_SIZE + MSG_LEN);
+        ByteBuffer cqBuffer2 = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);
+        cqBuffer2.putLong(1000 + MessageFormatUtil.COMMIT_LOG_CODA_SIZE + MSG_LEN);
         cqBuffer2.putInt(MSG_LEN);
         cqBuffer2.putLong(0);
         cqBuffer2.flip();
 
-        ByteBuffer cqBuffer3 = ByteBuffer.allocate(TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
+        ByteBuffer cqBuffer3 = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);
         cqBuffer3.putLong(1000 + MSG_LEN);
         cqBuffer3.putInt(MSG_LEN);
         cqBuffer3.putLong(0);
         cqBuffer3.flip();
 
-        ByteBuffer cqBuffer4 = ByteBuffer.allocate(TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-        cqBuffer4.putLong(1000 + TieredCommitLog.CODA_SIZE + MSG_LEN);
+        ByteBuffer cqBuffer4 = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);
+        cqBuffer4.putLong(1000 + MessageFormatUtil.COMMIT_LOG_CODA_SIZE + MSG_LEN);
         cqBuffer4.putInt(MSG_LEN - 10);
         cqBuffer4.putLong(0);
         cqBuffer4.flip();
 
-        ByteBuffer cqBuffer5 = ByteBuffer.allocate(TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
-        cqBuffer5.putLong(1000 + TieredCommitLog.CODA_SIZE + MSG_LEN);
+        ByteBuffer cqBuffer5 = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);
+        cqBuffer5.putLong(1000 + MessageFormatUtil.COMMIT_LOG_CODA_SIZE + MSG_LEN);
         cqBuffer5.putInt(MSG_LEN * 10);
         cqBuffer5.putLong(0);
         cqBuffer5.flip();
 
-        ByteBuffer cqBuffer = ByteBuffer.allocate(TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE * 2);
+        // Message buffer size is 0 or consume queue buffer size is 0
+        Assert.assertEquals(0,
+            MessageFormatUtil.splitMessageBuffer(null, ByteBuffer.allocate(0)).size());
+        Assert.assertEquals(0,
+            MessageFormatUtil.splitMessageBuffer(cqBuffer1, null).size());
+        Assert.assertEquals(0,
+            MessageFormatUtil.splitMessageBuffer(cqBuffer1, ByteBuffer.allocate(0)).size());
+        Assert.assertEquals(0,
+            MessageFormatUtil.splitMessageBuffer(ByteBuffer.allocate(0), msgBuffer).size());
+        Assert.assertEquals(0,
+            MessageFormatUtil.splitMessageBuffer(ByteBuffer.allocate(10), msgBuffer).size());
+
+        ByteBuffer cqBuffer = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE * 2);
         cqBuffer.put(cqBuffer1);
         cqBuffer.put(cqBuffer2);
         cqBuffer.flip();
         cqBuffer1.rewind();
         cqBuffer2.rewind();
-        List<SelectBufferResult> msgList = MessageBufferUtil.splitMessageBuffer(cqBuffer, msgBuffer);
+        List<SelectBufferResult> msgList = MessageFormatUtil.splitMessageBuffer(cqBuffer, msgBuffer);
         Assert.assertEquals(2, msgList.size());
         Assert.assertEquals(0, msgList.get(0).getStartOffset());
         Assert.assertEquals(MSG_LEN, msgList.get(0).getSize());
-        Assert.assertEquals(MSG_LEN + TieredCommitLog.CODA_SIZE, msgList.get(1).getStartOffset());
+        Assert.assertEquals(MSG_LEN + MessageFormatUtil.COMMIT_LOG_CODA_SIZE, msgList.get(1).getStartOffset());
         Assert.assertEquals(MSG_LEN, msgList.get(1).getSize());
 
-        cqBuffer = ByteBuffer.allocate(TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE * 2);
+        cqBuffer = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE * 2);
         cqBuffer.put(cqBuffer1);
         cqBuffer.put(cqBuffer4);
         cqBuffer.flip();
         cqBuffer1.rewind();
         cqBuffer4.rewind();
-        msgList = MessageBufferUtil.splitMessageBuffer(cqBuffer, msgBuffer);
+        msgList = MessageFormatUtil.splitMessageBuffer(cqBuffer, msgBuffer);
         Assert.assertEquals(1, msgList.size());
         Assert.assertEquals(0, msgList.get(0).getStartOffset());
         Assert.assertEquals(MSG_LEN, msgList.get(0).getSize());
 
-        cqBuffer = ByteBuffer.allocate(TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE * 3);
+        cqBuffer = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE * 3);
         cqBuffer.put(cqBuffer1);
         cqBuffer.put(cqBuffer3);
         cqBuffer.flip();
-        msgList = MessageBufferUtil.splitMessageBuffer(cqBuffer, msgBuffer);
+        cqBuffer1.rewind();
+        cqBuffer3.rewind();
+        msgList = MessageFormatUtil.splitMessageBuffer(cqBuffer, msgBuffer);
         Assert.assertEquals(2, msgList.size());
         Assert.assertEquals(0, msgList.get(0).getStartOffset());
         Assert.assertEquals(MSG_LEN, msgList.get(0).getSize());
-        Assert.assertEquals(MSG_LEN + TieredCommitLog.CODA_SIZE, msgList.get(1).getStartOffset());
+        Assert.assertEquals(MSG_LEN + MessageFormatUtil.COMMIT_LOG_CODA_SIZE, msgList.get(1).getStartOffset());
         Assert.assertEquals(MSG_LEN, msgList.get(1).getSize());
 
-        cqBuffer = ByteBuffer.allocate(TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE);
+        cqBuffer = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE);
         cqBuffer.put(cqBuffer5);
         cqBuffer.flip();
-        msgList = MessageBufferUtil.splitMessageBuffer(cqBuffer, msgBuffer);
+        msgList = MessageFormatUtil.splitMessageBuffer(cqBuffer, msgBuffer);
         Assert.assertEquals(0, msgList.size());
-    }
-
-    @Test
-    public void testGetQueueOffset() {
-        ByteBuffer buffer = buildMockedMessageBuffer();
-        long queueOffset = MessageBufferUtil.getQueueOffset(buffer);
-        Assert.assertEquals(6, queueOffset);
-    }
-
-    @Test
-    public void testGetStoreTimeStamp() {
-        ByteBuffer buffer = buildMockedMessageBuffer();
-        long storeTimeStamp = MessageBufferUtil.getStoreTimeStamp(buffer);
-        Assert.assertEquals(11, storeTimeStamp);
-    }
-
-    @Test
-    public void testGetOffsetId() {
-        ByteBuffer buffer = buildMockedMessageBuffer();
-        InetSocketAddress inetSocketAddress = new InetSocketAddress("255.255.255.255", 65535);
-        ByteBuffer addr = ByteBuffer.allocate(Long.BYTES);
-        addr.put(inetSocketAddress.getAddress().getAddress(), 0, 4);
-        addr.putInt(inetSocketAddress.getPort());
-        addr.flip();
-        for (int i = 0; i < addr.remaining(); i++) {
-            buffer.put(MessageBufferUtil.STORE_HOST_POSITION + i, addr.get(i));
-        }
-        String excepted = MessageDecoder.createMessageId(ByteBuffer.allocate(TieredStoreUtil.MSG_ID_LENGTH), addr, 7);
-        String offsetId = MessageBufferUtil.getOffsetId(buffer);
-        Assert.assertEquals(excepted, offsetId);
-    }
 
-    @Test
-    public void testGetProperties() {
-        ByteBuffer buffer = buildMockedMessageBuffer();
-        Map<String, String> properties = MessageBufferUtil.getProperties(buffer);
-        Assert.assertEquals(2, properties.size());
-        Assert.assertTrue(properties.containsKey(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
-        Assert.assertEquals("uk", properties.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
-        Assert.assertTrue(properties.containsKey("userkey"));
-        Assert.assertEquals("uservalue0", properties.get("userkey"));
+        // Wrong magic code, it will destroy the mocked message buffer
+        msgBuffer.putInt(MessageFormatUtil.MAGIC_CODE_POSITION, -1);
+        cqBuffer = ByteBuffer.allocate(MessageFormatUtil.CONSUME_QUEUE_UNIT_SIZE * 2);
+        cqBuffer.put(cqBuffer1);
+        cqBuffer.put(cqBuffer2);
+        cqBuffer.flip();
+        cqBuffer1.rewind();
+        cqBuffer2.rewind();
+        Assert.assertEquals(1, MessageFormatUtil.splitMessageBuffer(cqBuffer, msgBuffer).size());
     }
 }
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/MessageStoreUtilTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/MessageStoreUtilTest.java
new file mode 100644
index 00000000000..cadaef8708f
--- /dev/null
+++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/MessageStoreUtilTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tieredstore.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.commons.io.FileUtils;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MessageStoreUtilTest {
+
+    private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME);
+    private static final String TIERED_STORE_PATH = "tiered_store_test";
+
+    public static String getRandomStorePath() {
+        return Paths.get(System.getProperty("user.home"), TIERED_STORE_PATH,
+            UUID.randomUUID().toString().replace("-", "").toUpperCase().substring(0, 16)).toString();
+    }
+
+    public static void deleteStoreDirectory(String storePath) {
+        try {
+            FileUtils.deleteDirectory(new File(storePath));
+        } catch (IOException e) {
+            log.error("Delete store directory failed, filePath: {}", storePath, e);
+        }
+    }
+
+    @Test
+    public void toHumanReadableTest() {
+        Map<Long, String> capacityTable = new HashMap<Long, String>() {
+            {
+                put(-1L, "-1");
+                put(0L, "0B");
+                put(1023L, "1023B");
+                put(1024L, "1KB");
+                put(12_345L, "12.06KB");
+                put(10_123_456L, "9.65MB");
+                put(10_123_456_798L, "9.43GB");
+                put(123 * 1024L * 1024L * 1024L * 1024L, "123TB");
+                put(123 * 1024L * 1024L * 1024L * 1024L * 1024L, "123PB");
+                put(1_777_777_777_777_777_777L, "1.54EB");
+            }
+        };
+        capacityTable.forEach((in, expected) ->
+            Assert.assertEquals(expected, MessageStoreUtil.toHumanReadable(in)));
+    }
+
+    @Test
+    public void getHashTest() {
+        Assert.assertEquals("161c08ff", MessageStoreUtil.getHash("TieredStorageDailyTest"));
+    }
+
+    @Test
+    public void filePathTest() {
+        MessageQueue mq = new MessageQueue();
+        mq.setBrokerName("BrokerName");
+        mq.setTopic("topicName");
+        mq.setQueueId(2);
+        Assert.assertEquals("BrokerName/topicName/2", MessageStoreUtil.toFilePath(mq));
+    }
+
+    @Test
+    public void offset2FileNameTest() {
+        Assert.assertEquals("cfcd208400000000000000000000", MessageStoreUtil.offset2FileName(0));
+        Assert.assertEquals("b10da56800000000004294937144", MessageStoreUtil.offset2FileName(4294937144L));
+    }
+
+    @Test
+    public void fileName2OffsetTest() {
+        Assert.assertEquals(0, MessageStoreUtil.fileName2Offset("cfcd208400000000000000000000"));
+        Assert.assertEquals(4294937144L, MessageStoreUtil.fileName2Offset("b10da56800000000004294937144"));
+    }
+
+    @Test
+    public void indexServicePathTest() {
+        Assert.assertEquals("brokerName/rmq_sys_INDEX/0", MessageStoreUtil.getIndexFilePath("brokerName"));
+    }
+}
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/TieredStoreUtilTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/TieredStoreUtilTest.java
deleted file mode 100644
index 82e11252485..00000000000
--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/util/TieredStoreUtilTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tieredstore.util;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TieredStoreUtilTest {
-
-    private static final Map<Long, String> DATA_MAP = new HashMap<Long, String>() {
-        {
-            put(0L, "0Bytes");
-            put(1023L, "1023Bytes");
-            put(1024L, "1KB");
-            put(12_345L, "12.06KB");
-            put(10_123_456L, "9.65MB");
-            put(10_123_456_798L, "9.43GB");
-            put(1_777_777_777_777_777_777L, "1.54EB");
-        }
-    };
-
-    @Test
-    public void getHash() {
-        Assert.assertEquals("161c08ff", TieredStoreUtil.getHash("TieredStorageDailyTest"));
-    }
-
-    @Test
-    public void testOffset2FileName() {
-        Assert.assertEquals("cfcd208400000000000000000000", TieredStoreUtil.offset2FileName(0));
-        Assert.assertEquals("b10da56800000000004294937144", TieredStoreUtil.offset2FileName(4294937144L));
-    }
-
-    @Test
-    public void testFileName2Offset() {
-        Assert.assertEquals(0, TieredStoreUtil.fileName2Offset("cfcd208400000000000000000000"));
-        Assert.assertEquals(4294937144L, TieredStoreUtil.fileName2Offset("b10da56800000000004294937144"));
-    }
-
-    @Test
-    public void testToHumanReadable() {
-        DATA_MAP.forEach((in, expected) -> Assert.assertEquals(expected, TieredStoreUtil.toHumanReadable(in)));
-    }
-}
diff --git a/tieredstore/src/test/resources/rmq.logback-test.xml b/tieredstore/src/test/resources/rmq.logback-test.xml
index ac0895e05e4..070bf134cb9 100644
--- a/tieredstore/src/test/resources/rmq.logback-test.xml
+++ b/tieredstore/src/test/resources/rmq.logback-test.xml
@@ -24,7 +24,7 @@
     </appender>
 
     <property name="CONSOLE_LOG_PATTERN"
-              value="%d{yyyy-MM-dd HH:mm:ss.SSS,GMT+8} ${LOG_LEVEL_PATTERN:-%5p} [%20.20thread] [%20.20logger{39}] %m%n"/>
+              value="%d{yyyy-MM-dd HH:mm:ss.SSS,GMT+8} ${LOG_LEVEL_PATTERN:-%5p} [%20.20thread] %m%n"/>
 
     <root level="info">
         <appender-ref ref="STDOUT"/>
diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel
index 05d88f7b002..ec6fa1eaacb 100644
--- a/tools/BUILD.bazel
+++ b/tools/BUILD.bazel
@@ -21,7 +21,6 @@ java_library(
     srcs = glob(["src/main/java/**/*.java"]),
     visibility = ["//visibility:public"],
     deps = [
-        "//acl",
         "//remoting",
         "//client",    
         "//common",   
@@ -40,6 +39,7 @@ java_library(
         "@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
         "@maven//:io_github_aliyunmq_rocketmq_logback_classic",
         "@maven//:org_apache_rocketmq_rocketmq_rocksdb",
+        "@maven//:com_alibaba_fastjson2_fastjson2",
     ],
 )
 
@@ -56,7 +56,8 @@ java_library(
         "//:test_deps",
         "@maven//:org_apache_commons_commons_lang3", 
         "@maven//:io_netty_netty_all",      
-        "@maven//:commons_cli_commons_cli",         
+        "@maven//:commons_cli_commons_cli",     
+        "@maven//:org_junit_jupiter_junit_jupiter_api",    
     ],
     resources = glob(["src/test/resources/*.xml"]),
 )
diff --git a/tools/pom.xml b/tools/pom.xml
index e1daa57a62f..8be122c6861 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>5.1.5-SNAPSHOT</version>
+        <version>5.3.3</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -38,7 +38,7 @@
         </dependency>
         <dependency>
             <groupId>${project.groupId}</groupId>
-            <artifactId>rocketmq-acl</artifactId>
+            <artifactId>rocketmq-auth</artifactId>
         </dependency>
         <dependency>
             <groupId>${project.groupId}</groupId>
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
index 40bd5d56d30..9780df13dda 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
@@ -16,23 +16,18 @@
  */
 package org.apache.rocketmq.tools.admin;
 
-import java.io.UnsupportedEncodingException;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.QueryResult;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.BoundaryType;
+import org.apache.rocketmq.common.CheckRocksdbCqWriteResult;
 import org.apache.rocketmq.common.Pair;
-import org.apache.rocketmq.common.PlainAccessConfig;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.message.MessageRequestMode;
 import org.apache.rocketmq.common.topic.TopicValidator;
-import org.apache.rocketmq.common.BoundaryType;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.exception.RemotingConnectException;
@@ -42,9 +37,10 @@
 import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;
 import org.apache.rocketmq.remoting.protocol.admin.RollbackStats;
 import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;
+import org.apache.rocketmq.remoting.protocol.body.AclInfo;
 import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;
+import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;
 import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData;
-import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo;
 import org.apache.rocketmq.remoting.protocol.body.ClusterInfo;
 import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;
 import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList;
@@ -53,7 +49,6 @@
 import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache;
 import org.apache.rocketmq.remoting.protocol.body.GroupList;
 import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
-import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;
 import org.apache.rocketmq.remoting.protocol.body.KVTable;
 import org.apache.rocketmq.remoting.protocol.body.ProducerConnection;
 import org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo;
@@ -62,6 +57,8 @@
 import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;
 import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
 import org.apache.rocketmq.remoting.protocol.body.TopicList;
+import org.apache.rocketmq.remoting.protocol.body.UserInfo;
+import org.apache.rocketmq.remoting.protocol.header.ExportRocksDBConfigToJsonRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;
 import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
@@ -73,6 +70,12 @@
 import org.apache.rocketmq.tools.admin.api.MessageTrack;
 import org.apache.rocketmq.tools.admin.common.AdminToolResult;
 
+import java.io.UnsupportedEncodingException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
 public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt {
     private final DefaultMQAdminExtImpl defaultMQAdminExtImpl;
     private String adminExtGroup = "admin_ext_group";
@@ -145,18 +148,18 @@ public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {
         return defaultMQAdminExtImpl.earliestMsgStoreTime(mq);
     }
 
-    @Override
-    public MessageExt viewMessage(
-        String offsetMsgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        return defaultMQAdminExtImpl.viewMessage(offsetMsgId);
-    }
-
     @Override
     public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end)
         throws MQClientException, InterruptedException {
         return defaultMQAdminExtImpl.queryMessage(topic, key, maxNum, begin, end);
     }
 
+
+    public QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, long end)
+        throws MQClientException, InterruptedException, RemotingException {
+        return defaultMQAdminExtImpl.queryMessage(clusterName, topic, key, maxNum, begin, end);
+    }
+
     @Override
     public void start() throws MQClientException {
         defaultMQAdminExtImpl.start();
@@ -200,34 +203,9 @@ public void createAndUpdateTopicConfig(String addr, TopicConfig config) throws R
     }
 
     @Override
-    public void createAndUpdatePlainAccessConfig(String addr,
-        PlainAccessConfig config) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        defaultMQAdminExtImpl.createAndUpdatePlainAccessConfig(addr, config);
-    }
-
-    @Override
-    public void deletePlainAccessConfig(String addr,
-        String accessKey) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        defaultMQAdminExtImpl.deletePlainAccessConfig(addr, accessKey);
-    }
-
-    @Override
-    public void updateGlobalWhiteAddrConfig(String addr,
-        String globalWhiteAddrs) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        defaultMQAdminExtImpl.updateGlobalWhiteAddrConfig(addr, globalWhiteAddrs);
-    }
-
-    @Override
-    public void updateGlobalWhiteAddrConfig(String addr,
-        String globalWhiteAddrs,
-        String aclFileFullPath) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        defaultMQAdminExtImpl.updateGlobalWhiteAddrConfig(addr, globalWhiteAddrs, aclFileFullPath);
-    }
-
-    @Override
-    public ClusterAclVersionInfo examineBrokerClusterAclVersionInfo(
-        String addr) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        return defaultMQAdminExtImpl.examineBrokerClusterAclVersionInfo(addr);
+    public void createAndUpdateTopicConfigList(String addr,
+        List<TopicConfig> topicConfigList) throws InterruptedException, RemotingException, MQClientException {
+        defaultMQAdminExtImpl.createAndUpdateTopicConfigList(addr, topicConfigList);
     }
 
     @Override
@@ -237,6 +215,12 @@ public void createAndUpdateSubscriptionGroupConfig(String addr,
         defaultMQAdminExtImpl.createAndUpdateSubscriptionGroupConfig(addr, config);
     }
 
+    @Override
+    public void createAndUpdateSubscriptionGroupConfigList(String brokerAddr,
+        List<SubscriptionGroupConfig> configs) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+        defaultMQAdminExtImpl.createAndUpdateSubscriptionGroupConfigList(brokerAddr, configs);
+    }
+
     @Override
     public SubscriptionGroupConfig examineSubscriptionGroupConfig(String addr, String group)
         throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
@@ -293,6 +277,12 @@ public ConsumeStats examineConsumeStats(
         return examineConsumeStats(consumerGroup, null);
     }
 
+    @Override
+    public ConsumeStats examineConsumeStats(String clusterName, String consumerGroup,
+        String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
+        return defaultMQAdminExtImpl.examineConsumeStats(clusterName, consumerGroup, topic);
+    }
+
     @Override
     public ConsumeStats examineConsumeStats(String consumerGroup,
         String topic) throws RemotingException, MQClientException,
@@ -452,16 +442,35 @@ public List<RollbackStats> resetOffsetByTimestampOld(String consumerGroup, Strin
         return defaultMQAdminExtImpl.resetOffsetByTimestampOld(consumerGroup, topic, timestamp, force);
     }
 
+    public List<RollbackStats> resetOffsetByTimestampOld(String clusterName, String consumerGroup, String topic, long timestamp,
+        boolean force)
+        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+        return defaultMQAdminExtImpl.resetOffsetByTimestampOld(clusterName, consumerGroup, topic, timestamp, force);
+    }
+
+    @Override
+    public Map<MessageQueue, Long> resetOffsetByTimestamp(String clusterName, String topic, String group,
+        long timestamp, boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+        return defaultMQAdminExtImpl.resetOffsetByTimestamp(clusterName, topic, group, timestamp, isForce);
+    }
+
     @Override
     public Map<MessageQueue, Long> resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce)
         throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
         return resetOffsetByTimestamp(topic, group, timestamp, isForce, false);
     }
 
+    public Map<MessageQueue, Long> resetOffsetByTimestamp(String clusterName, String topic, String group,
+        long timestamp, boolean isForce, boolean isC)
+        throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+        return defaultMQAdminExtImpl.resetOffsetByTimestamp(clusterName, topic, group, timestamp, isForce, isC);
+    }
+
+
     public Map<MessageQueue, Long> resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce,
         boolean isC)
         throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        return defaultMQAdminExtImpl.resetOffsetByTimestamp(topic, group, timestamp, isForce, isC);
+        return defaultMQAdminExtImpl.resetOffsetByTimestamp(null, topic, group, timestamp, isForce, isC);
     }
 
     @Override
@@ -581,16 +590,18 @@ public ConsumerRunningInfo getConsumerRunningInfo(String consumerGroup, String c
     }
 
     @Override
-    public ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup, String clientId, String msgId)
-        throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
-        return defaultMQAdminExtImpl.consumeMessageDirectly(consumerGroup, clientId, msgId);
+    public ConsumeMessageDirectlyResult consumeMessageDirectly(final String consumerGroup, final String clientId,
+        final String topic,
+        final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
+        return defaultMQAdminExtImpl.consumeMessageDirectly(consumerGroup, clientId, topic, msgId);
     }
 
     @Override
-    public ConsumeMessageDirectlyResult consumeMessageDirectly(final String consumerGroup, final String clientId,
+    public ConsumeMessageDirectlyResult consumeMessageDirectly(final String clusterName, final String consumerGroup,
+        final String clientId,
         final String topic,
         final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
-        return defaultMQAdminExtImpl.consumeMessageDirectly(consumerGroup, clientId, topic, msgId);
+        return defaultMQAdminExtImpl.consumeMessageDirectly(clusterName, consumerGroup, clientId, topic, msgId);
     }
 
     @Override
@@ -731,9 +742,16 @@ public QueryConsumeQueueResponseBody queryConsumeQueue(String brokerAddr, String
     }
 
     @Override
-    public boolean resumeCheckHalfMessage(String msgId)
-        throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
-        return this.defaultMQAdminExtImpl.resumeCheckHalfMessage(msgId);
+    public CheckRocksdbCqWriteResult checkRocksdbCqWriteProgress(String brokerAddr, String topic, long checkStoreTime)
+        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {
+        return this.defaultMQAdminExtImpl.checkRocksdbCqWriteProgress(brokerAddr, topic, checkStoreTime);
+    }
+
+    @Override
+    public void exportRocksDBConfigToJson(String brokerAddr,
+        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configType)
+        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {
+        this.defaultMQAdminExtImpl.exportRocksDBConfigToJson(brokerAddr, configType);
     }
 
     @Override
@@ -802,10 +820,10 @@ public void resetMasterFlushOffset(String brokerAddr, long masterFlushOffset)
         this.defaultMQAdminExtImpl.resetMasterFlushOffset(brokerAddr, masterFlushOffset);
     }
 
-    public QueryResult queryMessageByUniqKey(String topic, String key, int maxNum, long begin, long end)
+    public QueryResult queryMessageByUniqKey(String clusterName, String topic, String key, int maxNum, long begin,
+        long end)
         throws MQClientException, InterruptedException {
-
-        return defaultMQAdminExtImpl.queryMessageByUniqKey(topic, key, maxNum, begin, end);
+        return defaultMQAdminExtImpl.queryMessageByUniqKey(clusterName, topic, key, maxNum, begin, end);
     }
 
     public DefaultMQAdminExtImpl getDefaultMQAdminExtImpl() {
@@ -837,13 +855,14 @@ public void updateControllerConfig(Properties properties,
 
     @Override
     public Pair<ElectMasterResponseHeader, BrokerMemberGroup> electMaster(String controllerAddr, String clusterName,
-                                                                          String brokerName, Long brokerId) throws RemotingException, InterruptedException, MQBrokerException {
+        String brokerName, Long brokerId) throws RemotingException, InterruptedException, MQBrokerException {
         return this.defaultMQAdminExtImpl.electMaster(controllerAddr, clusterName, brokerName, brokerId);
     }
 
     @Override
     public void cleanControllerBrokerData(String controllerAddr, String clusterName, String brokerName,
-        String brokerControllerIdsToClean, boolean isCleanLivingBroker) throws RemotingException, InterruptedException, MQBrokerException {
+        String brokerControllerIdsToClean,
+        boolean isCleanLivingBroker) throws RemotingException, InterruptedException, MQBrokerException {
         this.defaultMQAdminExtImpl.cleanControllerBrokerData(controllerAddr, clusterName, brokerName, brokerControllerIdsToClean, isCleanLivingBroker);
     }
 
@@ -874,4 +893,97 @@ public String setCommitLogReadAheadMode(String brokerAddr, String mode)
         InterruptedException, MQBrokerException {
         return defaultMQAdminExtImpl.setCommitLogReadAheadMode(brokerAddr, mode);
     }
+
+    @Override
+    public void createUser(String brokerAddr,
+        UserInfo userInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        defaultMQAdminExtImpl.createUser(brokerAddr, userInfo);
+    }
+
+    @Override
+    public void createUser(String brokerAddr, String username, String password,
+        String userType) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        defaultMQAdminExtImpl.createUser(brokerAddr, username, password, userType);
+    }
+
+    @Override
+    public void updateUser(String brokerAddr, String username,
+        String password, String userType,
+        String userStatus) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        defaultMQAdminExtImpl.updateUser(brokerAddr, username, password, userType, userStatus);
+    }
+
+    @Override
+    public void updateUser(String brokerAddr,
+        UserInfo userInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        defaultMQAdminExtImpl.updateUser(brokerAddr, userInfo);
+    }
+
+    @Override
+    public void deleteUser(String brokerAddr,
+        String username) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        defaultMQAdminExtImpl.deleteUser(brokerAddr, username);
+    }
+
+    @Override
+    public UserInfo getUser(String brokerAddr,
+        String username) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        return defaultMQAdminExtImpl.getUser(brokerAddr, username);
+    }
+
+    @Override
+    public List<UserInfo> listUser(String brokerAddr,
+        String filter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        return defaultMQAdminExtImpl.listUser(brokerAddr, filter);
+    }
+
+    @Override
+    public void createAcl(String brokerAddr, String subject, List<String> resources, List<String> actions,
+        List<String> sourceIps,
+        String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        defaultMQAdminExtImpl.createAcl(brokerAddr, subject, resources, actions, sourceIps, decision);
+    }
+
+    @Override
+    public void createAcl(String brokerAddr,
+        AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        defaultMQAdminExtImpl.createAcl(brokerAddr, aclInfo);
+    }
+
+    @Override
+    public void updateAcl(String brokerAddr, String subject, List<String> resources, List<String> actions,
+        List<String> sourceIps,
+        String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        defaultMQAdminExtImpl.updateAcl(brokerAddr, subject, resources, actions, sourceIps, decision);
+    }
+
+    @Override
+    public void updateAcl(String brokerAddr,
+        AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        defaultMQAdminExtImpl.updateAcl(brokerAddr, aclInfo);
+    }
+
+    @Override
+    public void deleteAcl(String brokerAddr, String subject,
+        String resource) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        defaultMQAdminExtImpl.deleteAcl(brokerAddr, subject, resource);
+    }
+
+    @Override
+    public AclInfo getAcl(String brokerAddr,
+        String subject) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        return defaultMQAdminExtImpl.getAcl(brokerAddr, subject);
+    }
+
+    @Override
+    public List<AclInfo> listAcl(String brokerAddr, String subjectFilter,
+        String resourceFilter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        return defaultMQAdminExtImpl.listAcl(brokerAddr, subjectFilter, resourceFilter);
+    }
+
+    @Override
+    public void exportPopRecords(String brokerAddr, long timeout) throws RemotingConnectException,
+        RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        defaultMQAdminExtImpl.exportPopRecords(brokerAddr, timeout);
+    }
 }
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
index 9447d95bb57..1bdcc765d61 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
@@ -17,26 +17,6 @@
 package org.apache.rocketmq.tools.admin;
 
 import com.alibaba.fastjson.JSON;
-import java.io.UnsupportedEncodingException;
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.client.QueryResult;
@@ -45,10 +25,11 @@
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.impl.MQClientManager;
 import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+import org.apache.rocketmq.common.BoundaryType;
+import org.apache.rocketmq.common.CheckRocksdbCqWriteResult;
 import org.apache.rocketmq.common.KeyBuilder;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.Pair;
-import org.apache.rocketmq.common.PlainAccessConfig;
 import org.apache.rocketmq.common.ServiceState;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.TopicConfig;
@@ -64,7 +45,6 @@
 import org.apache.rocketmq.common.namesrv.NamesrvUtil;
 import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.common.utils.NetworkUtil;
-import org.apache.rocketmq.common.BoundaryType;
 import org.apache.rocketmq.common.utils.ThreadUtils;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
@@ -80,9 +60,10 @@
 import org.apache.rocketmq.remoting.protocol.admin.RollbackStats;
 import org.apache.rocketmq.remoting.protocol.admin.TopicOffset;
 import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;
+import org.apache.rocketmq.remoting.protocol.body.AclInfo;
 import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;
+import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;
 import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData;
-import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo;
 import org.apache.rocketmq.remoting.protocol.body.ClusterInfo;
 import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;
 import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList;
@@ -91,7 +72,6 @@
 import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache;
 import org.apache.rocketmq.remoting.protocol.body.GroupList;
 import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
-import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;
 import org.apache.rocketmq.remoting.protocol.body.KVTable;
 import org.apache.rocketmq.remoting.protocol.body.ProducerConnection;
 import org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo;
@@ -100,6 +80,8 @@
 import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;
 import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
 import org.apache.rocketmq.remoting.protocol.body.TopicList;
+import org.apache.rocketmq.remoting.protocol.body.UserInfo;
+import org.apache.rocketmq.remoting.protocol.header.ExportRocksDBConfigToJsonRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
@@ -121,27 +103,30 @@
 import org.apache.rocketmq.tools.admin.common.AdminToolsResultCodeEnum;
 import org.apache.rocketmq.tools.command.CommandUtil;
 
+import java.io.UnsupportedEncodingException;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
 public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner {
 
     private static final String SOCKS_PROXY_JSON = "socksProxyJson";
-    private static final Set<String> SYSTEM_GROUP_SET = new HashSet<>();
-
-    static {
-        SYSTEM_GROUP_SET.add(MixAll.DEFAULT_CONSUMER_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.DEFAULT_PRODUCER_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.TOOLS_CONSUMER_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.SCHEDULE_CONSUMER_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.FILTERSRV_CONSUMER_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.MONITOR_CONSUMER_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.CLIENT_INNER_PRODUCER_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.SELF_TEST_PRODUCER_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.SELF_TEST_CONSUMER_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.ONS_HTTP_PROXY_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.CID_ONSAPI_PERMISSION_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.CID_ONSAPI_OWNER_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.CID_ONSAPI_PULL_GROUP);
-        SYSTEM_GROUP_SET.add(MixAll.CID_SYS_RMQ_TRANS);
-    }
 
     private final Logger logger = LoggerFactory.getLogger(DefaultMQAdminExtImpl.class);
     private final DefaultMQAdminExt defaultMQAdminExt;
@@ -274,34 +259,9 @@ public void createAndUpdateTopicConfig(String addr,
     }
 
     @Override
-    public void createAndUpdatePlainAccessConfig(String addr,
-        PlainAccessConfig config) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        this.mqClientInstance.getMQClientAPIImpl().createPlainAccessConfig(addr, config, timeoutMillis);
-    }
-
-    @Override
-    public void deletePlainAccessConfig(String addr,
-        String accessKey) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        this.mqClientInstance.getMQClientAPIImpl().deleteAccessConfig(addr, accessKey, timeoutMillis);
-    }
-
-    @Override
-    public void updateGlobalWhiteAddrConfig(String addr,
-        String globalWhiteAddrs) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        this.mqClientInstance.getMQClientAPIImpl().updateGlobalWhiteAddrsConfig(addr, globalWhiteAddrs, null, timeoutMillis);
-    }
-
-    @Override
-    public void updateGlobalWhiteAddrConfig(String addr,
-        String globalWhiteAddrs,
-        String aclFileFullPath) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        this.mqClientInstance.getMQClientAPIImpl().updateGlobalWhiteAddrsConfig(addr, globalWhiteAddrs, aclFileFullPath, timeoutMillis);
-    }
-
-    @Override
-    public ClusterAclVersionInfo examineBrokerClusterAclVersionInfo(
-        String addr) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        return this.mqClientInstance.getMQClientAPIImpl().getBrokerClusterAclInfo(addr, timeoutMillis);
+    public void createAndUpdateTopicConfigList(final String brokerAddr,
+        final List<TopicConfig> topicConfigList) throws RemotingException, InterruptedException, MQClientException {
+        this.mqClientInstance.getMQClientAPIImpl().createTopicList(brokerAddr, topicConfigList, timeoutMillis);
     }
 
     @Override
@@ -310,6 +270,12 @@ public void createAndUpdateSubscriptionGroupConfig(String addr,
         this.mqClientInstance.getMQClientAPIImpl().createSubscriptionGroup(addr, config, timeoutMillis);
     }
 
+    @Override
+    public void createAndUpdateSubscriptionGroupConfigList(String brokerAddr,
+        List<SubscriptionGroupConfig> configs) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+        this.mqClientInstance.getMQClientAPIImpl().createSubscriptionGroupList(brokerAddr, configs, timeoutMillis);
+    }
+
     @Override
     public SubscriptionGroupConfig examineSubscriptionGroupConfig(String addr,
         String group) throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
@@ -369,6 +335,7 @@ public void run() {
                                 if (addr != null) {
                                     TopicStatsTable tst = mqClientInstance.getMQClientAPIImpl().getTopicStatsInfo(addr, topic, timeoutMillis);
                                     topicStatsTable.getOffsetTable().putAll(tst.getOffsetTable());
+                                    topicStatsTable.setTopicPutTps(topicStatsTable.getTopicPutTps() + tst.getTopicPutTps());
                                 }
                             } catch (Exception e) {
                                 logger.error("getTopicStatsInfo error. topic={}", topic, e);
@@ -408,14 +375,21 @@ public KVTable fetchBrokerRuntimeStats(
         return this.mqClientInstance.getMQClientAPIImpl().getBrokerRuntimeInfo(brokerAddr, timeoutMillis);
     }
 
+    @Override
+    public ConsumeStats examineConsumeStats(
+        String consumerGroup,
+        String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
+        return examineConsumeStats(null, consumerGroup, topic);
+    }
+
     @Override
     public ConsumeStats examineConsumeStats(
         String consumerGroup) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
-        return examineConsumeStats(consumerGroup, null);
+        return examineConsumeStats(null, consumerGroup, null);
     }
 
     @Override
-    public ConsumeStats examineConsumeStats(String consumerGroup,
+    public ConsumeStats examineConsumeStats(String clusterName, String consumerGroup,
         String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
         TopicRouteData topicRouteData = null;
         List<String> routeTopics = new ArrayList<>();
@@ -424,6 +398,12 @@ public ConsumeStats examineConsumeStats(String consumerGroup,
             routeTopics.add(topic);
             routeTopics.add(KeyBuilder.buildPopRetryTopic(topic, consumerGroup));
         }
+
+        // Use clusterName topic to get topic route for lmq or rmq_sys_wheel_timer
+        if (!StringUtils.isEmpty(topic) && (MixAll.isLmq(topic) || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer")) && !StringUtils.isEmpty(clusterName)) {
+            routeTopics.add(clusterName);
+        }
+
         for (int i = 0; i < routeTopics.size(); i++) {
             try {
                 topicRouteData = this.examineTopicRouteInfo(routeTopics.get(i));
@@ -453,25 +433,33 @@ public ConsumeStats examineConsumeStats(String consumerGroup,
             topics.add(messageQueue.getTopic());
         }
 
-        ConsumeStats staticResult = new ConsumeStats();
-        staticResult.setConsumeTps(result.getConsumeTps());
-        // for topic, we put the physical stats, how about group?
-        // staticResult.getOffsetTable().putAll(result.getOffsetTable());
-
-        for (String currentTopic : topics) {
-            TopicRouteData currentRoute = this.examineTopicRouteInfo(currentTopic);
-            if (currentRoute.getTopicQueueMappingByBroker() == null
-                || currentRoute.getTopicQueueMappingByBroker().isEmpty()) {
-                //normal topic
-                for (Map.Entry<MessageQueue, OffsetWrapper> entry : result.getOffsetTable().entrySet()) {
-                    if (entry.getKey().getTopic().equals(currentTopic)) {
-                        staticResult.getOffsetTable().put(entry.getKey(), entry.getValue());
+        ConsumeStats staticResult = null;
+
+        if (StringUtils.isEmpty(clusterName)) {
+
+            staticResult = new ConsumeStats();
+            staticResult.setConsumeTps(result.getConsumeTps());
+            // for topic, we put the physical stats, how about group?
+            // staticResult.getOffsetTable().putAll(result.getOffsetTable());
+
+            for (String currentTopic : topics) {
+                TopicRouteData currentRoute = this.examineTopicRouteInfo(currentTopic);
+                if (currentRoute.getTopicQueueMappingByBroker() == null
+                    || currentRoute.getTopicQueueMappingByBroker().isEmpty()) {
+                    //normal topic
+                    for (Map.Entry<MessageQueue, OffsetWrapper> entry : result.getOffsetTable().entrySet()) {
+                        if (entry.getKey().getTopic().equals(currentTopic)) {
+                            staticResult.getOffsetTable().put(entry.getKey(), entry.getValue());
+                        }
                     }
                 }
+                Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigFromRoute(currentTopic, currentRoute, defaultMQAdminExt);
+                ConsumeStats consumeStats = MQAdminUtils.convertPhysicalConsumeStats(brokerConfigMap, result);
+                staticResult.getOffsetTable().putAll(consumeStats.getOffsetTable());
             }
-            Map<String, TopicConfigAndQueueMapping> brokerConfigMap = MQAdminUtils.examineTopicConfigFromRoute(currentTopic, currentRoute, defaultMQAdminExt);
-            ConsumeStats consumeStats = MQAdminUtils.convertPhysicalConsumeStats(brokerConfigMap, result);
-            staticResult.getOffsetTable().putAll(consumeStats.getOffsetTable());
+
+        } else {
+            staticResult = result;
         }
 
         if (staticResult.getOffsetTable().isEmpty()) {
@@ -588,7 +576,7 @@ public MessageExt viewMessage(String topic,
         String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
         try {
             MessageDecoder.decodeMessageId(msgId);
-            return this.viewMessage(msgId);
+            return this.mqClientInstance.getMQAdminImpl().viewMessage(topic, msgId);
         } catch (Exception e) {
             logger.warn("the msgId maybe created by new client. msgId={}", msgId, e);
         }
@@ -600,7 +588,7 @@ public MessageExt queryMessage(String clusterName, String topic,
         String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
         try {
             MessageDecoder.decodeMessageId(msgId);
-            return this.viewMessage(msgId);
+            return this.mqClientInstance.getMQAdminImpl().viewMessage(topic, msgId);
         } catch (Exception e) {
             logger.warn("the msgId maybe created by new client. msgId={}", msgId, e);
         }
@@ -797,10 +785,16 @@ public void deleteKvConfig(String namespace,
         this.mqClientInstance.getMQClientAPIImpl().deleteKVConfigValue(namespace, key, timeoutMillis);
     }
 
-    @Override
-    public List<RollbackStats> resetOffsetByTimestampOld(String consumerGroup, String topic, long timestamp,
+    public List<RollbackStats> resetOffsetByTimestampOld(String clusterName, String consumerGroup, String topic,
+        long timestamp,
         boolean force) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic);
+        String routeTopic = topic;
+        // Use clusterName topic to get topic route for lmq or rmq_sys_wheel_timer
+        if (!StringUtils.isEmpty(topic) && (MixAll.isLmq(topic) || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer"))
+            && !StringUtils.isEmpty(clusterName)) {
+            routeTopic = clusterName;
+        }
+        TopicRouteData topicRouteData = this.examineTopicRouteInfo(routeTopic);
         List<RollbackStats> rollbackStatsList = new ArrayList<>();
         Map<String, QueueData> topicRouteMap = new HashMap<>();
         for (QueueData queueData : topicRouteData.getQueueDatas()) {
@@ -815,6 +809,12 @@ public List<RollbackStats> resetOffsetByTimestampOld(String consumerGroup, Strin
         return rollbackStatsList;
     }
 
+    @Override
+    public List<RollbackStats> resetOffsetByTimestampOld(String consumerGroup, String topic, long timestamp,
+        boolean force) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+        return resetOffsetByTimestampOld(null, consumerGroup, topic, timestamp, force);
+    }
+
     private List<RollbackStats> resetOffsetByTimestampOld(String brokerAddr, QueueData queueData, String consumerGroup,
         String topic, long timestamp,
         boolean force) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
@@ -850,7 +850,7 @@ private List<RollbackStats> resetOffsetByTimestampOld(String brokerAddr, QueueDa
     @Override
     public Map<MessageQueue, Long> resetOffsetByTimestamp(String topic, String group, long timestamp,
         boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        return resetOffsetByTimestamp(topic, group, timestamp, isForce, false);
+        return resetOffsetByTimestamp(null, topic, group, timestamp, isForce, false);
     }
 
     @Override
@@ -937,9 +937,16 @@ public void run() {
         });
     }
 
-    public Map<MessageQueue, Long> resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce,
+    public Map<MessageQueue, Long> resetOffsetByTimestamp(String clusterName, String topic, String group,
+        long timestamp, boolean isForce,
         boolean isC) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic);
+        String routeTopic = topic;
+        // Use clusterName topic to get topic route for lmq or rmq_sys_wheel_timer
+        if (!StringUtils.isEmpty(topic) && (MixAll.isLmq(topic) || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer"))
+            && !StringUtils.isEmpty(clusterName)) {
+            routeTopic = clusterName;
+        }
+        TopicRouteData topicRouteData = this.examineTopicRouteInfo(routeTopic);
         List<BrokerData> brokerDatas = topicRouteData.getBrokerDatas();
         Map<MessageQueue, Long> allOffsetTable = new HashMap<>();
         if (brokerDatas != null) {
@@ -1310,18 +1317,24 @@ public ConsumerRunningInfo getConsumerRunningInfo(String consumerGroup, String c
     }
 
     @Override
-    public ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup, String clientId,
-        String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
-        MessageExt msg = this.viewMessage(msgId);
-
-        return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(NetworkUtil.socketAddress2String(msg.getStoreHost()), consumerGroup, clientId, msg.getTopic(), msgId, timeoutMillis);
+    public ConsumeMessageDirectlyResult consumeMessageDirectly(final String consumerGroup, final String clientId,
+        final String topic,
+        final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
+        MessageExt msg = this.viewMessage(topic, msgId);
+        if (msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX) == null) {
+            return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(NetworkUtil.socketAddress2String(msg.getStoreHost()), consumerGroup, clientId, topic, msgId, timeoutMillis);
+        } else {
+            MessageClientExt msgClient = (MessageClientExt) msg;
+            return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(NetworkUtil.socketAddress2String(msg.getStoreHost()), consumerGroup, clientId, topic, msgClient.getOffsetMsgId(), timeoutMillis);
+        }
     }
 
     @Override
-    public ConsumeMessageDirectlyResult consumeMessageDirectly(final String consumerGroup, final String clientId,
+    public ConsumeMessageDirectlyResult consumeMessageDirectly(final String clusterName, final String consumerGroup,
+        final String clientId,
         final String topic,
         final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
-        MessageExt msg = this.viewMessage(topic, msgId);
+        MessageExt msg = this.queryMessage(clusterName, topic, msgId);
         if (msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX) == null) {
             return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(NetworkUtil.socketAddress2String(msg.getStoreHost()), consumerGroup, clientId, topic, msgId, timeoutMillis);
         } else {
@@ -1636,7 +1649,7 @@ public SubscriptionGroupWrapper getUserSubscriptionGroup(final String brokerAddr
         Iterator<Entry<String, SubscriptionGroupConfig>> iterator = subscriptionGroupWrapper.getSubscriptionGroupTable().entrySet().iterator();
         while (iterator.hasNext()) {
             Map.Entry<String, SubscriptionGroupConfig> configEntry = iterator.next();
-            if (MixAll.isSysConsumerGroup(configEntry.getKey()) || SYSTEM_GROUP_SET.contains(configEntry.getKey())) {
+            if (MixAll.isSysConsumerGroup(configEntry.getKey()) || MixAll.isPredefinedGroup(configEntry.getKey())) {
                 iterator.remove();
             }
         }
@@ -1659,10 +1672,10 @@ public TopicConfigSerializeWrapper getUserTopicConfig(final String brokerAddr, f
         while (iterator.hasNext()) {
             TopicConfig topicConfig = iterator.next().getValue();
             if (topicList.getTopicList().contains(topicConfig.getTopicName())
-                    || TopicValidator.isSystemTopic(topicConfig.getTopicName())) {
+                || TopicValidator.isSystemTopic(topicConfig.getTopicName())) {
                 iterator.remove();
             } else if (!specialTopic && StringUtils.startsWithAny(topicConfig.getTopicName(),
-                    MixAll.RETRY_GROUP_TOPIC_PREFIX, MixAll.DLQ_GROUP_TOPIC_PREFIX)) {
+                MixAll.RETRY_GROUP_TOPIC_PREFIX, MixAll.DLQ_GROUP_TOPIC_PREFIX)) {
                 iterator.remove();
             } else if (!PermName.isValid(topicConfig.getPerm())) {
                 iterator.remove();
@@ -1714,12 +1727,6 @@ public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {
         return this.mqClientInstance.getMQAdminImpl().earliestMsgStoreTime(mq);
     }
 
-    @Override
-    public MessageExt viewMessage(
-        String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
-        return this.mqClientInstance.getMQAdminImpl().viewMessage(msgId);
-    }
-
     @Override
     public QueryResult queryMessage(String topic, String key, int maxNum, long begin,
         long end) throws MQClientException, InterruptedException {
@@ -1727,6 +1734,11 @@ public QueryResult queryMessage(String topic, String key, int maxNum, long begin
         return this.mqClientInstance.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end);
     }
 
+    public QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin,
+        long end) throws MQClientException, InterruptedException, RemotingException {
+        return this.mqClientInstance.getMQAdminImpl().queryMessage(clusterName, topic, key, maxNum, begin, end, false);
+    }
+
     @Override
     public void updateConsumeOffset(String brokerAddr, String consumeGroup, MessageQueue mq,
         long offset) throws RemotingException, InterruptedException, MQBrokerException {
@@ -1759,11 +1771,16 @@ public QueryConsumeQueueResponseBody queryConsumeQueue(String brokerAddr, String
     }
 
     @Override
-    public boolean resumeCheckHalfMessage(
-        String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
-        MessageExt msg = this.viewMessage(msgId);
+    public CheckRocksdbCqWriteResult checkRocksdbCqWriteProgress(String brokerAddr, String topic, long checkStoreTime)
+        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {
+        return this.mqClientInstance.getMQClientAPIImpl().checkRocksdbCqWriteProgress(brokerAddr, topic, checkStoreTime, timeoutMillis);
+    }
 
-        return this.mqClientInstance.getMQClientAPIImpl().resumeCheckHalfMessage(NetworkUtil.socketAddress2String(msg.getStoreHost()), msgId, timeoutMillis);
+    @Override
+    public void exportRocksDBConfigToJson(String brokerAddr,
+        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configType)
+        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {
+        this.mqClientInstance.getMQClientAPIImpl().exportRocksDBConfigToJson(brokerAddr, configType, timeoutMillis);
     }
 
     @Override
@@ -1771,10 +1788,10 @@ public boolean resumeCheckHalfMessage(final String topic,
         final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
         MessageExt msg = this.viewMessage(topic, msgId);
         if (msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX) == null) {
-            return this.mqClientInstance.getMQClientAPIImpl().resumeCheckHalfMessage(NetworkUtil.socketAddress2String(msg.getStoreHost()), msgId, timeoutMillis);
+            return this.mqClientInstance.getMQClientAPIImpl().resumeCheckHalfMessage(NetworkUtil.socketAddress2String(msg.getStoreHost()), topic, msgId, timeoutMillis);
         } else {
             MessageClientExt msgClient = (MessageClientExt) msg;
-            return this.mqClientInstance.getMQClientAPIImpl().resumeCheckHalfMessage(NetworkUtil.socketAddress2String(msg.getStoreHost()), msgClient.getOffsetMsgId(), timeoutMillis);
+            return this.mqClientInstance.getMQClientAPIImpl().resumeCheckHalfMessage(NetworkUtil.socketAddress2String(msg.getStoreHost()), topic, msgClient.getOffsetMsgId(), timeoutMillis);
         }
     }
 
@@ -1792,10 +1809,9 @@ public long searchOffset(final String brokerAddr, final String topicName, final
         return this.mqClientInstance.getMQClientAPIImpl().searchOffset(brokerAddr, topicName, queueId, timestamp, timeoutMillis);
     }
 
-    public QueryResult queryMessageByUniqKey(String topic, String key, int maxNum, long begin,
+    public QueryResult queryMessageByUniqKey(String clusterName, String topic, String key, int maxNum, long begin,
         long end) throws MQClientException, InterruptedException {
-
-        return this.mqClientInstance.getMQAdminImpl().queryMessageByUniqKey(topic, key, maxNum, begin, end);
+        return this.mqClientInstance.getMQAdminImpl().queryMessageByUniqKey(clusterName, topic, key, maxNum, begin, end);
     }
 
     @Override
@@ -1821,6 +1837,12 @@ public void resetOffsetByQueueId(final String brokerAddr, final String consumeGr
         }
     }
 
+    @Override
+    public Map<MessageQueue, Long> resetOffsetByTimestamp(String clusterName, String topic, String group,
+        long timestamp, boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+        return resetOffsetByTimestamp(clusterName, topic, group, timestamp, isForce, false);
+    }
+
     @Override
     public HARuntimeInfo getBrokerHAStatus(
         String brokerAddr) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
@@ -1853,7 +1875,7 @@ public void resetMasterFlushOffset(String brokerAddr,
 
     @Override
     public Pair<ElectMasterResponseHeader, BrokerMemberGroup> electMaster(String controllerAddr, String clusterName,
-                                                                          String brokerName, Long brokerId) throws RemotingException, InterruptedException, MQBrokerException {
+        String brokerName, Long brokerId) throws RemotingException, InterruptedException, MQBrokerException {
         return this.mqClientInstance.getMQClientAPIImpl().electMaster(controllerAddr, clusterName, brokerName, brokerId);
     }
 
@@ -1931,4 +1953,101 @@ public String setCommitLogReadAheadMode(String brokerAddr, String mode)
         throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException {
         return this.mqClientInstance.getMQClientAPIImpl().setCommitLogReadAheadMode(brokerAddr, mode, timeoutMillis);
     }
+
+    @Override
+    public void createUser(String brokerAddr,
+        UserInfo userInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        this.mqClientInstance.getMQClientAPIImpl().createUser(brokerAddr, userInfo, timeoutMillis);
+    }
+
+    @Override
+    public void createUser(String brokerAddr, String username, String password,
+        String userType) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        UserInfo userInfo = UserInfo.of(username, password, userType);
+        this.createUser(brokerAddr, userInfo);
+    }
+
+    @Override
+    public void updateUser(String brokerAddr, String username,
+        String password, String userType,
+        String userStatus) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        UserInfo userInfo = UserInfo.of(username, password, userType, userStatus);
+        this.mqClientInstance.getMQClientAPIImpl().updateUser(brokerAddr, userInfo, timeoutMillis);
+    }
+
+    @Override
+    public void updateUser(String brokerAddr,
+        UserInfo userInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        this.mqClientInstance.getMQClientAPIImpl().updateUser(brokerAddr, userInfo, timeoutMillis);
+    }
+
+    @Override
+    public void deleteUser(String brokerAddr,
+        String username) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        this.mqClientInstance.getMQClientAPIImpl().deleteUser(brokerAddr, username, timeoutMillis);
+    }
+
+    @Override
+    public UserInfo getUser(String brokerAddr,
+        String username) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        return this.mqClientInstance.getMQClientAPIImpl().getUser(brokerAddr, username, timeoutMillis);
+    }
+
+    @Override
+    public List<UserInfo> listUser(String brokerAddr,
+        String filter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        return this.mqClientInstance.getMQClientAPIImpl().listUser(brokerAddr, filter, timeoutMillis);
+    }
+
+    @Override
+    public void createAcl(String brokerAddr, String subject, List<String> resources, List<String> actions,
+        List<String> sourceIps,
+        String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        AclInfo aclInfo = AclInfo.of(subject, resources, actions, sourceIps, decision);
+        this.createAcl(brokerAddr, aclInfo);
+    }
+
+    @Override
+    public void createAcl(String brokerAddr,
+        AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        this.mqClientInstance.getMQClientAPIImpl().createAcl(brokerAddr, aclInfo, timeoutMillis);
+    }
+
+    @Override
+    public void updateAcl(String brokerAddr, String subject, List<String> resources, List<String> actions,
+        List<String> sourceIps,
+        String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        AclInfo aclInfo = AclInfo.of(subject, resources, actions, sourceIps, decision);
+        this.updateAcl(brokerAddr, aclInfo);
+    }
+
+    @Override
+    public void updateAcl(String brokerAddr,
+        AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        this.mqClientInstance.getMQClientAPIImpl().updateAcl(brokerAddr, aclInfo, timeoutMillis);
+    }
+
+    @Override
+    public void deleteAcl(String brokerAddr, String subject,
+        String resource) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        this.mqClientInstance.getMQClientAPIImpl().deleteAcl(brokerAddr, subject, resource, timeoutMillis);
+    }
+
+    @Override
+    public AclInfo getAcl(String brokerAddr,
+        String subject) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        return this.mqClientInstance.getMQClientAPIImpl().getAcl(brokerAddr, subject, timeoutMillis);
+    }
+
+    @Override
+    public List<AclInfo> listAcl(String brokerAddr, String subjectFilter,
+        String resourceFilter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        return this.mqClientInstance.getMQClientAPIImpl().listAcl(brokerAddr, subjectFilter, resourceFilter, timeoutMillis);
+    }
+
+    @Override
+    public void exportPopRecords(String brokerAddr, long timeout) throws RemotingConnectException,
+        RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException {
+        this.mqClientInstance.getMQClientAPIImpl().exportPopRecord(brokerAddr, timeout);
+    }
 }
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
index 3148fc0987e..a10a58950d3 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
@@ -16,16 +16,11 @@
  */
 package org.apache.rocketmq.tools.admin;
 
-import java.io.UnsupportedEncodingException;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
 import org.apache.rocketmq.client.MQAdmin;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.CheckRocksdbCqWriteResult;
 import org.apache.rocketmq.common.Pair;
-import org.apache.rocketmq.common.PlainAccessConfig;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageQueue;
@@ -38,9 +33,10 @@
 import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats;
 import org.apache.rocketmq.remoting.protocol.admin.RollbackStats;
 import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;
+import org.apache.rocketmq.remoting.protocol.body.AclInfo;
 import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup;
+import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;
 import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData;
-import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo;
 import org.apache.rocketmq.remoting.protocol.body.ClusterInfo;
 import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;
 import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList;
@@ -49,7 +45,6 @@
 import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache;
 import org.apache.rocketmq.remoting.protocol.body.GroupList;
 import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
-import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo;
 import org.apache.rocketmq.remoting.protocol.body.KVTable;
 import org.apache.rocketmq.remoting.protocol.body.ProducerConnection;
 import org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo;
@@ -58,6 +53,8 @@
 import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper;
 import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
 import org.apache.rocketmq.remoting.protocol.body.TopicList;
+import org.apache.rocketmq.remoting.protocol.body.UserInfo;
+import org.apache.rocketmq.remoting.protocol.header.ExportRocksDBConfigToJsonRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;
 import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
@@ -69,6 +66,12 @@
 import org.apache.rocketmq.tools.admin.api.MessageTrack;
 import org.apache.rocketmq.tools.admin.common.AdminToolResult;
 
+import java.io.UnsupportedEncodingException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
 public interface MQAdminExt extends MQAdmin {
     void start() throws MQClientException;
 
@@ -90,29 +93,17 @@ void createAndUpdateTopicConfig(final String addr,
         final TopicConfig config) throws RemotingException, MQBrokerException,
         InterruptedException, MQClientException;
 
-    void createAndUpdatePlainAccessConfig(final String addr,
-        final PlainAccessConfig plainAccessConfig) throws RemotingException, MQBrokerException,
-        InterruptedException, MQClientException;
-
-    void deletePlainAccessConfig(final String addr, final String accessKey) throws RemotingException, MQBrokerException,
-        InterruptedException, MQClientException;
-
-    void updateGlobalWhiteAddrConfig(final String addr,
-        final String globalWhiteAddrs) throws RemotingException, MQBrokerException,
-        InterruptedException, MQClientException;
-
-    void updateGlobalWhiteAddrConfig(final String addr, final String globalWhiteAddrs,
-        String aclFileFullPath) throws RemotingException, MQBrokerException,
-        InterruptedException, MQClientException;
-
-    ClusterAclVersionInfo examineBrokerClusterAclVersionInfo(
-        final String addr) throws RemotingException, MQBrokerException,
-        InterruptedException, MQClientException;
+    void createAndUpdateTopicConfigList(final String addr,
+        final List<TopicConfig> topicConfigList) throws InterruptedException, RemotingException, MQClientException;
 
     void createAndUpdateSubscriptionGroupConfig(final String addr,
         final SubscriptionGroupConfig config) throws RemotingException,
         MQBrokerException, InterruptedException, MQClientException;
 
+    void createAndUpdateSubscriptionGroupConfigList(String brokerAddr,
+        List<SubscriptionGroupConfig> configs) throws RemotingException,
+        MQBrokerException, InterruptedException, MQClientException;
+
     SubscriptionGroupConfig examineSubscriptionGroupConfig(final String addr,
         final String group) throws InterruptedException, RemotingException, MQClientException, MQBrokerException;
 
@@ -139,10 +130,17 @@ ConsumeStats examineConsumeStats(
         final String consumerGroup) throws RemotingException, MQClientException, InterruptedException,
         MQBrokerException;
 
+    CheckRocksdbCqWriteResult checkRocksdbCqWriteProgress(String brokerAddr, String topic, long checkStoreTime)
+        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException;
+
     ConsumeStats examineConsumeStats(final String consumerGroup,
         final String topic) throws RemotingException, MQClientException,
         InterruptedException, MQBrokerException;
 
+    ConsumeStats examineConsumeStats(final String clusterName, final String consumerGroup,
+        final String topic) throws RemotingException, MQClientException,
+        InterruptedException, MQBrokerException;
+
     ConsumeStats examineConsumeStats(final String brokerAddr, final String consumerGroup, final String topicName,
         final long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException,
         RemotingConnectException, MQBrokerException;
@@ -223,6 +221,9 @@ List<RollbackStats> resetOffsetByTimestampOld(String consumerGroup, String topic
     Map<MessageQueue, Long> resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce)
         throws RemotingException, MQBrokerException, InterruptedException, MQClientException;
 
+    Map<MessageQueue, Long> resetOffsetByTimestamp(String clusterName, String topic, String group, long timestamp, boolean isForce)
+        throws RemotingException, MQBrokerException, InterruptedException, MQClientException;
+
     void resetOffsetNew(String consumerGroup, String topic, long timestamp) throws RemotingException, MQBrokerException,
         InterruptedException, MQClientException;
 
@@ -281,9 +282,10 @@ ConsumerRunningInfo getConsumerRunningInfo(final String consumerGroup, final Str
 
     ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup,
         String clientId,
+        String topic,
         String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException;
 
-    ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup,
+    ConsumeMessageDirectlyResult consumeMessageDirectly(String clusterName, String consumerGroup,
         String clientId,
         String topic,
         String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException;
@@ -371,8 +373,9 @@ QueryConsumeQueueResponseBody queryConsumeQueue(final String brokerAddr,
         final long index, final int count, final String consumerGroup)
         throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException;
 
-    boolean resumeCheckHalfMessage(String msgId)
-        throws RemotingException, MQClientException, InterruptedException, MQBrokerException;
+    void exportRocksDBConfigToJson(String brokerAddr,
+        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configType)
+        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException;
 
     boolean resumeCheckHalfMessage(final String topic,
         final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException;
@@ -481,4 +484,34 @@ String getColdDataFlowCtrInfo(final String brokerAddr)
     String setCommitLogReadAheadMode(final String brokerAddr, String mode)
         throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, UnsupportedEncodingException, InterruptedException, MQBrokerException;
 
+    void createUser(String brokerAddr, String username, String password, String userType) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    void createUser(String brokerAddr, UserInfo userInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    void updateUser(String brokerAddr, String username, String password, String userType, String userStatus) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    void updateUser(String brokerAddr, UserInfo userInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    void deleteUser(String brokerAddr, String username) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    UserInfo getUser(String brokerAddr, String username) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    List<UserInfo> listUser(String brokerAddr, String filter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    void createAcl(String brokerAddr, String subject, List<String> resources, List<String> actions, List<String> sourceIps, String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    void createAcl(String brokerAddr, AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    void updateAcl(String brokerAddr, String subject, List<String> resources, List<String> actions, List<String> sourceIps, String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    void updateAcl(String brokerAddr, AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    void deleteAcl(String brokerAddr, String subject, String resource) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    AclInfo getAcl(String brokerAddr, String subject) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    List<AclInfo> listAcl(String brokerAddr, String subjectFilter, String resourceFilter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
+
+    void exportPopRecords(String brokerAddr, long timeout) throws RemotingConnectException,
+        RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException;
 }
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
index 35f00748228..b5caf069080 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
@@ -16,8 +16,6 @@
  */
 package org.apache.rocketmq.tools.command;
 
-import java.util.ArrayList;
-import java.util.List;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.DefaultParser;
 import org.apache.commons.cli.Options;
@@ -27,10 +25,18 @@
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.srvutil.ServerUtil;
-import org.apache.rocketmq.tools.command.acl.ClusterAclConfigVersionListSubCommand;
-import org.apache.rocketmq.tools.command.acl.DeleteAccessConfigSubCommand;
-import org.apache.rocketmq.tools.command.acl.UpdateAccessConfigSubCommand;
-import org.apache.rocketmq.tools.command.acl.UpdateGlobalWhiteAddrSubCommand;
+import org.apache.rocketmq.tools.command.auth.CopyAclsSubCommand;
+import org.apache.rocketmq.tools.command.auth.CopyUsersSubCommand;
+import org.apache.rocketmq.tools.command.auth.CreateAclSubCommand;
+import org.apache.rocketmq.tools.command.auth.CreateUserSubCommand;
+import org.apache.rocketmq.tools.command.auth.DeleteAclSubCommand;
+import org.apache.rocketmq.tools.command.auth.DeleteUserSubCommand;
+import org.apache.rocketmq.tools.command.auth.GetAclSubCommand;
+import org.apache.rocketmq.tools.command.auth.GetUserSubCommand;
+import org.apache.rocketmq.tools.command.auth.ListAclSubCommand;
+import org.apache.rocketmq.tools.command.auth.ListUserSubCommand;
+import org.apache.rocketmq.tools.command.auth.UpdateAclSubCommand;
+import org.apache.rocketmq.tools.command.auth.UpdateUserSubCommand;
 import org.apache.rocketmq.tools.command.broker.BrokerConsumeStatsSubCommad;
 import org.apache.rocketmq.tools.command.broker.BrokerStatusSubCommand;
 import org.apache.rocketmq.tools.command.broker.CleanExpiredCQSubCommand;
@@ -55,6 +61,7 @@
 import org.apache.rocketmq.tools.command.consumer.GetConsumerConfigSubCommand;
 import org.apache.rocketmq.tools.command.consumer.SetConsumeModeSubCommand;
 import org.apache.rocketmq.tools.command.consumer.StartMonitoringSubCommand;
+import org.apache.rocketmq.tools.command.consumer.UpdateSubGroupListSubCommand;
 import org.apache.rocketmq.tools.command.consumer.UpdateSubGroupSubCommand;
 import org.apache.rocketmq.tools.command.container.AddBrokerSubCommand;
 import org.apache.rocketmq.tools.command.container.RemoveBrokerSubCommand;
@@ -65,7 +72,9 @@
 import org.apache.rocketmq.tools.command.controller.UpdateControllerConfigSubCommand;
 import org.apache.rocketmq.tools.command.export.ExportConfigsCommand;
 import org.apache.rocketmq.tools.command.export.ExportMetadataCommand;
+import org.apache.rocketmq.tools.command.export.ExportMetadataInRocksDBCommand;
 import org.apache.rocketmq.tools.command.export.ExportMetricsCommand;
+import org.apache.rocketmq.tools.command.export.ExportPopRecordCommand;
 import org.apache.rocketmq.tools.command.ha.GetSyncStateSetSubCommand;
 import org.apache.rocketmq.tools.command.ha.HAStatusSubCommand;
 import org.apache.rocketmq.tools.command.message.CheckMsgSendRTCommand;
@@ -79,7 +88,7 @@
 import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand;
 import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand;
 import org.apache.rocketmq.tools.command.message.SendMessageCommand;
-import org.apache.rocketmq.tools.command.export.ExportMetadataInRocksDBCommand;
+import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand;
 import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand;
 import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand;
 import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand;
@@ -90,6 +99,7 @@
 import org.apache.rocketmq.tools.command.offset.ResetOffsetByTimeCommand;
 import org.apache.rocketmq.tools.command.offset.SkipAccumulationSubCommand;
 import org.apache.rocketmq.tools.command.producer.ProducerSubCommand;
+import org.apache.rocketmq.tools.command.queue.CheckRocksdbCqWriteProgressCommand;
 import org.apache.rocketmq.tools.command.queue.QueryConsumeQueueCommand;
 import org.apache.rocketmq.tools.command.stats.StatsAllSubCommand;
 import org.apache.rocketmq.tools.command.topic.AllocateMQSubCommand;
@@ -101,9 +111,13 @@
 import org.apache.rocketmq.tools.command.topic.TopicStatusSubCommand;
 import org.apache.rocketmq.tools.command.topic.UpdateOrderConfCommand;
 import org.apache.rocketmq.tools.command.topic.UpdateStaticTopicSubCommand;
+import org.apache.rocketmq.tools.command.topic.UpdateTopicListSubCommand;
 import org.apache.rocketmq.tools.command.topic.UpdateTopicPermSubCommand;
 import org.apache.rocketmq.tools.command.topic.UpdateTopicSubCommand;
 
+import java.util.ArrayList;
+import java.util.List;
+
 public class MQAdminStartup {
     protected static final List<SubCommand> SUB_COMMANDS = new ArrayList<>();
 
@@ -175,8 +189,10 @@ public static void main0(String[] args, RPCHook rpcHook) {
 
     public static void initCommand() {
         initCommand(new UpdateTopicSubCommand());
+        initCommand(new UpdateTopicListSubCommand());
         initCommand(new DeleteTopicSubCommand());
         initCommand(new UpdateSubGroupSubCommand());
+        initCommand(new UpdateSubGroupListSubCommand());
         initCommand(new SetConsumeModeSubCommand());
         initCommand(new DeleteSubscriptionGroupCommand());
         initCommand(new UpdateBrokerConfigSubCommand());
@@ -242,12 +258,6 @@ public static void initCommand() {
         initCommand(new SendMessageCommand());
         initCommand(new ConsumeMessageCommand());
 
-        //for acl command
-        initCommand(new UpdateAccessConfigSubCommand());
-        initCommand(new DeleteAccessConfigSubCommand());
-        initCommand(new ClusterAclConfigVersionListSubCommand());
-        initCommand(new UpdateGlobalWhiteAddrSubCommand());
-
         initCommand(new UpdateStaticTopicSubCommand());
         initCommand(new RemappingStaticTopicSubCommand());
 
@@ -255,6 +265,7 @@ public static void initCommand() {
         initCommand(new ExportConfigsCommand());
         initCommand(new ExportMetricsCommand());
         initCommand(new ExportMetadataInRocksDBCommand());
+        initCommand(new ExportPopRecordCommand());
 
         initCommand(new HAStatusSubCommand());
 
@@ -272,6 +283,22 @@ public static void initCommand() {
         initCommand(new UpdateColdDataFlowCtrGroupConfigSubCommand());
         initCommand(new RemoveColdDataFlowCtrGroupConfigSubCommand());
         initCommand(new CommitLogSetReadAheadSubCommand());
+
+        initCommand(new CreateUserSubCommand());
+        initCommand(new UpdateUserSubCommand());
+        initCommand(new DeleteUserSubCommand());
+        initCommand(new GetUserSubCommand());
+        initCommand(new ListUserSubCommand());
+        initCommand(new CopyUsersSubCommand());
+
+        initCommand(new CreateAclSubCommand());
+        initCommand(new UpdateAclSubCommand());
+        initCommand(new DeleteAclSubCommand());
+        initCommand(new GetAclSubCommand());
+        initCommand(new ListAclSubCommand());
+        initCommand(new CopyAclsSubCommand());
+        initCommand(new RocksDBConfigToJsonCommand());
+        initCommand(new CheckRocksdbCqWriteProgressCommand());
     }
 
     private static void printHelp() {
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java
deleted file mode 100644
index 26ed028fb33..00000000000
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tools.command.acl;
-
-import java.sql.Timestamp;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Map;
-import java.util.Set;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.cli.Options;
-import org.apache.rocketmq.client.exception.MQBrokerException;
-import org.apache.rocketmq.client.exception.MQClientException;
-import org.apache.rocketmq.common.UtilAll;
-import org.apache.rocketmq.remoting.RPCHook;
-import org.apache.rocketmq.remoting.exception.RemotingException;
-import org.apache.rocketmq.remoting.protocol.DataVersion;
-import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo;
-import org.apache.rocketmq.srvutil.ServerUtil;
-import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
-import org.apache.rocketmq.tools.command.CommandUtil;
-import org.apache.rocketmq.tools.command.SubCommand;
-import org.apache.rocketmq.tools.command.SubCommandException;
-
-public class ClusterAclConfigVersionListSubCommand implements SubCommand {
-
-    @Override
-    public String commandName() {
-        return "clusterAclConfigVersion";
-    }
-
-    @Override
-    public String commandDesc() {
-        return "List all of acl config version information in cluster.";
-    }
-
-    @Override
-    public Options buildCommandlineOptions(Options options) {
-        OptionGroup optionGroup = new OptionGroup();
-
-        Option opt = new Option("b", "brokerAddr", true, "query acl config version for which broker");
-        optionGroup.addOption(opt);
-
-        opt = new Option("c", "clusterName", true, "query acl config version for specified cluster");
-        optionGroup.addOption(opt);
-
-        optionGroup.setRequired(true);
-        options.addOptionGroup(optionGroup);
-
-        return options;
-    }
-
-    @Override
-    public void execute(CommandLine commandLine, Options options,
-        RPCHook rpcHook) throws SubCommandException {
-
-        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
-        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
-
-        try {
-
-            if (commandLine.hasOption('b')) {
-                String addr = commandLine.getOptionValue('b').trim();
-                defaultMQAdminExt.start();
-                printClusterBaseInfo(defaultMQAdminExt, addr);
-
-                System.out.printf("get broker's plain access config version success. Address:%s %n", addr);
-                return;
-
-            } else if (commandLine.hasOption('c')) {
-                String clusterName = commandLine.getOptionValue('c').trim();
-
-                defaultMQAdminExt.start();
-
-                Set<String> masterSet =
-                    CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
-                System.out.printf("%-16s  %-22s  %-22s  %-20s  %-22s  %-22s%n",
-                    "#Cluster Name",
-                    "#Broker Name",
-                    "#Broker Addr",
-                    "#AclFilePath",
-                    "#AclConfigVersionNum",
-                    "#AclLastUpdateTime"
-                );
-                for (String addr : masterSet) {
-                    printClusterBaseInfo(defaultMQAdminExt, addr);
-                }
-                System.out.printf("get cluster's plain access config version success.%n");
-
-                return;
-            }
-
-            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
-        } catch (Exception e) {
-            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
-        } finally {
-            defaultMQAdminExt.shutdown();
-        }
-    }
-
-    private void printClusterBaseInfo(
-        final DefaultMQAdminExt defaultMQAdminExt, final String addr) throws
-        InterruptedException, MQBrokerException, RemotingException, MQClientException {
-
-        ClusterAclVersionInfo clusterAclVersionInfo = defaultMQAdminExt.examineBrokerClusterAclVersionInfo(addr);
-        Map<String, DataVersion> aclDataVersion = clusterAclVersionInfo.getAllAclConfigDataVersion();
-        DateFormat sdf = new SimpleDateFormat(UtilAll.YYYY_MM_DD_HH_MM_SS);
-        if (aclDataVersion.size() > 0) {
-            for (Map.Entry<String, DataVersion> entry : aclDataVersion.entrySet()) {
-                System.out.printf("%-16s  %-22s  %-22s  %-20s  %-22s  %-22s%n",
-                    clusterAclVersionInfo.getClusterName(),
-                    clusterAclVersionInfo.getBrokerName(),
-                    clusterAclVersionInfo.getBrokerAddr(),
-                    entry.getKey(),
-                    String.valueOf(entry.getValue().getCounter()),
-                    sdf.format(new Timestamp(entry.getValue().getTimestamp()))
-                );
-            }
-        }
-    }
-}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java
deleted file mode 100644
index d8a06f92d14..00000000000
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tools.command.acl;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.cli.Options;
-import org.apache.rocketmq.common.PlainAccessConfig;
-import org.apache.rocketmq.remoting.RPCHook;
-import org.apache.rocketmq.srvutil.ServerUtil;
-import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
-import org.apache.rocketmq.tools.command.CommandUtil;
-import org.apache.rocketmq.tools.command.SubCommand;
-import org.apache.rocketmq.tools.command.SubCommandException;
-
-public class UpdateAccessConfigSubCommand implements SubCommand {
-
-    @Override
-    public String commandName() {
-        return "updateAclConfig";
-    }
-
-    @Override
-    public String commandDesc() {
-        return "Update acl config yaml file in broker.";
-    }
-
-    @Override
-    public Options buildCommandlineOptions(Options options) {
-        OptionGroup optionGroup = new OptionGroup();
-
-        Option opt = new Option("b", "brokerAddr", true, "update acl config file to which broker");
-        optionGroup.addOption(opt);
-
-        opt = new Option("c", "clusterName", true, "update acl config file to which cluster");
-        optionGroup.addOption(opt);
-
-        optionGroup.setRequired(true);
-        options.addOptionGroup(optionGroup);
-
-        opt = new Option("a", "accessKey", true, "set accessKey in acl config file");
-        opt.setRequired(true);
-        options.addOption(opt);
-
-        opt = new Option("s", "secretKey", true, "set secretKey in acl config file");
-        opt.setRequired(true);
-        options.addOption(opt);
-
-        opt = new Option("w", "whiteRemoteAddress", true, "set white ip Address for account in acl config file");
-        opt.setRequired(false);
-        options.addOption(opt);
-
-        opt = new Option("i", "defaultTopicPerm", true, "set default topicPerm in acl config file");
-        opt.setRequired(false);
-        options.addOption(opt);
-
-        opt = new Option("u", "defaultGroupPerm", true, "set default GroupPerm in acl config file");
-        opt.setRequired(false);
-        options.addOption(opt);
-
-        opt = new Option("t", "topicPerms", true, "set topicPerms list,eg: topicA=DENY,topicD=SUB");
-        opt.setRequired(false);
-        options.addOption(opt);
-
-        opt = new Option("g", "groupPerms", true, "set groupPerms list,eg: groupD=DENY,groupD=SUB");
-        opt.setRequired(false);
-        options.addOption(opt);
-
-        opt = new Option("m", "admin", true, "set admin flag in acl config file");
-        opt.setRequired(false);
-        options.addOption(opt);
-
-        return options;
-    }
-
-    @Override
-    public void execute(CommandLine commandLine, Options options,
-        RPCHook rpcHook) throws SubCommandException {
-
-        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
-        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
-
-        try {
-            PlainAccessConfig accessConfig = new PlainAccessConfig();
-            accessConfig.setAccessKey(commandLine.getOptionValue('a').trim());
-            // Secretkey
-            if (commandLine.hasOption('s')) {
-                accessConfig.setSecretKey(commandLine.getOptionValue('s').trim());
-            }
-
-            // Admin
-            if (commandLine.hasOption('m')) {
-                accessConfig.setAdmin(Boolean.parseBoolean(commandLine.getOptionValue('m').trim()));
-            }
-
-            // DefaultTopicPerm
-            if (commandLine.hasOption('i')) {
-                accessConfig.setDefaultTopicPerm(commandLine.getOptionValue('i').trim());
-            }
-
-            // DefaultGroupPerm
-            if (commandLine.hasOption('u')) {
-                accessConfig.setDefaultGroupPerm(commandLine.getOptionValue('u').trim());
-            }
-
-            // WhiteRemoteAddress
-            if (commandLine.hasOption('w')) {
-                accessConfig.setWhiteRemoteAddress(commandLine.getOptionValue('w').trim());
-            }
-
-            // TopicPerms list value
-            if (commandLine.hasOption('t')) {
-                String[] topicPerms = commandLine.getOptionValue('t').trim().split(",");
-                List<String> topicPermList = new ArrayList<>();
-                if (topicPerms != null) {
-                    for (String topicPerm : topicPerms) {
-                        topicPermList.add(topicPerm);
-                    }
-                }
-                accessConfig.setTopicPerms(topicPermList);
-            }
-
-            // GroupPerms list value
-            if (commandLine.hasOption('g')) {
-                String[] groupPerms = commandLine.getOptionValue('g').trim().split(",");
-                List<String> groupPermList = new ArrayList<>();
-                if (groupPerms != null) {
-                    for (String groupPerm : groupPerms) {
-                        groupPermList.add(groupPerm);
-                    }
-                }
-                accessConfig.setGroupPerms(groupPermList);
-            }
-
-            if (commandLine.hasOption('b')) {
-                String addr = commandLine.getOptionValue('b').trim();
-
-                defaultMQAdminExt.start();
-                defaultMQAdminExt.createAndUpdatePlainAccessConfig(addr, accessConfig);
-
-                System.out.printf("create or update plain access config to %s success.%n", addr);
-                System.out.printf("%s", accessConfig);
-                return;
-
-            } else if (commandLine.hasOption('c')) {
-                String clusterName = commandLine.getOptionValue('c').trim();
-
-                defaultMQAdminExt.start();
-                Set<String> brokerAddrSet =
-                    CommandUtil.fetchMasterAndSlaveAddrByClusterName(defaultMQAdminExt, clusterName);
-                for (String addr : brokerAddrSet) {
-                    defaultMQAdminExt.createAndUpdatePlainAccessConfig(addr, accessConfig);
-                    System.out.printf("create or update plain access config to %s success.%n", addr);
-                }
-
-                System.out.printf("%s", accessConfig);
-                return;
-            }
-
-            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
-        } catch (Exception e) {
-            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
-        } finally {
-            defaultMQAdminExt.shutdown();
-        }
-    }
-}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/auth/CopyAclsSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/CopyAclsSubCommand.java
new file mode 100644
index 00000000000..16dd5e0382f
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/CopyAclsSubCommand.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tools.command.auth;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.protocol.body.AclInfo;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class CopyAclsSubCommand implements SubCommand {
+
+    @Override
+    public String commandName() {
+        return "copyAcl";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "Copy acl to cluster.";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+
+        Option opt = new Option("f", "fromBroker", true, "the source broker that the acls copy from");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("t", "toBroker", true, "the target broker that the acls copy to");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("s", "subjects", true, "the subject list of acl to copy.");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options,
+        RPCHook rpcHook) throws SubCommandException {
+
+        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        try {
+            if (commandLine.hasOption("f") && commandLine.hasOption("t")) {
+                String sourceBroker = StringUtils.trim(commandLine.getOptionValue("f"));
+                String targetBroker = StringUtils.trim(commandLine.getOptionValue("t"));
+                String subjects = StringUtils.trim(commandLine.getOptionValue('s'));
+
+                defaultMQAdminExt.start();
+
+                List<AclInfo> aclInfos = new ArrayList<>();
+                if (StringUtils.isNotBlank(subjects)) {
+                    for (String subject : StringUtils.split(subjects, ",")) {
+                        AclInfo aclInfo = defaultMQAdminExt.getAcl(sourceBroker, subject);
+                        if (aclInfo != null) {
+                            aclInfos.add(aclInfo);
+                        }
+                    }
+                } else {
+                    aclInfos = defaultMQAdminExt.listAcl(sourceBroker, null, null);
+                }
+
+                if (CollectionUtils.isEmpty(aclInfos)) {
+                    return;
+                }
+
+                for (AclInfo aclInfo : aclInfos) {
+                    if (defaultMQAdminExt.getAcl(targetBroker, aclInfo.getSubject()) == null) {
+                        defaultMQAdminExt.createAcl(targetBroker, aclInfo);
+                    } else {
+                        defaultMQAdminExt.updateAcl(targetBroker, aclInfo);
+                    }
+                    System.out.printf("copy acl of %s from %s to %s success.%n", aclInfo.getSubject(), sourceBroker, targetBroker);
+                }
+
+                return;
+            }
+
+            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/auth/CopyUsersSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/CopyUsersSubCommand.java
new file mode 100644
index 00000000000..7f2c224ad88
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/CopyUsersSubCommand.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tools.command.auth;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.protocol.body.UserInfo;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class CopyUsersSubCommand implements SubCommand {
+
+    @Override
+    public String commandName() {
+        return "copyUser";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "Copy user to cluster.";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+
+        Option opt = new Option("f", "fromBroker", true, "the source broker that the users copy from");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("t", "toBroker", true, "the target broker that the users copy to");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("u", "usernames", true, "the username list of user to copy.");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options,
+        RPCHook rpcHook) throws SubCommandException {
+
+        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        try {
+            if (commandLine.hasOption("f") && commandLine.hasOption("t")) {
+                String sourceBroker = StringUtils.trim(commandLine.getOptionValue("f"));
+                String targetBroker = StringUtils.trim(commandLine.getOptionValue("t"));
+                String usernames = StringUtils.trim(commandLine.getOptionValue('u'));
+
+                defaultMQAdminExt.start();
+
+                List<UserInfo> userInfos = new ArrayList<>();
+                if (StringUtils.isNotBlank(usernames)) {
+                    for (String username : StringUtils.split(usernames, ",")) {
+                        UserInfo userInfo = defaultMQAdminExt.getUser(sourceBroker, username);
+                        if (userInfo != null) {
+                            userInfos.add(userInfo);
+                        }
+                    }
+                } else {
+                    userInfos = defaultMQAdminExt.listUser(sourceBroker, null);
+                }
+
+                if (CollectionUtils.isEmpty(userInfos)) {
+                    return;
+                }
+
+                for (UserInfo userInfo : userInfos) {
+                    if (defaultMQAdminExt.getUser(targetBroker, userInfo.getUsername()) == null) {
+                        defaultMQAdminExt.createUser(targetBroker, userInfo);
+                    } else {
+                        defaultMQAdminExt.updateUser(targetBroker, userInfo);
+                    }
+                    System.out.printf("copy user of %s from %s to %s success.%n", userInfo.getUsername(), sourceBroker, targetBroker);
+                }
+
+                return;
+            }
+
+            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/auth/CreateAclSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/CreateAclSubCommand.java
new file mode 100644
index 00000000000..3dcfbcc8d94
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/CreateAclSubCommand.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tools.command.auth;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.CommandUtil;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class CreateAclSubCommand implements SubCommand {
+
+    @Override
+    public String commandName() {
+        return "createAcl";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "Create acl to cluster.";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+        OptionGroup optionGroup = new OptionGroup();
+
+        Option opt = new Option("c", "clusterName", true, "create acl to which cluster");
+        optionGroup.addOption(opt);
+
+        opt = new Option("b", "brokerAddr", true, "create acl to which broker");
+        optionGroup.addOption(opt);
+
+        optionGroup.setRequired(true);
+        options.addOptionGroup(optionGroup);
+
+        opt = new Option("s", "subject", true, "the subject of acl to create.");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("r", "resources", true, "the resources of acl to create");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("a", "actions", true, "the actions of acl to create");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("i", "sourceIp", true, "the sourceIps of acl to create");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("d", "decision", true, "the decision of acl to create");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options,
+        RPCHook rpcHook) throws SubCommandException {
+
+        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        try {
+            String subject = null;
+            if (commandLine.hasOption('s')) {
+                subject = StringUtils.trim(commandLine.getOptionValue('s'));
+            }
+            List<String> resources = null;
+            if (commandLine.hasOption('r')) {
+                resources = Arrays.stream(StringUtils.split(commandLine.getOptionValue('r'), ','))
+                    .map(StringUtils::trim).collect(Collectors.toList());
+            }
+            List<String> actions = null;
+            if (commandLine.hasOption('a')) {
+                actions = Arrays.stream(StringUtils.split(commandLine.getOptionValue('a'), ','))
+                    .map(StringUtils::trim).collect(Collectors.toList());
+            }
+
+            List<String> sourceIps = null;
+            if (commandLine.hasOption('i')) {
+                sourceIps = Arrays.stream(StringUtils.split(commandLine.getOptionValue('i'), ','))
+                    .map(StringUtils::trim).collect(Collectors.toList());
+            }
+
+            String decision = null;
+            if (commandLine.hasOption('d')) {
+                decision = StringUtils.trim(commandLine.getOptionValue('d'));
+            }
+
+            if (commandLine.hasOption('b')) {
+                String addr = StringUtils.trim(commandLine.getOptionValue('b'));
+
+                defaultMQAdminExt.start();
+                defaultMQAdminExt.createAcl(addr, subject, resources, actions, sourceIps, decision);
+
+                System.out.printf("create acl to %s success.%n", addr);
+                return;
+            } else if (commandLine.hasOption('c')) {
+                String clusterName = StringUtils.trim(commandLine.getOptionValue('c'));
+
+                defaultMQAdminExt.start();
+                Set<String> brokerAddrSet =
+                    CommandUtil.fetchMasterAndSlaveAddrByClusterName(defaultMQAdminExt, clusterName);
+                for (String addr : brokerAddrSet) {
+                    defaultMQAdminExt.createAcl(addr, subject, resources, actions, sourceIps, decision);
+                    System.out.printf("create acl to %s success.%n", addr);
+                }
+                return;
+            }
+
+            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/auth/CreateUserSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/CreateUserSubCommand.java
new file mode 100644
index 00000000000..7dad3d6dfcc
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/CreateUserSubCommand.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tools.command.auth;
+
+import java.util.Set;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.CommandUtil;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class CreateUserSubCommand implements SubCommand {
+
+    @Override
+    public String commandName() {
+        return "createUser";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "Create user to cluster.";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+        OptionGroup optionGroup = new OptionGroup();
+
+        Option opt = new Option("c", "clusterName", true, "create user to which cluster");
+        optionGroup.addOption(opt);
+
+        opt = new Option("b", "brokerAddr", true, "create user to which broker");
+        optionGroup.addOption(opt);
+
+        optionGroup.setRequired(true);
+        options.addOptionGroup(optionGroup);
+
+        opt = new Option("u", "username", true, "the username of user to create.");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("p", "password", true, "the password of user to create");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("t", "userType", true, "the userType of user to create");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options,
+        RPCHook rpcHook) throws SubCommandException {
+
+        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        try {
+            String username = StringUtils.trim(commandLine.getOptionValue('u'));
+            String password = StringUtils.trim(commandLine.getOptionValue('p'));
+            String userType = StringUtils.trim(commandLine.getOptionValue('t'));
+
+            if (commandLine.hasOption('b')) {
+                String addr = StringUtils.trim(commandLine.getOptionValue('b'));
+
+                defaultMQAdminExt.start();
+                defaultMQAdminExt.createUser(addr, username, password, userType);
+
+                System.out.printf("create user to %s success.%n", addr);
+                return;
+
+            } else if (commandLine.hasOption('c')) {
+                String clusterName = StringUtils.trim(commandLine.getOptionValue('c'));
+
+                defaultMQAdminExt.start();
+                Set<String> brokerAddrSet =
+                    CommandUtil.fetchMasterAndSlaveAddrByClusterName(defaultMQAdminExt, clusterName);
+                for (String addr : brokerAddrSet) {
+                    defaultMQAdminExt.createUser(addr, username, password, userType);
+                    System.out.printf("create user to %s success.%n", addr);
+                }
+                return;
+            }
+
+            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/auth/DeleteAclSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/DeleteAclSubCommand.java
new file mode 100644
index 00000000000..a3553e0cb39
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/DeleteAclSubCommand.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tools.command.auth;
+
+import java.util.Set;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.CommandUtil;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class DeleteAclSubCommand implements SubCommand {
+
+    @Override
+    public String commandName() {
+        return "deleteAcl";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "Delete acl from cluster.";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+        OptionGroup optionGroup = new OptionGroup();
+
+        Option opt = new Option("c", "clusterName", true, "delete acl from which cluster");
+        optionGroup.addOption(opt);
+
+        opt = new Option("b", "brokerAddr", true, "delete acl from which broker");
+        optionGroup.addOption(opt);
+
+        optionGroup.setRequired(true);
+        options.addOptionGroup(optionGroup);
+
+        opt = new Option("s", "subject", true, "the subject of acl to delete.");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("r", "resources", true, "the resources of acl to delete");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options,
+        RPCHook rpcHook) throws SubCommandException {
+
+        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        try {
+            String subject = null;
+            if (commandLine.hasOption("s")) {
+                subject = StringUtils.trim(commandLine.getOptionValue("s"));
+            }
+            String resource = null;
+            if (commandLine.hasOption('r')) {
+                resource = StringUtils.trim(commandLine.getOptionValue("r"));
+            }
+
+            if (commandLine.hasOption('b')) {
+                String addr = StringUtils.trim(commandLine.getOptionValue('b'));
+
+                defaultMQAdminExt.start();
+                defaultMQAdminExt.deleteAcl(addr, subject, resource);
+
+                System.out.printf("delete acl to %s success.%n", addr);
+                return;
+            } else if (commandLine.hasOption('c')) {
+                String clusterName = StringUtils.trim(commandLine.getOptionValue('c'));
+
+                defaultMQAdminExt.start();
+                Set<String> brokerAddrSet =
+                    CommandUtil.fetchMasterAndSlaveAddrByClusterName(defaultMQAdminExt, clusterName);
+                for (String addr : brokerAddrSet) {
+                    defaultMQAdminExt.deleteAcl(addr, subject, resource);
+                    System.out.printf("delete acl to %s success.%n", addr);
+                }
+                return;
+            }
+
+            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/DeleteUserSubCommand.java
similarity index 68%
rename from tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java
rename to tools/src/main/java/org/apache/rocketmq/tools/command/auth/DeleteUserSubCommand.java
index a7f3d295a72..88344a65e94 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/DeleteUserSubCommand.java
@@ -14,13 +14,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.tools.command.acl;
+package org.apache.rocketmq.tools.command.auth;
 
 import java.util.Set;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionGroup;
 import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.srvutil.ServerUtil;
 import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
@@ -28,37 +29,32 @@
 import org.apache.rocketmq.tools.command.SubCommand;
 import org.apache.rocketmq.tools.command.SubCommandException;
 
-public class DeleteAccessConfigSubCommand implements SubCommand {
+public class DeleteUserSubCommand implements SubCommand {
 
     @Override
     public String commandName() {
-        return "deleteAclConfig";
-    }
-
-    @Override
-    public String commandAlias() {
-        return "deleteAccessConfig";
+        return "deleteUser";
     }
 
     @Override
     public String commandDesc() {
-        return "Delete Acl Config Account in broker.";
+        return "Delete user from cluster.";
     }
 
     @Override
     public Options buildCommandlineOptions(Options options) {
         OptionGroup optionGroup = new OptionGroup();
 
-        Option opt = new Option("b", "brokerAddr", true, "delete acl config account from which broker");
+        Option opt = new Option("c", "clusterName", true, "delete acl from which cluster");
         optionGroup.addOption(opt);
 
-        opt = new Option("c", "clusterName", true, "delete acl config account from which cluster");
+        opt = new Option("b", "brokerAddr", true, "delete user from which broker");
         optionGroup.addOption(opt);
 
         optionGroup.setRequired(true);
         options.addOptionGroup(optionGroup);
 
-        opt = new Option("a", "accessKey", true, "set accessKey in acl config file for deleting which account");
+        opt = new Option("u", "username", true, "the username of user to delete.");
         opt.setRequired(true);
         options.addOption(opt);
 
@@ -73,32 +69,26 @@ public void execute(CommandLine commandLine, Options options,
         defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
 
         try {
-
-            String accessKey = commandLine.getOptionValue('a').trim();
+            String username = StringUtils.trim(commandLine.getOptionValue('u'));
 
             if (commandLine.hasOption('b')) {
-                String addr = commandLine.getOptionValue('b').trim();
+                String addr = StringUtils.trim(commandLine.getOptionValue('b'));
 
                 defaultMQAdminExt.start();
-                defaultMQAdminExt.deletePlainAccessConfig(addr, accessKey);
+                defaultMQAdminExt.deleteUser(addr, username);
 
-                System.out.printf("delete plain access config account from %s success.%n", addr);
-                System.out.printf("account's accessKey is:%s", accessKey);
+                System.out.printf("delete user to %s success.%n", addr);
                 return;
-
             } else if (commandLine.hasOption('c')) {
-                String clusterName = commandLine.getOptionValue('c').trim();
+                String clusterName = StringUtils.trim(commandLine.getOptionValue('c'));
 
                 defaultMQAdminExt.start();
-
                 Set<String> brokerAddrSet =
                     CommandUtil.fetchMasterAndSlaveAddrByClusterName(defaultMQAdminExt, clusterName);
                 for (String addr : brokerAddrSet) {
-                    defaultMQAdminExt.deletePlainAccessConfig(addr, accessKey);
-                    System.out.printf("delete plain access config account from %s success.%n", addr);
+                    defaultMQAdminExt.deleteUser(addr, username);
+                    System.out.printf("delete user to %s success.%n", addr);
                 }
-
-                System.out.printf("account's accessKey is:%s", accessKey);
                 return;
             }
 
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/auth/GetAclSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/GetAclSubCommand.java
new file mode 100644
index 00000000000..1697bfb3f13
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/GetAclSubCommand.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tools.command.auth;
+
+import java.util.List;
+import java.util.Set;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.protocol.body.AclInfo;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.CommandUtil;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class GetAclSubCommand implements SubCommand {
+
+    private static final String FORMAT = "%-16s  %-10s  %-22s  %-20s  %-24s  %-10s%n";
+
+    @Override
+    public String commandName() {
+        return "getAcl";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "Get acl from cluster.";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+        OptionGroup optionGroup = new OptionGroup();
+
+        Option opt = new Option("b", "brokerAddr", true, "get acl for which broker");
+        optionGroup.addOption(opt);
+
+        opt = new Option("c", "clusterName", true, "get acl for specified cluster");
+        optionGroup.addOption(opt);
+
+        optionGroup.setRequired(true);
+        options.addOptionGroup(optionGroup);
+
+        opt = new Option("s", "subject", true, "the subject of acl to get");
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options,
+        RPCHook rpcHook) throws SubCommandException {
+
+        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        try {
+            String subject = StringUtils.trim(commandLine.getOptionValue('s'));
+
+            if (commandLine.hasOption('b')) {
+                String addr = StringUtils.trim(commandLine.getOptionValue('b'));
+                defaultMQAdminExt.start();
+
+                AclInfo aclInfo = defaultMQAdminExt.getAcl(addr, subject);
+                if (aclInfo != null) {
+                    printAcl(aclInfo);
+                }
+                return;
+            } else if (commandLine.hasOption('c')) {
+                String clusterName = StringUtils.trim(commandLine.getOptionValue('c'));
+
+                defaultMQAdminExt.start();
+
+                Set<String> masterSet =
+                    CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
+                if (CollectionUtils.isEmpty(masterSet)) {
+                    throw new SubCommandException(this.getClass().getSimpleName() + " command failed, there is no broker in cluster.");
+                }
+                for (String masterAddr : masterSet) {
+                    AclInfo aclInfo = defaultMQAdminExt.getAcl(masterAddr, subject);
+                    if (aclInfo != null) {
+                        printAcl(aclInfo);
+                    }
+                }
+                return;
+            }
+
+            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+
+    private void printAcl(AclInfo acl) {
+        if (acl == null) {
+            return;
+        }
+        System.out.printf(FORMAT, "#Subject", "#PolicyType", "#Resource", "#Actions", "#SourceIp", "#Decision");
+        List<AclInfo.PolicyInfo> policyInfos = acl.getPolicies();
+        if (CollectionUtils.isEmpty(policyInfos)) {
+            System.out.printf(FORMAT, acl.getSubject(), "", "", "", "", "");
+        }
+        policyInfos.forEach(policy -> {
+            List<AclInfo.PolicyEntryInfo> entries = policy.getEntries();
+            if (CollectionUtils.isEmpty(entries)) {
+                return;
+            }
+            entries.forEach(entry -> {
+                System.out.printf(FORMAT, acl.getSubject(), policy.getPolicyType(), entry.getResource(),
+                    entry.getActions(), entry.getSourceIps(), entry.getDecision());
+            });
+        });
+    }
+}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/auth/GetUserSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/GetUserSubCommand.java
new file mode 100644
index 00000000000..061155db706
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/GetUserSubCommand.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tools.command.auth;
+
+import java.util.Set;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.protocol.body.UserInfo;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.CommandUtil;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class GetUserSubCommand implements SubCommand {
+
+    private static final String FORMAT = "%-16s  %-22s  %-22s  %-22s%n";
+
+    @Override
+    public String commandName() {
+        return "getUser";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "Get user from cluster.";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+        OptionGroup optionGroup = new OptionGroup();
+
+        Option opt = new Option("b", "brokerAddr", true, "get user for which broker");
+        optionGroup.addOption(opt);
+
+        opt = new Option("c", "clusterName", true, "get user for specified cluster");
+        optionGroup.addOption(opt);
+
+        optionGroup.setRequired(true);
+        options.addOptionGroup(optionGroup);
+
+        opt = new Option("u", "username", true, "the username of user to get");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options,
+        RPCHook rpcHook) throws SubCommandException {
+
+        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        try {
+            String username = StringUtils.trim(commandLine.getOptionValue('u'));
+
+            if (commandLine.hasOption('b')) {
+                String addr = StringUtils.trim(commandLine.getOptionValue('b'));
+                defaultMQAdminExt.start();
+
+                UserInfo userInfo = defaultMQAdminExt.getUser(addr, username);
+                if (userInfo != null) {
+                    printUser(userInfo);
+                }
+                return;
+            } else if (commandLine.hasOption('c')) {
+                String clusterName = StringUtils.trim(commandLine.getOptionValue('c'));
+
+                defaultMQAdminExt.start();
+
+                Set<String> masterSet =
+                    CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
+                if (CollectionUtils.isEmpty(masterSet)) {
+                    throw new SubCommandException(this.getClass().getSimpleName() + " command failed, there is no broker in cluster.");
+                }
+                for (String masterAddr : masterSet) {
+                    UserInfo userInfo = defaultMQAdminExt.getUser(masterAddr, username);
+                    if (userInfo != null) {
+                        printUser(userInfo);
+                        break;
+                    }
+                }
+                return;
+            }
+
+            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+
+    private void printUser(UserInfo user) {
+        if (user == null) {
+            return;
+        }
+        System.out.printf(FORMAT, "#UserName", "#Password", "#UserType", "#UserStatus");
+        System.out.printf(FORMAT, user.getUsername(), user.getPassword(), user.getUserType(), user.getUserStatus());
+    }
+}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/auth/ListAclSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/ListAclSubCommand.java
new file mode 100644
index 00000000000..cfd7322f636
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/ListAclSubCommand.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tools.command.auth;
+
+import java.util.List;
+import java.util.Set;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.protocol.body.AclInfo;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.CommandUtil;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class ListAclSubCommand implements SubCommand {
+
+    private static final String FORMAT = "%-16s  %-10s  %-22s  %-20s  %-24s  %-10s%n";
+
+    @Override
+    public String commandName() {
+        return "listAcl";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "List acl from cluster.";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+        OptionGroup optionGroup = new OptionGroup();
+
+        Option opt = new Option("b", "brokerAddr", true, "list acl for which broker.");
+        optionGroup.addOption(opt);
+
+        opt = new Option("c", "clusterName", true, "list acl for specified cluster.");
+        optionGroup.addOption(opt);
+
+        optionGroup.setRequired(true);
+        options.addOptionGroup(optionGroup);
+
+        opt = new Option("s", "subject", true, "the subject of acl to filter.");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("r", "resource", true, "the resource of acl to filter.");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options,
+        RPCHook rpcHook) throws SubCommandException {
+
+        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        try {
+            String subjectFilter = StringUtils.trim(commandLine.getOptionValue('s'));
+            String resourceFilter = StringUtils.trim(commandLine.getOptionValue('r'));
+
+            if (commandLine.hasOption('b')) {
+                String addr = StringUtils.trim(commandLine.getOptionValue('b'));
+                defaultMQAdminExt.start();
+
+                List<AclInfo> aclInfos = defaultMQAdminExt.listAcl(addr, subjectFilter, resourceFilter);
+                if (CollectionUtils.isNotEmpty(aclInfos)) {
+                    printAcl(aclInfos);
+                }
+                return;
+            } else if (commandLine.hasOption('c')) {
+                String clusterName = StringUtils.trim(commandLine.getOptionValue('c'));
+
+                defaultMQAdminExt.start();
+
+                Set<String> masterSet =
+                    CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
+                if (CollectionUtils.isEmpty(masterSet)) {
+                    throw new SubCommandException(this.getClass().getSimpleName() + " command failed, there is no broker in cluster.");
+                }
+                for (String masterAddr : masterSet) {
+                    List<AclInfo> aclInfos = defaultMQAdminExt.listAcl(masterAddr, subjectFilter, resourceFilter);
+                    if (CollectionUtils.isNotEmpty(aclInfos)) {
+                        printAcl(aclInfos);
+                    }
+                }
+                return;
+            }
+
+            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+
+    private void printAcl(List<AclInfo> acls) {
+        System.out.printf(FORMAT, "#Subject", "#PolicyType", "#Resource", "#Actions", "#SourceIp", "#Decision");
+        acls.forEach(acl -> {
+            List<AclInfo.PolicyInfo> policyInfos = acl.getPolicies();
+            if (CollectionUtils.isEmpty(policyInfos)) {
+                System.out.printf(FORMAT, acl.getSubject(), "", "", "", "", "");
+            }
+            policyInfos.forEach(policy -> {
+                List<AclInfo.PolicyEntryInfo> entries = policy.getEntries();
+                if (CollectionUtils.isEmpty(entries)) {
+                    return;
+                }
+                entries.forEach(entry -> System.out.printf(FORMAT, acl.getSubject(), policy.getPolicyType(), entry.getResource(),
+                    entry.getActions(), entry.getSourceIps(), entry.getDecision()));
+            });
+        });
+    }
+}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/auth/ListUserSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/ListUserSubCommand.java
new file mode 100644
index 00000000000..24eb62427ff
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/ListUserSubCommand.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tools.command.auth;
+
+import java.util.List;
+import java.util.Set;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.protocol.body.UserInfo;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.CommandUtil;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class ListUserSubCommand implements SubCommand {
+
+    private static final String FORMAT = "%-16s  %-22s  %-22s  %-22s%n";
+
+    @Override
+    public String commandName() {
+        return "listUser";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "List user from cluster.";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+        OptionGroup optionGroup = new OptionGroup();
+
+        Option opt = new Option("b", "brokerAddr", true, "list user for which broker");
+        optionGroup.addOption(opt);
+
+        opt = new Option("c", "clusterName", true, "list user for specified cluster");
+        optionGroup.addOption(opt);
+
+        optionGroup.setRequired(true);
+        options.addOptionGroup(optionGroup);
+
+        opt = new Option("f", "filter", true, "the filter to list users");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options,
+        RPCHook rpcHook) throws SubCommandException {
+
+        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        try {
+            String filter = StringUtils.trim(commandLine.getOptionValue('f'));
+
+            if (commandLine.hasOption('b')) {
+                String addr = StringUtils.trim(commandLine.getOptionValue('b'));
+                defaultMQAdminExt.start();
+
+                List<UserInfo> userInfos = defaultMQAdminExt.listUser(addr, filter);
+                if (CollectionUtils.isNotEmpty(userInfos)) {
+                    printUsers(userInfos);
+                }
+                return;
+            } else if (commandLine.hasOption('c')) {
+                String clusterName = StringUtils.trim(commandLine.getOptionValue('c'));
+
+                defaultMQAdminExt.start();
+
+                Set<String> masterSet =
+                    CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
+                if (CollectionUtils.isEmpty(masterSet)) {
+                    throw new SubCommandException(this.getClass().getSimpleName() + " command failed, there is no broker in cluster.");
+                }
+                for (String masterAddr : masterSet) {
+                    List<UserInfo> userInfos = defaultMQAdminExt.listUser(masterAddr, filter);
+                    if (CollectionUtils.isNotEmpty(userInfos)) {
+                        printUsers(userInfos);
+                        System.out.printf("get user from %s success.%n", masterAddr);
+                        break;
+                    }
+                }
+                return;
+            }
+
+            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+
+    private void printUsers(List<UserInfo> users) {
+        System.out.printf(FORMAT, "#UserName", "#Password", "#UserType", "#UserStatus");
+        users.forEach(user -> System.out.printf(FORMAT, user.getUsername(), user.getPassword(), user.getUserType(), user.getUserStatus()));
+    }
+}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/auth/UpdateAclSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/UpdateAclSubCommand.java
new file mode 100644
index 00000000000..bccef4f2dd2
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/UpdateAclSubCommand.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tools.command.auth;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.CommandUtil;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class UpdateAclSubCommand implements SubCommand {
+
+    @Override
+    public String commandName() {
+        return "updateAcl";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "Update acl to cluster.";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+        OptionGroup optionGroup = new OptionGroup();
+
+        Option opt = new Option("c", "clusterName", true, "update acl to which cluster");
+        optionGroup.addOption(opt);
+
+        opt = new Option("b", "brokerAddr", true, "update acl to which broker");
+        optionGroup.addOption(opt);
+
+        optionGroup.setRequired(true);
+        options.addOptionGroup(optionGroup);
+
+        opt = new Option("s", "subject", true, "the subject of acl to update.");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("r", "resources", true, "the resources of acl to update");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("a", "actions", true, "the actions of acl to update");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("d", "decision", true, "the decision of acl to update");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("i", "sourceIp", true, "the sourceIps of acl to update");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options,
+        RPCHook rpcHook) throws SubCommandException {
+
+        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        try {
+            String subject = null;
+            if (commandLine.hasOption('s')) {
+                subject = StringUtils.trim(commandLine.getOptionValue('s'));
+            }
+            List<String> resources = null;
+            if (commandLine.hasOption('r')) {
+                resources = Arrays.stream(StringUtils.split(commandLine.getOptionValue('r'), ','))
+                    .map(StringUtils::trim).collect(Collectors.toList());
+            }
+            List<String> actions = null;
+            if (commandLine.hasOption('a')) {
+                actions = Arrays.stream(StringUtils.split(commandLine.getOptionValue('a'), ','))
+                    .map(StringUtils::trim).collect(Collectors.toList());
+            }
+
+            List<String> sourceIps = null;
+            if (commandLine.hasOption('i')) {
+                sourceIps = Arrays.stream(StringUtils.split(commandLine.getOptionValue('i'), ','))
+                    .map(StringUtils::trim).collect(Collectors.toList());
+            }
+
+            String decision = null;
+            if (commandLine.hasOption('d')) {
+                decision = StringUtils.trim(commandLine.getOptionValue('d'));
+            }
+
+            if (commandLine.hasOption('b')) {
+                String addr = StringUtils.trim(commandLine.getOptionValue('b'));
+
+                defaultMQAdminExt.start();
+                defaultMQAdminExt.updateAcl(addr, subject, resources, actions, sourceIps, decision);
+
+                System.out.printf("update acl to %s success.%n", addr);
+                return;
+
+            } else if (commandLine.hasOption('c')) {
+                String clusterName = StringUtils.trim(commandLine.getOptionValue('c'));
+
+                defaultMQAdminExt.start();
+                Set<String> brokerAddrSet =
+                    CommandUtil.fetchMasterAndSlaveAddrByClusterName(defaultMQAdminExt, clusterName);
+                for (String addr : brokerAddrSet) {
+                    defaultMQAdminExt.updateAcl(addr, subject, resources, actions, sourceIps, decision);
+                    System.out.printf("update acl to %s success.%n", addr);
+                }
+                return;
+            }
+
+            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/UpdateUserSubCommand.java
similarity index 65%
rename from tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java
rename to tools/src/main/java/org/apache/rocketmq/tools/command/auth/UpdateUserSubCommand.java
index 9dacf1fae64..ef8544f2c03 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/auth/UpdateUserSubCommand.java
@@ -14,13 +14,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.tools.command.acl;
+package org.apache.rocketmq.tools.command.auth;
 
 import java.util.Set;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionGroup;
 import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.srvutil.ServerUtil;
 import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
@@ -28,39 +29,47 @@
 import org.apache.rocketmq.tools.command.SubCommand;
 import org.apache.rocketmq.tools.command.SubCommandException;
 
-public class UpdateGlobalWhiteAddrSubCommand implements SubCommand {
+public class UpdateUserSubCommand implements SubCommand {
 
     @Override
     public String commandName() {
-        return "updateGlobalWhiteAddr";
+        return "updateUser";
     }
 
     @Override
     public String commandDesc() {
-        return "Update global white address for acl Config File in broker.";
+        return "Update user to cluster.";
     }
 
     @Override
     public Options buildCommandlineOptions(Options options) {
-
         OptionGroup optionGroup = new OptionGroup();
 
-        Option opt = new Option("b", "brokerAddr", true, "update global white address to which broker");
+        Option opt = new Option("c", "clusterName", true, "update user to which cluster");
         optionGroup.addOption(opt);
 
-        opt = new Option("c", "clusterName", true, "update global white address to which cluster");
+        opt = new Option("b", "brokerAddr", true, "update user to which broker");
         optionGroup.addOption(opt);
 
         optionGroup.setRequired(true);
         options.addOptionGroup(optionGroup);
 
-        opt = new Option("g", "globalWhiteRemoteAddresses", true, "set globalWhiteRemoteAddress list,eg: 10.10.103.*,192.168.0.*");
+        opt = new Option("u", "username", true, "the username of user to update.");
         opt.setRequired(true);
         options.addOption(opt);
 
-        opt = new Option("p", "aclFileFullPath", true, "update global white address of specified acl file,eg: /xxx/plain_test.yml");
-        opt.setRequired(false);
-        options.addOption(opt);
+        optionGroup = new OptionGroup();
+        opt = new Option("p", "password", true, "the password of user to update");
+        optionGroup.addOption(opt);
+
+        opt = new Option("t", "userType", true, "the userType of user to update");
+        optionGroup.addOption(opt);
+
+        opt = new Option("s", "userStatus", true, "the userStatus of user to update");
+        optionGroup.addOption(opt);
+        optionGroup.setRequired(true);
+
+        options.addOptionGroup(optionGroup);
 
         return options;
     }
@@ -73,26 +82,19 @@ public void execute(CommandLine commandLine, Options options,
         defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
 
         try {
-            // GlobalWhiteRemoteAddresses list value
-            String globalWhiteRemoteAddresses = commandLine.getOptionValue('g').trim();
-
-            String aclFileFullPath;
-            if (commandLine.hasOption('p')) {
-                aclFileFullPath = commandLine.getOptionValue('p').trim();
-            } else {
-                aclFileFullPath = null;
-            }
-
+            String username = StringUtils.trim(commandLine.getOptionValue('u'));
+            String password = StringUtils.trim(commandLine.getOptionValue('p'));
+            String userType = StringUtils.trim(commandLine.getOptionValue('t'));
+            String userStatus = StringUtils.trim(commandLine.getOptionValue('s'));
 
             if (commandLine.hasOption('b')) {
                 String addr = commandLine.getOptionValue('b').trim();
 
                 defaultMQAdminExt.start();
-                defaultMQAdminExt.updateGlobalWhiteAddrConfig(addr, globalWhiteRemoteAddresses, aclFileFullPath);
+                defaultMQAdminExt.updateUser(addr, username, password, userType, userStatus);
 
-                System.out.printf("update global white remote addresses to %s success.%n", addr);
+                System.out.printf("update user to %s success.%n", addr);
                 return;
-
             } else if (commandLine.hasOption('c')) {
                 String clusterName = commandLine.getOptionValue('c').trim();
 
@@ -100,8 +102,8 @@ public void execute(CommandLine commandLine, Options options,
                 Set<String> brokerAddrSet =
                     CommandUtil.fetchMasterAndSlaveAddrByClusterName(defaultMQAdminExt, clusterName);
                 for (String addr : brokerAddrSet) {
-                    defaultMQAdminExt.updateGlobalWhiteAddrConfig(addr, globalWhiteRemoteAddresses, aclFileFullPath);
-                    System.out.printf("update global white remote addresses to %s success.%n", addr);
+                    defaultMQAdminExt.updateUser(addr, username, password, userType, userStatus);
+                    System.out.printf("update user to %s success.%n", addr);
                 }
                 return;
             }
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java
index ede0fa5cf4e..8103f4c7f89 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java
@@ -177,9 +177,9 @@ private void printClusterMoreStats(final Set<String> clusterNames,
     }
 
     private void printClusterBaseInfo(final Set<String> clusterNames,
-                                      final DefaultMQAdminExt defaultMQAdminExt,
-                                      final ClusterInfo clusterInfo) {
-        System.out.printf("%-22s  %-22s  %-4s  %-22s %-16s  %16s  %16s  %-22s  %-11s  %-12s  %-8s  %-10s%n",
+        final DefaultMQAdminExt defaultMQAdminExt,
+        final ClusterInfo clusterInfo) {
+        System.out.printf("%-22s  %-22s  %-4s  %-22s %-16s  %16s  %30s  %-22s  %-11s  %-12s  %-8s  %-10s%n",
             "#Cluster Name",
             "#Broker Name",
             "#BID",
@@ -212,8 +212,10 @@ private void printClusterBaseInfo(final Set<String> clusterNames,
                         String version = "";
                         String sendThreadPoolQueueSize = "";
                         String pullThreadPoolQueueSize = "";
+                        String ackThreadPoolQueueSize = "";
                         String sendThreadPoolQueueHeadWaitTimeMills = "";
                         String pullThreadPoolQueueHeadWaitTimeMills = "";
+                        String ackThreadPoolQueueHeadWaitTimeMills = "";
                         String pageCacheLockTimeMills = "";
                         String earliestMessageTimeStamp = "";
                         String commitLogDiskRatio = "";
@@ -228,14 +230,14 @@ private void printClusterBaseInfo(final Set<String> clusterNames,
                             isBrokerActive = Boolean.parseBoolean(kvTable.getTable().get("brokerActive"));
                             String putTps = kvTable.getTable().get("putTps");
                             String getTransferredTps = kvTable.getTable().get("getTransferredTps");
-                            sendThreadPoolQueueSize = kvTable.getTable().get("sendThreadPoolQueueSize");
-                            pullThreadPoolQueueSize = kvTable.getTable().get("pullThreadPoolQueueSize");
 
                             sendThreadPoolQueueSize = kvTable.getTable().get("sendThreadPoolQueueSize");
                             pullThreadPoolQueueSize = kvTable.getTable().get("pullThreadPoolQueueSize");
+                            ackThreadPoolQueueSize = kvTable.getTable().getOrDefault("ackThreadPoolQueueSize", "N");
 
                             sendThreadPoolQueueHeadWaitTimeMills = kvTable.getTable().get("sendThreadPoolQueueHeadWaitTimeMills");
                             pullThreadPoolQueueHeadWaitTimeMills = kvTable.getTable().get("pullThreadPoolQueueHeadWaitTimeMills");
+                            ackThreadPoolQueueHeadWaitTimeMills = kvTable.getTable().getOrDefault("ackThreadPoolQueueHeadWaitTimeMills", "N");
                             pageCacheLockTimeMills = kvTable.getTable().get("pageCacheLockTimeMills");
                             earliestMessageTimeStamp = kvTable.getTable().get("earliestMessageTimeStamp");
                             commitLogDiskRatio = kvTable.getTable().get("commitLogDiskRatio");
@@ -280,14 +282,14 @@ private void printClusterBaseInfo(final Set<String> clusterNames,
                             space = Double.parseDouble(commitLogDiskRatio);
                         }
 
-                        System.out.printf("%-22s  %-22s  %-4s  %-22s %-16s  %16s  %16s  %-22s  %11s  %-12s  %-8s  %10s%n",
+                        System.out.printf("%-22s  %-22s  %-4s  %-22s %-16s  %16s  %30s  %-22s  %11s  %-12s  %-8s  %10s%n",
                             clusterName,
                             brokerName,
                             next1.getKey(),
                             next1.getValue(),
                             version,
                             String.format("%9.2f(%s,%sms)", in, sendThreadPoolQueueSize, sendThreadPoolQueueHeadWaitTimeMills),
-                            String.format("%9.2f(%s,%sms)", out, pullThreadPoolQueueSize, pullThreadPoolQueueHeadWaitTimeMills),
+                            String.format("%9.2f(%s,%sms|%s,%sms)", out, pullThreadPoolQueueSize, pullThreadPoolQueueHeadWaitTimeMills, ackThreadPoolQueueSize, ackThreadPoolQueueHeadWaitTimeMills),
                             String.format("%d-%d(%.1fw, %.1f, %.1f)", timerReadBehind, timerOffsetBehind, timerCongestNum / 10000.0f, timerEnqueueTps, timerDequeueTps),
                             pageCacheLockTimeMills,
                             String.format("%2.2f", hour),
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java
index c489cad6849..b638dcf61f3 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java
@@ -72,6 +72,10 @@ public Options buildCommandlineOptions(Options options) {
         optionShowClientIP.setRequired(false);
         options.addOption(optionShowClientIP);
 
+        opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route.");
+        opt.setRequired(false);
+        options.addOption(opt);
+
         return options;
     }
 
@@ -109,6 +113,8 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
             boolean showClientIP = commandLine.hasOption('s')
                 && "true".equalsIgnoreCase(commandLine.getOptionValue('s'));
 
+            String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null;
+
             if (commandLine.hasOption('g')) {
                 String consumerGroup = commandLine.getOptionValue('g').trim();
                 String topicName = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : null;
@@ -116,7 +122,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
                 if (topicName == null) {
                     consumeStats = defaultMQAdminExt.examineConsumeStats(consumerGroup);
                 } else {
-                    consumeStats = defaultMQAdminExt.examineConsumeStats(consumerGroup, topicName);
+                    consumeStats = defaultMQAdminExt.examineConsumeStats(clusterName, consumerGroup, topicName);
                 }
                 List<MessageQueue> mqList = new LinkedList<>(consumeStats.getOffsetTable().keySet());
                 Collections.sort(mqList);
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommand.java
new file mode 100644
index 00000000000..a36f50bd1b0
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommand.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.tools.command.consumer;
+
+import com.alibaba.fastjson2.JSON;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Set;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.CommandUtil;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class UpdateSubGroupListSubCommand implements SubCommand {
+    @Override
+    public String commandName() {
+        return "updateSubGroupList";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "Update or create subscription group in batch";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+        final OptionGroup optionGroup = new OptionGroup();
+        Option opt = new Option("b", "brokerAddr", true, "create groups to which broker");
+        optionGroup.addOption(opt);
+
+        opt = new Option("c", "clusterName", true, "create groups to which cluster");
+        optionGroup.addOption(opt);
+        optionGroup.setRequired(true);
+        options.addOptionGroup(optionGroup);
+
+        opt = new Option("f", "filename", true,
+            "Path to a file with a list of org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig in json format");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options,
+        RPCHook rpcHook) throws SubCommandException {
+        final DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        final String fileName = commandLine.getOptionValue('f').trim();
+
+        try {
+            final Path filePath = Paths.get(fileName);
+            if (!Files.exists(filePath)) {
+                System.out.printf("the file path %s does not exists%n", fileName);
+                return;
+            }
+            final byte[] groupConfigListBytes = Files.readAllBytes(filePath);
+            final List<SubscriptionGroupConfig> groupConfigs = JSON.parseArray(groupConfigListBytes, SubscriptionGroupConfig.class);
+            if (null == groupConfigs || groupConfigs.isEmpty()) {
+                return;
+            }
+
+            if (commandLine.hasOption('b')) {
+                String brokerAddress = commandLine.getOptionValue('b').trim();
+                defaultMQAdminExt.start();
+                defaultMQAdminExt.createAndUpdateSubscriptionGroupConfigList(brokerAddress, groupConfigs);
+
+                System.out.printf("submit batch of group config to %s success, please check the result later.%n",
+                    brokerAddress);
+                return;
+
+            } else if (commandLine.hasOption('c')) {
+                final String clusterName = commandLine.getOptionValue('c').trim();
+
+                defaultMQAdminExt.start();
+
+                Set<String> masterSet =
+                    CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
+                for (String brokerAddress : masterSet) {
+                    defaultMQAdminExt.createAndUpdateSubscriptionGroupConfigList(brokerAddress, groupConfigs);
+
+                    System.out.printf("submit batch of subscription group config to %s success, please check the result later.%n",
+                        brokerAddress);
+                }
+            }
+
+            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java
index 1ecb1fa2cd9..438d17d6689 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java
@@ -14,9 +14,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.rocketmq.tools.command.export;
 
 import com.alibaba.fastjson.JSONObject;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiConsumer;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
@@ -29,11 +34,6 @@
 import org.apache.rocketmq.tools.command.SubCommandException;
 import org.rocksdb.RocksIterator;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.BiConsumer;
-
 public class ExportMetadataInRocksDBCommand implements SubCommand {
     private static final String TOPICS_JSON_CONFIG = "topics";
     private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups";
@@ -45,7 +45,7 @@ public String commandName() {
 
     @Override
     public String commandDesc() {
-        return "export RocksDB kv config (topics/subscriptionGroups)";
+        return "export RocksDB kv config (topics/subscriptionGroups). Recommend to use [mqadmin rocksDBConfigToJson]";
     }
 
     @Override
@@ -76,7 +76,11 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
             return;
         }
 
-        String configType = commandLine.getOptionValue("configType").trim().toLowerCase();
+        String configType = commandLine.getOptionValue("configType").trim();
+        if (!path.endsWith("/")) {
+            path += "/";
+        }
+        path += configType;
 
         boolean jsonEnable = false;
         if (commandLine.hasOption("jsonEnable")) {
@@ -86,7 +90,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
 
         ConfigRocksDBStorage kvStore = new ConfigRocksDBStorage(path, true /* readOnly */);
         if (!kvStore.start()) {
-            System.out.print("RocksDB load error, path=" + path + "\n");
+            System.out.printf("RocksDB load error, path=%s\n" , path);
             return;
         }
 
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportPopRecordCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportPopRecordCommand.java
new file mode 100644
index 00000000000..f8b67c97af3
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportPopRecordCommand.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.tools.command.export;
+
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.protocol.body.ClusterInfo;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class ExportPopRecordCommand implements SubCommand {
+
+    @Override
+    public String commandName() {
+        return "exportPopRecord";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "Export pop consumer record";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+        Option opt = new Option(
+            "c", "clusterName", true, "choose one cluster to export");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("b", "brokerAddr", true, "choose one broker to export");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("d", "dryRun", true, "no actual changes will be made");
+        opt.setRequired(false);
+        options.addOption(opt);
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException {
+
+        DefaultMQAdminExt adminExt = new DefaultMQAdminExt(rpcHook);
+        adminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        try {
+            adminExt.start();
+            boolean dryRun = commandLine.hasOption('d') &&
+                Boolean.FALSE.toString().equalsIgnoreCase(commandLine.getOptionValue('d'));
+            if (commandLine.hasOption('b')) {
+                String brokerAddr = commandLine.getOptionValue('b').trim();
+                String brokerName = adminExt.getBrokerConfig(brokerAddr).getProperty("brokerName");
+                export(adminExt, brokerAddr, brokerName, dryRun);
+            } else if (commandLine.hasOption('c')) {
+                String clusterName = commandLine.getOptionValue('c').trim();
+                ClusterInfo clusterInfo = adminExt.examineBrokerClusterInfo();
+                if (clusterInfo != null) {
+                    Set<String> brokerNameSet = clusterInfo.getClusterAddrTable().get(clusterName);
+                    if (brokerNameSet != null) {
+                        brokerNameSet.forEach(brokerName -> {
+                            BrokerData brokerData = clusterInfo.getBrokerAddrTable().get(brokerName);
+                            if (brokerData != null) {
+                                brokerData.getBrokerAddrs().forEach(
+                                    (brokerId, brokerAddr) -> export(adminExt, brokerAddr, brokerName, dryRun));
+                            }
+                        });
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            adminExt.shutdown();
+        }
+    }
+
+    private void export(DefaultMQAdminExt adminExt, String brokerAddr, String brokerName, boolean dryRun) {
+        try {
+            if (!dryRun) {
+                adminExt.exportPopRecords(brokerAddr, TimeUnit.SECONDS.toMillis(30));
+            }
+            System.out.printf("Export broker records, " +
+                "brokerName=%s, brokerAddr=%s, dryRun=%s%n", brokerName, brokerAddr, dryRun);
+        } catch (Exception e) {
+            System.out.printf("Export broker records error, " +
+                "brokerName=%s, brokerAddr=%s, dryRun=%s%n%s", brokerName, brokerAddr, dryRun, e);
+        }
+    }
+}
+
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java
index bb82f5079e5..97e101d813c 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java
@@ -24,6 +24,7 @@
 import org.apache.commons.cli.Options;
 import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
 import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.impl.FindBrokerResult;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.message.MessageExt;
@@ -97,6 +98,12 @@ public Options buildCommandlineOptions(Options options) {
         opt.setRequired(false);
         options.addOption(opt);
 
+        opt =
+            new Option("l", "lmqParentTopic", true,
+                "Lmq parent topic, lmq is used to find the route.");
+        opt.setRequired(false);
+        options.addOption(opt);
+
         return options;
     }
 
@@ -113,11 +120,20 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
             String subExpression =
                 !commandLine.hasOption('s') ? "*" : commandLine.getOptionValue('s').trim();
 
+            String lmqParentTopic =
+                !commandLine.hasOption('l') ? null : commandLine.getOptionValue('l').trim();
+
             boolean printBody = !commandLine.hasOption('d') || Boolean.parseBoolean(commandLine.getOptionValue('d').trim());
 
             consumer.start();
 
-            Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(topic);
+            Set<MessageQueue> mqs;
+            if (lmqParentTopic != null) {
+                mqs = consumer.fetchSubscribeMessageQueues(lmqParentTopic);
+                mqs.forEach(mq -> mq.setTopic(topic));
+            } else {
+                mqs = consumer.fetchSubscribeMessageQueues(topic);
+            }
             for (MessageQueue mq : mqs) {
                 long minOffset = consumer.minOffset(mq);
                 long maxOffset = consumer.maxOffset(mq);
@@ -139,6 +155,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
                 READQ:
                 for (long offset = minOffset; offset < maxOffset; ) {
                     try {
+                        fillBrokerAddrIfNotExist(consumer, mq, lmqParentTopic);
                         PullResult pullResult = consumer.pull(mq, subExpression, offset, 32);
                         offset = pullResult.getNextBeginOffset();
                         switch (pullResult.getPullStatus()) {
@@ -167,4 +184,17 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
             consumer.shutdown();
         }
     }
+
+    public void fillBrokerAddrIfNotExist(DefaultMQPullConsumer defaultMQPullConsumer, MessageQueue messageQueue,
+        String routeTopic) {
+
+        FindBrokerResult findBrokerResult = defaultMQPullConsumer.getDefaultMQPullConsumerImpl().getRebalanceImpl().getmQClientFactory()
+            .findBrokerAddressInSubscribe(messageQueue.getBrokerName(), 0, false);
+        if (findBrokerResult == null) {
+            // use lmq parent topic to fill up broker addr table
+            defaultMQPullConsumer.getDefaultMQPullConsumerImpl().getRebalanceImpl().getmQClientFactory()
+                .updateTopicRouteInfoFromNameServer(routeTopic);
+        }
+
+    }
 }
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java
index b42612150a3..e83029eed31 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java
@@ -44,9 +44,10 @@
 import org.apache.rocketmq.tools.command.SubCommandException;
 
 public class QueryMsgByIdSubCommand implements SubCommand {
-    public static void queryById(final DefaultMQAdminExt admin, final String msgId, final Charset msgBodyCharset) throws MQClientException,
+    public static void queryById(final DefaultMQAdminExt admin, final String clusterName, final String topic,
+        final String msgId, final Charset msgBodyCharset) throws MQClientException,
         RemotingException, MQBrokerException, InterruptedException, IOException {
-        MessageExt msg = admin.viewMessage(msgId);
+        MessageExt msg = admin.queryMessage(clusterName, topic, msgId);
 
         printMsg(admin, msg, msgBodyCharset);
     }
@@ -55,7 +56,8 @@ public static void printMsg(final DefaultMQAdminExt admin, final MessageExt msg)
         printMsg(admin, msg, null);
     }
 
-    public static void printMsg(final DefaultMQAdminExt admin, final MessageExt msg, final Charset msgBodyCharset) throws IOException {
+    public static void printMsg(final DefaultMQAdminExt admin, final MessageExt msg,
+        final Charset msgBodyCharset) throws IOException {
         if (msg == null) {
             System.out.printf("%nMessage not found!");
             return;
@@ -191,7 +193,11 @@ public String commandDesc() {
 
     @Override
     public Options buildCommandlineOptions(Options options) {
-        Option opt = new Option("i", "msgId", true, "Message Id");
+        Option opt = new Option("t", "topic", true, "topic name");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("i", "msgId", true, "Message Id");
         opt.setRequired(true);
         options.addOption(opt);
 
@@ -215,6 +221,10 @@ public Options buildCommandlineOptions(Options options) {
         opt.setRequired(false);
         options.addOption(opt);
 
+        opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route.");
+        opt.setRequired(false);
+        options.addOption(opt);
+
         return options;
     }
 
@@ -227,6 +237,9 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
 
         try {
             defaultMQAdminExt.start();
+
+            String topic = commandLine.getOptionValue('t').trim();
+
             if (commandLine.hasOption('s')) {
                 if (commandLine.hasOption('u')) {
                     String unitName = commandLine.getOptionValue('u').trim();
@@ -237,13 +250,14 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
 
             final String msgIds = commandLine.getOptionValue('i').trim();
             final String[] msgIdArr = StringUtils.split(msgIds, ",");
+            String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null;
 
             if (commandLine.hasOption('g') && commandLine.hasOption('d')) {
                 final String consumerGroup = commandLine.getOptionValue('g').trim();
                 final String clientId = commandLine.getOptionValue('d').trim();
                 for (String msgId : msgIdArr) {
                     if (StringUtils.isNotBlank(msgId)) {
-                        pushMsg(defaultMQAdminExt, consumerGroup, clientId, msgId.trim());
+                        pushMsg(defaultMQAdminExt, clusterName, consumerGroup, clientId, topic, msgId.trim());
                     }
                 }
             } else if (commandLine.hasOption('s')) {
@@ -251,7 +265,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
                 if (resend) {
                     for (String msgId : msgIdArr) {
                         if (StringUtils.isNotBlank(msgId)) {
-                            sendMsg(defaultMQAdminExt, defaultMQProducer, msgId.trim());
+                            sendMsg(defaultMQAdminExt, clusterName, defaultMQProducer, topic, msgId.trim());
                         }
                     }
                 }
@@ -262,7 +276,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
                 }
                 for (String msgId : msgIdArr) {
                     if (StringUtils.isNotBlank(msgId)) {
-                        queryById(defaultMQAdminExt, msgId.trim(), msgBodyCharset);
+                        queryById(defaultMQAdminExt, clusterName, topic, msgId.trim(), msgBodyCharset);
                     }
                 }
 
@@ -275,13 +289,14 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
         }
     }
 
-    private void pushMsg(final DefaultMQAdminExt defaultMQAdminExt, final String consumerGroup, final String clientId,
-        final String msgId) {
+    private void pushMsg(final DefaultMQAdminExt defaultMQAdminExt, final String clusterName,
+        final String consumerGroup, final String clientId,
+        final String topic, final String msgId) {
         try {
             ConsumerRunningInfo consumerRunningInfo = defaultMQAdminExt.getConsumerRunningInfo(consumerGroup, clientId, false, false);
             if (consumerRunningInfo != null && ConsumerRunningInfo.isPushType(consumerRunningInfo)) {
                 ConsumeMessageDirectlyResult result =
-                        defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, msgId);
+                    defaultMQAdminExt.consumeMessageDirectly(clusterName, consumerGroup, clientId, topic, msgId);
                 System.out.printf("%s", result);
             } else {
                 System.out.printf("this %s client is not push consumer ,not support direct push \n", clientId);
@@ -291,10 +306,11 @@ private void pushMsg(final DefaultMQAdminExt defaultMQAdminExt, final String con
         }
     }
 
-    private void sendMsg(final DefaultMQAdminExt defaultMQAdminExt, final DefaultMQProducer defaultMQProducer,
-        final String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
+    private void sendMsg(final DefaultMQAdminExt defaultMQAdminExt, final String clusterName,
+        final DefaultMQProducer defaultMQProducer,
+        final String topic, final String msgId) {
         try {
-            MessageExt msg = defaultMQAdminExt.viewMessage(msgId);
+            MessageExt msg = defaultMQAdminExt.queryMessage(clusterName, topic, msgId);
             if (msg != null) {
                 // resend msg by id
                 System.out.printf("prepare resend msg. originalMsgId=%s", msgId);
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java
index 64627fd19fa..02961c3bb50 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java
@@ -23,6 +23,8 @@
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+
 import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
 import org.apache.rocketmq.tools.command.SubCommand;
 import org.apache.rocketmq.tools.command.SubCommandException;
@@ -41,7 +43,7 @@ public String commandDesc() {
 
     @Override
     public Options buildCommandlineOptions(Options options) {
-        Option opt = new Option("t", "topic", true, "topic name");
+        Option opt = new Option("t", "topic", true, "Topic name");
         opt.setRequired(true);
         options.addOption(opt);
 
@@ -57,7 +59,11 @@ public Options buildCommandlineOptions(Options options) {
         opt.setRequired(false);
         options.addOption(opt);
 
-        opt = new Option("c", "maxNum", true, "The maximum number of messages returned by the query, default:64");
+        opt = new Option("m", "maxNum", true, "The maximum number of messages returned by the query, default:64");
+        opt.setRequired(false);
+        options.addOption(opt);
+
+        opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route.");
         opt.setRequired(false);
         options.addOption(opt);
 
@@ -77,16 +83,20 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
             long beginTimestamp = 0;
             long endTimestamp = Long.MAX_VALUE;
             int maxNum = 64;
+            String clusterName = null;
             if (commandLine.hasOption("b")) {
                 beginTimestamp = Long.parseLong(commandLine.getOptionValue("b").trim());
             }
             if (commandLine.hasOption("e")) {
                 endTimestamp = Long.parseLong(commandLine.getOptionValue("e").trim());
             }
+            if (commandLine.hasOption("m")) {
+                maxNum = Integer.parseInt(commandLine.getOptionValue("m").trim());
+            }
             if (commandLine.hasOption("c")) {
-                maxNum = Integer.parseInt(commandLine.getOptionValue("c").trim());
+                clusterName = commandLine.getOptionValue("c").trim();
             }
-            this.queryByKey(defaultMQAdminExt, topic, key, maxNum, beginTimestamp, endTimestamp);
+            this.queryByKey(defaultMQAdminExt, clusterName, topic, key, maxNum, beginTimestamp, endTimestamp);
         } catch (Exception e) {
             throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
         } finally {
@@ -94,12 +104,13 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
         }
     }
 
-    private void queryByKey(final DefaultMQAdminExt admin, final String topic, final String key, int maxNum, long begin,
+    private void queryByKey(final DefaultMQAdminExt admin, final String cluster, final String topic, final String key, int maxNum, long begin,
         long end)
-        throws MQClientException, InterruptedException {
+        throws MQClientException, InterruptedException, RemotingException {
         admin.start();
 
-        QueryResult queryResult = admin.queryMessage(topic, key, maxNum, begin, end);
+        QueryResult queryResult = admin.queryMessage(cluster, topic, key, maxNum, begin, end);
+
         System.out.printf("%-50s %4s %40s%n",
             "#Message ID",
             "#QID",
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java
index b71cee90160..5295d91cc30 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java
@@ -25,13 +25,11 @@
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.rocketmq.client.QueryResult;
-import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
-import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult;
 import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo;
 import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
@@ -51,19 +49,18 @@ private DefaultMQAdminExt createMQAdminExt(RPCHook rpcHook) throws SubCommandExc
             defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
             try {
                 defaultMQAdminExt.start();
-            }
-            catch (Exception e) {
+            } catch (Exception e) {
                 throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
             }
             return defaultMQAdminExt;
         }
     }
 
-    public static void queryById(final DefaultMQAdminExt admin, final String topic, final String msgId,
-                                 final boolean showAll) throws MQClientException,
-            RemotingException, MQBrokerException, InterruptedException, IOException {
+    public static void queryById(final DefaultMQAdminExt admin, final String clusterName, final String topic,
+        final String msgId,
+        final boolean showAll) throws MQClientException, InterruptedException, IOException {
 
-        QueryResult queryResult = admin.queryMessageByUniqKey(topic, msgId, 32, 0, Long.MAX_VALUE);
+        QueryResult queryResult = admin.queryMessageByUniqKey(clusterName, topic, msgId, 32, 0, Long.MAX_VALUE);
         assert queryResult != null;
         List<MessageExt> list = queryResult.getMessageList();
         if (list == null || list.size() == 0) {
@@ -94,7 +91,7 @@ private static void showMessage(final DefaultMQAdminExt admin, MessageExt msg, i
         System.out.printf(strFormat, "Store Host:", RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()));
         System.out.printf(intFormat, "System Flag:", msg.getSysFlag());
         System.out.printf(strFormat, "Properties:",
-                msg.getProperties() != null ? msg.getProperties().toString() : "");
+            msg.getProperties() != null ? msg.getProperties().toString() : "");
         System.out.printf(strFormat, "Message Body Path:", bodyTmpFilePath);
 
         try {
@@ -166,6 +163,10 @@ public Options buildCommandlineOptions(Options options) {
         opt.setRequired(false);
         options.addOption(opt);
 
+        opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route.");
+        opt.setRequired(false);
+        options.addOption(opt);
+
         return options;
     }
 
@@ -173,10 +174,11 @@ public Options buildCommandlineOptions(Options options) {
     public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException {
 
         try {
-            defaultMQAdminExt =  createMQAdminExt(rpcHook);
+            defaultMQAdminExt = createMQAdminExt(rpcHook);
 
             final String msgId = commandLine.getOptionValue('i').trim();
             final String topic = commandLine.getOptionValue('t').trim();
+            String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null;
             final boolean showAll = commandLine.hasOption('a');
             if (commandLine.hasOption('g') && commandLine.hasOption('d')) {
                 final String consumerGroup = commandLine.getOptionValue('g').trim();
@@ -189,14 +191,14 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
                 }
                 if (consumerRunningInfo != null && ConsumerRunningInfo.isPushType(consumerRunningInfo)) {
                     ConsumeMessageDirectlyResult result =
-                            defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, topic, msgId);
+                        defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, topic, msgId);
                     System.out.printf("%s", result);
                 } else {
-                    System.out.printf("get consumer info failed or this %s client is not push consumer ,not support direct push \n", clientId);
+                    System.out.printf("get consumer info failed or this %s client is not push consumer, not support direct push \n", clientId);
                 }
 
             } else {
-                queryById(defaultMQAdminExt, topic, msgId, showAll);
+                queryById(defaultMQAdminExt, clusterName, topic, msgId, showAll);
             }
         } catch (Exception e) {
             throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
index b987ad873be..48bc163678b 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java
@@ -14,26 +14,42 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.rocketmq.tools.command.metadata;
 
 import com.alibaba.fastjson.JSONObject;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.common.config.RocksDBConfigManager;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.config.ConfigRocksDBStorage;
 import org.apache.rocketmq.common.utils.DataConverter;
 import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.protocol.body.ClusterInfo;
+import org.apache.rocketmq.remoting.protocol.header.ExportRocksDBConfigToJsonRequestHeader;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
 import org.apache.rocketmq.tools.command.SubCommand;
 import org.apache.rocketmq.tools.command.SubCommandException;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
+import org.rocksdb.RocksIterator;
 
 public class RocksDBConfigToJsonCommand implements SubCommand {
-    private static final String TOPICS_JSON_CONFIG = "topics";
-    private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups";
 
     @Override
     public String commandName() {
@@ -42,77 +58,267 @@ public String commandName() {
 
     @Override
     public String commandDesc() {
-        return "Convert RocksDB kv config (topics/subscriptionGroups) to json";
+        return "Convert RocksDB kv config (topics/subscriptionGroups/consumerOffsets) to json. " +
+            "[rpc mode] Use [-n, -c, -b, -t] to send Request to broker ( version >= 5.3.2 ) or [local mode] use [-p, -t, -j, -e] to load RocksDB. " +
+            "If -e is provided, tools will export json file instead of std print";
     }
 
     @Override
     public Options buildCommandlineOptions(Options options) {
-        Option pathOption = new Option("p", "path", true,
-                "Absolute path to the metadata directory");
-        pathOption.setRequired(true);
-        options.addOption(pathOption);
-
         Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " +
-                "topics/subscriptionGroups");
-        configTypeOption.setRequired(true);
+            "topics/subscriptionGroups/consumerOffsets. Required in local mode and default all in rpc mode.");
         options.addOption(configTypeOption);
 
+        // [local mode] options
+        Option pathOption = new Option("p", "configPath", true,
+            "[local mode] Absolute path to the metadata config directory");
+        options.addOption(pathOption);
+
+        Option exportPathOption = new Option("e", "exportFile", true,
+            "[local mode] Absolute file path for exporting, auto backup existing file, not directory. If exportFile is provided, will export Json file and ignore [-j].");
+        options.addOption(exportPathOption);
+
+        Option jsonEnableOption = new Option("j", "jsonEnable", true,
+            "[local mode] Json format enable, Default: true. If exportFile is provided, will export Json file and ignore [-j].");
+        options.addOption(jsonEnableOption);
+
+        // [rpc mode] options
+        Option nameserverOption = new Option("n", "nameserverAddr", true,
+            "[rpc mode] nameserverAddr. If nameserverAddr and clusterName are provided, will ignore [-p, -e, -j, -b] args");
+        options.addOption(nameserverOption);
+
+        Option clusterOption = new Option("c", "cluster", true,
+            "[rpc mode] Cluster name. If nameserverAddr and clusterName are provided, will ignore [-p, -e, -j, -b] args");
+        options.addOption(clusterOption);
+
+        Option brokerAddrOption = new Option("b", "brokerAddr", true,
+            "[rpc mode] Broker address. If brokerAddr is provided, will ignore [-p, -e, -j] args");
+        options.addOption(brokerAddrOption);
+
         return options;
     }
 
     @Override
     public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException {
-        String path = commandLine.getOptionValue("path").trim();
+        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> typeList = getConfigTypeList(commandLine);
+
+        if (commandLine.hasOption("nameserverAddr")) {
+            // [rpc mode] call all brokers in cluster to export to json file
+            System.out.print("Use [rpc mode] call all brokers in cluster to export to json file \n");
+            checkRequiredArgsProvided(commandLine, "rpc mode", "cluster");
+            handleRpcMode(commandLine, rpcHook, typeList);
+        } else if (commandLine.hasOption("brokerAddr")) {
+            // [rpc mode] call broker to export to json file
+            System.out.print("Use [rpc mode] call broker to export to json file \n");
+            handleRpcMode(commandLine, rpcHook, typeList);
+        } else if (commandLine.hasOption("configPath")) {
+            // [local mode] load rocksdb to print or export file
+            System.out.print("Use [local mode] load rocksdb to print or export file \n");
+            checkRequiredArgsProvided(commandLine, "local mode", "configType");
+            handleLocalMode(commandLine);
+        } else {
+            System.out.print(commandDesc() + "\n");
+        }
+    }
+
+    private void handleLocalMode(CommandLine commandLine) {
+        ExportRocksDBConfigToJsonRequestHeader.ConfigType type = Objects.requireNonNull(getConfigTypeList(commandLine)).get(0);
+        String path = commandLine.getOptionValue("configPath").trim();
         if (StringUtils.isEmpty(path) || !new File(path).exists()) {
             System.out.print("Rocksdb path is invalid.\n");
             return;
         }
+        path = Paths.get(path, type.toString()).toString();
+        String exportFile = commandLine.hasOption("exportFile") ? commandLine.getOptionValue("exportFile").trim() : null;
+        Map<String, JSONObject> configMap = getConfigMapFromRocksDB(path, type);
+        if (configMap != null) {
+            if (exportFile == null) {
+                if (commandLine.hasOption("jsonEnable") && "false".equalsIgnoreCase(commandLine.getOptionValue("jsonEnable").trim())) {
+                    printConfigMapJsonDisable(configMap);
+                } else {
+                    System.out.print(JSONObject.toJSONString(configMap, true) + "\n");
+                }
+            } else {
+                String jsonString = JSONObject.toJSONString(configMap, true);
+                try {
+                    MixAll.string2File(jsonString, exportFile);
+                } catch (IOException e) {
+                    System.out.print("persist file " + exportFile + " exception" + e);
+                }
+            }
+        }
+    }
 
-        String configType = commandLine.getOptionValue("configType").trim().toLowerCase();
+    private void checkRequiredArgsProvided(CommandLine commandLine, String mode,
+        String... args) throws SubCommandException {
+        for (String arg : args) {
+            if (!commandLine.hasOption(arg)) {
+                System.out.printf("%s Invalid args, please input %s\n", mode, String.join(",", args));
+                throw new SubCommandException("Invalid args");
+            }
+        }
+    }
+
+    private List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> getConfigTypeList(CommandLine commandLine) {
+        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> typeList = new ArrayList<>();
+        if (commandLine.hasOption("configType")) {
+            String configType = commandLine.getOptionValue("configType").trim();
+            try {
+                typeList.addAll(ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString(configType));
+            } catch (IllegalArgumentException e) {
+                System.out.print("Invalid configType: " + configType + " please input topics/subscriptionGroups/consumerOffsets \n");
+                return null;
+            }
+        } else {
+            typeList.addAll(Arrays.asList(ExportRocksDBConfigToJsonRequestHeader.ConfigType.values()));
+        }
+        return typeList;
+    }
 
-        final long memTableFlushInterval = 60 * 60 * 1000L;
-        RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval);
+    private static void printConfigMapJsonDisable(Map<String, JSONObject> configMap) {
+        AtomicLong count = new AtomicLong(0);
+        for (Map.Entry<String, JSONObject> entry : configMap.entrySet()) {
+            String configKey = entry.getKey();
+            System.out.printf("type: %s", configKey);
+            JSONObject jsonObject = entry.getValue();
+            jsonObject.forEach((k, v) -> System.out.printf("%d, Key: %s, Value: %s%n", count.incrementAndGet(), k, v));
+        }
+    }
+
+    private static Map<String, JSONObject> getConfigMapFromRocksDB(String path,
+        ExportRocksDBConfigToJsonRequestHeader.ConfigType configType) {
+
+        if (ExportRocksDBConfigToJsonRequestHeader.ConfigType.CONSUMER_OFFSETS.equals(configType)) {
+            return loadConsumerOffsets(path);
+        }
+
+        ConfigRocksDBStorage configRocksDBStorage = new ConfigRocksDBStorage(path, true);
+        configRocksDBStorage.start();
+        RocksIterator iterator = configRocksDBStorage.iterator();
         try {
-            if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) {
-                // for topics.json
-                final Map<String, JSONObject> topicsJsonConfig = new HashMap<>();
-                final Map<String, JSONObject> topicConfigTable = new HashMap<>();
-                boolean isLoad = kvConfigManager.load(path, (key, value) -> {
-                    final String topic = new String(key, DataConverter.CHARSET_UTF8);
-                    final String topicConfig = new String(value, DataConverter.CHARSET_UTF8);
-                    final JSONObject jsonObject = JSONObject.parseObject(topicConfig);
-                    topicConfigTable.put(topic, jsonObject);
-                });
-
-                if (isLoad) {
-                    topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable));
-                    final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true);
-                    System.out.print(topicsJsonStr + "\n");
-                    return;
-                }
+            final Map<String, JSONObject> configMap = new HashMap<>();
+            final JSONObject configTable = new JSONObject();
+            iterator.seekToFirst();
+            while (iterator.isValid()) {
+                final byte[] key = iterator.key();
+                final byte[] value = iterator.value();
+                final String name = new String(key, DataConverter.CHARSET_UTF8);
+                final String config = new String(value, DataConverter.CHARSET_UTF8);
+                final JSONObject jsonObject = JSONObject.parseObject(config);
+                configTable.put(name, jsonObject);
+                iterator.next();
+            }
+            byte[] kvDataVersion = configRocksDBStorage.getKvDataVersion();
+            if (kvDataVersion != null) {
+                configMap.put("dataVersion",
+                    JSONObject.parseObject(new String(kvDataVersion, DataConverter.CHARSET_UTF8)));
+            }
+
+            if (ExportRocksDBConfigToJsonRequestHeader.ConfigType.TOPICS.equals(configType)) {
+                configMap.put("topicConfigTable", configTable);
             }
-            if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) {
-                // for subscriptionGroup.json
-                final Map<String, JSONObject> subscriptionGroupJsonConfig = new HashMap<>();
-                final Map<String, JSONObject> subscriptionGroupTable = new HashMap<>();
-                boolean isLoad = kvConfigManager.load(path, (key, value) -> {
-                    final String subscriptionGroup = new String(key, DataConverter.CHARSET_UTF8);
-                    final String subscriptionGroupConfig = new String(value, DataConverter.CHARSET_UTF8);
-                    final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig);
-                    subscriptionGroupTable.put(subscriptionGroup, jsonObject);
-                });
-
-                if (isLoad) {
-                    subscriptionGroupJsonConfig.put("subscriptionGroupTable",
-                            (JSONObject) JSONObject.toJSON(subscriptionGroupTable));
-                    final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true);
-                    System.out.print(subscriptionGroupJsonStr + "\n");
+            if (ExportRocksDBConfigToJsonRequestHeader.ConfigType.SUBSCRIPTION_GROUPS.equals(configType)) {
+                configMap.put("subscriptionGroupTable", configTable);
+            }
+            return configMap;
+        } catch (Exception e) {
+            System.out.print("Error occurred while converting RocksDB kv config to json, " + "configType=" + configType + ", " + e.getMessage() + "\n");
+        } finally {
+            configRocksDBStorage.shutdown();
+        }
+        return null;
+    }
+
+    private void handleRpcMode(CommandLine commandLine, RPCHook rpcHook,
+        List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> type) {
+        String nameserverAddr = commandLine.hasOption('n') ? commandLine.getOptionValue("nameserverAddr").trim() : null;
+        String inputBrokerAddr = commandLine.hasOption('b') ? commandLine.getOptionValue('b').trim() : null;
+        String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null;
+
+        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook, 30 * 1000);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+        defaultMQAdminExt.setNamesrvAddr(nameserverAddr);
+
+        List<CompletableFuture<Void>> futureList = new ArrayList<>();
+
+        try {
+            defaultMQAdminExt.start();
+            if (clusterName != null) {
+                ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo();
+                Map<String, Set<String>> clusterAddrTable = clusterInfo.getClusterAddrTable();
+                Map<String, BrokerData> brokerAddrTable = clusterInfo.getBrokerAddrTable();
+                if (clusterAddrTable.get(clusterName) == null) {
+                    System.out.print("clusterAddrTable is empty");
                     return;
                 }
+                for (Map.Entry<String, BrokerData> entry : brokerAddrTable.entrySet()) {
+                    String brokerName = entry.getKey();
+                    BrokerData brokerData = entry.getValue();
+                    String brokerAddr = brokerData.getBrokerAddrs().get(0L);
+                    futureList.add(sendRequest(type, defaultMQAdminExt, brokerAddr, brokerName));
+                }
+            } else if (inputBrokerAddr != null) {
+                futureList.add(sendRequest(type, defaultMQAdminExt, inputBrokerAddr, null));
             }
-            System.out.print("Config type was not recognized, configType=" + configType + "\n");
+            CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).whenComplete(
+                (v, t) -> System.out.print("broker export done.")
+            ).join();
+        } catch (Exception e) {
+            throw new RuntimeException(this.getClass().getSimpleName() + " command failed", e);
         } finally {
-            kvConfigManager.stop();
+            defaultMQAdminExt.shutdown();
+        }
+    }
+
+    private CompletableFuture<Void> sendRequest(List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> type,
+        DefaultMQAdminExt defaultMQAdminExt, String brokerAddr, String brokerName) {
+        return CompletableFuture.supplyAsync(() -> {
+            try {
+                defaultMQAdminExt.exportRocksDBConfigToJson(brokerAddr, type);
+            } catch (Throwable t) {
+                System.out.print((brokerName != null) ? brokerName : brokerAddr + " export error");
+                throw new CompletionException(this.getClass().getSimpleName() + " command failed", t);
+            }
+            return null;
+        });
+    }
+
+    private static Map<String, JSONObject> loadConsumerOffsets(String path) {
+        ConfigRocksDBStorage configRocksDBStorage = new ConfigRocksDBStorage(path, true);
+        configRocksDBStorage.start();
+        RocksIterator iterator = configRocksDBStorage.iterator();
+        try {
+            final Map<String, JSONObject> configMap = new HashMap<>();
+            final JSONObject configTable = new JSONObject();
+            iterator.seekToFirst();
+            while (iterator.isValid()) {
+                final byte[] key = iterator.key();
+                final byte[] value = iterator.value();
+                final String name = new String(key, DataConverter.CHARSET_UTF8);
+                final String config = new String(value, DataConverter.CHARSET_UTF8);
+                final RocksDBOffsetSerializeWrapper jsonObject = JSONObject.parseObject(config, RocksDBOffsetSerializeWrapper.class);
+                configTable.put(name, jsonObject.getOffsetTable());
+                iterator.next();
+            }
+            configMap.put("offsetTable", configTable);
+            return configMap;
+        } catch (Exception e) {
+            System.out.print("Error occurred while converting RocksDB kv config to json, " + "configType=consumerOffsets, " + e.getMessage() + "\n");
+        } finally {
+            configRocksDBStorage.shutdown();
+        }
+        return null;
+    }
+
+    static class RocksDBOffsetSerializeWrapper {
+        private ConcurrentMap<Integer, Long> offsetTable = new ConcurrentHashMap<>(16);
+
+        public ConcurrentMap<Integer, Long> getOffsetTable() {
+            return offsetTable;
+        }
+
+        public void setOffsetTable(ConcurrentMap<Integer, Long> offsetTable) {
+            this.offsetTable = offsetTable;
         }
     }
-}
+}
\ No newline at end of file
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java
index 993fa501875..84a301bd60c 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java
@@ -77,6 +77,10 @@ public Options buildCommandlineOptions(Options options) {
         opt.setRequired(false);
         options.addOption(opt);
 
+        opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route.");
+        opt.setRequired(false);
+        options.addOption(opt);
+
         return options;
     }
 
@@ -88,6 +92,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
             String group = commandLine.getOptionValue("g").trim();
             String topic = commandLine.getOptionValue("t").trim();
             String timeStampStr = commandLine.getOptionValue("s").trim();
+            String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null;
             long timestamp = "now".equals(timeStampStr) ? System.currentTimeMillis() : 0;
 
             try {
@@ -129,7 +134,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
             if (brokerAddr != null && queueId >= 0) {
                 System.out.printf("start reset consumer offset by specified, " +
                         "group[%s], topic[%s], queueId[%s], broker[%s], timestamp(string)[%s], timestamp(long)[%s]%n",
-                        group, topic, queueId, brokerAddr, timeStampStr, timestamp);
+                    group, topic, queueId, brokerAddr, timeStampStr, timestamp);
 
                 long resetOffset = null != offset ? offset :
                     defaultMQAdminExt.searchOffset(brokerAddr, topic, queueId, timestamp, 3000);
@@ -143,11 +148,11 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
 
             Map<MessageQueue, Long> offsetTable;
             try {
-                offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(topic, group, timestamp, force, isC);
+                offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(clusterName, topic, group, timestamp, force, isC);
             } catch (MQClientException e) {
-                // if consumer not online, use old command to reset reset
+                // if consumer not online, use old command to reset
                 if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) {
-                    ResetOffsetByTimeOldCommand.resetOffset(defaultMQAdminExt, group, topic, timestamp, force, timeStampStr);
+                    ResetOffsetByTimeOldCommand.resetOffset(defaultMQAdminExt, clusterName, group, topic, timestamp, force, timeStampStr);
                     return;
                 }
                 throw e;
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java
index 7984bb8c39f..c179c5c8051 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java
@@ -34,12 +34,13 @@
 
 public class ResetOffsetByTimeOldCommand implements SubCommand {
 
-    public static void resetOffset(DefaultMQAdminExt defaultMQAdminExt, String consumerGroup, String topic,
+    public static void resetOffset(DefaultMQAdminExt defaultMQAdminExt, String clusterName, String consumerGroup,
+        String topic,
         long timestamp, boolean force, String timeStampStr)
         throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
 
         List<RollbackStats> rollbackStatsList =
-            defaultMQAdminExt.resetOffsetByTimestampOld(consumerGroup, topic, timestamp, force);
+            defaultMQAdminExt.resetOffsetByTimestampOld(clusterName, consumerGroup, topic, timestamp, force);
 
         System.out.printf("reset consumer offset by specified " +
                 "consumerGroup[%s], topic[%s], force[%s], timestamp(string)[%s], timestamp(long)[%s]%n",
@@ -93,6 +94,11 @@ public Options buildCommandlineOptions(Options options) {
         opt = new Option("f", "force", true, "set the force rollback by timestamp switch[true|false]");
         opt.setRequired(false);
         options.addOption(opt);
+
+        opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route.");
+        opt.setRequired(false);
+        options.addOption(opt);
+
         return options;
     }
 
@@ -104,6 +110,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
             String consumerGroup = commandLine.getOptionValue("g").trim();
             String topic = commandLine.getOptionValue("t").trim();
             String timeStampStr = commandLine.getOptionValue("s").trim();
+            String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null;
             long timestamp = 0;
             try {
                 timestamp = Long.parseLong(timeStampStr);
@@ -123,7 +130,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
                 force = Boolean.parseBoolean(commandLine.getOptionValue("f").trim());
             }
             defaultMQAdminExt.start();
-            resetOffset(defaultMQAdminExt, consumerGroup, topic, timestamp, force, timeStampStr);
+            resetOffset(defaultMQAdminExt, clusterName, consumerGroup, topic, timestamp, force, timeStampStr);
 
         } catch (Exception e) {
             throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java
index b22491a5918..8f2ac2e1e14 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java
@@ -57,6 +57,10 @@ public Options buildCommandlineOptions(Options options) {
         opt = new Option("f", "force", true, "set the force rollback by timestamp switch[true|false]");
         opt.setRequired(false);
         options.addOption(opt);
+
+        opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route.");
+        opt.setRequired(false);
+        options.addOption(opt);
         return options;
     }
 
@@ -68,6 +72,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
         try {
             String group = commandLine.getOptionValue("g").trim();
             String topic = commandLine.getOptionValue("t").trim();
+            String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null;
             boolean force = true;
             if (commandLine.hasOption('f')) {
                 force = Boolean.valueOf(commandLine.getOptionValue("f").trim());
@@ -76,7 +81,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t
             defaultMQAdminExt.start();
             Map<MessageQueue, Long> offsetTable;
             try {
-                offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(topic, group, timestamp, force);
+                offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(clusterName, topic, group, timestamp, force);
             } catch (MQClientException e) {
                 if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) {
                     List<RollbackStats> rollbackStatsList = defaultMQAdminExt.resetOffsetByTimestampOld(group, topic, timestamp, force);
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java
new file mode 100644
index 00000000000..a0fc9fce1fb
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.tools.command.queue;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.CheckRocksdbCqWriteResult;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.protocol.body.ClusterInfo;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.SubCommand;
+
+public class CheckRocksdbCqWriteProgressCommand implements SubCommand {
+
+    @Override
+    public String commandName() {
+        return "checkRocksdbCqWriteProgress";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "check if rocksdb cq is same as file cq";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+        Option opt = new Option("c", "cluster", true, "cluster name");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("n", "nameserverAddr", true, "nameserverAddr");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        opt = new Option("t", "topic", true, "topic name");
+        options.addOption(opt);
+
+        opt = new Option("cf", "checkFrom", true, "check from time");
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) {
+        DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+        defaultMQAdminExt.setNamesrvAddr(StringUtils.trim(commandLine.getOptionValue('n')));
+        String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : "";
+        String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : "";
+        // The default check is 30 days
+        long checkStoreTime = commandLine.hasOption("cf")
+            ? Long.parseLong(commandLine.getOptionValue("cf").trim())
+            : System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30L);
+
+        try {
+            defaultMQAdminExt.start();
+            ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo();
+            Map<String, Set<String>> clusterAddrTable = clusterInfo.getClusterAddrTable();
+            Map<String, BrokerData> brokerAddrTable = clusterInfo.getBrokerAddrTable();
+            if (clusterAddrTable.get(clusterName) == null) {
+                System.out.print("clusterAddrTable is empty");
+                return;
+            }
+            for (Map.Entry<String, BrokerData> entry : brokerAddrTable.entrySet()) {
+                String brokerName = entry.getKey();
+                BrokerData brokerData = entry.getValue();
+                String brokerAddr = brokerData.getBrokerAddrs().get(0L);
+                CheckRocksdbCqWriteResult result = defaultMQAdminExt.checkRocksdbCqWriteProgress(brokerAddr, topic, checkStoreTime);
+                if (result.getCheckStatus() == CheckRocksdbCqWriteResult.CheckStatus.CHECK_ERROR.getValue()) {
+                    System.out.print(brokerName + " check error, please check log... errInfo: " + result.getCheckResult());
+                } else {
+                    System.out.print(brokerName + " check doing, please wait and get the result from log... \n");
+                }
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+}
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java
index a1619ecedfd..ff91399f1c1 100644
--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java
@@ -27,6 +27,8 @@
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.protocol.admin.TopicOffset;
 import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable;
+import org.apache.rocketmq.remoting.protocol.route.BrokerData;
+import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
 import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
 import org.apache.rocketmq.tools.command.SubCommand;
 import org.apache.rocketmq.tools.command.SubCommandException;
@@ -48,6 +50,10 @@ public Options buildCommandlineOptions(Options options) {
         Option opt = new Option("t", "topic", true, "topic name");
         opt.setRequired(true);
         options.addOption(opt);
+
+        opt = new Option("c", "cluster", true, "cluster name or lmq parent topic, lmq is used to find the route.");
+        opt.setRequired(false);
+        options.addOption(opt);
         return options;
     }
 
@@ -58,10 +64,26 @@ public void execute(final CommandLine commandLine, final Options options,
 
         defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
 
+
         try {
+            TopicStatsTable topicStatsTable = new TopicStatsTable();
             defaultMQAdminExt.start();
             String topic = commandLine.getOptionValue('t').trim();
-            TopicStatsTable topicStatsTable = defaultMQAdminExt.examineTopicStats(topic);
+
+            if (commandLine.hasOption('c')) {
+                String cluster = commandLine.getOptionValue('c').trim();
+                TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(cluster);
+
+                for (BrokerData bd : topicRouteData.getBrokerDatas()) {
+                    String addr = bd.selectBrokerAddr();
+                    if (addr != null) {
+                        TopicStatsTable tst = defaultMQAdminExt.examineTopicStats(addr, topic);
+                        topicStatsTable.getOffsetTable().putAll(tst.getOffsetTable());
+                    }
+                }
+            } else {
+                topicStatsTable = defaultMQAdminExt.examineTopicStats(topic);
+            }
 
             List<MessageQueue> mqList = new LinkedList<>();
             mqList.addAll(topicStatsTable.getOffsetTable().keySet());
@@ -91,6 +113,8 @@ public void execute(final CommandLine commandLine, final Options options,
                     humanTimestamp
                 );
             }
+            System.out.printf("%n");
+            System.out.printf("Topic Put TPS: %s%n", topicStatsTable.getTopicPutTps());
         } catch (Exception e) {
             throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
         } finally {
diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommand.java
new file mode 100644
index 00000000000..a246059e117
--- /dev/null
+++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommand.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.tools.command.topic;
+
+import com.alibaba.fastjson2.JSON;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Set;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.srvutil.ServerUtil;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.CommandUtil;
+import org.apache.rocketmq.tools.command.SubCommand;
+import org.apache.rocketmq.tools.command.SubCommandException;
+
+public class UpdateTopicListSubCommand implements SubCommand {
+    @Override
+    public String commandName() {
+        return "updateTopicList";
+    }
+
+    @Override
+    public String commandDesc() {
+        return "create or update topic in batch";
+    }
+
+    @Override
+    public Options buildCommandlineOptions(Options options) {
+        final OptionGroup optionGroup = new OptionGroup();
+        Option opt = new Option("b", "brokerAddr", true, "create topic to which broker");
+        optionGroup.addOption(opt);
+        opt = new Option("c", "clusterName", true, "create topic to which cluster");
+        optionGroup.addOption(opt);
+        optionGroup.setRequired(true);
+        options.addOptionGroup(optionGroup);
+
+        opt = new Option("f", "filename", true, "Path to a file with list of org.apache.rocketmq.common.TopicConfig in json format");
+        opt.setRequired(true);
+        options.addOption(opt);
+
+        return options;
+    }
+
+    @Override
+    public void execute(CommandLine commandLine, Options options,
+        RPCHook rpcHook) throws SubCommandException {
+        final DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+        defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+
+        final String fileName = commandLine.getOptionValue('f').trim();
+
+
+        try {
+            final Path filePath = Paths.get(fileName);
+            if (!Files.exists(filePath)) {
+                System.out.printf("the file path %s does not exists%n", fileName);
+                return;
+            }
+            final byte[] topicConfigListBytes = Files.readAllBytes(filePath);
+            final List<TopicConfig> topicConfigs = JSON.parseArray(topicConfigListBytes, TopicConfig.class);
+            if (null == topicConfigs || topicConfigs.isEmpty()) {
+                return;
+            }
+
+            if (commandLine.hasOption('b')) {
+                String brokerAddress = commandLine.getOptionValue('b').trim();
+                defaultMQAdminExt.start();
+                defaultMQAdminExt.createAndUpdateTopicConfigList(brokerAddress, topicConfigs);
+
+                System.out.printf("submit batch of topic config to %s success, please check the result later.%n",
+                    brokerAddress);
+                return;
+
+            } else if (commandLine.hasOption('c')) {
+                final String clusterName = commandLine.getOptionValue('c').trim();
+
+                defaultMQAdminExt.start();
+
+                Set<String> masterSet =
+                    CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
+                for (String brokerAddress : masterSet) {
+                    defaultMQAdminExt.createAndUpdateTopicConfigList(brokerAddress, topicConfigs);
+
+                    System.out.printf("submit batch of topic config to %s success, please check the result later.%n",
+                        brokerAddress);
+                }
+            }
+
+            ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
+        } catch (Exception e) {
+            throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
+        } finally {
+            defaultMQAdminExt.shutdown();
+        }
+    }
+}
diff --git a/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java b/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java
index dc5642f88c2..ec5f7571d28 100644
--- a/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java
+++ b/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java
@@ -456,7 +456,7 @@ public void testMessageTrackDetail() throws InterruptedException, RemotingExcept
         connection.setConnectionSet(connections);
         when(mQClientAPIImpl.getConsumerConnectionList(anyString(), anyString(), anyLong())).thenReturn(connection);
         ConsumeStats consumeStats = new ConsumeStats();
-        when(mQClientAPIImpl.getConsumeStats(anyString(), anyString(), isNull(), anyLong())).thenReturn(consumeStats);
+        when(mQClientAPIImpl.getConsumeStats(anyString(), anyString(), (String) isNull(), anyLong())).thenReturn(consumeStats);
         List<MessageTrack> broadcastMessageTracks = defaultMQAdminExt.messageTrackDetail(messageExt);
         assertThat(broadcastMessageTracks.size()).isEqualTo(2);
         assertThat(broadcastMessageTracks.get(0).getTrackType()).isEqualTo(TrackType.CONSUME_BROADCASTING);
diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommandTest.java
deleted file mode 100644
index 28430b823c3..00000000000
--- a/tools/src/test/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommandTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tools.command.acl;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.Options;
-import org.apache.rocketmq.srvutil.ServerUtil;
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class DeleteAccessConfigSubCommandTest {
-
-    @Test
-    public void testExecute() {
-        DeleteAccessConfigSubCommand cmd = new DeleteAccessConfigSubCommand();
-        Options options = ServerUtil.buildCommandlineOptions(new Options());
-        String[] subargs = new String[] {"-a unit-test", "-c default-cluster"};
-        final CommandLine commandLine =
-            ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs,
-                cmd.buildCommandlineOptions(options), new DefaultParser());
-        assertThat(commandLine.getOptionValue('a').trim()).isEqualTo("unit-test");
-        assertThat(commandLine.getOptionValue('c').trim()).isEqualTo("default-cluster");
-    }
-}
diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommandTest.java
deleted file mode 100644
index 98646bb1613..00000000000
--- a/tools/src/test/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommandTest.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.tools.command.acl;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.Options;
-import org.apache.rocketmq.common.PlainAccessConfig;
-import org.apache.rocketmq.srvutil.ServerUtil;
-import org.junit.Assert;
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class UpdateAccessConfigSubCommandTest {
-
-    @Test
-    public void testExecute() {
-        UpdateAccessConfigSubCommand cmd = new UpdateAccessConfigSubCommand();
-        Options options = ServerUtil.buildCommandlineOptions(new Options());
-        String[] subargs = new String[] {
-            "-b","127.0.0.1:10911",
-            "-a","RocketMQ",
-            "-s","12345678",
-            "-w","192.168.0.*",
-            "-i","DENY",
-            "-u","SUB",
-            "-t","topicA=DENY;topicB=PUB|SUB",
-            "-g","groupA=DENY;groupB=SUB",
-            "-m","true"
-        };
-        // Note: Posix parser is capable of handling values that contains '='.
-        final CommandLine commandLine =
-            ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs,
-                cmd.buildCommandlineOptions(options), new DefaultParser());
-        assertThat(commandLine.getOptionValue('b').trim()).isEqualTo("127.0.0.1:10911");
-        assertThat(commandLine.getOptionValue('a').trim()).isEqualTo("RocketMQ");
-        assertThat(commandLine.getOptionValue('s').trim()).isEqualTo("12345678");
-        assertThat(commandLine.getOptionValue('w').trim()).isEqualTo("192.168.0.*");
-        assertThat(commandLine.getOptionValue('i').trim()).isEqualTo("DENY");
-        assertThat(commandLine.getOptionValue('u').trim()).isEqualTo("SUB");
-        assertThat(commandLine.getOptionValue('t').trim()).isEqualTo("topicA=DENY;topicB=PUB|SUB");
-        assertThat(commandLine.getOptionValue('g').trim()).isEqualTo("groupA=DENY;groupB=SUB");
-        assertThat(commandLine.getOptionValue('m').trim()).isEqualTo("true");
-
-        PlainAccessConfig accessConfig = new PlainAccessConfig();
-
-        // topicPerms list value
-        if (commandLine.hasOption('t')) {
-            String[] topicPerms = commandLine.getOptionValue('t').trim().split(";");
-            List<String> topicPermList = new ArrayList<>(Arrays.asList(topicPerms));
-            accessConfig.setTopicPerms(topicPermList);
-        }
-
-        // groupPerms list value
-        if (commandLine.hasOption('g')) {
-            String[] groupPerms = commandLine.getOptionValue('g').trim().split(";");
-            List<String> groupPermList = new ArrayList<>();
-            Collections.addAll(groupPermList, groupPerms);
-            accessConfig.setGroupPerms(groupPermList);
-        }
-
-        Assert.assertTrue(accessConfig.getTopicPerms().contains("topicB=PUB|SUB"));
-        Assert.assertTrue(accessConfig.getGroupPerms().contains("groupB=SUB"));
-
-    }
-}
diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommandTest.java
similarity index 67%
rename from tools/src/test/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommandTest.java
rename to tools/src/test/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommandTest.java
index 33e40a9cb81..0c23787709b 100644
--- a/tools/src/test/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommandTest.java
+++ b/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommandTest.java
@@ -14,7 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.tools.command.acl;
+
+package org.apache.rocketmq.tools.command.consumer;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.DefaultParser;
@@ -22,19 +23,23 @@
 import org.apache.rocketmq.srvutil.ServerUtil;
 import org.junit.Test;
 
-import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
 
-public class UpdateGlobalWhiteAddrSubCommandTest {
+public class UpdateSubGroupListSubCommandTest {
 
     @Test
-    public void testExecute() {
-        UpdateGlobalWhiteAddrSubCommand cmd = new UpdateGlobalWhiteAddrSubCommand();
+    public void testArguments() {
+        UpdateSubGroupListSubCommand cmd = new UpdateSubGroupListSubCommand();
         Options options = ServerUtil.buildCommandlineOptions(new Options());
-        String[] subargs = new String[] {"-g 10.10.103.*,192.168.0.*", "-c default-cluster"};
+
+        String brokerAddress = "127.0.0.1:10911";
+        String inputFileName = "groups.json";
+        String[] args = new String[] {"-b " + brokerAddress, "-f " + inputFileName};
         final CommandLine commandLine =
-            ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs,
+            ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), args,
                 cmd.buildCommandlineOptions(options), new DefaultParser());
-        assertThat(commandLine.getOptionValue('g').trim()).isEqualTo("10.10.103.*,192.168.0.*");
-        assertThat(commandLine.getOptionValue('c').trim()).isEqualTo("default-cluster");
+
+        assertEquals(brokerAddress, commandLine.getOptionValue('b').trim());
+        assertEquals(inputFileName, commandLine.getOptionValue('f').trim());
     }
-}
+}
\ No newline at end of file
diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java
index b722677daed..b24bd22db8f 100644
--- a/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java
+++ b/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java
@@ -122,12 +122,12 @@ public void before() throws NoSuchFieldException, IllegalAccessException, Interr
         retMsgExt.setReconsumeTimes(2);
         retMsgExt.setBornTimestamp(System.currentTimeMillis());
         retMsgExt.setStoreTimestamp(System.currentTimeMillis());
-        when(mQAdminImpl.viewMessage(anyString())).thenReturn(retMsgExt);
+        when(mQAdminImpl.viewMessage(anyString(), anyString())).thenReturn(retMsgExt);
 
         when(mQAdminImpl.queryMessageByUniqKey(anyString(), anyString())).thenReturn(retMsgExt);
 
         QueryResult queryResult = new QueryResult(0, Lists.newArrayList(retMsgExt));
-        when(defaultMQAdminExtImpl.queryMessageByUniqKey(anyString(), anyString(), anyInt(), anyLong(), anyLong())).thenReturn(queryResult);
+        when(mQAdminImpl.queryMessageByUniqKey(anyString(), anyString(), anyString(), anyInt(), anyLong(), anyLong())).thenReturn(queryResult);
 
         TopicRouteData topicRouteData = new TopicRouteData();
         List<BrokerData> brokerDataList = new ArrayList<>();
@@ -194,7 +194,7 @@ public void testExecuteConsumeActively() throws SubCommandException, Interrupted
 
         Options options = ServerUtil.buildCommandlineOptions(new Options());
 
-        String[] args = new String[] {"-t myTopicTest", "-i msgId"};
+        String[] args = new String[] {"-t myTopicTest", "-i msgId", "-c DefaultCluster"};
         CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin ", args,
             cmd.buildCommandlineOptions(options), new DefaultParser());
         cmd.execute(commandLine, options, null);
@@ -218,7 +218,7 @@ public void testExecuteConsumePassively() throws SubCommandException, Interrupte
 
         Options options = ServerUtil.buildCommandlineOptions(new Options());
 
-        String[] args = new String[] {"-t myTopicTest", "-i 7F000001000004D20000000000000066"};
+        String[] args = new String[] {"-t myTopicTest", "-i 7F000001000004D20000000000000066", "-c DefaultCluster"};
         CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin ", args,
             cmd.buildCommandlineOptions(options), new DefaultParser());
         cmd.execute(commandLine, options, null);
@@ -230,7 +230,7 @@ public void testExecuteWithConsumerGroupAndClientId() throws SubCommandException
 
         Options options = ServerUtil.buildCommandlineOptions(new Options());
 
-        String[] args = new String[] {"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-g producerGroupName", "-d clientId"};
+        String[] args = new String[] {"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-g producerGroupName", "-d clientId", "-c DefaultCluster"};
         CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin ", args,
             cmd.buildCommandlineOptions(options), new DefaultParser());
         cmd.execute(commandLine, options, null);
@@ -241,13 +241,13 @@ public void testExecute() throws SubCommandException {
 
         System.setProperty("rocketmq.namesrv.addr", "127.0.0.1:9876");
 
-        String[] args = new String[]{"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000"};
+        String[] args = new String[]{"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-c DefaultCluster"};
         Options options = ServerUtil.buildCommandlineOptions(new Options());
         CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin ", args,
             cmd.buildCommandlineOptions(options), new DefaultParser());
         cmd.execute(commandLine, options, null);
 
-        args = new String[] {"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-g producerGroupName", "-d clientId"};
+        args = new String[] {"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-g producerGroupName", "-d clientId", "-c DefaultCluster"};
         commandLine = ServerUtil.parseCmdLine("mqadmin ", args, cmd.buildCommandlineOptions(options),
             new DefaultParser());
         cmd.execute(commandLine, options, null);
diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommandTest.java
similarity index 71%
rename from tools/src/test/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommandTest.java
rename to tools/src/test/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommandTest.java
index 4298c78c38b..71704a7aa8c 100644
--- a/tools/src/test/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommandTest.java
+++ b/tools/src/test/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommandTest.java
@@ -14,7 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.tools.command.acl;
+
+package org.apache.rocketmq.tools.command.topic;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.DefaultParser;
@@ -22,18 +23,19 @@
 import org.apache.rocketmq.srvutil.ServerUtil;
 import org.junit.Test;
 
-import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
-public class ClusterAclConfigVersionListSubCommandTest {
+public class UpdateTopicListSubCommandTest {
 
     @Test
-    public void testExecute() {
-        ClusterAclConfigVersionListSubCommand cmd = new ClusterAclConfigVersionListSubCommand();
+    public void testArguments() {
+        UpdateTopicListSubCommand cmd = new UpdateTopicListSubCommand();
         Options options = ServerUtil.buildCommandlineOptions(new Options());
-        String[] subargs = new String[] {"-c default-cluster"};
+        String[] subargs = new String[] {"-b 127.0.0.1:10911", "-f topics.json"};
         final CommandLine commandLine =
             ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs,
                 cmd.buildCommandlineOptions(options), new DefaultParser());
-        assertThat(commandLine.getOptionValue('c').trim()).isEqualTo("default-cluster");
+        assertEquals("127.0.0.1:10911", commandLine.getOptionValue('b').trim());
+        assertEquals("topics.json", commandLine.getOptionValue('f').trim());
     }
-}
+}
\ No newline at end of file