feat(database-change): database change task adapt streaming read sql file

.github/workflows/build_image_for_test.yaml

@@ -0,0 +1,136 @@
+###
+# This workflow is used for testing new task framework
+###
+
+name: Build Docker Image For Test
+run-name: ${{ github.event_name }} triggered by ${{ github.actor }} 🚀
+
+on:
+  pull_request:
+    branches:
+      - "**"
+
+jobs:
+  calculate-version:
+    name: Calculate Version Number
+    runs-on: ubuntu-latest
+    outputs:
+      odc_rpm_release_number: ${{ steps.calculate_version.outputs.odc_rpm_release_number }}
+      odc_docker_image_tag: ${{ steps.calculate_version.outputs.odc_docker_image_tag }}
+    steps:
+      - name: Checkout workspace
+        uses: actions/checkout@v3
+      - name: Calculate version number
+        id: calculate_version
+        run: |
+          odc_rpm_release_number=$(date +%Y%m%d%H%M%S)
+          echo "odc_rpm_release_number=${odc_rpm_release_number}" >> $GITHUB_OUTPUT
+          echo "odc_rpm_release_number=${odc_rpm_release_number}"
+          odc_docker_image_tag="latest"
+          echo "odc_docker_image_tag=${odc_docker_image_tag}" >> $GITHUB_OUTPUT
+          echo "odc_docker_image_tag=${odc_docker_image_tag}"
+
+  build-docker-x86_64:
+    name: Build Docker Image (x86_64)
+    needs: [ calculate-version ]
+    runs-on: ubuntu-latest
+    env:
+      odc_rpm_release_number: ${{ needs.calculate-version.outputs.odc_rpm_release_number }}
+      odc_docker_image_tag: ${{ needs.calculate-version.outputs.odc_docker_image_tag }}
+    steps:
+      - name: Checkout workspace
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+      - name: Setup JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: "8"
+          distribution: "temurin"
+          cache: maven
+      - name: Setup node 16
+        uses: actions/setup-node@v3
+        with:
+          node-version: "16"
+      - name: Build front static resources
+        run: |
+          echo "Current directory: "`pwd`
+          echo "Start build front static resources"
+          pushd client
+          echo "Run npm install pnpm -g"
+          npm install pnpm -g
+          echo "Run pnpm install"
+          pnpm install
+          echo "Run npm run build:odc"
+          npm run build:odc
+          popd
+          echo "Build front static resources success"
+          echo "Start copy resources files"
+          static_resources_path="server/odc-server/src/main/resources/static"
+          if [ ! -d "${static_resources_path}" ]; then
+            echo "mkdir -p ${static_resources_path}"
+            mkdir -p "${static_resources_path}"
+          fi
+          rm --force --recursive --verbose ${static_resources_path}/*
+          cp --force --recursive --verbose client/dist/renderer/* ${static_resources_path}
+          echo "Copy resources files success"
+      - name: Set release version
+        id: set_release_version
+        run: |
+          main_version="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout | cut -d - -f 1)"
+          new_version="${main_version}-${odc_rpm_release_number}"
+          echo "new_version=${new_version}" >> $GITHUB_OUTPUT
+          echo "RPM's version is "${new_version}
+          mvn versions:set -DnewVersion="${new_version}"
+          mvn versions:commit
+      - name: Install ob-sql-parser
+        run: |
+          echo "Start install ob-sql-parser"
+          pushd libs/ob-sql-parser
+          echo "Current dir is "`pwd`
+          mvn clean install -Dmaven.test.skip=true
+          echo "Install ob-sql-parser success"
+          popd
+      - name: Install db-browser
+        run: |
+          echo "Start install db-browser"
+          pushd libs/db-browser
+          echo "Current dir is "`pwd`
+          mvn clean install -Dmaven.test.skip=true
+          echo "Install db-browser success"
+          popd
+      - name: Build rpm (x86_64)
+        run: |
+          echo "Start prepare oceanbase-client"
+          pushd import
+          echo "Current dir is "`pwd`
+          cp ../build-resource/obclient/2.2.4/linux_x86/obclient.tar.gz obclient.tar.gz
+          popd
+          echo "Prepare oceanbase-client success"
+          echo "Start build rpm package"
+          mvn help:system
+          mvn clean install -Dmaven.test.skip=true
+          mvn --file server/odc-server/pom.xml rpm:rpm -Drpm.prefix=/opt
+          echo "Build rpm package success"
+          rm --force --recursive --verbose distribution/docker/resources/odc-*.rpm
+          mkdir -p distribution/docker/resources/
+          mv --verbose server/odc-server/target/rpm/odc-server/RPMS/*/odc-*.rpm distribution/docker/resources/
+      - name: Build docker image (x86_64)
+        run: |
+          sed -e "s/DATE_CHANGE/$(date)/" -i distribution/docker/odc/Dockerfile
+          echo "odc_docker_image_tag=${odc_docker_image_tag}"
+          pushd distribution/docker
+          docker build -t docker.io/oceanbase/odc:${odc_docker_image_tag} -f odc/Dockerfile .
+          docker save -o resources/odc-${odc_docker_image_tag}.tar.gz docker.io/oceanbase/odc:${odc_docker_image_tag}
+          popd
+      - name: Push docker image (x86_64)
+        run: |
+          docker login -p${{ secrets.DOCKER_PASSWORD }} -u${{ secrets.DOCKER_USERNAME }}
+          docker load -i distribution/docker/resources/odc-${odc_docker_image_tag}.tar.gz
+          docker tag oceanbase/odc:${odc_docker_image_tag} ${{ secrets.DOCKER_USERNAME }}/odc:${odc_docker_image_tag}
+          docker push ${{ secrets.DOCKER_USERNAME }}/odc:${odc_docker_image_tag}
+      - name: Upload docker image (x86_64)
+        uses: actions/upload-artifact@v3
+        with:
+          name: odc-${{ env.odc_docker_image_tag }}.tar.gz
+          path: distribution/docker/resources/odc-*.tar.gz

server/integration-test/src/test/java/com/oceanbase/odc/service/task/BaseJobTest.java

@@ -23,6 +23,7 @@
 import org.junit.BeforeClass;
 import org.mockito.Mockito;
 
+import com.oceanbase.odc.common.security.PasswordUtils;
 import com.oceanbase.odc.service.common.model.HostProperties;
 import com.oceanbase.odc.service.task.caller.K8sJobClient;
 import com.oceanbase.odc.service.task.caller.NativeK8sJobClient;
@@ -67,6 +68,8 @@ public static void init() throws IOException {
         System.setProperty(JobEnvKeyConstants.ODC_BOOT_MODE, JobConstants.ODC_BOOT_MODE_EXECUTOR);
         System.setProperty(JobEnvKeyConstants.ODC_TASK_RUN_MODE, TaskRunModeEnum.K8S.name());
         System.setProperty(JobEnvKeyConstants.ODC_SERVER_PORT, "8990");
+        System.setProperty(JobEnvKeyConstants.ENCRYPT_KEY, PasswordUtils.random(32));
+        System.setProperty(JobEnvKeyConstants.ENCRYPT_SALT, PasswordUtils.random(8));
 
 
         DefaultJobConfiguration jc = new DefaultJobConfiguration() {};

server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java

@@ -24,6 +24,8 @@
 import org.junit.Test;
 
 import com.oceanbase.odc.TestConnectionUtil;
+import com.oceanbase.odc.common.crypto.Encryptors;
+import com.oceanbase.odc.common.crypto.TextEncryptor;
 import com.oceanbase.odc.common.json.JsonUtils;
 import com.oceanbase.odc.core.shared.PreConditions;
 import com.oceanbase.odc.core.shared.constant.ConnectType;
@@ -63,7 +65,9 @@ public void test_executeDatabaseChangeTask_run() {
 
         JobDefinition jd = buildJobDefinition();
         JobContext jc = new DefaultJobContextBuilder().build(jobIdentity, jd);
-        System.setProperty(JobEnvKeyConstants.ODC_JOB_CONTEXT, JobUtils.toJson(jc));
+        TextEncryptor textEncryptor = Encryptors.aesBase64(System.getProperty(JobEnvKeyConstants.ENCRYPT_KEY),
+                System.getProperty(JobEnvKeyConstants.ENCRYPT_SALT));
+        System.setProperty(JobEnvKeyConstants.ODC_JOB_CONTEXT, textEncryptor.encrypt(JobUtils.toJson(jc)));
         startTaskApplication();
         assertRunningResult(jc);
     }
@@ -75,15 +79,17 @@ public void test_executeDatabaseChangeTask_stop() {
 
         JobDefinition jd = buildJobDefinition();
         JobContext jc = new DefaultJobContextBuilder().build(jobIdentity, jd);
-        System.setProperty(JobEnvKeyConstants.ODC_JOB_CONTEXT, JobUtils.toJson(jc));
+        TextEncryptor textEncryptor = Encryptors.aesBase64(System.getProperty(JobEnvKeyConstants.ENCRYPT_KEY),
+                System.getProperty(JobEnvKeyConstants.ENCRYPT_SALT));
+        System.setProperty(JobEnvKeyConstants.ODC_JOB_CONTEXT, textEncryptor.encrypt(JobUtils.toJson(jc)));
         startTaskApplication();
         assertCancelResult(jc);
     }
 
     private void assertRunningResult(JobContext jc) {
 
         try {
-            Thread.sleep(600 * 1000L);
+            Thread.sleep(30 * 1000L);
             Task<?> task = ThreadPoolTaskExecutor.getInstance().getTask(jc.getJobIdentity());
             Assert.assertSame(JobStatus.DONE, task.getStatus());
         } catch (InterruptedException e) {
@@ -95,7 +101,7 @@ private void assertRunningResult(JobContext jc) {
     private void assertCancelResult(JobContext jc) {
 
         try {
-            Thread.sleep(5 * 1000L);
+            Thread.sleep(10 * 1000L);
             boolean result = ThreadPoolTaskExecutor.getInstance().cancel(jc.getJobIdentity());
             Assert.assertTrue(result);
         } catch (InterruptedException e) {