fix(taskframework): running task be canceled incorrect due to heartbeat timeout (#2763)

* add stop and failed

* format code

* rename method

* fix for comment

Author: krihy

server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskMonitor.java

@@ -69,7 +69,7 @@ public void monitor() {
         this.reportScheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory);
         reportScheduledExecutor.scheduleAtFixedRate(() -> {
             if (isTimeout() && !getTask().getStatus().isTerminated()) {
-                getTask().stop();
+                getTask().abort();
             }
             try {
                 if (JobUtils.getExecutorPort().isPresent()) {

server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/BaseTask.java

@@ -71,12 +71,21 @@ public void start(JobContext context) {
 
     @Override
     public boolean stop() {
+        return stop(JobStatus.CANCELED);
+    }
+
+    @Override
+    public boolean abort() {
+        return stop(JobStatus.FAILED);
+    }
+
+    private synchronized boolean stop(JobStatus status) {
         try {
             if (getStatus().isTerminated()) {
-                log.warn("Task is already finished and cannot be canceled, id={}, status={}.", getJobId(), getStatus());
+                log.warn("Task is already finished and cannot be stopped, id={}, status={}.", getJobId(), getStatus());
             } else {
                 doStop();
-                updateStatus(JobStatus.CANCELED);
+                updateStatus(status);
             }
             return true;
         } catch (Throwable e) {

server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/Task.java

@@ -21,6 +21,7 @@
 import com.oceanbase.odc.service.task.caller.JobContext;
 import com.oceanbase.odc.service.task.enums.JobStatus;
 import com.oceanbase.odc.service.task.executor.server.TaskExecutor;
+import com.oceanbase.odc.service.task.executor.server.TaskMonitor;
 
 /**
  * Task interface. Each task should implement this interface
@@ -40,6 +41,13 @@ public interface Task<RESULT> {
      */
     boolean stop();
 
+
+    /**
+     * Stop current task and set status to failed. This method will be called by {@link TaskMonitor} for
+     * stop a timeout task
+     */
+    boolean abort();
+
     /**
      * Modify current task parameters
      */

server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java

@@ -305,6 +305,10 @@ public void handleResult(TaskResult taskResult) {
             log.warn("Job identity is null");
             return;
         }
+        if (taskResult.getStatus() == JobStatus.CANCELED) {
+            log.warn("Job is canceled by odc server, this result is ignored.");
+            return;
+        }
         JobEntity je = find(taskResult.getJobIdentity().getId());
         if (je == null) {
             log.warn("Job identity is not exists by id {}", taskResult.getJobIdentity().getId());