Debug

Author: serban300

.gitlab-ci.yml

@@ -409,6 +409,7 @@ paras:
   #   - job:                         publish-docker-pr
 
   variables:
+    DEBUG_DEPTH: 10
     GH_DIR: "https://github.com/paritytech/zombienet/tree/${CI_COMMIT_SHORT_SHA}/tests/paras"
 
   before_script:

javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts

@@ -63,7 +63,7 @@ export class KubeClient extends Client {
     this.configPath = configPath;
     this.namespace = namespace;
     this.debug = true;
-    this.timeout = 300; // secs
+    this.timeout = 600; // secs
     this.tmpDir = tmpDir;
     this.localMagicFilepath = `${tmpDir}/finished.txt`;
     this.remoteDir = DEFAULT_REMOTE_DIR;
@@ -128,7 +128,7 @@ export class KubeClient extends Client {
 
     logTable.print();
 
-    await this.createResource(podDef, true, false);
+    await this.createResource(podDef, true);
     await this.wait_transfer_container(name);
 
     if (dbSnapshot) {
@@ -200,6 +200,8 @@ export class KubeClient extends Client {
       );
     }
 
+    debug("spawnFromDef: Before putLocalMagicFile");
+    await this.wait_container(name, name, "waiting");
     await this.putLocalMagicFile(name);
     await this.wait_pod_ready(name);
     logTable = new CreateLogTable({
@@ -230,7 +232,6 @@ export class KubeClient extends Client {
   async createResource(
     resourseDef: any,
     scoped: boolean = false,
-    waitReady: boolean = false,
   ): Promise<void> {
     await this.runCommand(["apply", "-f", "-"], {
       resourceDef: JSON.stringify(resourseDef),
@@ -240,27 +241,6 @@ export class KubeClient extends Client {
     debug(resourseDef);
     const name = resourseDef.metadata.name;
     const kind: string = resourseDef.kind.toLowerCase();
-
-    if (waitReady) {
-      // loop until ready
-      let t = this.timeout;
-      const args = ["get", kind, name, "-o", "jsonpath={.status}"];
-      do {
-        const result = await this.runCommand(args);
-        const status = JSON.parse(result.stdout);
-        if (["Running", "Succeeded"].includes(status.phase)) return;
-
-        // check if we are waiting init container
-        for (const s of status.initContainerStatuses) {
-          if (s.name === TRANSFER_CONTAINER_NAME && s.state.running) return;
-        }
-
-        await new Promise((resolve) => setTimeout(resolve, 3000));
-        t -= 3;
-      } while (t > 0);
-
-      throw new Error(`Timeout(${this.timeout}) for ${kind} : ${name}`);
-    }
   }
 
   async wait_pod_ready(podName: string): Promise<void> {
@@ -279,27 +259,61 @@ export class KubeClient extends Client {
 
     throw new Error(`Timeout(${this.timeout}) for pod : ${podName}`);
   }
-  async wait_transfer_container(podName: string): Promise<void> {
+
+  async wait_container(
+    pod: string,
+    container: string,
+    status: string,
+  ): Promise<void> {
     // loop until ready
     let t = this.timeout;
-    const args = ["get", "pod", podName, "-o", "jsonpath={.status}"];
+    const args = ["get", "pod", pod, "-o", "jsonpath={.status}"];
     do {
+      debug("wait_container_ready: loop until ready");
       const result = await this.runCommand(args);
-      const status = JSON.parse(result.stdout);
+      const json = JSON.parse(result.stdout);
+      debug(json);
+
+      const other_result = await this.runCommand([
+        "get",
+        "pod",
+        pod,
+        "--no-headers",
+      ]);
+      debug(other_result.stdout);
 
-      // check if we are waiting init container
-      if (status.initContainerStatuses) {
-        for (const s of status.initContainerStatuses) {
-          if (s.name === TRANSFER_CONTAINER_NAME && s.state.running) return;
-        }
+      let containerStatuses = json?.containerStatuses ?? [];
+      let initContainerStatuses = json?.initContainerStatuses ?? [];
+      for (const s of containerStatuses.concat(initContainerStatuses)) {
+        if (s.name === container && s.state[status]) return;
       }
 
       await new Promise((resolve) => setTimeout(resolve, 3000));
       t -= 3;
     } while (t > 0);
 
     throw new Error(
-      `Timeout(${this.timeout}) for transfer container for pod : ${podName}`,
+      `Timeout(${this.timeout}) for ${container} container for pod : ${pod}`,
+    );
+  }
+
+  async wait_transfer_container(pod: string): Promise<void> {
+    await this.wait_container(pod, TRANSFER_CONTAINER_NAME, "running");
+
+    // loop until ready
+    let t = this.timeout;
+    const args = ["logs", "--tail=1", pod, "-c", `${TRANSFER_CONTAINER_NAME}`];
+    do {
+      const result = await this.runCommand(args);
+
+      if (result.stdout == "waiting for tar to finish") return;
+
+      await new Promise((resolve) => setTimeout(resolve, 3000));
+      t -= 3;
+    } while (t > 0);
+
+    throw new Error(
+      `Timeout(${this.timeout}) for pod : ${pod}, container: ${TRANSFER_CONTAINER_NAME}`,
     );
   }
 
@@ -386,7 +400,7 @@ export class KubeClient extends Client {
     if (unique) {
       if (container === TRANSFER_CONTAINER_NAME) {
         const args = ["cp", localFilePath, `${identifier}:${podFilePath}`];
-        if (container) args.push("-c", container);
+        args.push("-c", container);
         await this.runCommand(args);
         debug("copyFileToPod", args);
       } else {
@@ -723,7 +737,7 @@ export class KubeClient extends Client {
     opts?: RunCommandOptions,
   ): Promise<RunCommandResponse> {
     try {
-      const augmentedCmd: string[] = ["--kubeconfig", this.configPath];
+      const augmentedCmd: string[] = [];
       if (opts?.scoped === undefined || opts?.scoped)
         augmentedCmd.push("--namespace", this.namespace);