include MetricsUnavailable condition to Complete in Trial

It is not easy for users to find why Trial failed when training code output incorrect format logs
since the trial-controller sets Succeeded condition with Failed to Trial if there are unavailable metrics in Katib DB as described in #1343.
So we also include MetricsUnavailable condition to Complete in Trial.

Author: tenzen-y

Makefile

@@ -24,7 +24,6 @@ ifndef HAS_SETUP_ENVTEST
 endif
 	echo "setup-envtest has already installed"
 
-
 check: generate fmt vet lint
 
 fmt:

pkg/apis/controller/experiments/v1beta1/experiment_types.go

@@ -100,6 +100,9 @@ type ExperimentStatus struct {
 	// List of trial names which have been early stopped.
 	EarlyStoppedTrialList []string `json:"earlyStoppedTrialList,omitempty"`
 
+	// List of trial names which have been metrics unavailable
+	MetricsUnavailableTrialList []string `json:"metricsUnavailableTrialList,omitempty"`
+
 	// Trials is the total number of trials owned by the experiment.
 	Trials int32 `json:"trials,omitempty"`
 
@@ -120,6 +123,9 @@ type ExperimentStatus struct {
 
 	// How many trials are currently early stopped.
 	TrialsEarlyStopped int32 `json:"trialsEarlyStopped,omitempty"`
+
+	// How many trials are currently metrics unavailable.
+	TrialMetricsUnavailable int32 `json:"trialMetricsUnavailable,omitempty"`
 }
 
 // OptimalTrial is the metrics and assignments of the best trial.

pkg/apis/controller/experiments/v1beta1/zz_generated.deepcopy.go

@@ -113,12 +113,13 @@ type TrialCondition struct {
 type TrialConditionType string
 
 const (
-	TrialCreated      TrialConditionType = "Created"
-	TrialRunning      TrialConditionType = "Running"
-	TrialSucceeded    TrialConditionType = "Succeeded"
-	TrialKilled       TrialConditionType = "Killed"
-	TrialFailed       TrialConditionType = "Failed"
-	TrialEarlyStopped TrialConditionType = "EarlyStopped"
+	TrialCreated            TrialConditionType = "Created"
+	TrialRunning            TrialConditionType = "Running"
+	TrialSucceeded          TrialConditionType = "Succeeded"
+	TrialKilled             TrialConditionType = "Killed"
+	TrialFailed             TrialConditionType = "Failed"
+	TrialMetricsUnavailable TrialConditionType = "MetricsUnavailable"
+	TrialEarlyStopped       TrialConditionType = "EarlyStopped"
 )
 
 // +genclient

pkg/apis/controller/trials/v1beta1/trial_types.go

@@ -86,15 +86,11 @@ func (trial *Trial) IsKilled() bool {
 
 // IsMetricsUnavailable returns true if Trial metrics are not available
 func (trial *Trial) IsMetricsUnavailable() bool {
-	cond := getCondition(trial, TrialSucceeded)
-	if cond != nil && cond.Status == v1.ConditionFalse {
-		return true
-	}
-	return false
+	return hasCondition(trial, TrialMetricsUnavailable)
 }
 
 func (trial *Trial) IsCompleted() bool {
-	return trial.IsSucceeded() || trial.IsFailed() || trial.IsKilled() || trial.IsEarlyStopped()
+	return trial.IsSucceeded() || trial.IsFailed() || trial.IsKilled() || trial.IsEarlyStopped() || trial.IsMetricsUnavailable()
 }
 
 func (trial *Trial) IsEarlyStopped() bool {
@@ -158,3 +154,11 @@ func (trial *Trial) MarkTrialStatusKilled(reason, message string) {
 	}
 	trial.setCondition(TrialKilled, v1.ConditionTrue, reason, message)
 }
+
+func (trial *Trial) MarkTrialStatusMetricsUnavailable(reason, message string) {
+	currentCond := getCondition(trial, TrialRunning)
+	if currentCond != nil {
+		trial.setCondition(TrialRunning, v1.ConditionFalse, currentCond.Reason, currentCond.Message)
+	}
+	trial.setCondition(TrialMetricsUnavailable, v1.ConditionTrue, reason, message)
+}

pkg/apis/controller/trials/v1beta1/util.go

@@ -226,16 +226,17 @@ message ParameterAssignment {
  * Current Trial status. It contains Trial's latest condition, start time, completion time, observation.
  */
 message TrialStatus {
-    // Trial can be in one of 6 conditions.
+    // Trial can be in one of 8 conditions.
     // TODO (andreyvelich): Remove unused conditions.
     enum TrialConditionType {
         CREATED = 0;
         RUNNING = 1;
         SUCCEEDED = 2;
         KILLED = 3;
         FAILED = 4;
-        EARLYSTOPPED = 5;
-        UNKNOWN = 6;
+        METRICSUNAVAILABLE = 5;
+        EARLYSTOPPED = 6;
+        UNKNOWN = 7;
     }
     string start_time = 1; // Trial start time in RFC3339 format
     string completion_time = 2; // Trial completion time in RFC3339 format

pkg/apis/manager/v1beta1/api.pb.go

@@ -771,7 +771,7 @@ Types of value for HyperParameter.
 <a name="api-v1-beta1-TrialStatus-TrialConditionType"></a>
 
 ### TrialStatus.TrialConditionType
-Trial can be in one of 6 conditions.
+Trial can be in one of 8 conditions.
 TODO (andreyvelich): Remove unused conditions.
 
 | Name | Number | Description |
@@ -781,8 +781,9 @@ TODO (andreyvelich): Remove unused conditions.
 | SUCCEEDED | 2 |  |
 | KILLED | 3 |  |
 | FAILED | 4 |  |
-| EARLYSTOPPED | 5 |  |
-| UNKNOWN | 6 |  |
+| METRICSUNAVAILABLE | 5 |  |
+| EARLYSTOPPED | 6 |  |
+| UNKNOWN | 7 |  |
 
 
 

pkg/apis/manager/v1beta1/api.proto

@@ -1777,7 +1777,7 @@ <h3 id="api.v1.beta1.ParameterType">ParameterType</h3>
         </table>
 
         <h3 id="api.v1.beta1.TrialStatus.TrialConditionType">TrialStatus.TrialConditionType</h3>
-        <p>Trial can be in one of 6 conditions.</p><p>TODO (andreyvelich): Remove unused conditions.</p>
+        <p>Trial can be in one of 8 conditions.</p><p>TODO (andreyvelich): Remove unused conditions.</p>
         <table class="enum-table">
           <thead>
             <tr><td>Name</td><td>Number</td><td>Description</td></tr>
@@ -1815,17 +1815,23 @@ <h3 id="api.v1.beta1.TrialStatus.TrialConditionType">TrialStatus.TrialConditionT
               </tr>
 
               <tr>
-                <td>EARLYSTOPPED</td>
+                <td>METRICSUNAVAILABLE</td>
                 <td>5</td>
                 <td><p></p></td>
               </tr>
 
               <tr>
-                <td>UNKNOWN</td>
+                <td>EARLYSTOPPED</td>
                 <td>6</td>
                 <td><p></p></td>
               </tr>
 
+              <tr>
+                <td>UNKNOWN</td>
+                <td>7</td>
+                <td><p></p></td>
+              </tr>
+            
           </tbody>
         </table>