Skip to content

Commit 396aa51

Browse files
fenzhuGitHub Enterprise
authored andcommitted
[CARMEL-6085] Show metrics after running analyze event command (#1012)
1 parent 1e14765 commit 396aa51

1 file changed

Lines changed: 11 additions & 2 deletions

File tree

sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzeEventCommand.scala

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,11 @@ case class AnalyzeEventCommand(file: String, prefix: String,
5757

5858
override lazy val metrics: Map[String, SQLMetric] = getMetrics
5959

60+
override val output: Seq[Attribute] = {
61+
AttributeReference("analyze_event_item", StringType, nullable = false)() ::
62+
AttributeReference("analyze_event_value", StringType, nullable = false)() :: Nil
63+
}
64+
6065
private def getMetrics: Map[String, SQLMetric] = {
6166
Map[String, SQLMetric](
6267
BasicWriteJobStatsTracker.NUM_FILES_KEY ->
@@ -75,6 +80,8 @@ case class AnalyzeEventCommand(file: String, prefix: String,
7580
throw new SparkException(s"Analyze event command should only be executed in " +
7681
s"reserved-test queue, current queue is $currentQueue.")
7782
}
83+
val result = new ArrayBuffer[Row]
84+
result += Row("Input Parameter", s"file=$file, prefix=$prefix, from=$from, to=$to")
7885

7986
val eventLogDir = sparkSession.sparkContext.getConf.get(EVENT_LOG_DIR)
8087
val hadoopConf = sparkSession.sessionState.newHadoopConf()
@@ -85,7 +92,9 @@ case class AnalyzeEventCommand(file: String, prefix: String,
8592
val candidateLogFiles: Seq[FileStatus] = AnalyzeEventCommand.getCandidateEventLogFiles(
8693
sparkSession, fs, qualifiedEventLogParentPath, file, prefix, from, to)
8794
logInfo(s"Analyze ${candidateLogFiles.length} Candidate EventLog Files.")
88-
if (candidateLogFiles.isEmpty) return ArrayBuffer.empty
95+
result += Row("Analyzed Files", s"${candidateLogFiles.length}")
96+
if (candidateLogFiles.isEmpty) return result
97+
8998
val skipTaskParsing = sparkSession.sessionState.conf.getConfString(
9099
"spark.sql.workload.task.parse.skip", "true").toBoolean
91100
val maxPartition = sparkSession.sessionState.conf.getConfString(
@@ -130,7 +139,7 @@ case class AnalyzeEventCommand(file: String, prefix: String,
130139
rdd.unpersist()
131140
sparkContext.setJobDescription(previousDescription)
132141
}
133-
Seq.empty[Row]
142+
result
134143
}
135144

136145
private def analyzeHugeLog(pathStr: String, task_event_count: Long, partitions: Int,

0 commit comments

Comments
 (0)