Skip to content

BIGTOP-4146: Reorganize task log files #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kevinw66 merged 2 commits into from
Jun 24, 2024
Merged

BIGTOP-4146: Reorganize task log files #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7500084
BIGTOP-4146: Reorganize task log files
kevinw66 Jun 20, 2024
7efcb9e
rollback mysql config
kevinw66 Jun 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ build/

### Log ###
logs
tasklogs
*.log
npm-debug.log*
yarn-debug.log*
Expand Down
12 changes: 0 additions & 12 deletions ...agent/src/main/java/org/apache/bigtop/manager/agent/executor/AbstractCommandExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

import org.apache.bigtop.manager.agent.holder.SpringContextHolder;
import org.apache.bigtop.manager.common.constants.MessageConstants;
import org.apache.bigtop.manager.common.message.entity.command.CommandLogMessage;
import org.apache.bigtop.manager.common.message.entity.command.CommandRequestMessage;
import org.apache.bigtop.manager.common.message.entity.command.CommandResponseMessage;
import org.apache.bigtop.manager.common.shell.ShellResult;
Expand Down Expand Up @@ -68,15 +67,4 @@ protected void doExecuteOnDevMode() {
}

protected abstract void doExecute();

protected void writeBackCommandLog(String log) {
CommandLogMessage logMessage = new CommandLogMessage();
logMessage.setLog(log);
logMessage.setMessageId(commandRequestMessage.getMessageId());
logMessage.setHostname(commandRequestMessage.getHostname());
logMessage.setTaskId(commandRequestMessage.getTaskId());
logMessage.setStageId(commandRequestMessage.getStageId());
logMessage.setJobId(commandRequestMessage.getJobId());
SpringContextHolder.getAgentWebSocket().sendMessage(logMessage);
}
}
2 changes: 1 addition & 1 deletion ...gent/src/main/java/org/apache/bigtop/manager/agent/executor/ComponentCommandExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public void doExecute() {
CommandPayload commandPayload =
JsonUtils.readFromString(commandRequestMessage.getMessagePayload(), CommandPayload.class);
log.info("[agent executeTask] taskEvent is: {}", commandRequestMessage);
ShellResult shellResult = StackExecutor.execute(commandPayload, this::writeBackCommandLog);
ShellResult shellResult = StackExecutor.execute(commandPayload);

commandResponseMessage.setCode(shellResult.getExitCode());
commandResponseMessage.setResult(shellResult.getResult());
Expand Down
4 changes: 4 additions & 0 deletions ...gent/src/main/java/org/apache/bigtop/manager/agent/scheduler/DefaultCommandScheduler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.bigtop.manager.agent.executor.CommandExecutors;
import org.apache.bigtop.manager.common.message.entity.command.CommandRequestMessage;

import org.slf4j.MDC;
import org.springframework.stereotype.Component;

import lombok.extern.slf4j.Slf4j;
Expand All @@ -45,10 +46,13 @@ public class DefaultCommandScheduler implements CommandScheduler {
public void submit(CommandRequestMessage message) {
queue.offer(() -> {
try {
MDC.put("taskId", message.getTaskId().toString());
CommandExecutor commandExecutor = CommandExecutors.getCommandExecutor(message.getCommandMessageType());
commandExecutor.execute(message);
} catch (Exception e) {
log.error("Error when running command", e);
} finally {
MDC.clear();
}
});
}
Expand Down
76 changes: 29 additions & 47 deletions bigtop-manager-agent/src/main/resources/logback-spring.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,76 +19,58 @@
-->

<configuration scan="true" scanPeriod="120 seconds">
<property name="log.base" value="logs"/>
<property scope="context" name="log.base.ctx" value="${log.base}"/>
<property name="log.base" value="logs" />
<property scope="context" name="log.base.ctx" value="${log.base}" />

<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>[%-5level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} [%thread] %logger{96}:[%line] - %msg%n</pattern>
<pattern>[%-5level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} [%thread] %logger{1}:[%line] - %m%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>

<appender name="WSLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/ws.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${log.base}/ws.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<maxHistory>30</maxHistory>
<maxFileSize>200MB</maxFileSize>
<totalSizeCap>10GB</totalSizeCap>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder>
<pattern>[%-5level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} [%thread] %logger{96}:[%line] - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<appender name="STACKLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/bigtop-manager-stack.log</file>
<appender name="AGENT_LOG_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/bigtop-manager-agent.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${log.base}/bigtop-manager-stack.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<fileNamePattern>${log.base}/bigtop-manager-agent.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<maxHistory>30</maxHistory>
<maxFileSize>200MB</maxFileSize>
<totalSizeCap>10GB</totalSizeCap>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder>
<pattern>[%-5level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} [%thread] %logger{96}:[%line] - %msg%n</pattern>
<pattern>[%-5level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} [%thread] %logger{1}:[%line] - %m%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<appender name="AGENTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/bigtop-manager-agent.log</file>

<appender name="TASK_LOG_FILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="ch.qos.logback.core.filter.EvaluatorFilter">
<evaluator>
<expression>logger.contains("org.apache.bigtop.manager.stack")</expression>
<expression>mdc.get("taskId") != null</expression>
</evaluator>
<onMatch>DENY</onMatch>
<onMismatch>ACCEPT</onMismatch>
<onMatch>ACCEPT</onMatch>
<onMismatch>DENY</onMismatch>
</filter>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${log.base}/bigtop-manager-agent.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<maxHistory>30</maxHistory>
<maxFileSize>200MB</maxFileSize>
<totalSizeCap>10GB</totalSizeCap>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder>
<pattern>[%-5level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} [%thread] %logger{96}:[%line] - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
<discriminator class="ch.qos.logback.classic.sift.MDCBasedDiscriminator">
<key>taskId</key>
<defaultValue>0</defaultValue>
</discriminator>
<sift>
<appender name="task-${taskId}" class="ch.qos.logback.core.FileAppender">
<file>tasklogs/task-${taskId}.log</file>
<encoder>
<pattern>[%-5level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} - %m%n</pattern>
<charset>UTF-8</charset>
</encoder>
<append>true</append>
</appender>
</sift>
</appender>

<logger name="org.apache.bigtop.manager.agent.ws.AgentWebSocketHandler" level="INFO" additivity="true">
<appender-ref ref="WSLOGFILE"/>
</logger>

<logger name="org.apache.bigtop.manager.stack" level="INFO" additivity="true">
<appender-ref ref="STACKLOGFILE"/>
</logger>

<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="AGENTLOGFILE"/>
<appender-ref ref="STDOUT" />
<appender-ref ref="AGENT_LOG_FILE" />
<appender-ref ref="TASK_LOG_FILE" />
</root>
</configuration>
45 changes: 0 additions & 45 deletions .../main/java/org/apache/bigtop/manager/common/message/entity/command/CommandLogMessage.java
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions ...mon/src/main/java/org/apache/bigtop/manager/common/message/serializer/KryoPoolHolder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import org.apache.bigtop.manager.common.message.entity.BaseRequestMessage;
import org.apache.bigtop.manager.common.message.entity.BaseResponseMessage;
import org.apache.bigtop.manager.common.message.entity.HeartbeatMessage;
import org.apache.bigtop.manager.common.message.entity.command.CommandLogMessage;
import org.apache.bigtop.manager.common.message.entity.command.CommandMessageType;
import org.apache.bigtop.manager.common.message.entity.command.CommandRequestMessage;
import org.apache.bigtop.manager.common.message.entity.command.CommandResponseMessage;
Expand Down Expand Up @@ -62,7 +61,6 @@ protected Kryo create() {
kryo.register(HeartbeatMessage.class);
kryo.register(CommandResponseMessage.class);
kryo.register(CommandRequestMessage.class);
kryo.register(CommandLogMessage.class);

// message pojo
kryo.register(HostInfo.class);
Expand Down
81 changes: 0 additions & 81 deletions bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/entity/CommandLog.java
View file
Open in desktop

This file was deleted.

28 changes: 0 additions & 28 deletions ...ager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/CommandLogRepository.java
View file
Open in desktop

This file was deleted.

2 changes: 2 additions & 0 deletions .../main/java/org/apache/bigtop/manager/server/command/stage/runner/AbstractStageRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,10 @@ public void run() {
boolean taskSuccess = res != null && res.getCode() == MessageConstants.SUCCESS_CODE;

if (taskSuccess) {
commandLogService.onLogReceived(task.getId(), task.getHostname(), "Success!");
onTaskSuccess(task);
} else {
commandLogService.onLogReceived(task.getId(), task.getHostname(), "Failed!");
onTaskFailure(task);
}

Expand Down
Loading