Skip to content

BIGTOP-4187: Upload tez.tar.tz to hdfs after installed #42

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kevinw66 merged 1 commit into from
Aug 14, 2024
Merged

BIGTOP-4187: Upload tez.tar.tz to hdfs after installed #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 39 additions & 2 deletions ...ger-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import lombok.Data;
import lombok.extern.slf4j.Slf4j;

import java.io.File;
import java.net.URI;
import java.security.PrivilegedAction;
import java.text.MessageFormat;
Expand All @@ -38,6 +39,13 @@
@Data
@Slf4j
public class HdfsUtil {

/**
* Create directory on hdfs if not exist
*
* @param user the system user to create the directory, which will infect the directory permission
* @param directory the directory path on hdfs
*/
public static void createDirectory(String user, String directory) {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
try {
Expand All @@ -46,6 +54,7 @@ public static void createDirectory(String user, String directory) {
// Create dest dir if not exist
Path destDirPath = new Path(directory);
if (!fs.exists(destDirPath)) {
log.info("Creating directory [{}] on hdfs", destDirPath);
fs.mkdirs(destDirPath);
}
} catch (Exception e) {
Expand All @@ -61,10 +70,25 @@ public static void createDirectory(String user, String directory) {
}
}

/**
* Upload file to hdfs, this will keep original filename on hdfs
*
* @param user the system user to upload the file, which will infect the file permission
* @param localFilePath the local file path
* @param destDir the destination directory on hdfs
*/
public static void uploadFile(String user, String localFilePath, String destDir) {
uploadFile(user, localFilePath, destDir, null);
}

/**
* Upload file to hdfs
*
* @param user the system user to upload the file, which will infect the file permission
* @param localFilePath the local file path
* @param destDir the destination directory on hdfs
* @param destFilename the destination filename on hdfs, if null, use the original filename
*/
public static void uploadFile(String user, String localFilePath, String destDir, String destFilename) {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
try {
Expand All @@ -73,12 +97,19 @@ public static void uploadFile(String user, String localFilePath, String destDir,
// Create dest dir if not exist
Path destDirPath = new Path(destDir);
if (!fs.exists(destDirPath)) {
log.info("Creating directory [{}] on hdfs", destDirPath);
fs.mkdirs(destDirPath);
}

// upload file
Path destFilePath = destFilename == null ? new Path(destDir) : new Path(destDir, destFilename);
fs.copyFromLocalFile(new Path(localFilePath), destFilePath);
String filename = destFilename == null
? localFilePath.substring(localFilePath.lastIndexOf(File.separator) + 1)
: destFilename;
Path destFilePath = new Path(destDir, filename);
if (!fs.exists(destFilePath)) {
log.info("Uploading [{}] to hdfs [{}]", localFilePath, destFilePath);
fs.copyFromLocalFile(new Path(localFilePath), destFilePath);
}
} catch (Exception e) {
log.error("Error while uploading file to hdfs", e);
throw new StackException(e);
Expand All @@ -92,6 +123,12 @@ public static void uploadFile(String user, String localFilePath, String destDir,
}
}

/**
* Get the hdfs FileSystem instance
*
* @return the hdfs FileSystem instance
* @throws Exception if any error occurs
*/
private static FileSystem getFileSystem() throws Exception {
Configuration conf = new Configuration();
conf.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
Expand Down
3 changes: 0 additions & 3 deletions ...ck-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,6 @@ public static ShellResult config(Params params, String componentName) {
Constants.PERMISSION_644,
hdfsParams.getGlobalParamsMap());

// log.info("Creating /apps on hdfs");
// HdfsUtil.createDirectory(hdfsUser, "/apps");

log.info("Successfully configured HDFS");
return ShellResult.success("HDFS Configure success!");
}
Expand Down
8 changes: 5 additions & 3 deletions ...tack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@

import org.apache.bigtop.manager.common.constants.Constants;
import org.apache.bigtop.manager.common.shell.ShellResult;
import org.apache.bigtop.manager.stack.bigtop.utils.HdfsUtil;
import org.apache.bigtop.manager.stack.core.enums.ConfigType;
import org.apache.bigtop.manager.stack.core.param.Params;
import org.apache.bigtop.manager.stack.core.utils.LocalSettings;
import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils;

import lombok.AccessLevel;
Expand All @@ -40,6 +42,7 @@ public static ShellResult config(Params params) {
TezParams tezParams = (TezParams) params;

String confDir = tezParams.confDir();
String hdfsUser = LocalSettings.users().get("hdfs");
String tezUser = tezParams.user();
String tezGroup = tezParams.group();

Expand All @@ -64,9 +67,8 @@ public static ShellResult config(Params params) {
PERMISSION_755,
tezParams.getGlobalParamsMap());

// maybe we should upload tez.tar.gz to HDFS here?
// log.info("Uploading tez.tar.gz to HDFS");
// HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/tez.tar.gz", "/apps/tez");
HdfsUtil.createDirectory(hdfsUser, "/apps");
HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/lib/tez.tar.gz", "/apps/tez");

log.info("Successfully configured Tez");
return ShellResult.success("Tez Configure success!");
Expand Down
9 changes: 9 additions & 0 deletions ...stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.bigtop.manager.common.message.entity.pojo.CustomCommandInfo;
import org.apache.bigtop.manager.common.shell.ShellResult;
import org.apache.bigtop.manager.common.utils.CaseUtils;
import org.apache.bigtop.manager.common.utils.Environments;
import org.apache.bigtop.manager.stack.core.exception.StackException;
import org.apache.bigtop.manager.stack.core.param.Params;
import org.apache.bigtop.manager.stack.core.spi.PrioritySPIFactory;
Expand Down Expand Up @@ -67,13 +68,21 @@ private static Script getCustomScript(String customCommand, List<CustomCommandIn
}

private static void runBeforeHook(String command, Params params) {
if (Environments.isDevMode()) {
return;
}

Hook hook = HOOK_MAP.get(command.toLowerCase());
if (hook != null) {
hook.before(params);
}
}

private static void runAfterHook(String command, Params params) {
if (Environments.isDevMode()) {
return;
}

Hook hook = HOOK_MAP.get(command.toLowerCase());
if (hook != null) {
hook.after(params);
Expand Down
4 changes: 2 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@
<java.version>17</java.version>
<flatten-maven-plugin.version>1.4.0</flatten-maven-plugin.version>
<maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version>
<maven-assembly-plugin.version>3.6.0</maven-assembly-plugin.version>
<maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
<build-helper-maven-plugin.version>3.6.0</build-helper-maven-plugin.version>
<maven-jar-plugin.version>3.2.0</maven-jar-plugin.version>
<spring-boot-maven-plugin.version>3.1.1</spring-boot-maven-plugin.version>
<spotless-maven-plugin.version>2.43.0</spotless-maven-plugin.version>
Expand Down Expand Up @@ -177,7 +177,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>${maven-assembly-plugin.version}</version>
<version>${build-helper-maven-plugin.version}</version>
</plugin>

<plugin>
Expand Down