Skip to content

Commit 83a4f04

Browse files
slfan1989slfan1989
authored
HDDS-11268. Add --table mode for OM/SCM Roles CLI (apache#7016)
(cherry picked from commit eb26677)
1 parent 4276ccf commit 83a4f04

12 files changed

Lines changed: 529 additions & 5 deletions

File tree

‎hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmInfo.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ public final class ScmInfo {
3030
private final String clusterId;
3131
private final String scmId;
3232
private final List<String> peerRoles;
33+
private final boolean scmRatisEnabled;
3334

3435
/**
3536
* Builder for ScmInfo.
@@ -38,6 +39,7 @@ public static class Builder {
3839
private String clusterId;
3940
private String scmId;
4041
private final List<String> peerRoles;
42+
private boolean scmRatisEnabled;
4143

4244
public Builder() {
4345
peerRoles = new ArrayList<>();
@@ -73,15 +75,28 @@ public Builder setRatisPeerRoles(List<String> roles) {
7375
return this;
7476
}
7577

78+
/**
79+
* Set whether SCM enables Ratis.
80+
*
81+
* @param ratisEnabled If it is true, it means that the Ratis mode is turned on.
82+
* If it is false, it means that the Ratis mode is not turned on.
83+
* @return Builder for scmInfo
84+
*/
85+
public Builder setScmRatisEnabled(boolean ratisEnabled) {
86+
scmRatisEnabled = ratisEnabled;
87+
return this;
88+
}
89+
7690
public ScmInfo build() {
77-
return new ScmInfo(clusterId, scmId, peerRoles);
91+
return new ScmInfo(clusterId, scmId, peerRoles, scmRatisEnabled);
7892
}
7993
}
8094

81-
private ScmInfo(String clusterId, String scmId, List<String> peerRoles) {
95+
private ScmInfo(String clusterId, String scmId, List<String> peerRoles, boolean ratisEnabled) {
8296
this.clusterId = clusterId;
8397
this.scmId = scmId;
8498
this.peerRoles = Collections.unmodifiableList(peerRoles);
99+
this.scmRatisEnabled = ratisEnabled;
85100
}
86101

87102
/**
@@ -107,4 +122,8 @@ public String getScmId() {
107122
public List<String> getRatisPeerRoles() {
108123
return peerRoles;
109124
}
125+
126+
public boolean getScmRatisEnabled() {
127+
return scmRatisEnabled;
128+
}
110129
}

‎hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,15 @@ StartContainerBalancerResponseProto startContainerBalancer(
392392
*/
393393
List<String> getScmRatisRoles() throws IOException;
394394

395+
/**
396+
* Get the current SCM mode.
397+
*
398+
* @return `true` indicates that it is in RATIS mode,
399+
* while `false` indicates that it is in STANDALONE mode.
400+
* @throws IOException an I/O exception of some sort has occurred.
401+
*/
402+
boolean isScmRatisEnable() throws IOException;
403+
395404
/**
396405
* Force generates new secret keys (rotate).
397406
*

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@
128128

129129
import java.io.Closeable;
130130
import java.io.IOException;
131+
import java.util.Arrays;
131132
import java.util.ArrayList;
132133
import java.util.HashMap;
133134
import java.util.List;
@@ -157,6 +158,12 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
157158
private final StorageContainerLocationProtocolPB rpcProxy;
158159
private final SCMContainerLocationFailoverProxyProvider fpp;
159160

161+
/**
162+
* This is used to check if 'leader' or 'follower' exists,
163+
* in order to confirm whether we have enabled Ratis.
164+
*/
165+
private final List<String> scmRatisRolesToCheck = Arrays.asList("leader", "follower");
166+
160167
/**
161168
* Creates a new StorageContainerLocationProtocolClientSideTranslatorPB.
162169
*
@@ -760,8 +767,23 @@ public ScmInfo getScmInfo() throws IOException {
760767
.setScmId(resp.getScmId())
761768
.setRatisPeerRoles(resp.getPeerRolesList());
762769

763-
return builder.build();
770+
// By default, we assume that SCM Ratis is not enabled.
764771

772+
// If the response contains the `ScmRatisEnabled` field,
773+
// we will set it directly; otherwise,
774+
// we will determine if Ratis is enabled based on
775+
// whether the `peerRolesList` contains the keywords 'leader' or 'follower'.
776+
if (resp.hasScmRatisEnabled()) {
777+
builder.setScmRatisEnabled(resp.getScmRatisEnabled());
778+
} else {
779+
List<String> peerRolesList = resp.getPeerRolesList();
780+
if (!peerRolesList.isEmpty()) {
781+
boolean containsScmRoles = peerRolesList.stream().map(String::toLowerCase)
782+
.anyMatch(scmRatisRolesToCheck::contains);
783+
builder.setScmRatisEnabled(containsScmRoles);
784+
}
785+
}
786+
return builder.build();
765787
}
766788

767789
@Override

‎hadoop-hdds/interface-client/src/main/proto/hdds.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,7 @@ message GetScmInfoResponseProto {
257257
required string clusterId = 1;
258258
required string scmId = 2;
259259
repeated string peerRoles = 3;
260+
optional bool scmRatisEnabled = 4;
260261
}
261262

262263
message AddScmRequestProto {

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -837,13 +837,15 @@ public ScmInfo getScmInfo() {
837837
if (scm.getScmHAManager().getRatisServer() != null) {
838838
builder.setRatisPeerRoles(
839839
scm.getScmHAManager().getRatisServer().getRatisRoles());
840+
builder.setScmRatisEnabled(true);
840841
} else {
841842
// In case, there is no ratis, there is no ratis role.
842843
// This will just print the hostname with ratis port as the default
843844
// behaviour.
844845
String address = scm.getSCMHANodeDetails().getLocalNodeDetails()
845846
.getRatisHostPortStr();
846847
builder.setRatisPeerRoles(Arrays.asList(address));
848+
builder.setScmRatisEnabled(false);
847849
}
848850
return builder.build();
849851
} catch (Exception ex) {

‎hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -519,6 +519,11 @@ public List<String> getScmRatisRoles() throws IOException {
519519
return storageContainerLocationClient.getScmInfo().getRatisPeerRoles();
520520
}
521521

522+
@Override
523+
public boolean isScmRatisEnable() throws IOException {
524+
return storageContainerLocationClient.getScmInfo().getScmRatisEnabled();
525+
}
526+
522527
@Override
523528
public boolean rotateSecretKeys(boolean force) throws IOException {
524529
return secretKeyClient.checkAndRotate(force);

‎hadoop-ozone/dist/src/main/smoketest/admincli/scmrole.robot‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,8 @@ Run scm roles
3030
List scm roles as JSON
3131
${output} = Execute ozone admin scm roles --json
3232
${leader} = Execute echo '${output}' | jq -r '.[] | select(.raftPeerRole == "LEADER")'
33-
Should Not Be Equal ${leader} ${EMPTY}
33+
Should Not Be Equal ${leader} ${EMPTY}
34+
35+
List scm roles as TABLE
36+
${output} = Execute ozone admin scm roles --table
37+
Should Match Regexp ${output} \\|.*LEADER.*

‎hadoop-ozone/dist/src/main/smoketest/omha/om-roles.robot‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ Assert Leader Present in JSON
2828
[Arguments] ${output}
2929
${leader} = Execute echo '${output}' | jq '.[] | select(.[] | .serverRole == "LEADER")'
3030
Should Not Be Equal ${leader} ${EMPTY}
31+
Assert Leader Present in TABLE
32+
[Arguments] ${output}
33+
Should Match Regexp ${output} \\|.*LEADER.*
3134

3235
*** Test Cases ***
3336
List om roles with OM service ID passed
@@ -53,3 +56,15 @@ List om roles as JSON without OM service ID passed
5356
Assert Leader Present in JSON ${output_without_id_passed}
5457
${output_without_id_passed} = Execute And Ignore Error ozone admin --set=ozone.om.service.ids=omservice,omservice2 om roles --json
5558
Should Contain ${output_without_id_passed} no Ozone Manager service ID specified
59+
60+
List om roles as TABLE with OM service ID passed
61+
${output_with_id_passed} = Execute ozone admin om roles --service-id=omservice --table
62+
Assert Leader Present in TABLE ${output_with_id_passed}
63+
${output_with_id_passed} = Execute ozone admin --set=ozone.om.service.ids=omservice,omservice2 om roles --service-id=omservice --table
64+
Assert Leader Present in TABLE ${output_with_id_passed}
65+
66+
List om roles as TABLE without OM service ID passed
67+
${output_without_id_passed} = Execute ozone admin om roles --table
68+
Assert Leader Present in TABLE ${output_without_id_passed}
69+
${output_without_id_passed} = Execute And Ignore Error ozone admin --set=ozone.om.service.ids=omservice,omservice2 om roles --table
70+
Should Contain ${output_without_id_passed} no Ozone Manager service ID specified

‎hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/GetServiceRolesSubcommand.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,20 @@
1818

1919
package org.apache.hadoop.ozone.admin.om;
2020

21+
import org.apache.hadoop.classification.VisibleForTesting;
2122
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
2223
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
2324
import org.apache.hadoop.hdds.server.JsonUtils;
2425
import org.apache.hadoop.ozone.client.OzoneClientException;
2526
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
2627
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
2728
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
29+
import org.apache.hadoop.ozone.utils.FormattingCLIUtils;
2830
import picocli.CommandLine;
2931

3032
import java.io.IOException;
3133
import java.util.ArrayList;
34+
import java.util.Arrays;
3235
import java.util.HashMap;
3336
import java.util.List;
3437
import java.util.Map;
@@ -57,14 +60,37 @@ public class GetServiceRolesSubcommand implements Callable<Void> {
5760
description = "Format output as JSON")
5861
private boolean json;
5962

63+
@CommandLine.Option(names = { "--table" },
64+
defaultValue = "false",
65+
description = "Format output as Table")
66+
private boolean table;
67+
6068
private OzoneManagerProtocol ozoneManagerClient;
6169

70+
private static final String OM_ROLES_TITLE = "Ozone Manager Roles";
71+
72+
private static final List<String> OM_ROLES_HEADER = Arrays.asList(
73+
"Host Name", "Node ID", "Role");
74+
6275
@Override
6376
public Void call() throws Exception {
6477
try {
6578
ozoneManagerClient = parent.createOmClient(omServiceId);
6679
if (json) {
6780
printOmServerRolesAsJson(ozoneManagerClient.getServiceList());
81+
} else if (table) {
82+
FormattingCLIUtils formattingCLIUtils = new FormattingCLIUtils(OM_ROLES_TITLE)
83+
.addHeaders(OM_ROLES_HEADER);
84+
List<ServiceInfo> serviceList = ozoneManagerClient.getServiceList();
85+
for (ServiceInfo serviceInfo : serviceList) {
86+
OMRoleInfo omRoleInfo = serviceInfo.getOmRoleInfo();
87+
if (omRoleInfo != null &&
88+
serviceInfo.getNodeType() == HddsProtos.NodeType.OM) {
89+
formattingCLIUtils.addLine(new String[]{serviceInfo.getHostname(),
90+
omRoleInfo.getNodeId(), omRoleInfo.getServerRole()});
91+
}
92+
}
93+
System.out.println(formattingCLIUtils.render());
6894
} else {
6995
printOmServerRoles(ozoneManagerClient.getServiceList());
7096
}
@@ -110,4 +136,14 @@ private void printOmServerRolesAsJson(List<ServiceInfo> serviceList)
110136
System.out.print(
111137
JsonUtils.toJsonStringWithDefaultPrettyPrinter(omServiceList));
112138
}
139+
140+
@VisibleForTesting
141+
public void setOzoneManagerClient(OzoneManagerProtocol ozoneManagerClient) {
142+
this.ozoneManagerClient = ozoneManagerClient;
143+
}
144+
145+
@VisibleForTesting
146+
public void setParent(OMAdmin parent) {
147+
this.parent = parent;
148+
}
113149
}

‎hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/GetScmRatisRolesSubcommand.java‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.hadoop.ozone.admin.scm;
1919

2020
import java.io.IOException;
21+
import java.util.Arrays;
2122
import java.util.Collections;
2223
import java.util.HashMap;
2324
import java.util.List;
@@ -27,6 +28,7 @@
2728
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
2829
import org.apache.hadoop.hdds.scm.client.ScmClient;
2930
import org.apache.hadoop.hdds.server.JsonUtils;
31+
import org.apache.hadoop.ozone.utils.FormattingCLIUtils;
3032
import picocli.CommandLine;
3133

3234
import static java.lang.System.err;
@@ -50,13 +52,44 @@ public class GetScmRatisRolesSubcommand extends ScmSubcommand {
5052
description = "Format output as JSON")
5153
private boolean json;
5254

55+
@CommandLine.Option(names = { "--table" },
56+
defaultValue = "false",
57+
description = "Format output as Table")
58+
private boolean table;
59+
60+
private static final String SCM_ROLES_TITLE = "Storage Container Manager Roles";
61+
62+
private static final List<String> RATIS_SCM_ROLES_HEADER = Arrays.asList(
63+
"Host Name", "Ratis Port", "Role", "Node ID", "Host Address");
64+
65+
private static final List<String> STANDALONE_SCM_ROLES_HEADER = Arrays.asList("Host Name", "Port");
66+
5367
@Override
54-
protected void execute(ScmClient scmClient) throws IOException {
68+
public void execute(ScmClient scmClient) throws IOException {
5569
List<String> ratisRoles = scmClient.getScmRatisRoles();
70+
boolean isRatisEnabled = scmClient.isScmRatisEnable();
5671
if (json) {
5772
Map<String, Map<String, String>> scmRoles = parseScmRoles(ratisRoles);
5873
System.out.print(
5974
JsonUtils.toJsonStringWithDefaultPrettyPrinter(scmRoles));
75+
} else if (table) {
76+
FormattingCLIUtils formattingCLIUtils = new FormattingCLIUtils(SCM_ROLES_TITLE);
77+
78+
// Determine which header to use based on whether Ratis is enabled or not.
79+
if (isRatisEnabled) {
80+
formattingCLIUtils.addHeaders(RATIS_SCM_ROLES_HEADER);
81+
} else {
82+
formattingCLIUtils.addHeaders(STANDALONE_SCM_ROLES_HEADER);
83+
}
84+
85+
for (String role : ratisRoles) {
86+
String[] roleItems = role.split(":");
87+
if (roleItems.length < 2) {
88+
err.println("Invalid response received for ScmRatisRoles.");
89+
}
90+
formattingCLIUtils.addLine(roleItems);
91+
}
92+
System.out.println(formattingCLIUtils.render());
6093
} else {
6194
for (String role: ratisRoles) {
6295
System.out.println(role);

0 commit comments

Comments
 (0)