Skip to content

HADOOP-17612. Upgrade Zookeeper to 3.6.3 and Curator to 5.2.0 #5067

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
melissayou wants to merge 13 commits into from
Closed

HADOOP-17612. Upgrade Zookeeper to 3.6.3 and Curator to 5.2.0 #5067

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
d5982b1
HADOOP-17612. Upgrade Zookeeper to 3.6.3 and Curator to 5.2.0 (#3241)
virajjasani Aug 3, 2021
faa80dd
HADOOP-17835. Use CuratorCache implementation instead of PathChildren…
virajjasani Aug 7, 2021
a9a9910
remove dangling lines of merge conflict
Oct 23, 2022
42a4f8e
HDFS-15383. RBF: Add support for router delegation token without watc…
fengnanli Jun 23, 2020
b0f6cc1
fix checkstyle
Oct 23, 2022
7f8300c
HADOOP-18427. Improve ZKDelegationTokenSecretManager#startThead With …
slfan1989 Sep 8, 2022
bf18e6d
HADOOP-18452. Fix TestKMS#testKMSHAZooKeeperDelegationToken Failed By…
slfan1989 Sep 14, 2022
51e995e
fix checkstyle and add supress warning for unchecked
Oct 25, 2022
8c681fb
Trigger Build
Oct 25, 2022
0c088e8
checkstyle fix
Oct 27, 2022
97dac96
space fix
Oct 27, 2022
7880263
Trigger Build
Oct 28, 2022
9429be9
Trigger Build
Oct 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 24 additions & 15 deletions ...main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import java.io.UncheckedIOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;

Expand Down Expand Up @@ -285,7 +284,8 @@ public void startThreads() throws IOException {
// by calling the incrSharedCount
currentSeqNum = incrSharedCount(delTokSeqCounter, seqNumBatchSize);
currentMaxSeqNum = currentSeqNum + seqNumBatchSize;
LOG.info("Fetched initial range of seq num, from {} to {} ", currentSeqNum + 1, currentMaxSeqNum);
LOG.info("Fetched initial range of seq num, from {} to {} ",
currentSeqNum + 1, currentMaxSeqNum);
} catch (Exception e) {
throw new IOException("Could not start Sequence Counter", e);
}
Expand All @@ -305,14 +305,18 @@ public void startThreads() throws IOException {
}
try {
keyCache = CuratorCache.bridgeBuilder(zkClient, ZK_DTSM_MASTER_KEY_ROOT).build();
CuratorCacheListener keyCacheListener = CuratorCacheListener.builder().forCreatesAndChanges((oldNode, node) -> {
try {
processKeyAddOrUpdate(node.getData());
} catch (IOException e) {
LOG.error("Error while processing Curator keyCacheListener " + "NODE_CREATED / NODE_CHANGED event");
throw new UncheckedIOException(e);
}
}).forDeletes(childData -> processKeyRemoved(childData.getPath())).build();
CuratorCacheListener keyCacheListener = CuratorCacheListener.builder()
.forCreatesAndChanges((oldNode, node) -> {
try {
processKeyAddOrUpdate(node.getData());
} catch (IOException e) {
LOG.error("Error while processing Curator keyCacheListener "
+ "NODE_CREATED / NODE_CHANGED event");
throw new UncheckedIOException(e);
}
})
.forDeletes(childData -> processKeyRemoved(childData.getPath()))
.build();
keyCache.listenable().addListener(keyCacheListener);
keyCache.start();
loadFromZKCache(false);
Expand All @@ -323,21 +327,26 @@ public void startThreads() throws IOException {
LOG.info("TokenCache is enabled");
try {
tokenCache = CuratorCache.bridgeBuilder(zkClient, ZK_DTSM_TOKENS_ROOT).build();
CuratorCacheListener tokenCacheListener = CuratorCacheListener.builder().forCreatesAndChanges((oldNode, node) -> {
CuratorCacheListener tokenCacheListener = CuratorCacheListener.builder()
.forCreatesAndChanges((oldNode, node) -> {
try {
processTokenAddOrUpdate(node.getData());
} catch (IOException e) {
LOG.error("Error while processing Curator tokenCacheListener " + "NODE_CREATED / NODE_CHANGED event");
LOG.error("Error while processing Curator tokenCacheListener "
+ "NODE_CREATED / NODE_CHANGED event");
throw new UncheckedIOException(e);
}
}).forDeletes(childData -> {
})
.forDeletes(childData -> {
try {
processTokenRemoved(childData);
} catch (IOException e) {
LOG.error("Error while processing Curator tokenCacheListener " + "NODE_DELETED event");
LOG.error("Error while processing Curator tokenCacheListener "
+ "NODE_DELETED event");
throw new UncheckedIOException(e);
}
}).build();
})
.build();
tokenCache.listenable().addListener(tokenCacheListener);
tokenCache.start();
loadFromZKCache(true);
Expand Down
2 changes: 1 addition & 1 deletion ...-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ protected void stopServer() throws Exception {


protected static ZooKeeperServer getServer(ServerCnxnFactory fac) {
return fac.getZooKeeperServer();
return fac.getZooKeeperServer();
}

protected void tearDownAll() throws Exception {
Expand Down
4 changes: 1 addition & 3 deletions ...doop/hdfs/server/federation/router/security/token/ZKDelegationTokenSecretManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;

import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -193,7 +191,7 @@ private void rebuildTokenCache(boolean initial) throws IOException {
// 2) remove tokens in local cache but not in zk anymore
for (AbstractDelegationTokenIdentifier ident : currentTokens.keySet()) {
if (!localTokenCache.contains(ident)) {
currentTokens.remove(ident);
currentTokens.remove(ident);
}
}
}
Expand Down
1 change: 1 addition & 0 deletions .../hadoop/hdfs/server/federation/security/token/TestZKDelegationTokenSecretManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ public void testMultiNodeTokenRemovalShortSyncWithoutWatch()

// This is very unlikely to happen in real case, but worth putting
// the case out
@SuppressWarnings("unchecked")
Copy link
Contributor

@xinglin xinglin Oct 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

trunk does not have suppress here.

Copy link
Contributor Author

@melissayou melissayou Oct 28, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes it was added by me to fix failed javac problem reported in earlier builds. There're two similar places(line135&201) and either will fail javac.

So two ways:

  1. Keep it in sync with trunk and let javac fail for this PR.
  2. (What I did here) fix it by adding suppress warning here as I'm pretty confident those types are correct. This test file is inconsistent - you can see the untouched method testMultiNodeOperationWithoutWatch() has the exact same problem and has suppress warning. So I think it's applicable to the other two methods as well.

Copy link
Contributor

@xinglin xinglin Oct 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree with you. Either way is fine. Will defer to @omalley for which approach to take on here.

@Test
public void testMultiNodeTokenRemovalLongSyncWithoutWatch()
throws Exception {
Expand Down