Skip to content

Commit ca902d1

Browse files
ZanderXuzengqiang.xu
ZanderXu
authored and
zengqiang.xu
committed
HDFS-17387. [FGL] Abstract the configuration locking mode (#6572)
* HDFS-17387. [FGL] Abstract the configurable locking mode
1 parent c4ba19c commit ca902d1

9 files changed

Lines changed: 525 additions & 51 deletions

File tree

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
3030
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker;
3131
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.ReservedSpaceCalculator;
32+
import org.apache.hadoop.hdfs.server.namenode.fgl.FSNLockManager;
33+
import org.apache.hadoop.hdfs.server.namenode.fgl.GlobalFSNamesystemLock;
3234
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
3335
import org.apache.hadoop.http.HttpConfig;
3436

@@ -1051,6 +1053,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
10511053
public static final String DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_KEY = "dfs.namenode.inode.attributes.provider.class";
10521054
public static final String DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_BYPASS_USERS_KEY = "dfs.namenode.inode.attributes.provider.bypass.users";
10531055
public static final String DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_BYPASS_USERS_DEFAULT = "";
1056+
public static final String DFS_NAMENODE_LOCK_MODEL_PROVIDER_KEY =
1057+
"dfs.namenode.lock.model.provider.class";
1058+
public static final Class<? extends FSNLockManager> DFS_NAMENODE_LOCK_MODEL_PROVIDER_DEFAULT =
1059+
GlobalFSNamesystemLock.class;
10541060

10551061
public static final String DFS_DATANODE_BP_READY_TIMEOUT_KEY = "dfs.datanode.bp-ready.timeout";
10561062
public static final long DFS_DATANODE_BP_READY_TIMEOUT_DEFAULT = 20;

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java‎

Lines changed: 60 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
3535
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_DEFAULT;
3636
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY;
37+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_MODEL_PROVIDER_DEFAULT;
38+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_MODEL_PROVIDER_KEY;
3739
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_DEFAULT;
3840
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_PERMISSIONS_SUPERUSER_ONLY_DEFAULT;
3941
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_PERMISSIONS_SUPERUSER_ONLY_KEY;
@@ -96,6 +98,7 @@
9698
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_LISTING_LIMIT_DEFAULT;
9799
import static org.apache.hadoop.hdfs.DFSUtil.isParentEntry;
98100

101+
import java.lang.reflect.Constructor;
99102
import java.nio.charset.StandardCharsets;
100103
import java.util.concurrent.atomic.AtomicLong;
101104

@@ -114,6 +117,8 @@
114117
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
115118

116119
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
120+
import org.apache.hadoop.hdfs.server.namenode.fgl.FSNLockManager;
121+
import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode;
117122
import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
118123
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotDeletionGc;
119124
import org.apache.hadoop.thirdparty.protobuf.ByteString;
@@ -621,7 +626,7 @@ private boolean isFromProxyUser(CallerContext ctx) {
621626
private final int numCommittedAllowed;
622627

623628
/** Lock to protect FSNamesystem. */
624-
private final FSNamesystemLock fsLock;
629+
private final FSNLockManager fsLock;
625630

626631
/**
627632
* Checkpoint lock to protect FSNamesystem modification on standby NNs.
@@ -871,7 +876,10 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException {
871876
this.contextFieldSeparator =
872877
conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY,
873878
HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT);
874-
fsLock = new FSNamesystemLock(conf, detailedLockHoldTimeMetrics);
879+
Class<? extends FSNLockManager> lockKlass = conf.getClass(
880+
DFS_NAMENODE_LOCK_MODEL_PROVIDER_KEY, DFS_NAMENODE_LOCK_MODEL_PROVIDER_DEFAULT,
881+
FSNLockManager.class);
882+
fsLock = createLock(lockKlass, conf, detailedLockHoldTimeMetrics);
875883
cpLock = new ReentrantLock();
876884

877885
this.fsImage = fsImage;
@@ -1077,6 +1085,18 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException {
10771085
}
10781086
}
10791087

1088+
private <T> T createLock(Class<T> theClass, Configuration conf,
1089+
MutableRatesWithAggregation mutableRatesMetrics) {
1090+
try {
1091+
Constructor<T> meth = theClass.getDeclaredConstructor(
1092+
Configuration.class, MutableRatesWithAggregation.class);
1093+
meth.setAccessible(true);
1094+
return meth.newInstance(conf, mutableRatesMetrics);
1095+
} catch (Exception e) {
1096+
throw new RuntimeException(e);
1097+
}
1098+
}
1099+
10801100
private static void checkForAsyncLogEnabledByOldConfigs(Configuration conf) {
10811101
// dfs.namenode.audit.log.async is no longer in use. Use log4j properties instead.
10821102
if (conf.getBoolean("dfs.namenode.audit.log.async", false)) {
@@ -1787,70 +1807,74 @@ public static List<URI> getSharedEditsDirs(Configuration conf) {
17871807
}
17881808

17891809
@Override
1790-
public void readLock() {
1791-
this.fsLock.readLock();
1792-
}
1793-
1794-
@Override
1795-
public void readLockInterruptibly() throws InterruptedException {
1796-
this.fsLock.readLockInterruptibly();
1810+
public void readLock(FSNamesystemLockMode lockMode) {
1811+
this.fsLock.readLock(lockMode);
17971812
}
17981813

17991814
@Override
1800-
public void readUnlock() {
1801-
this.fsLock.readUnlock();
1815+
public void readLockInterruptibly(FSNamesystemLockMode lockMode) throws InterruptedException {
1816+
this.fsLock.readLockInterruptibly(lockMode);
18021817
}
18031818

18041819
@Override
1805-
public void readUnlock(String opName) {
1806-
this.fsLock.readUnlock(opName);
1820+
public void readUnlock(FSNamesystemLockMode lockMode, String opName) {
1821+
this.fsLock.readUnlock(lockMode, opName);
18071822
}
18081823

18091824
public void readUnlock(String opName,
18101825
Supplier<String> lockReportInfoSupplier) {
1811-
this.fsLock.readUnlock(opName, lockReportInfoSupplier);
1826+
readUnlock(FSNamesystemLockMode.GLOBAL, opName, lockReportInfoSupplier);
18121827
}
18131828

1814-
@Override
1815-
public void writeLock() {
1816-
this.fsLock.writeLock();
1829+
public void readUnlock(FSNamesystemLockMode lockMode, String opName,
1830+
Supplier<String> lockReportInfoSupplier) {
1831+
this.fsLock.readUnlock(lockMode, opName, lockReportInfoSupplier);
18171832
}
18181833

18191834
@Override
1820-
public void writeLockInterruptibly() throws InterruptedException {
1821-
this.fsLock.writeLockInterruptibly();
1835+
public void writeLock(FSNamesystemLockMode lockMode) {
1836+
this.fsLock.writeLock(lockMode);
18221837
}
18231838

18241839
@Override
1825-
public void writeUnlock() {
1826-
this.fsLock.writeUnlock();
1840+
public void writeLockInterruptibly(FSNamesystemLockMode lockMode) throws InterruptedException {
1841+
this.fsLock.writeLockInterruptibly(lockMode);
18271842
}
18281843

18291844
@Override
1830-
public void writeUnlock(String opName) {
1831-
this.fsLock.writeUnlock(opName);
1845+
public void writeUnlock(FSNamesystemLockMode lockMode, String opName) {
1846+
this.fsLock.writeUnlock(lockMode, opName);
18321847
}
18331848

18341849
public void writeUnlock(String opName, boolean suppressWriteLockReport) {
1835-
this.fsLock.writeUnlock(opName, suppressWriteLockReport);
1850+
writeUnlock(FSNamesystemLockMode.GLOBAL, opName, suppressWriteLockReport);
1851+
}
1852+
1853+
public void writeUnlock(FSNamesystemLockMode lockMode, String opName,
1854+
boolean suppressWriteLockReport) {
1855+
this.fsLock.writeUnlock(lockMode, opName, suppressWriteLockReport);
1856+
}
1857+
1858+
public void writeUnlock(String opName, Supplier<String> lockReportInfoSupplier) {
1859+
writeUnlock(FSNamesystemLockMode.GLOBAL, opName, lockReportInfoSupplier);
18361860
}
18371861

1838-
public void writeUnlock(String opName,
1862+
public void writeUnlock(FSNamesystemLockMode lockMode, String opName,
18391863
Supplier<String> lockReportInfoSupplier) {
1840-
this.fsLock.writeUnlock(opName, lockReportInfoSupplier);
1864+
this.fsLock.writeUnlock(lockMode, opName, lockReportInfoSupplier);
18411865
}
18421866

18431867
@Override
1844-
public boolean hasWriteLock() {
1845-
return this.fsLock.isWriteLockedByCurrentThread();
1868+
public boolean hasWriteLock(FSNamesystemLockMode lockMode) {
1869+
return this.fsLock.hasWriteLock(lockMode);
18461870
}
18471871
@Override
1848-
public boolean hasReadLock() {
1849-
return this.fsLock.getReadHoldCount() > 0 || hasWriteLock();
1872+
public boolean hasReadLock(FSNamesystemLockMode lockMode) {
1873+
return this.fsLock.hasReadLock(lockMode);
18501874
}
18511875

18521876
public int getReadHoldCount() {
1853-
return this.fsLock.getReadHoldCount();
1877+
return this.fsLock.getReadHoldCount(FSNamesystemLockMode.GLOBAL);
18541878
}
18551879

18561880
/** Lock the checkpoint lock */
@@ -4931,21 +4955,21 @@ public float getReconstructionQueuesInitProgress() {
49314955
@Metric({"LockQueueLength", "Number of threads waiting to " +
49324956
"acquire FSNameSystemLock"})
49334957
public int getFsLockQueueLength() {
4934-
return fsLock.getQueueLength();
4958+
return fsLock.getQueueLength(FSNamesystemLockMode.FS);
49354959
}
49364960

49374961
@Metric(value = {"ReadLockLongHoldCount", "The number of time " +
49384962
"the read lock has been held for longer than the threshold"},
49394963
type = Metric.Type.COUNTER)
49404964
public long getNumOfReadLockLongHold() {
4941-
return fsLock.getNumOfReadLockLongHold();
4965+
return fsLock.getNumOfReadLockLongHold(FSNamesystemLockMode.FS);
49424966
}
49434967

49444968
@Metric(value = {"WriteLockLongHoldCount", "The number of time " +
49454969
"the write lock has been held for longer than the threshold"},
49464970
type = Metric.Type.COUNTER)
49474971
public long getNumOfWriteLockLongHold() {
4948-
return fsLock.getNumOfWriteLockLongHold();
4972+
return fsLock.getNumOfWriteLockLongHold(FSNamesystemLockMode.FS);
49494973
}
49504974

49514975
int getNumberOfDatanodes(DatanodeReportType type) {
@@ -7097,12 +7121,12 @@ public void setEditLogTailerForTests(EditLogTailer tailer) {
70977121

70987122
@VisibleForTesting
70997123
void setFsLockForTests(ReentrantReadWriteLock lock) {
7100-
this.fsLock.coarseLock = lock;
7124+
this.fsLock.setLockForTests(lock);
71017125
}
71027126

71037127
@VisibleForTesting
71047128
public ReentrantReadWriteLock getFsLockForTests() {
7105-
return fsLock.coarseLock;
7129+
return fsLock.getLockForTests();
71067130
}
71077131

71087132
@VisibleForTesting

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@
6464
* {@link MutableRatesWithAggregation}. However since threads are re-used
6565
* between operations this should not generally be an issue.
6666
*/
67-
class FSNamesystemLock {
67+
public class FSNamesystemLock {
6868
@VisibleForTesting
6969
protected ReentrantReadWriteLock coarseLock;
7070

@@ -129,7 +129,7 @@ public Long initialValue() {
129129

130130
private static final String OVERALL_METRIC_NAME = "Overall";
131131

132-
FSNamesystemLock(Configuration conf,
132+
public FSNamesystemLock(Configuration conf,
133133
MutableRatesWithAggregation detailedHoldTimeMetrics) {
134134
this(conf, detailedHoldTimeMetrics, new Timer());
135135
}
@@ -489,6 +489,14 @@ public long getWriteLockReportingThresholdMs() {
489489
return writeLockReportingThresholdMs;
490490
}
491491

492+
public void setLockForTests(ReentrantReadWriteLock lock) {
493+
this.coarseLock = lock;
494+
}
495+
496+
public ReentrantReadWriteLock getLockForTests() {
497+
return this.coarseLock;
498+
}
499+
492500
/**
493501
* Read lock Held Info.
494502
*/

0 commit comments

Comments
 (0)