From 824b84ca6dd20791f7c766e1dfa0299a76dd6949 Mon Sep 17 00:00:00 2001 From: Kidd Gu Date: Sun, 18 Sep 2022 03:28:33 +0800 Subject: [PATCH 1/4] init --- .../server/datanode/erasurecode/StripedReconstructor.java | 8 ++++++++ .../hdfs/server/datanode/erasurecode/StripedWriter.java | 2 ++ 2 files changed, 10 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java index 466c7c4254c8b..1732ceff6d6c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java @@ -119,6 +119,7 @@ abstract class StripedReconstructor { private ErasureCodingWorker erasureCodingWorker; private final CachingStrategy cachingStrategy; private long maxTargetLength = 0L; + private int liveIndiceNum = 0; private final BitSet liveBitSet; private final BitSet excludeBitSet; @@ -136,6 +137,9 @@ abstract class StripedReconstructor { liveBitSet = new BitSet( ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()); for (int i = 0; i < stripedReconInfo.getLiveIndices().length; i++) { + if(!liveBitSet.get(stripedReconInfo.getLiveIndices()[i])){ + liveIndiceNum++; + } liveBitSet.set(stripedReconInfo.getLiveIndices()[i]); } excludeBitSet = new BitSet( @@ -288,6 +292,10 @@ RawErasureDecoder getDecoder() { return decoder; } + int getIndicesNum(){ + return liveIndiceNum; + } + void cleanup() { if (decoder != null) { decoder.release(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java index ca06dfc120531..d6a2befd7b7d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java @@ -84,6 +84,8 @@ class StripedWriter { writers = new StripedBlockWriter[targets.length]; targetIndices = new short[targets.length]; + Preconditions.checkArgument(targetIndices.length <= reconstructor.getIndicesNum(), + "Reconstrutcion work gets too much targets."); Preconditions.checkArgument(targetIndices.length <= parityBlkNum, "Too much missed striped blocks."); initTargetIndices(); From 5b4abf4b7a804fdca0db78107d73ebe65cea719b Mon Sep 17 00:00:00 2001 From: Kidd Gu Date: Sun, 18 Sep 2022 18:47:07 +0800 Subject: [PATCH 2/4] fixed --- .../hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java index d6a2befd7b7d7..6384d24926c93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java @@ -84,7 +84,7 @@ class StripedWriter { writers = new StripedBlockWriter[targets.length]; targetIndices = new short[targets.length]; - Preconditions.checkArgument(targetIndices.length <= reconstructor.getIndicesNum(), + Preconditions.checkArgument(targetIndices.length <= dataBlkNum + parityBlkNum - reconstructor.getIndicesNum(), "Reconstrutcion work gets too much targets."); Preconditions.checkArgument(targetIndices.length <= parityBlkNum, "Too much missed striped blocks."); From 77586f5dde4506bc4382b6b78690b37b0d731f42 Mon Sep 17 00:00:00 2001 From: Kidd Gu Date: Wed, 21 Sep 2022 20:25:38 +0800 Subject: [PATCH 3/4] commit suggestion --- .../server/datanode/erasurecode/StripedReconstructor.java | 8 ++------ .../hdfs/server/datanode/erasurecode/StripedWriter.java | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java index 1732ceff6d6c2..7acb679200118 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java @@ -119,7 +119,6 @@ abstract class StripedReconstructor { private ErasureCodingWorker erasureCodingWorker; private final CachingStrategy cachingStrategy; private long maxTargetLength = 0L; - private int liveIndiceNum = 0; private final BitSet liveBitSet; private final BitSet excludeBitSet; @@ -137,9 +136,6 @@ abstract class StripedReconstructor { liveBitSet = new BitSet( ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()); for (int i = 0; i < stripedReconInfo.getLiveIndices().length; i++) { - if(!liveBitSet.get(stripedReconInfo.getLiveIndices()[i])){ - liveIndiceNum++; - } liveBitSet.set(stripedReconInfo.getLiveIndices()[i]); } excludeBitSet = new BitSet( @@ -292,8 +288,8 @@ RawErasureDecoder getDecoder() { return decoder; } - int getIndicesNum(){ - return liveIndiceNum; + int getNumLiveBlocks(){ + return liveBitSet.cardinality(); } void cleanup() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java index 6384d24926c93..a1ae93da2a58c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java @@ -82,9 +82,9 @@ class StripedWriter { assert targetStorageIds != null; writers = new StripedBlockWriter[targets.length]; - targetIndices = new short[targets.length]; - Preconditions.checkArgument(targetIndices.length <= dataBlkNum + parityBlkNum - reconstructor.getIndicesNum(), + Preconditions.checkArgument( + targetIndices.length <= dataBlkNum + parityBlkNum - reconstructor.getNumLiveBlocks(), "Reconstrutcion work gets too much targets."); Preconditions.checkArgument(targetIndices.length <= parityBlkNum, "Too much missed striped blocks."); From c679261f247a39cbd1dc1dcca143a23008b90cc8 Mon Sep 17 00:00:00 2001 From: Kidd Gu Date: Thu, 22 Sep 2022 22:05:38 +0800 Subject: [PATCH 4/4] typo --- .../hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java index a1ae93da2a58c..00be1279c8179 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java @@ -85,7 +85,7 @@ class StripedWriter { targetIndices = new short[targets.length]; Preconditions.checkArgument( targetIndices.length <= dataBlkNum + parityBlkNum - reconstructor.getNumLiveBlocks(), - "Reconstrutcion work gets too much targets."); + "Reconstruction work gets too much targets."); Preconditions.checkArgument(targetIndices.length <= parityBlkNum, "Too much missed striped blocks."); initTargetIndices();