From 89c75dbe2b1d195676d9f0a17a2dbd4074e66f02 Mon Sep 17 00:00:00 2001 From: stevenxi Date: Wed, 6 Jan 2021 18:41:10 +0800 Subject: [PATCH 1/4] HBASE-25459 WAL can't be cleaned in some scenes --- .../hbase/regionserver/wal/SequenceIdAccounting.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java index 5c6fcd2d1aa6..2f414e8df144 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java @@ -250,7 +250,12 @@ ConcurrentMap getOrCreateLowestSequenceIds(byte[] enco */ private static long getLowestSequenceId(Map sequenceids) { long lowest = HConstants.NO_SEQNUM; - for (Long sid: sequenceids.values()) { + for (Map.Entry entry : sequenceids.entrySet()){ + if (entry.getKey() instanceof ImmutableByteArray && entry.getKey().toString().equals("METAFAMILY")){ + LOG.info("ColumnFamily is METAFAMILY! continue ..."); + continue; + } + Long sid = entry.getValue(); if (lowest == HConstants.NO_SEQNUM || sid.longValue() < lowest) { lowest = sid.longValue(); } From ece491f05506f0a55b3e0fbccf2606f8db74d2b2 Mon Sep 17 00:00:00 2001 From: stevenxi Date: Thu, 7 Jan 2021 10:51:19 +0800 Subject: [PATCH 2/4] HBASE-25459 fix code after reviewed --- .../regionserver/wal/SequenceIdAccounting.java | 3 +-- .../wal/TestSequenceIdAccounting.java | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java index 2f414e8df144..6be95391819b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java @@ -251,8 +251,7 @@ ConcurrentMap getOrCreateLowestSequenceIds(byte[] enco private static long getLowestSequenceId(Map sequenceids) { long lowest = HConstants.NO_SEQNUM; for (Map.Entry entry : sequenceids.entrySet()){ - if (entry.getKey() instanceof ImmutableByteArray && entry.getKey().toString().equals("METAFAMILY")){ - LOG.info("ColumnFamily is METAFAMILY! continue ..."); + if (entry.getKey().toString().equals("METAFAMILY")){ continue; } Long sid = entry.getValue(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSequenceIdAccounting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSequenceIdAccounting.java index 098dc86461b6..222a43bbdfc9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSequenceIdAccounting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSequenceIdAccounting.java @@ -44,10 +44,14 @@ public class TestSequenceIdAccounting { private static final byte [] ENCODED_REGION_NAME = Bytes.toBytes("r"); private static final byte [] FAMILY_NAME = Bytes.toBytes("cf"); + private static final byte [] META_FAMILY = Bytes.toBytes("METAFAMILY"); private static final Set FAMILIES; + private static final Set META_FAMILY_SET; static { FAMILIES = new HashSet<>(); FAMILIES.add(FAMILY_NAME); + META_FAMILY_SET = new HashSet<>(); + META_FAMILY_SET.add(META_FAMILY); } @Test @@ -123,6 +127,19 @@ public void testAreAllLower() { sida.update(ENCODED_REGION_NAME, FAMILIES, ++sequenceid, true); sida.update(ENCODED_REGION_NAME, FAMILIES, ++sequenceid, true); assertTrue(sida.areAllLower(m, null)); + m.put(ENCODED_REGION_NAME, sequenceid); + assertFalse(sida.areAllLower(m, null)); + // Test the METAFAMILY is filtered in SequenceIdAccounting.lowestUnflushedSequenceIds + SequenceIdAccounting meta_sida = new SequenceIdAccounting(); + Map meta_m = new HashMap<>(); + meta_sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME); + meta_m.put(ENCODED_REGION_NAME, sequenceid); + meta_sida.update(ENCODED_REGION_NAME, META_FAMILY_SET, ++sequenceid, true); + meta_sida.update(ENCODED_REGION_NAME, META_FAMILY_SET, ++sequenceid, true); + meta_sida.update(ENCODED_REGION_NAME, META_FAMILY_SET, ++sequenceid, true); + assertTrue(meta_sida.areAllLower(meta_m, null)); + meta_m.put(ENCODED_REGION_NAME, sequenceid); + assertTrue(meta_sida.areAllLower(meta_m, null)); } @Test From 791ff59bfbd4828d1588eba2377512fb8c090744 Mon Sep 17 00:00:00 2001 From: stevenxi Date: Fri, 8 Jan 2021 10:30:48 +0800 Subject: [PATCH 3/4] Trigger the UT --- .../hadoop/hbase/regionserver/wal/TestSequenceIdAccounting.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSequenceIdAccounting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSequenceIdAccounting.java index 222a43bbdfc9..8eb99b3a4772 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSequenceIdAccounting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSequenceIdAccounting.java @@ -129,6 +129,7 @@ public void testAreAllLower() { assertTrue(sida.areAllLower(m, null)); m.put(ENCODED_REGION_NAME, sequenceid); assertFalse(sida.areAllLower(m, null)); + // Test the METAFAMILY is filtered in SequenceIdAccounting.lowestUnflushedSequenceIds SequenceIdAccounting meta_sida = new SequenceIdAccounting(); Map meta_m = new HashMap<>(); From 971316bca244dec073ca1c06c08e0781441bd5be Mon Sep 17 00:00:00 2001 From: stevenxi Date: Tue, 12 Jan 2021 11:21:27 +0800 Subject: [PATCH 4/4] HBASE-25495 fix comment error of admin.rb --- hbase-shell/src/main/ruby/hbase/admin.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 6228ad78486d..d874d6337b84 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -1414,7 +1414,7 @@ def create_namespace(namespace_name, *args) #---------------------------------------------------------------------------------------------- # modify a namespace def alter_namespace(namespace_name, *args) - # Fail if table name is not a string + # Fail if namespace name is not a string raise(ArgumentError, 'Namespace name must be of type String') unless namespace_name.is_a?(String) nsd = @admin.getNamespaceDescriptor(namespace_name)