diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index 20ad1496d744..3d4ae6c00fee 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -756,7 +756,7 @@ public static boolean matchingRow(final Cell left, final Cell right) { /** * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. Instead use - * {@link #matchingRows(Cell, byte[]))} + * {@link #matchingRows(Cell, byte[])} */ @Deprecated public static boolean matchingRow(final Cell left, final byte[] buf) { @@ -894,8 +894,15 @@ public static boolean matchingQualifier(final Cell left, final byte[] buf, final } public static boolean matchingColumn(final Cell left, final byte[] fam, final byte[] qual) { - if (!matchingFamily(left, fam)) return false; - return matchingQualifier(left, qual); + return matchingFamily(left, fam) && matchingQualifier(left, qual); + } + + /** + * @return True if matching column family and the qualifier starts with qual + */ + public static boolean matchingColumnFamilyAndQualifierPrefix(final Cell left, final byte[] fam, + final byte[] qual) { + return matchingFamily(left, fam) && PrivateCellUtil.qualifierStartsWith(left, qual); } /** diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index 52518877c790..58e84489fa52 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -199,11 +199,10 @@ public void map(WALKey key, WALEdit value, Context context) Delete del = null; Cell lastCell = null; for (Cell cell : value.getCells()) { - // filtering WAL meta entries + // Filtering WAL meta marker entries. if (WALEdit.isMetaEditFamily(cell)) { continue; } - // Allow a subclass filter out this cell. if (filter(context, cell)) { // A WALEdit may contain multiple operations (HBASE-3584) and/or diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 8722b4f6538c..9e999d6a0c87 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -3485,7 +3485,7 @@ public List listReplicationPeers(String regex) if (cpHost != null) { cpHost.preListReplicationPeers(regex); } - LOG.info(getClientIdAuditPrefix() + " list replication peers, regex=" + regex); + LOG.debug("{} list replication peers, regex={}", getClientIdAuditPrefix(), regex); Pattern pattern = regex == null ? null : Pattern.compile(regex); List peers = this.replicationPeerManager.listPeers(pattern); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 28a8749f7026..d403205c1be5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -1725,7 +1725,7 @@ public Pair> call() throws IOException { status.setStatus("Writing region close event to WAL"); // Always write close marker to wal even for read only table. This is not a big problem as we - // do not write any data into the region. + // do not write any data into the region; it is just a meta edit in the WAL file. if (!abort && wal != null && getRegionServerServices() != null && RegionReplicaUtil.isDefaultReplica(getRegionInfo())) { writeRegionCloseMarker(wal); @@ -2691,7 +2691,8 @@ private void logFatLineOnFlush(Collection storesToFlush, long sequenceId } } MemStoreSize mss = this.memStoreSizing.getMemStoreSize(); - LOG.info("Flushing " + storesToFlush.size() + "/" + stores.size() + " column families," + + LOG.info("Flushing " + this.getRegionInfo().getEncodedName() + " " + + storesToFlush.size() + "/" + stores.size() + " column families," + " dataSize=" + StringUtils.byteDesc(mss.getDataSize()) + " heapSize=" + StringUtils.byteDesc(mss.getHeapSize()) + ((perCfExtras != null && perCfExtras.length() > 0)? perCfExtras.toString(): "") + @@ -4818,7 +4819,7 @@ private long replayRecoveredEdits(final Path edits, for (Cell cell: val.getCells()) { // Check this edit is for me. Also, guard against writing the special // METACOLUMN info such as HBASE::CACHEFLUSH entries - if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) { + if (WALEdit.isMetaEditFamily(cell)) { // if region names don't match, skipp replaying compaction marker if (!checkRowWithinBoundary) { //this is a special edit, we should handle it diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 6dc828abd08a..e61f2531bd26 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1928,7 +1928,7 @@ private void startServices() throws IOException { healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration()); } - this.walRoller = new LogRoller(this, this); + this.walRoller = new LogRoller(this); this.flushThroughputController = FlushThroughputControllerFactory.create(this, conf); this.procedureResultReporter = new RemoteProcedureResultReporter(this); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java index 464f51baef0f..c9bce71ca493 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java @@ -1,4 +1,4 @@ -/** +/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -28,7 +28,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; @@ -56,7 +55,6 @@ public class LogRoller extends HasThread implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(LogRoller.class); private final ConcurrentMap walNeedsRoll = new ConcurrentHashMap<>(); - private final Server server; protected final RegionServerServices services; private volatile long lastRollTime = System.currentTimeMillis(); // Period to roll log. @@ -99,16 +97,14 @@ public void requestRollAll() { } } - /** @param server */ - public LogRoller(final Server server, final RegionServerServices services) { + public LogRoller(RegionServerServices services) { super("LogRoller"); - this.server = server; this.services = services; - this.rollPeriod = this.server.getConfiguration(). + this.rollPeriod = this.services.getConfiguration(). getLong("hbase.regionserver.logroll.period", 3600000); - this.threadWakeFrequency = this.server.getConfiguration(). + this.threadWakeFrequency = this.services.getConfiguration(). getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000); - this.checkLowReplicationInterval = this.server.getConfiguration().getLong( + this.checkLowReplicationInterval = this.services.getConfiguration().getLong( "hbase.regionserver.hlog.check.lowreplication.interval", 30 * 1000); } @@ -144,7 +140,7 @@ private void abort(String reason, Throwable cause) { LOG.warn("Failed to shutdown wal", e); } } - server.abort(reason, cause); + this.services.abort(reason, cause); } @Override @@ -156,7 +152,7 @@ public void run() { periodic = (now - this.lastRollTime) > this.rollPeriod; if (periodic) { // Time for periodic roll, fall through - LOG.debug("Wal roll period {} ms elapsed", this.rollPeriod); + LOG.debug("WAL roll period {} ms elapsed", this.rollPeriod); } else { synchronized (this) { if (walNeedsRoll.values().stream().anyMatch(Boolean::booleanValue)) { @@ -183,9 +179,9 @@ public void run() { WAL wal = entry.getKey(); // reset the flag in front to avoid missing roll request before we return from rollWriter. walNeedsRoll.put(wal, Boolean.FALSE); - // Force the roll if the logroll.period is elapsed or if a roll was requested. - // The returned value is an array of actual region names. - byte[][] regionsToFlush = wal.rollWriter(periodic || entry.getValue().booleanValue()); + // Force the roll if the logroll.period is elapsed or if a roll was requested. + // The returned value is an array of actual region names. + byte[][] regionsToFlush = wal.rollWriter(periodic || entry.getValue().booleanValue()); if (regionsToFlush != null) { for (byte[] r : regionsToFlush) { scheduleFlush(Bytes.toString(r)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java index d5213a36545f..9564662d9d51 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java @@ -242,8 +242,8 @@ public abstract class AbstractFSWAL implements WAL { private static final class WalProps { /** - * Map the encoded region name to the highest sequence id. Contain all the regions it has - * entries of + * Map the encoded region name to the highest sequence id. + *

Contains all the regions it has an entry for. */ public final Map encodedName2HighestSequenceId; @@ -585,9 +585,9 @@ public int getNumLogFiles() { } /** - * If the number of un-archived WAL files is greater than maximum allowed, check the first - * (oldest) WAL file, and returns those regions which should be flushed so that it can be - * archived. + * If the number of un-archived WAL files ('live' WALs) is greater than maximum allowed, + * check the first (oldest) WAL, and return those regions which should be flushed so that + * it can be let-go/'archived'. * @return regions (encodedRegionNames) to flush in order to archive oldest WAL file. */ byte[][] findRegionsToForceFlush() throws IOException { @@ -860,10 +860,6 @@ public void close() throws IOException { /** * updates the sequence number of a specific store. depending on the flag: replaces current seq * number if the given seq id is bigger, or even if it is lower than existing one - * @param encodedRegionName - * @param familyName - * @param sequenceid - * @param onlyIfGreater */ @Override public void updateStore(byte[] encodedRegionName, byte[] familyName, Long sequenceid, @@ -973,7 +969,7 @@ protected final void postSync(final long timeInNanos, final int handlerSyncs) { } protected final long stampSequenceIdAndPublishToRingBuffer(RegionInfo hri, WALKeyImpl key, - WALEdit edits, boolean inMemstore, boolean closeRegion, RingBuffer ringBuffer) + WALEdit edits, boolean inMemstore, RingBuffer ringBuffer) throws IOException { if (this.closed) { throw new IOException( @@ -987,7 +983,7 @@ protected final long stampSequenceIdAndPublishToRingBuffer(RegionInfo hri, WALKe ServerCall rpcCall = RpcServer.getCurrentCall().filter(c -> c instanceof ServerCall) .filter(c -> c.getCellScanner() != null).map(c -> (ServerCall) c).orElse(null); try (TraceScope scope = TraceUtil.createTrace(implClassName + ".append")) { - FSWALEntry entry = new FSWALEntry(txid, key, edits, hri, inMemstore, closeRegion, rpcCall); + FSWALEntry entry = new FSWALEntry(txid, key, edits, hri, inMemstore, rpcCall); entry.stampRegionSequenceId(we); ringBuffer.get(txid).load(entry); } finally { @@ -1025,13 +1021,13 @@ public OptionalLong getLogFileSizeIfBeingWritten(Path path) { @Override public long appendData(RegionInfo info, WALKeyImpl key, WALEdit edits) throws IOException { - return append(info, key, edits, true, false); + return append(info, key, edits, true); } @Override - public long appendMarker(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean closeRegion) + public long appendMarker(RegionInfo info, WALKeyImpl key, WALEdit edits) throws IOException { - return append(info, key, edits, false, closeRegion); + return append(info, key, edits, false); } /** @@ -1055,17 +1051,17 @@ public long appendMarker(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean * @param key Modified by this call; we add to it this edits region edit/sequence id. * @param edits Edits to append. MAY CONTAIN NO EDITS for case where we want to get an edit * sequence id that is after all currently appended edits. - * @param inMemstore Always true except for case where we are writing a region event marker, for - * example, a compaction completion record into the WAL; in this case the entry is just - * so we can finish an unfinished compaction -- it is not an edit for memstore. - * @param closeRegion Whether this is a region close marker, i.e, the last wal edit for this - * region on this region server. The WAL implementation should remove all the related - * stuff, for example, the sequence id accounting. + * @param inMemstore Always true except for case where we are writing a region event meta + * marker edit, for example, a compaction completion record into the WAL or noting a + * Region Open event. In these cases the entry is just so we can finish an unfinished + * compaction after a crash when the new Server reads the WAL on recovery, etc. These + * transition event 'Markers' do not go via the memstore. When memstore is false, + * we presume a Marker event edit. * @return Returns a 'transaction id' and key will have the region edit/sequence id * in it. */ - protected abstract long append(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean inMemstore, - boolean closeRegion) throws IOException; + protected abstract long append(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean inMemstore) + throws IOException; protected abstract void doAppend(W writer, FSWALEntry entry) throws IOException; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java index 0307a882cac5..68f2db5c05f4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java @@ -558,9 +558,9 @@ private boolean shouldScheduleConsumer() { } @Override - protected long append(RegionInfo hri, WALKeyImpl key, WALEdit edits, boolean inMemstore, - boolean closeRegion) throws IOException { - long txid = stampSequenceIdAndPublishToRingBuffer(hri, key, edits, inMemstore, closeRegion, + protected long append(RegionInfo hri, WALKeyImpl key, WALEdit edits, boolean inMemstore) + throws IOException { + long txid = stampSequenceIdAndPublishToRingBuffer(hri, key, edits, inMemstore, waitingConsumePayloads); if (shouldScheduleConsumer()) { consumeExecutor.execute(consumer); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index 810267113076..78259d1ba1a5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -431,8 +431,8 @@ protected void doShutdown() throws IOException { @Override protected long append(final RegionInfo hri, final WALKeyImpl key, final WALEdit edits, - final boolean inMemstore, boolean closeRegion) throws IOException { - return stampSequenceIdAndPublishToRingBuffer(hri, key, edits, inMemstore, closeRegion, + final boolean inMemstore) throws IOException { + return stampSequenceIdAndPublishToRingBuffer(hri, key, edits, inMemstore, disruptor.getRingBuffer()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java index c174650194ee..44c96dee7619 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java @@ -40,7 +40,7 @@ /** * A WAL Entry for {@link AbstractFSWAL} implementation. Immutable. * A subclass of {@link Entry} that carries extra info across the ring buffer such as - * region sequence id (we want to use this later, just before we write the WAL to ensure region + * region sequenceid (we want to use this later, just before we write the WAL to ensure region * edits maintain order). The extra info added here is not 'serialized' as part of the WALEdit * hence marked 'transient' to underline this fact. It also adds mechanism so we can wait on * the assign of the region sequence id. See #stampRegionSequenceId(). @@ -50,17 +50,32 @@ class FSWALEntry extends Entry { // The below data members are denoted 'transient' just to highlight these are not persisted; // they are only in memory and held here while passing over the ring buffer. private final transient long txid; + + /** + * If false, means this is a meta edit written by the hbase system itself. It was not in + * memstore. HBase uses these edit types to note in the log operational transitions such + * as compactions, flushes, or region open/closes. + */ private final transient boolean inMemstore; + + /** + * Set if this is a meta edit and it is of close region type. + */ private final transient boolean closeRegion; + private final transient RegionInfo regionInfo; private final transient Set familyNames; private final transient ServerCall rpcCall; + /** + * @param inMemstore If true, then this is a data edit, one that came from client. If false, it + * is a meta edit made by the hbase system itself and is for the WAL only. + */ FSWALEntry(final long txid, final WALKeyImpl key, final WALEdit edit, final RegionInfo regionInfo, - final boolean inMemstore, boolean closeRegion, ServerCall rpcCall) { + final boolean inMemstore, ServerCall rpcCall) { super(key, edit); this.inMemstore = inMemstore; - this.closeRegion = closeRegion; + this.closeRegion = !inMemstore && edit.isRegionCloseMarker(); this.regionInfo = regionInfo; this.txid = txid; if (inMemstore) { @@ -68,7 +83,7 @@ class FSWALEntry extends Entry { Set families = edit.getFamilies(); this.familyNames = families != null ? families : collectFamilies(edit.getCells()); } else { - this.familyNames = Collections. emptySet(); + this.familyNames = Collections.emptySet(); } this.rpcCall = rpcCall; if (rpcCall != null) { @@ -83,7 +98,7 @@ static Set collectFamilies(List cells) { } else { Set set = new TreeSet<>(Bytes.BYTES_COMPARATOR); for (Cell cell: cells) { - if (!CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) { + if (!WALEdit.isMetaEditFamily(cell)) { set.add(CellUtil.cloneFamily(cell)); } } @@ -94,7 +109,7 @@ static Set collectFamilies(List cells) { @Override public String toString() { return "sequence=" + this.txid + ", " + super.toString(); - }; + } boolean isInMemStore() { return this.inMemstore; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java index 8e87b40ac446..9b5a4d7a4ca4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java @@ -38,13 +38,11 @@ import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; /** - *

- * Accounting of sequence ids per region and then by column family. So we can our accounting + * Accounting of sequence ids per region and then by column family. So we can keep our accounting * current, call startCacheFlush and then finishedCacheFlush or abortCacheFlush so this instance can * keep abreast of the state of sequence id persistence. Also call update per append. - *

*

- * For the implementation, we assume that all the {@code encodedRegionName} passed in is gotten by + * For the implementation, we assume that all the {@code encodedRegionName} passed in are gotten by * {@link org.apache.hadoop.hbase.client.RegionInfo#getEncodedNameAsBytes()}. So it is safe to use * it as a hash key. And for family name, we use {@link ImmutableByteArray} as key. This is because * hash based map is much faster than RBTree or CSLM and here we are on the critical write path. See @@ -53,8 +51,8 @@ */ @InterfaceAudience.Private class SequenceIdAccounting { - private static final Logger LOG = LoggerFactory.getLogger(SequenceIdAccounting.class); + /** * This lock ties all operations on {@link SequenceIdAccounting#flushingSequenceIds} and * {@link #lowestUnflushedSequenceIds} Maps. {@link #lowestUnflushedSequenceIds} has the @@ -110,7 +108,6 @@ class SequenceIdAccounting { /** * Returns the lowest unflushed sequence id for the region. - * @param encodedRegionName * @return Lowest outstanding unflushed sequenceid for encodedRegionName. Will * return {@link HConstants#NO_SEQNUM} when none. */ @@ -125,8 +122,6 @@ long getLowestSequenceId(final byte[] encodedRegionName) { } /** - * @param encodedRegionName - * @param familyName * @return Lowest outstanding unflushed sequenceid for encodedRegionname and * familyName. Returned sequenceid may be for an edit currently being * flushed. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java index e8c7404d606f..09e1a39cb8d0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java @@ -1,5 +1,4 @@ -/** - * +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -66,7 +65,7 @@ public static WALKeyImpl writeCompactionMarker(WAL wal, NavigableMap replicationScope, RegionInfo hri, final CompactionDescriptor c, MultiVersionConcurrencyControl mvcc) throws IOException { WALKeyImpl walKey = - writeMarker(wal, replicationScope, hri, WALEdit.createCompaction(hri, c), false, mvcc); + writeMarker(wal, replicationScope, hri, WALEdit.createCompaction(hri, c), mvcc); if (LOG.isTraceEnabled()) { LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c)); } @@ -82,7 +81,7 @@ public static WALKeyImpl writeFlushMarker(WAL wal, NavigableMap RegionInfo hri, final FlushDescriptor f, boolean sync, MultiVersionConcurrencyControl mvcc) throws IOException { WALKeyImpl walKey = doFullMarkerAppendTransaction(wal, replicationScope, hri, - WALEdit.createFlushWALEdit(hri, f), false, mvcc, sync); + WALEdit.createFlushWALEdit(hri, f), mvcc, sync); if (LOG.isTraceEnabled()) { LOG.trace("Appended flush marker " + TextFormat.shortDebugString(f)); } @@ -94,11 +93,11 @@ public static WALKeyImpl writeFlushMarker(WAL wal, NavigableMap * only. Not for external client consumption. */ public static WALKeyImpl writeRegionEventMarker(WAL wal, - NavigableMap replicationScope, RegionInfo hri, final RegionEventDescriptor r, - final MultiVersionConcurrencyControl mvcc) throws IOException { + NavigableMap replicationScope, RegionInfo hri, RegionEventDescriptor r, + MultiVersionConcurrencyControl mvcc) + throws IOException { WALKeyImpl walKey = - writeMarker(wal, replicationScope, hri, WALEdit.createRegionEventWALEdit(hri, r), - r.getEventType() == RegionEventDescriptor.EventType.REGION_CLOSE, mvcc); + writeMarker(wal, replicationScope, hri, WALEdit.createRegionEventWALEdit(hri, r), mvcc); if (LOG.isTraceEnabled()) { LOG.trace("Appended region event marker " + TextFormat.shortDebugString(r)); } @@ -116,22 +115,22 @@ public static WALKeyImpl writeRegionEventMarker(WAL wal, * @throws IOException We will throw an IOException if we can not append to the HLog. */ public static WALKeyImpl writeBulkLoadMarkerAndSync(final WAL wal, - final NavigableMap replicationScope, final RegionInfo hri, - final WALProtos.BulkLoadDescriptor desc, final MultiVersionConcurrencyControl mvcc) + final NavigableMap replicationScope, final RegionInfo hri, + final WALProtos.BulkLoadDescriptor desc, final MultiVersionConcurrencyControl mvcc) throws IOException { WALKeyImpl walKey = writeMarker(wal, replicationScope, hri, - WALEdit.createBulkLoadEvent(hri, desc), false, mvcc); + WALEdit.createBulkLoadEvent(hri, desc), mvcc); if (LOG.isTraceEnabled()) { LOG.trace("Appended Bulk Load marker " + TextFormat.shortDebugString(desc)); } return walKey; } - private static WALKeyImpl writeMarker(final WAL wal, - final NavigableMap replicationScope, final RegionInfo hri, final WALEdit edit, - boolean closeRegion, final MultiVersionConcurrencyControl mvcc) throws IOException { + private static WALKeyImpl writeMarker(final WAL wal, NavigableMap replicationScope, RegionInfo hri, WALEdit edit, MultiVersionConcurrencyControl mvcc) + throws IOException { // If sync == true in below, then timeout is not used; safe to pass UNSPECIFIED_TIMEOUT - return doFullMarkerAppendTransaction(wal, replicationScope, hri, edit, closeRegion, mvcc, true); + return doFullMarkerAppendTransaction(wal, replicationScope, hri, edit, mvcc, true); } /** @@ -144,14 +143,14 @@ private static WALKeyImpl writeMarker(final WAL wal, */ private static WALKeyImpl doFullMarkerAppendTransaction(final WAL wal, final NavigableMap replicationScope, final RegionInfo hri, final WALEdit edit, - boolean closeRegion, final MultiVersionConcurrencyControl mvcc, final boolean sync) + final MultiVersionConcurrencyControl mvcc, final boolean sync) throws IOException { // TODO: Pass in current time to use? WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), System.currentTimeMillis(), mvcc, replicationScope); long trx = MultiVersionConcurrencyControl.NONE; try { - trx = wal.appendMarker(hri, walKey, edit, closeRegion); + trx = wal.appendMarker(hri, walKey, edit); if (sync) { wal.sync(trx); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALActionListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALActionListener.java index 27b25c45a753..d25ab072eb1e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALActionListener.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALActionListener.java @@ -20,7 +20,6 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; import org.apache.hadoop.hbase.replication.ReplicationUtils; import org.apache.hadoop.hbase.wal.WALEdit; @@ -74,7 +73,7 @@ static void scopeWALEdits(WALKey logKey, WALEdit logEdit, Configuration conf) { } // For replay, or if all the cells are markers, do not need to store replication scope. if (logEdit.isReplay() || - logEdit.getCells().stream().allMatch(c -> CellUtil.matchingFamily(c, WALEdit.METAFAMILY))) { + logEdit.getCells().stream().allMatch(c -> WALEdit.isMetaEditFamily(c))) { ((WALKeyImpl) logKey).clearReplicationScope(); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java index 249ced7bf03f..558f686f97ac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java @@ -162,17 +162,17 @@ public void close() { @Override public long appendData(RegionInfo info, WALKeyImpl key, WALEdit edits) throws IOException { - return append(info, key, edits, true, false); + return append(info, key, edits, true); } @Override - public long appendMarker(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean closeRegion) + public long appendMarker(RegionInfo info, WALKeyImpl key, WALEdit edits) throws IOException { - return append(info, key, edits, false, closeRegion); + return append(info, key, edits, false); } - private long append(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean inMemstore, - boolean closeRegion) throws IOException { + private long append(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean inMemstore) + throws IOException { WriteEntry writeEntry = key.getMvcc().begin(); if (!edits.isReplay()) { for (Cell cell : edits.getCells()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/LogRecoveredEditsOutputSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/LogRecoveredEditsOutputSink.java index aa649e42b362..9fc43b13e0d8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/LogRecoveredEditsOutputSink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/LogRecoveredEditsOutputSink.java @@ -362,7 +362,7 @@ void filterCellByStore(WAL.Entry logEntry) { // We make the assumption that most cells will be kept. ArrayList keptCells = new ArrayList<>(logEntry.getEdit().getCells().size()); for (Cell cell : logEntry.getEdit().getCells()) { - if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) { + if (WALEdit.isMetaEditFamily(cell)) { keptCells.add(cell); } else { byte[] family = CellUtil.cloneFamily(cell); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java index 64adbcc2569b..7f7c4124b931 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -81,7 +81,7 @@ public interface WAL extends Closeable, WALFileLengthProvider { * can clean logs. Returns null if nothing to flush. Names are actual * region names as returned by {@link RegionInfo#getEncodedName()} */ - byte[][] rollWriter(boolean force) throws FailedLogCloseException, IOException; + byte[][] rollWriter(boolean force) throws IOException; /** * Stop accepting new writes. If we have unsynced writes still in buffer, sync them. @@ -99,7 +99,7 @@ public interface WAL extends Closeable, WALFileLengthProvider { /** * Append a set of data edits to the WAL. 'Data' here means that the content in the edits will - * also be added to memstore. + * also have transitioned through the memstore. *

* The WAL is not flushed/sync'd after this transaction completes BUT on return this edit must * have its region edit/sequence id assigned else it messes up our unification of mvcc and @@ -110,13 +110,16 @@ public interface WAL extends Closeable, WALFileLengthProvider { * sequence id that is after all currently appended edits. * @return Returns a 'transaction id' and key will have the region edit/sequence id * in it. - * @see #appendMarker(RegionInfo, WALKeyImpl, WALEdit, boolean) + * @see #appendMarker(RegionInfo, WALKeyImpl, WALEdit) */ long appendData(RegionInfo info, WALKeyImpl key, WALEdit edits) throws IOException; /** - * Append a marker edit to the WAL. A marker could be a FlushDescriptor, a compaction marker, or - * region event marker. The difference here is that, a marker will not be added to memstore. + * Append an operational 'meta' event marker edit to the WAL. A marker meta edit could + * be a FlushDescriptor, a compaction marker, or a region event marker; e.g. region open + * or region close. The difference between a 'marker' append and a 'data' append as in + * {@link #appendData(RegionInfo, WALKeyImpl, WALEdit)}is that a marker will not have + * transitioned through the memstore. *

* The WAL is not flushed/sync'd after this transaction completes BUT on return this edit must * have its region edit/sequence id assigned else it messes up our unification of mvcc and @@ -125,15 +128,11 @@ public interface WAL extends Closeable, WALFileLengthProvider { * @param key Modified by this call; we add to it this edits region edit/sequence id. * @param edits Edits to append. MAY CONTAIN NO EDITS for case where we want to get an edit * sequence id that is after all currently appended edits. - * @param closeRegion Whether this is a region close marker, i.e, the last wal edit for this - * region on this region server. The WAL implementation should remove all the related - * stuff, for example, the sequence id accounting. * @return Returns a 'transaction id' and key will have the region edit/sequence id * in it. * @see #appendData(RegionInfo, WALKeyImpl, WALEdit) */ - long appendMarker(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean closeRegion) - throws IOException; + long appendMarker(RegionInfo info, WALKeyImpl key, WALEdit edits) throws IOException; /** * updates the seuence number of a specific store. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEdit.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEdit.java index 214000200cba..cfa414d3f05a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEdit.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEdit.java @@ -1,5 +1,4 @@ -/** - * +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -37,8 +36,6 @@ import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; @@ -52,45 +49,103 @@ * single record, in PB format, followed (optionally) by Cells written via the WALCellEncoder. *

This class is LimitedPrivate for CPs to read-only. The {@link #add} methods are * classified as private methods, not for use by CPs.

+ * + *

A particular WALEdit 'type' is the 'meta' type used to mark key operational + * events in the WAL such as compaction, flush, or region open. These meta types do not traverse + * hbase memstores. They are edits made by the hbase system rather than edit data submitted by + * clients. They only show in the WAL. These 'Meta' types have not been formally specified + * (or made into an explicit class type). They evolved organically. HBASE-8457 suggests codifying + * a WALEdit 'type' by adding a type field to WALEdit that gets serialized into the WAL. TODO. + * Would have to work on the consumption-side. Reading WALs on replay we seem to consume + * a Cell-at-a-time rather than by WALEdit. We are already in the below going out of our + * way to figure particular types -- e.g. if a compaction, replay, or close meta Marker -- during + * normal processing so would make sense to do this. Current system is an awkward marking of Cell + * columnfamily as {@link #METAFAMILY} and then setting qualifier based off meta edit type. For + * replay-time where we read Cell-at-a-time, there are utility methods below for figuring + * meta type. See also + * {@link #createBulkLoadEvent(RegionInfo, WALProtos.BulkLoadDescriptor)}, etc., for where we + * create meta WALEdit instances.

+ * *

WALEdit will accumulate a Set of all column family names referenced by the Cells * {@link #add(Cell)}'d. This is an optimization. Usually when loading a WALEdit, we have the * column family name to-hand.. just shove it into the WALEdit if available. Doing this, we can * save on a parse of each Cell to figure column family down the line when we go to add the * WALEdit to the WAL file. See the hand-off in FSWALEntry Constructor. + * @see WALKey */ // TODO: Do not expose this class to Coprocessors. It has set methods. A CP might meddle. @InterfaceAudience.LimitedPrivate({ HBaseInterfaceAudience.REPLICATION, HBaseInterfaceAudience.COPROC }) public class WALEdit implements HeapSize { - private static final Logger LOG = LoggerFactory.getLogger(WALEdit.class); - - // TODO: Get rid of this; see HBASE-8457 + // Below defines are for writing WALEdit 'meta' Cells.. + // TODO: Get rid of this system of special 'meta' Cells. See HBASE-8457. It suggests + // adding a type to WALEdit itself for use denoting meta Edits and their types. public static final byte [] METAFAMILY = Bytes.toBytes("METAFAMILY"); - @VisibleForTesting + + /** + * @deprecated Since 2.3.0. Not used. + */ + @Deprecated public static final byte [] METAROW = Bytes.toBytes("METAROW"); + + /** + * @deprecated Since 2.3.0. Make it protected, internal-use only. Use + * {@link #isCompactionMarker(Cell)} + */ + @Deprecated @VisibleForTesting public static final byte[] COMPACTION = Bytes.toBytes("HBASE::COMPACTION"); + + /** + * @deprecated Since 2.3.0. Make it protected, internal-use only. + */ + @Deprecated @VisibleForTesting public static final byte [] FLUSH = Bytes.toBytes("HBASE::FLUSH"); - @VisibleForTesting - public static final byte [] REGION_EVENT = Bytes.toBytes("HBASE::REGION_EVENT"); + + /** + * Qualifier for region event meta 'Marker' WALEdits start with the + * {@link #REGION_EVENT_PREFIX} prefix ('HBASE::REGION_EVENT::'). After the prefix, + * we note the type of the event which we get from the RegionEventDescriptor protobuf + * instance type (A RegionEventDescriptor protobuf instance is written as the meta Marker + * Cell value). Adding a type suffix means we do not have to deserialize the protobuf to + * figure out what type of event this is.. .just read the qualifier suffix. For example, + * a close region event descriptor will have a qualifier of HBASE::REGION_EVENT::REGION_CLOSE. + * See WAL.proto and the EventType in RegionEventDescriptor protos for all possible + * event types. + */ + private static final String REGION_EVENT_STR = "HBASE::REGION_EVENT"; + private static final String REGION_EVENT_PREFIX_STR = REGION_EVENT_STR + "::"; + private static final byte [] REGION_EVENT_PREFIX = Bytes.toBytes(REGION_EVENT_PREFIX_STR); + + /** + * @deprecated Since 2.3.0. Remove. Not for external use. Not used. + */ + @Deprecated + public static final byte [] REGION_EVENT = Bytes.toBytes(REGION_EVENT_STR); + + /** + * We use this define figuring if we are carrying a close event. + */ + private static final byte [] REGION_EVENT_CLOSE = + createRegionEventDescriptorQualifier(RegionEventDescriptor.EventType.REGION_CLOSE); + @VisibleForTesting public static final byte [] BULK_LOAD = Bytes.toBytes("HBASE::BULK_LOAD"); - private final boolean replay; + private final transient boolean replay; - private ArrayList cells = null; + private ArrayList cells; /** * All the Cell families in cells. Updated by {@link #add(Cell)} and * {@link #add(Map)}. This Set is passed to the FSWALEntry so it does not have * to recalculate the Set of families in a transaction; makes for a bunch of CPU savings. - * An optimization that saves on CPU-expensive Cell-parsing. */ private Set families = null; public WALEdit() { - this(false); + this(1, false); } /** @@ -100,8 +155,8 @@ public WALEdit() { * @see HBASE-20781 */ @Deprecated - public WALEdit(boolean isReplay) { - this(1, isReplay); + public WALEdit(boolean replay) { + this(1, replay); } /** @@ -125,7 +180,7 @@ public WALEdit(int cellCount, boolean isReplay) { private Set getOrCreateFamilies() { if (this.families == null) { - this.families = new TreeSet(Bytes.BYTES_COMPARATOR); + this.families = new TreeSet<>(Bytes.BYTES_COMPARATOR); } return this.families; } @@ -140,22 +195,26 @@ private Set getOrCreateFamilies() { /** * @return True is f is {@link #METAFAMILY} + * @deprecated Since 2.3.0. Do not expose. Make protected. */ + @Deprecated public static boolean isMetaEditFamily(final byte [] f) { return Bytes.equals(METAFAMILY, f); } + /** + * Replaying WALs can read Cell-at-a-time so need this method in those cases. + */ public static boolean isMetaEditFamily(Cell cell) { return CellUtil.matchingFamily(cell, METAFAMILY); } + /** + * @return True if this is a meta edit; has one edit only and its columnfamily + * is {@link #METAFAMILY}. + */ public boolean isMetaEdit() { - for (Cell cell: cells) { - if (!isMetaEditFamily(cell)) { - return false; - } - } - return true; + return this.families != null && this.families.size() == 1 && this.families.contains(METAFAMILY); } /** @@ -215,7 +274,7 @@ public int readFromCells(Codec.Decoder cellDecoder, int expectedCount) throws IO cells.clear(); cells.ensureCapacity(expectedCount); while (cells.size() < expectedCount && cellDecoder.advance()) { - cells.add(cellDecoder.current()); + add(cellDecoder.current()); } return cells.size(); } @@ -241,7 +300,7 @@ public long estimatedSerializedSizeOf() { public String toString() { StringBuilder sb = new StringBuilder(); - sb.append("[#edits: " + cells.size() + " = <"); + sb.append("[#edits: ").append(cells.size()).append(" = <"); for (Cell cell : cells) { sb.append(cell); sb.append("; "); @@ -257,30 +316,61 @@ public static WALEdit createFlushWALEdit(RegionInfo hri, FlushDescriptor f) { } public static FlushDescriptor getFlushDescriptor(Cell cell) throws IOException { - if (CellUtil.matchingColumn(cell, METAFAMILY, FLUSH)) { - return FlushDescriptor.parseFrom(CellUtil.cloneValue(cell)); - } - return null; + return CellUtil.matchingColumn(cell, METAFAMILY, FLUSH)? + FlushDescriptor.parseFrom(CellUtil.cloneValue(cell)): null; } + /** + * @return A meta Marker WALEdit that has a single Cell whose value is the passed in + * regionEventDesc serialized and whose row is this region, + * columnfamily is {@link #METAFAMILY} and qualifier is + * {@link #REGION_EVENT_PREFIX} + {@link RegionEventDescriptor#getEventType()}; + * for example HBASE::REGION_EVENT::REGION_CLOSE. + */ public static WALEdit createRegionEventWALEdit(RegionInfo hri, RegionEventDescriptor regionEventDesc) { - KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, REGION_EVENT, - EnvironmentEdgeManager.currentTime(), regionEventDesc.toByteArray()); + return createRegionEventWALEdit(getRowForRegion(hri), regionEventDesc); + } + + @VisibleForTesting + public static WALEdit createRegionEventWALEdit(byte [] rowForRegion, + RegionEventDescriptor regionEventDesc) { + KeyValue kv = new KeyValue(rowForRegion, METAFAMILY, + createRegionEventDescriptorQualifier(regionEventDesc.getEventType()), + EnvironmentEdgeManager.currentTime(), regionEventDesc.toByteArray()); return new WALEdit().add(kv, METAFAMILY); } + /** + * @return Cell qualifier for the passed in RegionEventDescriptor Type; e.g. we'll + * return something like a byte array with HBASE::REGION_EVENT::REGION_OPEN in it. + */ + @VisibleForTesting + public static byte [] createRegionEventDescriptorQualifier(RegionEventDescriptor.EventType t) { + return Bytes.toBytes(REGION_EVENT_PREFIX_STR + t.toString()); + } + + /** + * Public so can be accessed from regionserver.wal package. + * @return True if this is a Marker Edit and it is a RegionClose type. + */ + public boolean isRegionCloseMarker() { + return isMetaEdit() && PrivateCellUtil.matchingQualifier(this.cells.get(0), + REGION_EVENT_CLOSE, 0, REGION_EVENT_CLOSE.length); + } + + /** + * @return Returns a RegionEventDescriptor made by deserializing the content of the + * passed in cell, IFF the cell is a RegionEventDescriptor + * type WALEdit. + */ public static RegionEventDescriptor getRegionEventDescriptor(Cell cell) throws IOException { - if (CellUtil.matchingColumn(cell, METAFAMILY, REGION_EVENT)) { - return RegionEventDescriptor.parseFrom(CellUtil.cloneValue(cell)); - } - return null; + return CellUtil.matchingColumnFamilyAndQualifierPrefix(cell, METAFAMILY, REGION_EVENT_PREFIX)? + RegionEventDescriptor.parseFrom(CellUtil.cloneValue(cell)): null; } /** - * Create a compaction WALEdit - * @param c - * @return A WALEdit that has c serialized as its value + * @return A Marker WALEdit that has c serialized as its value */ public static WALEdit createCompaction(final RegionInfo hri, final CompactionDescriptor c) { byte [] pbbytes = c.toByteArray(); @@ -305,10 +395,7 @@ public static byte[] getRowForRegion(RegionInfo hri) { * @return deserialized CompactionDescriptor or null. */ public static CompactionDescriptor getCompaction(Cell kv) throws IOException { - if (isCompactionMarker(kv)) { - return CompactionDescriptor.parseFrom(CellUtil.cloneValue(kv)); - } - return null; + return isCompactionMarker(kv)? CompactionDescriptor.parseFrom(CellUtil.cloneValue(kv)): null; } /** @@ -328,12 +415,9 @@ public static boolean isCompactionMarker(Cell cell) { * @return The WALEdit for the BulkLoad */ public static WALEdit createBulkLoadEvent(RegionInfo hri, - WALProtos.BulkLoadDescriptor bulkLoadDescriptor) { - KeyValue kv = new KeyValue(getRowForRegion(hri), - METAFAMILY, - BULK_LOAD, - EnvironmentEdgeManager.currentTime(), - bulkLoadDescriptor.toByteArray()); + WALProtos.BulkLoadDescriptor bulkLoadDescriptor) { + KeyValue kv = new KeyValue(getRowForRegion(hri), METAFAMILY, BULK_LOAD, + EnvironmentEdgeManager.currentTime(), bulkLoadDescriptor.toByteArray()); return new WALEdit().add(kv, METAFAMILY); } @@ -343,17 +427,16 @@ public static WALEdit createBulkLoadEvent(RegionInfo hri, * @return deserialized BulkLoadDescriptor or null. */ public static WALProtos.BulkLoadDescriptor getBulkLoadDescriptor(Cell cell) throws IOException { - if (CellUtil.matchingColumn(cell, METAFAMILY, BULK_LOAD)) { - return WALProtos.BulkLoadDescriptor.parseFrom(CellUtil.cloneValue(cell)); - } - return null; + return CellUtil.matchingColumn(cell, METAFAMILY, BULK_LOAD)? + WALProtos.BulkLoadDescriptor.parseFrom(CellUtil.cloneValue(cell)): null; } /** * Append the given map of family->edits to a WALEdit data structure. * This does not write to the WAL itself. * Note that as an optimization, we will stamp the Set of column families into the WALEdit - * to save on our having to calculate it subsequently way down in the actual WAL writing. + * to save on our having to calculate column families subsequently down in the actual WAL + * writing. * * @param familyMap map of family->edits */ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java index 23f6c29fddc5..0e3fac9a79ce 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -113,20 +112,19 @@ public void verifyBulkLoadEvent() throws IOException { storeFileName = (new Path(storeFileName)).getName(); List storeFileNames = new ArrayList<>(); storeFileNames.add(storeFileName); - when(log.appendMarker(any(), any(), - argThat(bulkLogWalEdit(WALEdit.BULK_LOAD, tableName.toBytes(), - familyName, storeFileNames)), - anyBoolean())).thenAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocation) { - WALKeyImpl walKey = invocation.getArgument(1); - MultiVersionConcurrencyControl mvcc = walKey.getMvcc(); - if (mvcc != null) { - MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin(); - walKey.setWriteEntry(we); - } - return 01L; - }; + when(log.appendMarker(any(), any(), argThat( + bulkLogWalEdit(WALEdit.BULK_LOAD, tableName.toBytes(), familyName, storeFileNames)))). + thenAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) { + WALKeyImpl walKey = invocation.getArgument(1); + MultiVersionConcurrencyControl mvcc = walKey.getMvcc(); + if (mvcc != null) { + MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin(); + walKey.setWriteEntry(we); + } + return 01L; + }; }); testRegionWithFamiliesAndSpecifiedTableName(tableName, family1) .bulkLoadHFiles(familyPaths, false, null); @@ -141,8 +139,7 @@ public void bulkHLogShouldThrowNoErrorAndWriteMarkerWithBlankInput() throws IOEx @Test public void shouldBulkLoadSingleFamilyHLog() throws IOException { when(log.appendMarker(any(), - any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)), - anyBoolean())).thenAnswer(new Answer() { + any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))).thenAnswer(new Answer() { @Override public Object answer(InvocationOnMock invocation) { WALKeyImpl walKey = invocation.getArgument(1); @@ -161,8 +158,7 @@ public Object answer(InvocationOnMock invocation) { @Test public void shouldBulkLoadManyFamilyHLog() throws IOException { when(log.appendMarker(any(), - any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)), - anyBoolean())).thenAnswer(new Answer() { + any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))).thenAnswer(new Answer() { @Override public Object answer(InvocationOnMock invocation) { WALKeyImpl walKey = invocation.getArgument(1); @@ -182,8 +178,7 @@ public Object answer(InvocationOnMock invocation) { @Test public void shouldBulkLoadManyFamilyHLogEvenWhenTableNameNamespaceSpecified() throws IOException { when(log.appendMarker(any(), - any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)), - anyBoolean())).thenAnswer(new Answer() { + any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))).thenAnswer(new Answer() { @Override public Object answer(InvocationOnMock invocation) { WALKeyImpl walKey = invocation.getArgument(1); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java index 063949353c28..4e6a1fefc222 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -31,7 +31,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Put; @@ -162,18 +161,17 @@ public long getLength() { } // Make up mocked server and services. - Server server = mock(Server.class); - when(server.getConfiguration()).thenReturn(CONF); - when(server.isStopped()).thenReturn(false); - when(server.isAborted()).thenReturn(false); RegionServerServices services = mock(RegionServerServices.class); + when(services.getConfiguration()).thenReturn(CONF); + when(services.isStopped()).thenReturn(false); + when(services.isAborted()).thenReturn(false); // OK. Now I have my mocked up Server and RegionServerServices and my dodgy WAL, go ahead with // the test. FileSystem fs = FileSystem.get(CONF); Path rootDir = new Path(dir + getName()); DodgyFSLog dodgyWAL = new DodgyFSLog(fs, rootDir, getName(), CONF); dodgyWAL.init(); - LogRoller logRoller = new LogRoller(server, services); + LogRoller logRoller = new LogRoller(services); logRoller.addWAL(dodgyWAL); logRoller.start(); @@ -224,7 +222,7 @@ public long getLength() { // to just continue. // So, should be no abort at this stage. Verify. - Mockito.verify(server, Mockito.atLeast(0)). + Mockito.verify(services, Mockito.atLeast(0)). abort(Mockito.anyString(), (Throwable)Mockito.anyObject()); try { dodgyWAL.throwAppendException = false; @@ -240,7 +238,7 @@ public long getLength() { // happens. If it don't we'll timeout the whole test. That is fine. while (true) { try { - Mockito.verify(server, Mockito.atLeast(1)). + Mockito.verify(services, Mockito.atLeast(1)). abort(Mockito.anyString(), (Throwable)Mockito.anyObject()); break; } catch (WantedButNotInvoked t) { @@ -249,7 +247,7 @@ public long getLength() { } } finally { // To stop logRoller, its server has to say it is stopped. - Mockito.when(server.isStopped()).thenReturn(true); + Mockito.when(services.isStopped()).thenReturn(true); if (logRoller != null) logRoller.close(); if (region != null) { try { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index f178be5f0e3e..3339a5e1d3db 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -29,7 +29,6 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -5630,7 +5629,7 @@ public void testOpenRegionWrittenToWAL() throws Exception { TEST_UTIL.getConfiguration(), rss, null); verify(wal, times(1)).appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), - editCaptor.capture(), anyBoolean()); + editCaptor.capture()); WALEdit edit = editCaptor.getValue(); assertNotNull(edit); @@ -5712,16 +5711,16 @@ public Long answer(InvocationOnMock invocation) throws Throwable { return 1L; } }); - when(wal.appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class), - anyBoolean())).thenAnswer(new Answer() { - @Override - public Long answer(InvocationOnMock invocation) throws Throwable { - WALKeyImpl key = invocation.getArgument(1); - MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); - key.setWriteEntry(we); - return 1L; - } - }); + when(wal.appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class))). + thenAnswer(new Answer() { + @Override + public Long answer(InvocationOnMock invocation) throws Throwable { + WALKeyImpl key = invocation.getArgument(1); + MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); + key.setWriteEntry(we); + return 1L; + } + }); return wal; } @@ -5754,8 +5753,8 @@ public void testCloseRegionWrittenToWAL() throws Exception { region.close(false); // 2 times, one for region open, the other close region - verify(wal, times(2)).appendMarker(any(RegionInfo.class), (WALKeyImpl) any(WALKeyImpl.class), - editCaptor.capture(), anyBoolean()); + verify(wal, times(2)).appendMarker(any(RegionInfo.class), + (WALKeyImpl) any(WALKeyImpl.class), editCaptor.capture()); WALEdit edit = editCaptor.getAllValues().get(1); assertNotNull(edit); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java index 6212613fe2ec..10b9d8f6b081 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java @@ -30,7 +30,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparatorImpl; -import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -195,15 +194,18 @@ private int verifyAllEditsMadeItIn(final FileSystem fs, final Configuration conf WALEdit val = entry.getEdit(); count++; // Check this edit is for this region. - if (!Bytes - .equals(key.getEncodedRegionName(), region.getRegionInfo().getEncodedNameAsBytes())) { + if (!Bytes.equals(key.getEncodedRegionName(), + region.getRegionInfo().getEncodedNameAsBytes())) { continue; } Cell previous = null; for (Cell cell : val.getCells()) { - if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) continue; - if (previous != null && CellComparatorImpl.COMPARATOR.compareRows(previous, cell) == 0) + if (WALEdit.isMetaEditFamily(cell)) { continue; + } + if (previous != null && CellComparatorImpl.COMPARATOR.compareRows(previous, cell) == 0) { + continue; + } previous = cell; walCells.add(cell); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java index d9a76e828c73..a50ef7800452 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -204,11 +204,10 @@ public long getLength() { @Test public void testLockupWhenSyncInMiddleOfZigZagSetup() throws IOException { // Mocked up server and regionserver services. Needed below. - Server server = Mockito.mock(Server.class); - Mockito.when(server.getConfiguration()).thenReturn(CONF); - Mockito.when(server.isStopped()).thenReturn(false); - Mockito.when(server.isAborted()).thenReturn(false); RegionServerServices services = Mockito.mock(RegionServerServices.class); + Mockito.when(services.getConfiguration()).thenReturn(CONF); + Mockito.when(services.isStopped()).thenReturn(false); + Mockito.when(services.isAborted()).thenReturn(false); // OK. Now I have my mocked up Server & RegionServerServices and dodgy WAL, go ahead with test. FileSystem fs = FileSystem.get(CONF); @@ -217,7 +216,7 @@ public void testLockupWhenSyncInMiddleOfZigZagSetup() throws IOException { dodgyWAL.init(); Path originalWAL = dodgyWAL.getCurrentFileName(); // I need a log roller running. - LogRoller logRoller = new LogRoller(server, services); + LogRoller logRoller = new LogRoller(services); logRoller.addWAL(dodgyWAL); // There is no 'stop' once a logRoller is running.. it just dies. logRoller.start(); @@ -294,7 +293,7 @@ public void run() { } } finally { // To stop logRoller, its server has to say it is stopped. - Mockito.when(server.isStopped()).thenReturn(true); + Mockito.when(services.isStopped()).thenReturn(true); Closeables.close(logRoller, true); try { if (region != null) { @@ -380,11 +379,10 @@ public long getLength() { } // Mocked up server and regionserver services. Needed below. - final Server server = Mockito.mock(Server.class); - Mockito.when(server.getConfiguration()).thenReturn(CONF); - Mockito.when(server.isStopped()).thenReturn(false); - Mockito.when(server.isAborted()).thenReturn(false); RegionServerServices services = Mockito.mock(RegionServerServices.class); + Mockito.when(services.getConfiguration()).thenReturn(CONF); + Mockito.when(services.isStopped()).thenReturn(false); + Mockito.when(services.isAborted()).thenReturn(false); // OK. Now I have my mocked up Server & RegionServerServices and dodgy WAL, go ahead with test. FileSystem fs = FileSystem.get(CONF); @@ -392,7 +390,7 @@ public long getLength() { final DodgyFSLog dodgyWAL = new DodgyFSLog(fs, rootDir, getName(), CONF); dodgyWAL.init(); // I need a log roller running. - LogRoller logRoller = new LogRoller(server, services); + LogRoller logRoller = new LogRoller(services); logRoller.addWAL(dodgyWAL); // There is no 'stop' once a logRoller is running.. it just dies. logRoller.start(); @@ -433,7 +431,7 @@ public void run() { } finally { // To stop logRoller, its server has to say it is stopped. - Mockito.when(server.isStopped()).thenReturn(true); + Mockito.when(services.isStopped()).thenReturn(true); if (logRoller != null) { logRoller.close(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java index 3b1ffb526343..2f2351714821 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java @@ -428,7 +428,7 @@ public void run() { final RegionInfo info = region.getRegionInfo(); final WALKeyImpl logkey = new WALKeyImpl(info.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), clusterIds, -1, -1, region.getMVCC(), scopes); - wal.append(info, logkey, edits, true, false); + wal.append(info, logkey, edits, true); region.getMVCC().completeAndWait(logkey.getWriteEntry()); } region.flush(true); @@ -478,7 +478,7 @@ public void testWriteEntryCanBeNull() throws IOException { new WALKeyImpl(ri.getEncodedNameAsBytes(), td.getTableName(), SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes); try { - wal.append(ri, key, cols, true, false); + wal.append(ri, key, cols, true); fail("Should fail since the wal has already been closed"); } catch (IOException e) { // expected diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java index 704141cec4c2..d05fcce7ee95 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java @@ -1157,7 +1157,7 @@ private FSWALEntry createFSWALEntry(HTableDescriptor htd, HRegionInfo hri, long byte[] rowName, byte[] family, EnvironmentEdge ee, MultiVersionConcurrencyControl mvcc, int index, NavigableMap scopes) throws IOException { FSWALEntry entry = new FSWALEntry(sequence, createWALKey(htd.getTableName(), hri, mvcc, scopes), - createWALEdit(rowName, family, ee, index), hri, true, false, null); + createWALEdit(rowName, family, ee, index), hri, true, null); entry.stampRegionSequenceId(mvcc.begin()); return entry; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java index c43b9189083a..ee02e6bb35c4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; @@ -124,9 +123,8 @@ protected void atHeadOfRingBufferEventHandlerAppend() { @Test public void testBrokenWriter() throws Exception { - Server server = mock(Server.class); - when(server.getConfiguration()).thenReturn(CONF); RegionServerServices services = mock(RegionServerServices.class); + when(services.getConfiguration()).thenReturn(CONF); TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf("table")) .setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build(); RegionInfo ri = RegionInfoBuilder.newBuilder(td.getTableName()).build(); @@ -138,7 +136,7 @@ public void testBrokenWriter() throws Exception { long timestamp = System.currentTimeMillis(); String testName = currentTest.getMethodName(); AtomicInteger failedCount = new AtomicInteger(0); - try (LogRoller roller = new LogRoller(server, services); + try (LogRoller roller = new LogRoller(services); AsyncFSWAL wal = new AsyncFSWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName, CONF, null, true, null, null, GROUP, CHANNEL_CLASS) { @@ -196,7 +194,7 @@ public void run() { SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes); try { - wal.append(ri, key, cols, true, false); + wal.append(ri, key, cols, true); } catch (IOException e) { // should not happen throw new UncheckedIOException(e); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java index 01de1f47731d..f58d36428cf4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java @@ -63,12 +63,12 @@ public void sync(long txid, boolean forceSync) throws IOException { } @Override - protected long append(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean inMemstore, - boolean closeRegion) throws IOException { + protected long append(RegionInfo info, WALKeyImpl key, WALEdit edits, boolean inMemstore) + throws IOException { if (this.ft == FailureType.APPEND) { throw new IOException("append"); } - return super.append(info, key, edits, inMemstore, closeRegion); + return super.append(info, key, edits, inMemstore); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java index 8865de15a423..cfeaac8357dd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java @@ -1376,12 +1376,10 @@ private static void appendRegionEvent(Writer w, String region) throws IOExceptio 1, ServerName.parseServerName("ServerName:9099"), ImmutableMap.>of()); final long time = EnvironmentEdgeManager.currentTime(); - KeyValue kv = new KeyValue(Bytes.toBytes(region), WALEdit.METAFAMILY, WALEdit.REGION_EVENT, - time, regionOpenDesc.toByteArray()); final WALKeyImpl walKey = new WALKeyImpl(Bytes.toBytes(region), TABLE_NAME, 1, time, HConstants.DEFAULT_CLUSTER_ID); - w.append( - new Entry(walKey, new WALEdit().add(kv))); + WALEdit we = WALEdit.createRegionEventWALEdit(Bytes.toBytes(region), regionOpenDesc); + w.append(new Entry(walKey, we)); w.sync(false); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java index 7e6ed8fc4267..3578ccecbbeb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java @@ -321,7 +321,7 @@ public int run(String[] args) throws Exception { final HRegion[] regions = new HRegion[numRegions]; final Runnable[] benchmarks = new Runnable[numRegions]; final MockRegionServerServices mockServices = new MockRegionServerServices(getConf()); - final LogRoller roller = new LogRoller(mockServices, mockServices); + final LogRoller roller = new LogRoller(mockServices); Threads.setDaemonThreadRunning(roller.getThread(), "WALPerfEval.logRoller"); try {