-
Notifications
You must be signed in to change notification settings - Fork 3.4k
HBASE-27788 Skip family comparing when compare cells inner the store #5171
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 9 commits
03074de
d09ce01
bd02b76
b954066
dfb5ca6
73e0859
0a267d3
27358bb
bc5dcc7
449e07a
17536bb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.hadoop.hbase; | ||
|
|
||
| import org.apache.hadoop.conf.Configuration; | ||
| import org.apache.hadoop.hbase.util.Bytes; | ||
| import org.apache.yetus.audience.InterfaceAudience; | ||
| import org.apache.yetus.audience.InterfaceStability; | ||
|
|
||
| /** | ||
| * Compare two HBase cells inner store, skip compare family for better performance. Important!!! we | ||
| * should not make fake cell with fake family which length greater than zero inner store, otherwise | ||
| * this optimization cannot be used. | ||
| */ | ||
| @InterfaceAudience.Private | ||
| @InterfaceStability.Evolving | ||
| public class InnerStoreCellComparator extends CellComparatorImpl { | ||
|
|
||
| private static final long serialVersionUID = 8186411895799094989L; | ||
|
|
||
| public static final InnerStoreCellComparator INNER_STORE_COMPARATOR = | ||
| new InnerStoreCellComparator(); | ||
|
|
||
| @Override | ||
| protected int compareFamilies(Cell left, int leftFamilyLength, Cell right, | ||
| int rightFamilyLength) { | ||
| return leftFamilyLength - rightFamilyLength; | ||
| } | ||
|
|
||
| @Override | ||
| protected int compareFamilies(KeyValue left, int leftFamilyPosition, int leftFamilyLength, | ||
| KeyValue right, int rightFamilyPosition, int rightFamilyLength) { | ||
| return leftFamilyLength - rightFamilyLength; | ||
| } | ||
|
|
||
| @Override | ||
| protected int compareFamilies(ByteBufferKeyValue left, int leftFamilyPosition, | ||
| int leftFamilyLength, ByteBufferKeyValue right, int rightFamilyPosition, | ||
| int rightFamilyLength) { | ||
| return leftFamilyLength - rightFamilyLength; | ||
| } | ||
|
|
||
| @Override | ||
| protected int compareFamilies(KeyValue left, int leftFamilyPosition, int leftFamilyLength, | ||
| ByteBufferKeyValue right, int rightFamilyPosition, int rightFamilyLength) { | ||
| return leftFamilyLength - rightFamilyLength; | ||
| } | ||
|
|
||
| /** | ||
| * Utility method that makes a guess at comparator to use based off passed tableName. Use in | ||
| * extreme when no comparator specified. | ||
| * @return CellComparator to use going off the {@code tableName} passed. | ||
| */ | ||
| public static CellComparator getInnerStoreCellComparator(Configuration conf, | ||
| TableName tableName) { | ||
| return getInnerStoreCellComparator(conf, tableName.toBytes()); | ||
| } | ||
|
|
||
| /** | ||
| * Utility method that makes a guess at comparator to use based off passed tableName. Use in | ||
| * extreme when no comparator specified. | ||
| * @return CellComparator to use going off the {@code tableName} passed. | ||
| */ | ||
| public static CellComparator getInnerStoreCellComparator(byte[] tableName) { | ||
| return getInnerStoreCellComparator(null, tableName); | ||
| } | ||
|
|
||
| public static CellComparator getInnerStoreCellComparator(Configuration conf, byte[] tableName) { | ||
| if ( | ||
| conf != null && conf.getBoolean(HConstants.USE_META_CELL_COMPARATOR, | ||
| HConstants.DEFAULT_USE_META_CELL_COMPARATOR) | ||
| ) { | ||
| return MetaCellComparator.META_COMPARATOR; | ||
| } | ||
| return Bytes.equals(tableName, TableName.META_TABLE_NAME.toBytes()) | ||
| ? MetaCellComparator.META_COMPARATOR | ||
| : InnerStoreCellComparator.INNER_STORE_COMPARATOR; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -27,6 +27,7 @@ | |
| import org.apache.hadoop.fs.FSDataInputStream; | ||
| import org.apache.hadoop.hbase.CellComparator; | ||
| import org.apache.hadoop.hbase.CellComparatorImpl; | ||
| import org.apache.hadoop.hbase.InnerStoreCellComparator; | ||
| import org.apache.hadoop.hbase.KeyValue; | ||
| import org.apache.hadoop.hbase.MetaCellComparator; | ||
| import org.apache.hadoop.hbase.io.compress.Compression; | ||
|
|
@@ -121,7 +122,8 @@ public class FixedFileTrailer { | |
| * Raw key comparator class name in version 3 | ||
| */ | ||
| // We could write the actual class name from 2.0 onwards and handle BC | ||
| private String comparatorClassName = CellComparator.getInstance().getClass().getName(); | ||
| private String comparatorClassName = | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No compatibility issues?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Thought about it and i think so, because we do the conversion when we save and read the hfile, the actual store is the KVComparator, we can't change this since we want hbase1.x to be able to read the files generated in the new version. See the comment of getHBase1CompatibleName for more detail. |
||
| InnerStoreCellComparator.INNER_STORE_COMPARATOR.getClass().getName(); | ||
|
|
||
| /** | ||
| * The encryption key | ||
|
|
@@ -574,7 +576,10 @@ public void setComparatorClass(Class<? extends CellComparator> klass) { | |
| */ | ||
| @Deprecated | ||
| private String getHBase1CompatibleName(final String comparator) { | ||
| if (comparator.equals(CellComparatorImpl.class.getName())) { | ||
| if ( | ||
| comparator.equals(CellComparatorImpl.class.getName()) | ||
| || comparator.equals(InnerStoreCellComparator.class.getName()) | ||
| ) { | ||
| return KeyValue.COMPARATOR.getClass().getName(); | ||
| } | ||
| if (comparator.equals(MetaCellComparator.class.getName())) { | ||
|
|
@@ -593,7 +598,7 @@ private static Class<? extends CellComparator> getComparatorClass(String compara | |
| || comparatorClassName.equals(KeyValue.COMPARATOR.getClass().getName()) | ||
| || (comparatorClassName.equals("org.apache.hadoop.hbase.CellComparator")) | ||
| ) { | ||
| comparatorKlass = CellComparatorImpl.class; | ||
| comparatorKlass = InnerStoreCellComparator.class; | ||
| } else if ( | ||
| comparatorClassName.equals(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName()) | ||
| || comparatorClassName.equals(KeyValue.META_COMPARATOR.getClass().getName()) | ||
|
|
@@ -622,8 +627,11 @@ private static Class<? extends CellComparator> getComparatorClass(String compara | |
| } | ||
|
|
||
| static CellComparator createComparator(String comparatorClassName) throws IOException { | ||
| if (comparatorClassName.equals(CellComparatorImpl.COMPARATOR.getClass().getName())) { | ||
| return CellComparatorImpl.COMPARATOR; | ||
| if ( | ||
| comparatorClassName | ||
| .equals(InnerStoreCellComparator.INNER_STORE_COMPARATOR.getClass().getName()) | ||
| ) { | ||
| return InnerStoreCellComparator.INNER_STORE_COMPARATOR; | ||
| } else | ||
| if (comparatorClassName.equals(MetaCellComparator.META_COMPARATOR.getClass().getName())) { | ||
| return MetaCellComparator.META_COMPARATOR; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -277,14 +277,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi | |
| public static final String RECOVERED_EDITS_IGNORE_EOF = | ||
| "hbase.hregion.recovered.edits.ignore.eof"; | ||
|
|
||
| /** | ||
| * Whether to use {@link MetaCellComparator} even if we are not meta region. Used when creating | ||
| * master local region. | ||
| */ | ||
| public static final String USE_META_CELL_COMPARATOR = "hbase.region.use.meta.cell.comparator"; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not a huge deal, but why move this constant in this PR? Doesn't seem related?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Because the new InnerStoreCellComparator which placed in hbase-common module reference it too, and the hbase-common should not depend on hbase-server. |
||
|
|
||
| public static final boolean DEFAULT_USE_META_CELL_COMPARATOR = false; | ||
|
|
||
| final AtomicBoolean closed = new AtomicBoolean(false); | ||
|
|
||
| /* | ||
|
|
@@ -792,8 +784,8 @@ public HRegion(final HRegionFileSystem fs, final WAL wal, final Configuration co | |
| // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor | ||
| this.baseConf = confParam; | ||
| this.conf = new CompoundConfiguration().add(confParam).addBytesMap(htd.getValues()); | ||
| this.cellComparator = htd.isMetaTable() | ||
| || conf.getBoolean(USE_META_CELL_COMPARATOR, DEFAULT_USE_META_CELL_COMPARATOR) | ||
| this.cellComparator = htd.isMetaTable() || conf.getBoolean(HConstants.USE_META_CELL_COMPARATOR, | ||
| HConstants.DEFAULT_USE_META_CELL_COMPARATOR) | ||
| ? MetaCellComparator.META_COMPARATOR | ||
| : CellComparatorImpl.COMPARATOR; | ||
| this.lock = new ReentrantReadWriteLock( | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we move this judgement in upper layer so we do not need to move USE_META_CELL_COMPARATOR to HConstants? It is just for internal use, we'd better not put it into an IA.Public class...
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Let me try.