Skip to content

Commit 369adcd

Browse files
apurtellapurtell
authored
HBASE-27019 Minor compression performance improvements (#4420)
TRACE level logging is expensive enough to warrant removal. They were useful during development but are now just overhead. Also we unnecessarily create new compressor and decompressor instances in the reset() methods for the Aircompressor and Lz4 codecs. Remove. Signed-off-by: Viraj Jasani <vjasani@apache.org> Signed-off-by: Xiaolin Ha <haxiaolin@apache.org>
1 parent 27ced70 commit 369adcd

12 files changed

Lines changed: 18 additions & 248 deletions

File tree

‎hbase-compression/hbase-compression-aircompressor/src/main/java/org/apache/hadoop/hbase/io/compress/aircompressor/HadoopCompressor.java‎

Lines changed: 2 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424
import org.apache.hadoop.hbase.io.compress.CanReinit;
2525
import org.apache.hadoop.hbase.io.compress.CompressionUtil;
2626
import org.apache.yetus.audience.InterfaceAudience;
27-
import org.slf4j.Logger;
28-
import org.slf4j.LoggerFactory;
2927

3028
/**
3129
* Hadoop compressor glue for aircompressor compressors.
@@ -34,7 +32,6 @@
3432
public abstract class HadoopCompressor<T extends Compressor>
3533
implements CanReinit, org.apache.hadoop.io.compress.Compressor {
3634

37-
protected static final Logger LOG = LoggerFactory.getLogger(HadoopCompressor.class);
3835
protected T compressor;
3936
protected ByteBuffer inBuf, outBuf;
4037
protected int bufferSize;
@@ -56,7 +53,6 @@ public int compress(byte[] b, int off, int len) throws IOException {
5653
if (outBuf.hasRemaining()) {
5754
int remaining = outBuf.remaining(), n = Math.min(remaining, len);
5855
outBuf.get(b, off, n);
59-
LOG.trace("compress: read {} remaining bytes from outBuf", n);
6056
return n;
6157
}
6258
// We don't actually begin compression until our caller calls finish().
@@ -77,7 +73,6 @@ public int compress(byte[] b, int off, int len) throws IOException {
7773
} else {
7874
if (outBuf.capacity() < needed) {
7975
needed = CompressionUtil.roundInt2(needed);
80-
LOG.trace("compress: resize outBuf {}", needed);
8176
outBuf = ByteBuffer.allocate(needed);
8277
} else {
8378
outBuf.clear();
@@ -89,42 +84,34 @@ public int compress(byte[] b, int off, int len) throws IOException {
8984
final int written = writeBuffer.position() - oldPos;
9085
bytesWritten += written;
9186
inBuf.clear();
92-
LOG.trace("compress: compressed {} -> {}", uncompressed, written);
9387
finished = true;
9488
if (!direct) {
9589
outBuf.flip();
9690
int n = Math.min(written, len);
9791
outBuf.get(b, off, n);
98-
LOG.trace("compress: {} bytes", n);
9992
return n;
10093
} else {
101-
LOG.trace("compress: {} bytes direct", written);
10294
return written;
10395
}
10496
} else {
10597
finished = true;
10698
}
10799
}
108-
LOG.trace("No output");
109100
return 0;
110101
}
111102

112103
@Override
113104
public void end() {
114-
LOG.trace("end");
115105
}
116106

117107
@Override
118108
public void finish() {
119-
LOG.trace("finish");
120109
finish = true;
121110
}
122111

123112
@Override
124113
public boolean finished() {
125-
boolean b = finished && !outBuf.hasRemaining();
126-
LOG.trace("finished: {}", b);
127-
return b;
114+
return finished && !outBuf.hasRemaining();
128115
}
129116

130117
@Override
@@ -139,14 +126,11 @@ public long getBytesWritten() {
139126

140127
@Override
141128
public boolean needsInput() {
142-
boolean b = !finished();
143-
LOG.trace("needsInput: {}", b);
144-
return b;
129+
return !finished();
145130
}
146131

147132
@Override
148133
public void reinit(Configuration conf) {
149-
LOG.trace("reinit");
150134
if (conf != null) {
151135
// Buffer size might have changed
152136
int newBufferSize = getBufferSize(conf);
@@ -159,15 +143,8 @@ public void reinit(Configuration conf) {
159143
reset();
160144
}
161145

162-
@SuppressWarnings("unchecked")
163146
@Override
164147
public void reset() {
165-
LOG.trace("reset");
166-
try {
167-
compressor = (T) compressor.getClass().getDeclaredConstructor().newInstance();
168-
} catch (Exception e) {
169-
throw new RuntimeException(e);
170-
}
171148
inBuf.clear();
172149
outBuf.clear();
173150
outBuf.position(outBuf.capacity());
@@ -184,13 +161,11 @@ public void setDictionary(byte[] b, int off, int len) {
184161

185162
@Override
186163
public void setInput(byte[] b, int off, int len) {
187-
LOG.trace("setInput: off={} len={}", off, len);
188164
if (inBuf.remaining() < len) {
189165
// Get a new buffer that can accomodate the accumulated input plus the additional
190166
// input that would cause a buffer overflow without reallocation.
191167
// This condition should be fortunately rare, because it is expensive.
192168
int needed = CompressionUtil.roundInt2(inBuf.capacity() + len);
193-
LOG.trace("setInput: resize inBuf {}", needed);
194169
ByteBuffer newBuf = ByteBuffer.allocate(needed);
195170
inBuf.flip();
196171
newBuf.put(inBuf);

‎hbase-compression/hbase-compression-aircompressor/src/main/java/org/apache/hadoop/hbase/io/compress/aircompressor/HadoopDecompressor.java‎

Lines changed: 1 addition & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@
2222
import java.nio.ByteBuffer;
2323
import org.apache.hadoop.hbase.io.compress.CompressionUtil;
2424
import org.apache.yetus.audience.InterfaceAudience;
25-
import org.slf4j.Logger;
26-
import org.slf4j.LoggerFactory;
2725

2826
/**
2927
* Hadoop decompressor glue for aircompressor decompressors.
@@ -32,7 +30,6 @@
3230
public class HadoopDecompressor<T extends Decompressor>
3331
implements org.apache.hadoop.io.compress.Decompressor {
3432

35-
protected static final Logger LOG = LoggerFactory.getLogger(HadoopDecompressor.class);
3633
protected T decompressor;
3734
protected ByteBuffer inBuf, outBuf;
3835
protected int inLen;
@@ -50,7 +47,6 @@ public int decompress(byte[] b, int off, int len) throws IOException {
5047
if (outBuf.hasRemaining()) {
5148
int remaining = outBuf.remaining(), n = Math.min(remaining, len);
5249
outBuf.get(b, off, n);
53-
LOG.trace("decompress: read {} remaining bytes from outBuf", n);
5450
return n;
5551
}
5652
if (inBuf.position() > 0) {
@@ -63,50 +59,36 @@ public int decompress(byte[] b, int off, int len) throws IOException {
6359
inBuf.rewind();
6460
inBuf.limit(inBuf.capacity());
6561
final int written = outBuf.position();
66-
LOG.trace("decompress: decompressed {} -> {}", remaining, written);
6762
outBuf.flip();
6863
int n = Math.min(written, len);
6964
outBuf.get(b, off, n);
70-
LOG.trace("decompress: {} bytes", n);
7165
return n;
7266
}
73-
LOG.trace("decompress: No output, finished");
7467
finished = true;
7568
return 0;
7669
}
7770

7871
@Override
7972
public void end() {
80-
LOG.trace("end");
8173
}
8274

8375
@Override
8476
public boolean finished() {
85-
LOG.trace("finished");
8677
return finished;
8778
}
8879

8980
@Override
9081
public int getRemaining() {
91-
LOG.trace("getRemaining: {}", inLen);
9282
return inLen;
9383
}
9484

9585
@Override
9686
public boolean needsDictionary() {
97-
LOG.trace("needsDictionary");
9887
return false;
9988
}
10089

101-
@SuppressWarnings("unchecked")
10290
@Override
10391
public void reset() {
104-
LOG.trace("reset");
105-
try {
106-
decompressor = (T) decompressor.getClass().getDeclaredConstructor().newInstance();
107-
} catch (Exception e) {
108-
throw new RuntimeException(e);
109-
}
11092
inBuf.rewind();
11193
inBuf.limit(inBuf.capacity());
11294
inLen = 0;
@@ -117,9 +99,7 @@ public void reset() {
11799

118100
@Override
119101
public boolean needsInput() {
120-
boolean b = (inBuf.position() == 0);
121-
LOG.trace("needsInput: {}", b);
122-
return b;
102+
return inBuf.position() == 0;
123103
}
124104

125105
@Override
@@ -129,13 +109,11 @@ public void setDictionary(byte[] b, int off, int len) {
129109

130110
@Override
131111
public void setInput(byte[] b, int off, int len) {
132-
LOG.trace("setInput: off={} len={}", off, len);
133112
if (inBuf.remaining() < len) {
134113
// Get a new buffer that can accomodate the accumulated input plus the additional
135114
// input that would cause a buffer overflow without reallocation.
136115
// This condition should be fortunately rare, because it is expensive.
137116
int needed = CompressionUtil.roundInt2(inBuf.capacity() + len);
138-
LOG.trace("setInput: resize inBuf {}", needed);
139117
ByteBuffer newBuf = ByteBuffer.allocate(needed);
140118
inBuf.flip();
141119
newBuf.put(inBuf);

‎hbase-compression/hbase-compression-brotli/src/main/java/org/apache/hadoop/hbase/io/compress/brotli/BrotliCompressor.java‎

Lines changed: 2 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,13 @@
2727
import org.apache.hadoop.hbase.io.compress.CompressionUtil;
2828
import org.apache.hadoop.io.compress.Compressor;
2929
import org.apache.yetus.audience.InterfaceAudience;
30-
import org.slf4j.Logger;
31-
import org.slf4j.LoggerFactory;
3230

3331
/**
3432
* Hadoop compressor glue for Brotli4j
3533
*/
3634
@InterfaceAudience.Private
3735
public class BrotliCompressor implements CanReinit, Compressor {
3836

39-
protected static final Logger LOG = LoggerFactory.getLogger(BrotliCompressor.class);
4037
protected ByteBuffer inBuf, outBuf;
4138
protected int bufferSize;
4239
protected boolean finish, finished;
@@ -64,7 +61,6 @@ public int compress(byte[] b, int off, int len) throws IOException {
6461
if (outBuf.hasRemaining()) {
6562
int remaining = outBuf.remaining(), n = Math.min(remaining, len);
6663
outBuf.get(b, off, n);
67-
LOG.trace("compress: read {} remaining bytes from outBuf", n);
6864
return n;
6965
}
7066
// We don't actually begin compression until our caller calls finish().
@@ -84,7 +80,6 @@ public int compress(byte[] b, int off, int len) throws IOException {
8480
} else {
8581
if (outBuf.capacity() < needed) {
8682
needed = CompressionUtil.roundInt2(needed);
87-
LOG.trace("compress: resize outBuf {}", needed);
8883
outBuf = ByteBuffer.allocate(needed);
8984
} else {
9085
outBuf.clear();
@@ -96,42 +91,34 @@ public int compress(byte[] b, int off, int len) throws IOException {
9691
final int written = writeBuf.position() - oldPos;
9792
bytesWritten += written;
9893
inBuf.clear();
99-
LOG.trace("compress: compressed {} -> {}", uncompressed, written);
10094
finished = true;
10195
if (!direct) {
10296
outBuf.flip();
10397
int n = Math.min(written, len);
10498
outBuf.get(b, off, n);
105-
LOG.trace("compress: {} bytes", n);
10699
return n;
107100
} else {
108-
LOG.trace("compress: {} bytes direct", written);
109101
return written;
110102
}
111103
} else {
112104
finished = true;
113105
}
114106
}
115-
LOG.trace("No output");
116107
return 0;
117108
}
118109

119110
@Override
120111
public void end() {
121-
LOG.trace("end");
122112
}
123113

124114
@Override
125115
public void finish() {
126-
LOG.trace("finish");
127116
finish = true;
128117
}
129118

130119
@Override
131120
public boolean finished() {
132-
boolean b = finished && !outBuf.hasRemaining();
133-
LOG.trace("finished: {}", b);
134-
return b;
121+
return finished && !outBuf.hasRemaining();
135122
}
136123

137124
@Override
@@ -146,14 +133,11 @@ public long getBytesWritten() {
146133

147134
@Override
148135
public boolean needsInput() {
149-
boolean b = !finished();
150-
LOG.trace("needsInput: {}", b);
151-
return b;
136+
return !finished();
152137
}
153138

154139
@Override
155140
public void reinit(Configuration conf) {
156-
LOG.trace("reinit");
157141
if (conf != null) {
158142
// Quality or window settings might have changed
159143
params.setQuality(BrotliCodec.getLevel(conf));
@@ -171,7 +155,6 @@ public void reinit(Configuration conf) {
171155

172156
@Override
173157
public void reset() {
174-
LOG.trace("reset");
175158
inBuf.clear();
176159
outBuf.clear();
177160
outBuf.position(outBuf.capacity());
@@ -188,13 +171,11 @@ public void setDictionary(byte[] b, int off, int len) {
188171

189172
@Override
190173
public void setInput(byte[] b, int off, int len) {
191-
LOG.trace("setInput: off={} len={}", off, len);
192174
if (inBuf.remaining() < len) {
193175
// Get a new buffer that can accomodate the accumulated input plus the additional
194176
// input that would cause a buffer overflow without reallocation.
195177
// This condition should be fortunately rare, because it is expensive.
196178
int needed = CompressionUtil.roundInt2(inBuf.capacity() + len);
197-
LOG.trace("setInput: resize inBuf {}", needed);
198179
ByteBuffer newBuf = ByteBuffer.allocate(needed);
199180
inBuf.flip();
200181
newBuf.put(inBuf);

0 commit comments

Comments
 (0)