HBASE-28647 Support streams in REST Client, RemoteHTable and RemoteAdmin #6010
+581
−342
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.hadoop.hbase.rest; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.io.OutputStream; | ||
| import org.apache.http.Header; | ||
| import org.apache.http.HttpEntity; | ||
| import org.apache.http.message.BasicHeader; | ||
| import org.apache.http.protocol.HTTP; | ||
| import org.apache.yetus.audience.InterfaceAudience; | ||
|
|
||
| /** | ||
| * For sending a Protobuf encoded object via Apache HttpClient efficiently without an interim byte | ||
| * array. This exposes the underlying Apache HttpClient types, but so do the other client classes. | ||
| */ | ||
| @InterfaceAudience.Public | ||
| public class ProtobufHttpEntity implements HttpEntity { | ||
|
|
||
| private ProtobufMessageHandler handler; | ||
|
|
||
| public ProtobufHttpEntity(ProtobufMessageHandler handler) { | ||
| this.handler = handler; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isRepeatable() { | ||
| return false; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isChunked() { | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| public long getContentLength() { | ||
| return -1; | ||
| } | ||
|
|
||
| @Override | ||
| public Header getContentType() { | ||
| return new BasicHeader(HTTP.CONTENT_TYPE, Constants.MIMETYPE_PROTOBUF_IETF); | ||
| } | ||
|
|
||
| @Override | ||
| public Header getContentEncoding() { | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public InputStream getContent() throws IOException, UnsupportedOperationException { | ||
| throw new UnsupportedOperationException("only writeTo is supported"); | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(OutputStream outStream) throws IOException { | ||
| handler.writeProtobufOutput(outStream); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isStreaming() { | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| public void consumeContent() throws IOException { | ||
| } | ||
|
|
||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -31,6 +31,7 @@ | |
| import org.apache.hadoop.hbase.CompatibilityFactory; | ||
| import org.apache.hadoop.hbase.HBaseClassTestRule; | ||
| import org.apache.hadoop.hbase.HConstants; | ||
| import org.apache.hadoop.hbase.rest.client.Client; | ||
| import org.apache.hadoop.hbase.rest.client.Response; | ||
| import org.apache.hadoop.hbase.rest.model.CellModel; | ||
| import org.apache.hadoop.hbase.rest.model.CellSetModel; | ||
|
|
@@ -41,6 +42,8 @@ | |
| import org.apache.hadoop.hbase.testclassification.RestTests; | ||
| import org.apache.hadoop.hbase.util.Bytes; | ||
| import org.apache.http.Header; | ||
| import org.apache.http.client.methods.CloseableHttpResponse; | ||
| import org.apache.http.client.methods.HttpPut; | ||
| import org.apache.http.message.BasicHeader; | ||
| import org.junit.ClassRule; | ||
| import org.junit.Test; | ||
|
|
@@ -167,6 +170,39 @@ public void testMultipleCellCheckPutPB() throws IOException { | |
| assertEquals(200, response.getCode()); | ||
| } | ||
|
|
||
| @Test | ||
| public void testMultipleCellPutPBEntity() throws IOException { | ||
| Response response = getValuePB(TABLE, ROW_1, COLUMN_1); | ||
| assertEquals(404, response.getCode()); | ||
|
|
||
| CellSetModel model = new CellSetModel(); | ||
| RowModel row1 = new RowModel(ROW_1); | ||
| row1.addCell(new CellModel(Bytes.toBytes(COLUMN_1), Bytes.toBytes(VALUE_1))); | ||
| row1.addCell(new CellModel(Bytes.toBytes(COLUMN_2), Bytes.toBytes(VALUE_2))); | ||
| row1.addCell(new CellModel(Bytes.toBytes(COLUMN_3), Bytes.toBytes(VALUE_3))); | ||
| model.addRow(row1); | ||
|
|
||
| // This illustrates how to use the lower level API to more efficiently marshall cells | ||
| // to Protobuf for mutations | ||
| StringBuilder path = new StringBuilder(); | ||
| path.append('/'); | ||
| path.append(TABLE); | ||
| path.append('/'); | ||
| path.append("dummy_row"); | ||
| HttpPut httpPut = new HttpPut(path.toString()); | ||
| httpPut.setEntity(new ProtobufHttpEntity(model)); | ||
|
Contributor
Author
There was a problem hiding this comment. This is how a normal client can use ProtobufHttpEntity, @Apache9 . |
||
| try (CloseableHttpResponse closableResponse = | ||
| client.execute(httpPut, Client.ACCEPT_PROTOBUF_HEADER_ARR)) { | ||
| assertEquals(200, closableResponse.getStatusLine().getStatusCode()); | ||
| } | ||
| checkValuePB(TABLE, ROW_1, COLUMN_1, VALUE_1); | ||
| checkValuePB(TABLE, ROW_1, COLUMN_2, VALUE_2); | ||
| checkValuePB(TABLE, ROW_1, COLUMN_3, VALUE_3); | ||
|
|
||
| response = deleteRow(TABLE, ROW_1); | ||
| assertEquals(200, response.getCode()); | ||
| } | ||
|
|
||
| @Test | ||
| public void testMultipleCellCheckPutXML() throws IOException, JAXBException { | ||
| Response response = getValuePB(TABLE, ROW_1, COLUMN_1); | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Does this need to be IA.Public?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, we expect the user to explicitly create these entities.
i have not updated the tests to use them, but the intentended usage is similar to how
org.apache.hadoop.hbase.rest.client.RemoteHTable.put(Put) is implemented.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
At least have a section in the javadoc to show how to use this class?
And since we have not reach an agreement on whether to move RemoteAdmin and RemoteHTable to src/main, I'm not sure whether we should make this class IA.Public...
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I only mentined RemoHTable as example where this is used, but this
is not limited to RemotHTable, it should also be used directly in user code.
I will add a standalone test case that uses this class as an illustration.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is not tied to RemoteAdmin / RemoteHTable.
This is just a more efficient way to use the existing public Client class (or any custom client based on Apache HttpClient)
The RemoteAdmin / RemoteHTable API does not even change, this new class only used internally in the RemoteHTable implementation.