From 384113ea6903dfe2889ed1ec980fe67fa3ac0f2e Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 20 May 2015 14:49:34 -0700 Subject: [PATCH 1/8] make chunk size configurable --- .../gcloud/examples/StorageExample.java | 2 +- .../gcloud/storage/BlobReadChannel.java | 7 +++++++ .../gcloud/storage/BlobReadChannelImpl.java | 10 ++++++++-- .../gcloud/storage/BlobWriteChannel.java | 5 +++++ .../gcloud/storage/BlobWriterChannelImpl.java | 20 ++++++++++++------- .../google/gcloud/storage/StorageService.java | 4 ++-- 6 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index a4550f19d2de..b0d44c292d2c 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -479,7 +479,7 @@ public String params() { } public static void printUsage() { - StringBuilder actionAndParams = new StringBuilder(""); + StringBuilder actionAndParams = new StringBuilder(); for (Map.Entry entry : ACTIONS.entrySet()) { actionAndParams.append("\n\t").append(entry.getKey()); diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java index 89f3420a2a28..ad1a385d9a83 100644 --- a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java +++ b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java @@ -39,4 +39,11 @@ public interface BlobReadChannel extends ReadableByteChannel, Serializable, Clos void close(); void seek(int position) throws IOException; + + /** + * Sets the minimum size that will be read by a single RPC. + * Read data will be locally buffered until consumed. + */ + void chunkSize(int chunkSize); + } diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java b/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java index 8ca8a01f2df3..27d37b127d55 100644 --- a/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java +++ b/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java @@ -33,7 +33,7 @@ */ class BlobReadChannelImpl implements BlobReadChannel { - private static final int MIN_BUFFER_SIZE = 2 * 1024 * 1024; + private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024; private static final long serialVersionUID = 4821762590742862669L; private final StorageServiceOptions serviceOptions; @@ -42,6 +42,7 @@ class BlobReadChannelImpl implements BlobReadChannel { private int position; private boolean isOpen; private boolean endOfStream; + private int chunkSize = DEFAULT_CHUNK_SIZE; private transient StorageRpc storageRpc; private transient StorageObject storageObject; @@ -105,6 +106,11 @@ public void seek(int position) throws IOException { endOfStream = false; } + @Override + public void chunkSize(int chunkSize) { + this.chunkSize = chunkSize <= 0 ? DEFAULT_CHUNK_SIZE : chunkSize; + } + @Override public int read(ByteBuffer byteBuffer) throws IOException { validateOpen(); @@ -112,7 +118,7 @@ public int read(ByteBuffer byteBuffer) throws IOException { if (endOfStream) { return -1; } - final int toRead = Math.max(byteBuffer.remaining(), MIN_BUFFER_SIZE); + final int toRead = Math.max(byteBuffer.remaining(), chunkSize); buffer = runWithRetries(new Callable() { @Override public byte[] call() { diff --git a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java index 77ce84a8ea7a..20b2ce087632 100644 --- a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java +++ b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java @@ -29,4 +29,9 @@ */ public interface BlobWriteChannel extends WritableByteChannel, Serializable, Closeable { + /** + * Sets the minimum size that will be written by a single RPC. + * Written data will be buffered and only flushed upon reaching this size or closing the channel. + */ + void chunkSize(int chunkSize); } diff --git a/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java b/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java index b7736346bba0..2b8e66cc33ce 100644 --- a/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java +++ b/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java @@ -35,8 +35,8 @@ class BlobWriterChannelImpl implements BlobWriteChannel { private static final long serialVersionUID = 8675286882724938737L; - private static final int CHUNK_SIZE = 256 * 1024; - private static final int MIN_BUFFER_SIZE = 8 * CHUNK_SIZE; + private static final int MIN_CHUNK_SIZE = 256 * 1024; + private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE; private final StorageServiceOptions options; private final Blob blob; @@ -45,6 +45,7 @@ class BlobWriterChannelImpl implements BlobWriteChannel { private byte[] buffer = new byte[0]; private int limit; private boolean isOpen = true; + private int chunkSize = DEFAULT_CHUNK_SIZE; private transient StorageRpc storageRpc; private transient StorageObject storageObject; @@ -65,8 +66,8 @@ private void writeObject(ObjectOutputStream out) throws IOException { } private void flush(boolean compact) { - if (limit >= MIN_BUFFER_SIZE || compact && limit >= CHUNK_SIZE) { - final int length = limit - limit % CHUNK_SIZE; + if (limit >= chunkSize || compact && limit >= MIN_CHUNK_SIZE) { + final int length = limit - limit % MIN_CHUNK_SIZE; runWithRetries(callable(new Runnable() { @Override public void run() { @@ -75,7 +76,7 @@ public void run() { }), options.retryParams(), StorageServiceImpl.EXCEPTION_HANDLER); position += length; limit -= length; - byte[] temp = new byte[compact ? limit : MIN_BUFFER_SIZE]; + byte[] temp = new byte[compact ? limit : chunkSize]; System.arraycopy(buffer, length, temp, 0, limit); buffer = temp; } @@ -107,8 +108,7 @@ public int write(ByteBuffer byteBuffer) throws IOException { if (spaceInBuffer >= toWrite) { byteBuffer.get(buffer, limit, toWrite); } else { - buffer = Arrays.copyOf(buffer, - Math.max(MIN_BUFFER_SIZE, buffer.length + toWrite - spaceInBuffer)); + buffer = Arrays.copyOf(buffer, Math.max(chunkSize, buffer.length + toWrite - spaceInBuffer)); byteBuffer.get(buffer, limit, toWrite); } limit += toWrite; @@ -135,4 +135,10 @@ public void run() { buffer = null; } } + + @Override + public void chunkSize(int chunkSize) { + chunkSize = (chunkSize / MIN_CHUNK_SIZE) * MIN_CHUNK_SIZE; + this.chunkSize = Math.max(MIN_CHUNK_SIZE, chunkSize); + } } diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 9e95b0f1281d..6fefb3af3b16 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -121,8 +121,8 @@ public static BlobTargetOption predefinedAcl(PredefinedAcl acl) { return new BlobTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry()); } - public static BlobTargetOption doesNotExists() { - return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0); + public static BlobTargetOption doesNotExist() { + return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0L); } public static BlobTargetOption generationMatch() { From 1ba5cb446833ec6d20d70481a2c50978ab1fe680 Mon Sep 17 00:00:00 2001 From: ozarov Date: Wed, 20 May 2015 22:50:24 -0700 Subject: [PATCH 2/8] work on signURL --- .../com/google/gcloud/AuthCredentials.java | 12 +- .../java/com/google/gcloud/storage/Cors.java | 18 +-- .../com/google/gcloud/storage/HttpMethod.java | 24 +++ .../google/gcloud/storage/StorageService.java | 148 ++++++++++++++++++ .../gcloud/storage/StorageServiceImpl.java | 8 +- .../com/google/gcloud/storage/CorsTest.java | 3 +- 6 files changed, 197 insertions(+), 16 deletions(-) create mode 100644 src/main/java/com/google/gcloud/storage/HttpMethod.java diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/src/main/java/com/google/gcloud/AuthCredentials.java index 839da54e62cf..6cdb737ddd91 100644 --- a/src/main/java/com/google/gcloud/AuthCredentials.java +++ b/src/main/java/com/google/gcloud/AuthCredentials.java @@ -62,7 +62,7 @@ private Object readResolve() throws ObjectStreamException { } } - private static class ServiceAccountAuthCredentials extends AuthCredentials { + public static class ServiceAccountAuthCredentials extends AuthCredentials { private static final long serialVersionUID = 8007708734318445901L; private final String account; @@ -94,6 +94,14 @@ protected HttpRequestInitializer httpRequestInitializer( return builder.build(); } + public String account() { + return account; + } + + public PrivateKey privateKey() { + return privateKey; + } + @Override public int hashCode() { return Objects.hash(account, privateKey); @@ -187,7 +195,7 @@ public static AuthCredentials createApplicationDefaults() throws IOException { return new ApplicationDefaultAuthCredentials(); } - public static AuthCredentials createFor(String account, PrivateKey privateKey) { + public static ServiceAccountAuthCredentials createFor(String account, PrivateKey privateKey) { return new ServiceAccountAuthCredentials(account, privateKey); } diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java index b1953aa5e0e4..ce8cfb95b6e9 100644 --- a/src/main/java/com/google/gcloud/storage/Cors.java +++ b/src/main/java/com/google/gcloud/storage/Cors.java @@ -53,14 +53,10 @@ public Bucket.Cors apply(Cors cors) { }; private final Integer maxAgeSeconds; - private final ImmutableList methods; + private final ImmutableList methods; private final ImmutableList origins; private final ImmutableList responseHeaders; - public enum Method { - ANY, GET, HEAD, PUT, POST, DELETE - } - public static final class Origin implements Serializable { private static final long serialVersionUID = -4447958124895577993L; @@ -118,7 +114,7 @@ public String value() { public static final class Builder { private Integer maxAgeSeconds; - private ImmutableList methods; + private ImmutableList methods; private ImmutableList origins; private ImmutableList responseHeaders; @@ -129,7 +125,7 @@ public Builder maxAgeSeconds(Integer maxAgeSeconds) { return this; } - public Builder methods(Iterable methods) { + public Builder methods(Iterable methods) { this.methods = methods != null ? ImmutableList.copyOf(methods) : null; return this; } @@ -160,7 +156,7 @@ public Integer maxAgeSeconds() { return maxAgeSeconds; } - public List methods() { + public List methods() { return methods; } @@ -217,10 +213,10 @@ Bucket.Cors toPb() { static Cors fromPb(Bucket.Cors cors) { Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds()); if (cors.getMethod() != null) { - builder.methods(transform(cors.getMethod(), new Function() { + builder.methods(transform(cors.getMethod(), new Function() { @Override - public Method apply(String name) { - return Method.valueOf(name.toUpperCase()); + public HttpMethod apply(String name) { + return HttpMethod.valueOf(name.toUpperCase()); } })); } diff --git a/src/main/java/com/google/gcloud/storage/HttpMethod.java b/src/main/java/com/google/gcloud/storage/HttpMethod.java new file mode 100644 index 000000000000..f5889aedae90 --- /dev/null +++ b/src/main/java/com/google/gcloud/storage/HttpMethod.java @@ -0,0 +1,24 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +/** + * + */ +public enum HttpMethod { + GET, HEAD, PUT, POST, DELETE +} diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 6fefb3af3b16..2e500b001a93 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -16,14 +16,19 @@ package com.google.gcloud.storage; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.collect.ImmutableList; +import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials; import com.google.gcloud.Service; import com.google.gcloud.spi.StorageRpc; +import org.joda.time.DateTime; + import java.io.Serializable; +import java.net.URL; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; @@ -407,6 +412,139 @@ public static Builder builder() { } } + /** + * A request for signing a URL. + */ + class SignUrlRequest implements Serializable { + + private final Blob blob; + private final HttpMethod httpMethod; + private final Long expiration; + private final boolean includeContentType; + private final boolean includeMd5; + private final String headers; + private final ServiceAccountAuthCredentials authCredentials; + + public static class Builder { + + private Blob blob; + private HttpMethod httpMethod; + private long expiration; + private boolean includeContentType; + private boolean includeMd5; + private String headers; + private ServiceAccountAuthCredentials authCredentials; + + private Builder() {} + + public Builder blob(Blob blob) { + this.blob = blob; + return this; + } + + /** + * The HTTP method to be used with the signed URL. + */ + public Builder httpMethod(HttpMethod httpMethod) { + this.httpMethod = httpMethod; + return this; + } + + /** + * Sets expiration time for the URL. + * Defaults to one day. + */ + public Builder expiration(long expiration) { + this.expiration = expiration; + return this; + } + + /** + * Indicate if signature should include the blob's content-type. + * If {@code true} users of the signed URL should include + * the same content-type with their request. + */ + public Builder includeContentType(boolean includeContentType) { + this.includeContentType = includeContentType; + return this; + } + + /** + * Indicate if signature should include the blob's md5. + * If {@code true} users of the signed URL should include it with their requests. + */ + public Builder includeMd5(boolean includeMd5) { + this.includeMd5 = includeMd5; + return this; + } + + /** + * If headers are provided, the server will check to make sure that the client + * provides matching values. + * For information about how to create canonical headers for signing, + * see About Canonical Extension Headers. + */ + public Builder canonicalizedExtensionHeaders(String headers) { + this.headers = headers; + return this; + } + + /** + * The service account credentials for signing the URL. + */ + public Builder serviceAccountAuthCredentials(ServiceAccountAuthCredentials authCredentials) { + this.authCredentials = authCredentials; + return this; + } + + public SignUrlRequest build() { + return new SignUrlRequest(this); + } + } + + private SignUrlRequest(Builder builder) { + blob = checkNotNull(builder.blob); + httpMethod = builder.httpMethod; + expiration = firstNonNull(builder.expiration, new DateTime().plusDays(1).getMillis()); + includeContentType = builder.includeContentType; // verify blob has content-type + includeMd5 = builder.includeMd5; // verify blob has md5 + authCredentials = builder.authCredentials; // default if null + headers = builder.headers; + } + + public Blob blob() { + return blob; + } + + public HttpMethod httpMethod() { + return httpMethod; + } + + public String canonicalizedExtensionHeaders() { + return headers; + } + + public long expiration() { + return expiration; + } + + public ServiceAccountAuthCredentials authCredentials() { + return authCredentials; + } + + public boolean includeContentType() { + return includeContentType; + } + + public boolean includeMd5() { + return includeMd5; + } + + public static Builder builder() { + return new Builder(); + } + } + /** * Create a new bucket. * @@ -528,4 +666,14 @@ public static Builder builder() { * @throws StorageServiceException upon failure */ BlobWriteChannel writer(Blob blob, BlobTargetOption... options); + + /** + * Generates a signed URL for a blob. + * If you have a blob that you want to allow access to for a set + * amount of time, you can use this method to generate a URL that + * is only valid within a certain time period. + * This is particularly useful if you don't want publicly + * accessible blobs, but don't want to require users to explicitly log in. + */ + URL signUrl(SignUrlRequest request); } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 95b95141be14..8dda8004c016 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -29,7 +29,6 @@ import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; -import static java.util.concurrent.Executors.callable; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.Function; @@ -47,6 +46,7 @@ import com.google.gcloud.spi.StorageRpc.Tuple; import java.io.Serializable; +import java.net.URL; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -429,6 +429,12 @@ public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) { return new BlobWriterChannelImpl(options(), blob, optionsMap); } + @Override + public URL signUrl(SignUrlRequest request) { + // todo: implement and add test + return null; + } + private Map optionMap(Long generation, Long metaGeneration, Iterable options) { return optionMap(generation, metaGeneration, options, false); diff --git a/src/test/java/com/google/gcloud/storage/CorsTest.java b/src/test/java/com/google/gcloud/storage/CorsTest.java index 8b0379f03583..f29020c6380f 100644 --- a/src/test/java/com/google/gcloud/storage/CorsTest.java +++ b/src/test/java/com/google/gcloud/storage/CorsTest.java @@ -19,7 +19,6 @@ import static org.junit.Assert.assertEquals; import com.google.common.collect.ImmutableList; -import com.google.gcloud.storage.Cors.Method; import com.google.gcloud.storage.Cors.Origin; import org.junit.Test; @@ -39,7 +38,7 @@ public void testOrigin() { public void corsTest() { List origins = ImmutableList.of(Origin.any(), Origin.of("o")); List headers = ImmutableList.of("h1", "h2"); - List methods = ImmutableList.of(Method.ANY); + List methods = ImmutableList.of(HttpMethod.ANY); Cors cors = Cors.builder() .maxAgeSeconds(100) .origins(origins) From ee47724fe124aca3c4fcd27611a53b7d14be1606 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 26 May 2015 15:18:24 -0700 Subject: [PATCH 3/8] Revert changes that were applied on a different branch. --- .../com/google/gcloud/AuthCredentials.java | 12 +- .../gcloud/examples/StorageExample.java | 2 +- .../gcloud/storage/BlobReadChannel.java | 7 + .../gcloud/storage/BlobReadChannelImpl.java | 10 +- .../gcloud/storage/BlobWriteChannel.java | 5 + .../gcloud/storage/BlobWriterChannelImpl.java | 20 ++- .../java/com/google/gcloud/storage/Cors.java | 18 +-- .../com/google/gcloud/storage/HttpMethod.java | 24 +++ .../google/gcloud/storage/StorageService.java | 152 +++++++++++++++++- .../gcloud/storage/StorageServiceImpl.java | 8 +- .../com/google/gcloud/storage/CorsTest.java | 3 +- 11 files changed, 233 insertions(+), 28 deletions(-) create mode 100644 src/main/java/com/google/gcloud/storage/HttpMethod.java diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/src/main/java/com/google/gcloud/AuthCredentials.java index 839da54e62cf..6cdb737ddd91 100644 --- a/src/main/java/com/google/gcloud/AuthCredentials.java +++ b/src/main/java/com/google/gcloud/AuthCredentials.java @@ -62,7 +62,7 @@ private Object readResolve() throws ObjectStreamException { } } - private static class ServiceAccountAuthCredentials extends AuthCredentials { + public static class ServiceAccountAuthCredentials extends AuthCredentials { private static final long serialVersionUID = 8007708734318445901L; private final String account; @@ -94,6 +94,14 @@ protected HttpRequestInitializer httpRequestInitializer( return builder.build(); } + public String account() { + return account; + } + + public PrivateKey privateKey() { + return privateKey; + } + @Override public int hashCode() { return Objects.hash(account, privateKey); @@ -187,7 +195,7 @@ public static AuthCredentials createApplicationDefaults() throws IOException { return new ApplicationDefaultAuthCredentials(); } - public static AuthCredentials createFor(String account, PrivateKey privateKey) { + public static ServiceAccountAuthCredentials createFor(String account, PrivateKey privateKey) { return new ServiceAccountAuthCredentials(account, privateKey); } diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index a4550f19d2de..b0d44c292d2c 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -479,7 +479,7 @@ public String params() { } public static void printUsage() { - StringBuilder actionAndParams = new StringBuilder(""); + StringBuilder actionAndParams = new StringBuilder(); for (Map.Entry entry : ACTIONS.entrySet()) { actionAndParams.append("\n\t").append(entry.getKey()); diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java index 89f3420a2a28..ad1a385d9a83 100644 --- a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java +++ b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java @@ -39,4 +39,11 @@ public interface BlobReadChannel extends ReadableByteChannel, Serializable, Clos void close(); void seek(int position) throws IOException; + + /** + * Sets the minimum size that will be read by a single RPC. + * Read data will be locally buffered until consumed. + */ + void chunkSize(int chunkSize); + } diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java b/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java index 8ca8a01f2df3..27d37b127d55 100644 --- a/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java +++ b/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java @@ -33,7 +33,7 @@ */ class BlobReadChannelImpl implements BlobReadChannel { - private static final int MIN_BUFFER_SIZE = 2 * 1024 * 1024; + private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024; private static final long serialVersionUID = 4821762590742862669L; private final StorageServiceOptions serviceOptions; @@ -42,6 +42,7 @@ class BlobReadChannelImpl implements BlobReadChannel { private int position; private boolean isOpen; private boolean endOfStream; + private int chunkSize = DEFAULT_CHUNK_SIZE; private transient StorageRpc storageRpc; private transient StorageObject storageObject; @@ -105,6 +106,11 @@ public void seek(int position) throws IOException { endOfStream = false; } + @Override + public void chunkSize(int chunkSize) { + this.chunkSize = chunkSize <= 0 ? DEFAULT_CHUNK_SIZE : chunkSize; + } + @Override public int read(ByteBuffer byteBuffer) throws IOException { validateOpen(); @@ -112,7 +118,7 @@ public int read(ByteBuffer byteBuffer) throws IOException { if (endOfStream) { return -1; } - final int toRead = Math.max(byteBuffer.remaining(), MIN_BUFFER_SIZE); + final int toRead = Math.max(byteBuffer.remaining(), chunkSize); buffer = runWithRetries(new Callable() { @Override public byte[] call() { diff --git a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java index 77ce84a8ea7a..20b2ce087632 100644 --- a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java +++ b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java @@ -29,4 +29,9 @@ */ public interface BlobWriteChannel extends WritableByteChannel, Serializable, Closeable { + /** + * Sets the minimum size that will be written by a single RPC. + * Written data will be buffered and only flushed upon reaching this size or closing the channel. + */ + void chunkSize(int chunkSize); } diff --git a/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java b/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java index b7736346bba0..2b8e66cc33ce 100644 --- a/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java +++ b/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java @@ -35,8 +35,8 @@ class BlobWriterChannelImpl implements BlobWriteChannel { private static final long serialVersionUID = 8675286882724938737L; - private static final int CHUNK_SIZE = 256 * 1024; - private static final int MIN_BUFFER_SIZE = 8 * CHUNK_SIZE; + private static final int MIN_CHUNK_SIZE = 256 * 1024; + private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE; private final StorageServiceOptions options; private final Blob blob; @@ -45,6 +45,7 @@ class BlobWriterChannelImpl implements BlobWriteChannel { private byte[] buffer = new byte[0]; private int limit; private boolean isOpen = true; + private int chunkSize = DEFAULT_CHUNK_SIZE; private transient StorageRpc storageRpc; private transient StorageObject storageObject; @@ -65,8 +66,8 @@ private void writeObject(ObjectOutputStream out) throws IOException { } private void flush(boolean compact) { - if (limit >= MIN_BUFFER_SIZE || compact && limit >= CHUNK_SIZE) { - final int length = limit - limit % CHUNK_SIZE; + if (limit >= chunkSize || compact && limit >= MIN_CHUNK_SIZE) { + final int length = limit - limit % MIN_CHUNK_SIZE; runWithRetries(callable(new Runnable() { @Override public void run() { @@ -75,7 +76,7 @@ public void run() { }), options.retryParams(), StorageServiceImpl.EXCEPTION_HANDLER); position += length; limit -= length; - byte[] temp = new byte[compact ? limit : MIN_BUFFER_SIZE]; + byte[] temp = new byte[compact ? limit : chunkSize]; System.arraycopy(buffer, length, temp, 0, limit); buffer = temp; } @@ -107,8 +108,7 @@ public int write(ByteBuffer byteBuffer) throws IOException { if (spaceInBuffer >= toWrite) { byteBuffer.get(buffer, limit, toWrite); } else { - buffer = Arrays.copyOf(buffer, - Math.max(MIN_BUFFER_SIZE, buffer.length + toWrite - spaceInBuffer)); + buffer = Arrays.copyOf(buffer, Math.max(chunkSize, buffer.length + toWrite - spaceInBuffer)); byteBuffer.get(buffer, limit, toWrite); } limit += toWrite; @@ -135,4 +135,10 @@ public void run() { buffer = null; } } + + @Override + public void chunkSize(int chunkSize) { + chunkSize = (chunkSize / MIN_CHUNK_SIZE) * MIN_CHUNK_SIZE; + this.chunkSize = Math.max(MIN_CHUNK_SIZE, chunkSize); + } } diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java index b1953aa5e0e4..ce8cfb95b6e9 100644 --- a/src/main/java/com/google/gcloud/storage/Cors.java +++ b/src/main/java/com/google/gcloud/storage/Cors.java @@ -53,14 +53,10 @@ public Bucket.Cors apply(Cors cors) { }; private final Integer maxAgeSeconds; - private final ImmutableList methods; + private final ImmutableList methods; private final ImmutableList origins; private final ImmutableList responseHeaders; - public enum Method { - ANY, GET, HEAD, PUT, POST, DELETE - } - public static final class Origin implements Serializable { private static final long serialVersionUID = -4447958124895577993L; @@ -118,7 +114,7 @@ public String value() { public static final class Builder { private Integer maxAgeSeconds; - private ImmutableList methods; + private ImmutableList methods; private ImmutableList origins; private ImmutableList responseHeaders; @@ -129,7 +125,7 @@ public Builder maxAgeSeconds(Integer maxAgeSeconds) { return this; } - public Builder methods(Iterable methods) { + public Builder methods(Iterable methods) { this.methods = methods != null ? ImmutableList.copyOf(methods) : null; return this; } @@ -160,7 +156,7 @@ public Integer maxAgeSeconds() { return maxAgeSeconds; } - public List methods() { + public List methods() { return methods; } @@ -217,10 +213,10 @@ Bucket.Cors toPb() { static Cors fromPb(Bucket.Cors cors) { Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds()); if (cors.getMethod() != null) { - builder.methods(transform(cors.getMethod(), new Function() { + builder.methods(transform(cors.getMethod(), new Function() { @Override - public Method apply(String name) { - return Method.valueOf(name.toUpperCase()); + public HttpMethod apply(String name) { + return HttpMethod.valueOf(name.toUpperCase()); } })); } diff --git a/src/main/java/com/google/gcloud/storage/HttpMethod.java b/src/main/java/com/google/gcloud/storage/HttpMethod.java new file mode 100644 index 000000000000..f5889aedae90 --- /dev/null +++ b/src/main/java/com/google/gcloud/storage/HttpMethod.java @@ -0,0 +1,24 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +/** + * + */ +public enum HttpMethod { + GET, HEAD, PUT, POST, DELETE +} diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 9e95b0f1281d..2e500b001a93 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -16,14 +16,19 @@ package com.google.gcloud.storage; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.collect.ImmutableList; +import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials; import com.google.gcloud.Service; import com.google.gcloud.spi.StorageRpc; +import org.joda.time.DateTime; + import java.io.Serializable; +import java.net.URL; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; @@ -121,8 +126,8 @@ public static BlobTargetOption predefinedAcl(PredefinedAcl acl) { return new BlobTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry()); } - public static BlobTargetOption doesNotExists() { - return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0); + public static BlobTargetOption doesNotExist() { + return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0L); } public static BlobTargetOption generationMatch() { @@ -407,6 +412,139 @@ public static Builder builder() { } } + /** + * A request for signing a URL. + */ + class SignUrlRequest implements Serializable { + + private final Blob blob; + private final HttpMethod httpMethod; + private final Long expiration; + private final boolean includeContentType; + private final boolean includeMd5; + private final String headers; + private final ServiceAccountAuthCredentials authCredentials; + + public static class Builder { + + private Blob blob; + private HttpMethod httpMethod; + private long expiration; + private boolean includeContentType; + private boolean includeMd5; + private String headers; + private ServiceAccountAuthCredentials authCredentials; + + private Builder() {} + + public Builder blob(Blob blob) { + this.blob = blob; + return this; + } + + /** + * The HTTP method to be used with the signed URL. + */ + public Builder httpMethod(HttpMethod httpMethod) { + this.httpMethod = httpMethod; + return this; + } + + /** + * Sets expiration time for the URL. + * Defaults to one day. + */ + public Builder expiration(long expiration) { + this.expiration = expiration; + return this; + } + + /** + * Indicate if signature should include the blob's content-type. + * If {@code true} users of the signed URL should include + * the same content-type with their request. + */ + public Builder includeContentType(boolean includeContentType) { + this.includeContentType = includeContentType; + return this; + } + + /** + * Indicate if signature should include the blob's md5. + * If {@code true} users of the signed URL should include it with their requests. + */ + public Builder includeMd5(boolean includeMd5) { + this.includeMd5 = includeMd5; + return this; + } + + /** + * If headers are provided, the server will check to make sure that the client + * provides matching values. + * For information about how to create canonical headers for signing, + * see About Canonical Extension Headers. + */ + public Builder canonicalizedExtensionHeaders(String headers) { + this.headers = headers; + return this; + } + + /** + * The service account credentials for signing the URL. + */ + public Builder serviceAccountAuthCredentials(ServiceAccountAuthCredentials authCredentials) { + this.authCredentials = authCredentials; + return this; + } + + public SignUrlRequest build() { + return new SignUrlRequest(this); + } + } + + private SignUrlRequest(Builder builder) { + blob = checkNotNull(builder.blob); + httpMethod = builder.httpMethod; + expiration = firstNonNull(builder.expiration, new DateTime().plusDays(1).getMillis()); + includeContentType = builder.includeContentType; // verify blob has content-type + includeMd5 = builder.includeMd5; // verify blob has md5 + authCredentials = builder.authCredentials; // default if null + headers = builder.headers; + } + + public Blob blob() { + return blob; + } + + public HttpMethod httpMethod() { + return httpMethod; + } + + public String canonicalizedExtensionHeaders() { + return headers; + } + + public long expiration() { + return expiration; + } + + public ServiceAccountAuthCredentials authCredentials() { + return authCredentials; + } + + public boolean includeContentType() { + return includeContentType; + } + + public boolean includeMd5() { + return includeMd5; + } + + public static Builder builder() { + return new Builder(); + } + } + /** * Create a new bucket. * @@ -528,4 +666,14 @@ public static Builder builder() { * @throws StorageServiceException upon failure */ BlobWriteChannel writer(Blob blob, BlobTargetOption... options); + + /** + * Generates a signed URL for a blob. + * If you have a blob that you want to allow access to for a set + * amount of time, you can use this method to generate a URL that + * is only valid within a certain time period. + * This is particularly useful if you don't want publicly + * accessible blobs, but don't want to require users to explicitly log in. + */ + URL signUrl(SignUrlRequest request); } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 95b95141be14..8dda8004c016 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -29,7 +29,6 @@ import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; -import static java.util.concurrent.Executors.callable; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.Function; @@ -47,6 +46,7 @@ import com.google.gcloud.spi.StorageRpc.Tuple; import java.io.Serializable; +import java.net.URL; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -429,6 +429,12 @@ public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) { return new BlobWriterChannelImpl(options(), blob, optionsMap); } + @Override + public URL signUrl(SignUrlRequest request) { + // todo: implement and add test + return null; + } + private Map optionMap(Long generation, Long metaGeneration, Iterable options) { return optionMap(generation, metaGeneration, options, false); diff --git a/src/test/java/com/google/gcloud/storage/CorsTest.java b/src/test/java/com/google/gcloud/storage/CorsTest.java index 8b0379f03583..f29020c6380f 100644 --- a/src/test/java/com/google/gcloud/storage/CorsTest.java +++ b/src/test/java/com/google/gcloud/storage/CorsTest.java @@ -19,7 +19,6 @@ import static org.junit.Assert.assertEquals; import com.google.common.collect.ImmutableList; -import com.google.gcloud.storage.Cors.Method; import com.google.gcloud.storage.Cors.Origin; import org.junit.Test; @@ -39,7 +38,7 @@ public void testOrigin() { public void corsTest() { List origins = ImmutableList.of(Origin.any(), Origin.of("o")); List headers = ImmutableList.of("h1", "h2"); - List methods = ImmutableList.of(Method.ANY); + List methods = ImmutableList.of(HttpMethod.ANY); Cors cors = Cors.builder() .maxAgeSeconds(100) .origins(origins) From 8c5d0dbb26148299d5042ead44b5a9109794f733 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 26 May 2015 16:05:15 -0700 Subject: [PATCH 4/8] add HttpMethod which was removed by the merge --- .../com/google/gcloud/storage/HttpMethod.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 gcloud-java-storage/src/main/java/com/google/gcloud/storage/HttpMethod.java diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/HttpMethod.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/HttpMethod.java new file mode 100644 index 000000000000..9d7944140915 --- /dev/null +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/HttpMethod.java @@ -0,0 +1,24 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +/** + * Http method supported by Storage service. + */ +public enum HttpMethod { + GET, HEAD, PUT, POST, DELETE +} From f25b08c7b1f31e46a293f43137f11a64ac63993c Mon Sep 17 00:00:00 2001 From: ozarov Date: Tue, 26 May 2015 20:31:13 -0700 Subject: [PATCH 5/8] work on sign url --- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../google/gcloud/storage/StorageService.java | 190 +++++------------- .../gcloud/storage/StorageServiceImpl.java | 2 +- 3 files changed, 53 insertions(+), 141 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java index ab1e9affbbce..5a99cce69aa5 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException; String open(StorageObject object, Map options) throws StorageServiceException; void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest, long destOffset, int length, boolean last) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { // These options are part of the Google Cloud storage header options enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException; String open(StorageObject object, Map options) throws StorageServiceException; void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest, long destOffset, int length, boolean last) throws StorageServiceException; } \ No newline at end of file diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java index 2e500b001a93..27c7f8636be6 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java @@ -16,19 +16,16 @@ package com.google.gcloud.storage; -import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.collect.ImmutableList; -import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials; import com.google.gcloud.Service; import com.google.gcloud.spi.StorageRpc; -import org.joda.time.DateTime; - import java.io.Serializable; import java.net.URL; +import java.security.PrivateKey; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; @@ -218,6 +215,51 @@ public static BlobListOption recursive(boolean recursive) { } } + class SignUrlOption implements Serializable { + + private static final long serialVersionUID = 7850569877451099267L; + + private final String name; + private final Object value; + + private SignUrlOption(String name, Object value) { + this.name = name; + this.value = value; + } + + /** + * The HTTP method to be used with the signed URL. + */ + public static SignUrlOption httpMethod(HttpMethod httpMethod) { + return new SignUrlOption("HTTP_METHOD", httpMethod.name()); + } + + /** + * Use it if signature should include the blob's content-type. + * When used, users of the signed URL should include the blob's content-type with their request. + */ + public static SignUrlOption withContentType() { + return new SignUrlOption("CONTENT_TYPE", true); + } + + /** + * Use it if signature should include the blob's md5. + * When used, users of the signed URL should include the blob's md5 with their request. + */ + public static SignUrlOption withMd5() { + return new SignUrlOption("MD5", true); + } + + /** + * The private key to use for signing the URL. + * A private key is required for signing. If not provided an attempt will be made to get + * it from the service credentials. + */ + public static SignUrlOption signWith(PrivateKey privateKey) { + return new SignUrlOption("PRIVATE_KEY", privateKey); + } + } + class ComposeRequest implements Serializable { private static final long serialVersionUID = -7385681353748590911L; @@ -412,139 +454,6 @@ public static Builder builder() { } } - /** - * A request for signing a URL. - */ - class SignUrlRequest implements Serializable { - - private final Blob blob; - private final HttpMethod httpMethod; - private final Long expiration; - private final boolean includeContentType; - private final boolean includeMd5; - private final String headers; - private final ServiceAccountAuthCredentials authCredentials; - - public static class Builder { - - private Blob blob; - private HttpMethod httpMethod; - private long expiration; - private boolean includeContentType; - private boolean includeMd5; - private String headers; - private ServiceAccountAuthCredentials authCredentials; - - private Builder() {} - - public Builder blob(Blob blob) { - this.blob = blob; - return this; - } - - /** - * The HTTP method to be used with the signed URL. - */ - public Builder httpMethod(HttpMethod httpMethod) { - this.httpMethod = httpMethod; - return this; - } - - /** - * Sets expiration time for the URL. - * Defaults to one day. - */ - public Builder expiration(long expiration) { - this.expiration = expiration; - return this; - } - - /** - * Indicate if signature should include the blob's content-type. - * If {@code true} users of the signed URL should include - * the same content-type with their request. - */ - public Builder includeContentType(boolean includeContentType) { - this.includeContentType = includeContentType; - return this; - } - - /** - * Indicate if signature should include the blob's md5. - * If {@code true} users of the signed URL should include it with their requests. - */ - public Builder includeMd5(boolean includeMd5) { - this.includeMd5 = includeMd5; - return this; - } - - /** - * If headers are provided, the server will check to make sure that the client - * provides matching values. - * For information about how to create canonical headers for signing, - * see About Canonical Extension Headers. - */ - public Builder canonicalizedExtensionHeaders(String headers) { - this.headers = headers; - return this; - } - - /** - * The service account credentials for signing the URL. - */ - public Builder serviceAccountAuthCredentials(ServiceAccountAuthCredentials authCredentials) { - this.authCredentials = authCredentials; - return this; - } - - public SignUrlRequest build() { - return new SignUrlRequest(this); - } - } - - private SignUrlRequest(Builder builder) { - blob = checkNotNull(builder.blob); - httpMethod = builder.httpMethod; - expiration = firstNonNull(builder.expiration, new DateTime().plusDays(1).getMillis()); - includeContentType = builder.includeContentType; // verify blob has content-type - includeMd5 = builder.includeMd5; // verify blob has md5 - authCredentials = builder.authCredentials; // default if null - headers = builder.headers; - } - - public Blob blob() { - return blob; - } - - public HttpMethod httpMethod() { - return httpMethod; - } - - public String canonicalizedExtensionHeaders() { - return headers; - } - - public long expiration() { - return expiration; - } - - public ServiceAccountAuthCredentials authCredentials() { - return authCredentials; - } - - public boolean includeContentType() { - return includeContentType; - } - - public boolean includeMd5() { - return includeMd5; - } - - public static Builder builder() { - return new Builder(); - } - } - /** * Create a new bucket. * @@ -669,11 +578,14 @@ public static Builder builder() { /** * Generates a signed URL for a blob. - * If you have a blob that you want to allow access to for a set + * If you have a blob that you want to allow access to for a fixed * amount of time, you can use this method to generate a URL that * is only valid within a certain time period. * This is particularly useful if you don't want publicly * accessible blobs, but don't want to require users to explicitly log in. + * + * @param blob the blob associated with the signed url + * @param expiration the signed URL expiration (epoch time in milliseconds) */ - URL signUrl(SignUrlRequest request); + URL signUrl(Blob blob, long expiration, SignUrlOption... options); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 8dda8004c016..17d603372a01 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -430,7 +430,7 @@ public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) { } @Override - public URL signUrl(SignUrlRequest request) { + public URL signUrl(Blob blob, long expiration, SignUrlOption... options) { // todo: implement and add test return null; } From df2b3505aaff4b28a88d684f62484b60a0c812aa Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 27 May 2015 18:00:40 -0700 Subject: [PATCH 6/8] sign url - work in progress --- .../google/gcloud/storage/StorageService.java | 16 ++++++++++++++-- .../gcloud/storage/StorageServiceImpl.java | 7 ++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java index 27c7f8636be6..5f65b66b8c20 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java @@ -222,11 +222,23 @@ class SignUrlOption implements Serializable { private final String name; private final Object value; + enum OPTIONS { + + } + private SignUrlOption(String name, Object value) { this.name = name; this.value = value; } + String name() { + return name; + } + + Object value() { + return value; + } + /** * The HTTP method to be used with the signed URL. */ @@ -585,7 +597,7 @@ public static Builder builder() { * accessible blobs, but don't want to require users to explicitly log in. * * @param blob the blob associated with the signed url - * @param expiration the signed URL expiration (epoch time in milliseconds) + * @param expirationTimeMillis the signed URL expiration (epoch time in milliseconds) */ - URL signUrl(Blob blob, long expiration, SignUrlOption... options); + URL signUrl(Blob blob, long expirationTimeMillis, SignUrlOption... options); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 17d603372a01..508092e78db5 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -47,6 +47,7 @@ import java.io.Serializable; import java.net.URL; +import java.security.PrivateKey; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -431,7 +432,11 @@ public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) { @Override public URL signUrl(Blob blob, long expiration, SignUrlOption... options) { - // todo: implement and add test + Map optionMap = Maps.newHashMapWithExpectedSize(options.length); + for (SignUrlOption option : options) { + optionMap.put(option.name(), option.value()); + } + PrivateKey key = (PrivateKey) optionMap.get(""); return null; } From 1f28b34aac0904d98eaead89e4d716d10c6bbd7d Mon Sep 17 00:00:00 2001 From: ozarov Date: Wed, 27 May 2015 22:11:32 -0700 Subject: [PATCH 7/8] sign url - work in progress --- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../google/gcloud/storage/StorageService.java | 42 ++++++----- .../gcloud/storage/StorageServiceImpl.java | 73 +++++++++++++++++-- 3 files changed, 89 insertions(+), 28 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index e27d837d7173..12c74b811ea9 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -81,10 +81,10 @@ public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; + // todo: validate what is returned by getRootURL and use that for host default (and use host()) storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); - // Todo: make sure nulls are being used as Data.asNull() } private static StorageServiceException translate(IOException exception) { diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java index 5f65b66b8c20..edf12c9f8feb 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java @@ -20,12 +20,12 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.collect.ImmutableList; +import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials; import com.google.gcloud.Service; import com.google.gcloud.spi.StorageRpc; import java.io.Serializable; import java.net.URL; -import java.security.PrivateKey; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; @@ -219,20 +219,20 @@ class SignUrlOption implements Serializable { private static final long serialVersionUID = 7850569877451099267L; - private final String name; + private final Option option; private final Object value; - enum OPTIONS { - + enum Option { + HTTP_METHOD, CONTENT_TYPE, MD5, SERVICE_ACCOUNT_CRED; } - - private SignUrlOption(String name, Object value) { - this.name = name; + + private SignUrlOption(Option option, Object value) { + this.option = option; this.value = value; } - String name() { - return name; + Option option() { + return option; } Object value() { @@ -243,7 +243,7 @@ Object value() { * The HTTP method to be used with the signed URL. */ public static SignUrlOption httpMethod(HttpMethod httpMethod) { - return new SignUrlOption("HTTP_METHOD", httpMethod.name()); + return new SignUrlOption(Option.HTTP_METHOD, httpMethod.name()); } /** @@ -251,7 +251,7 @@ public static SignUrlOption httpMethod(HttpMethod httpMethod) { * When used, users of the signed URL should include the blob's content-type with their request. */ public static SignUrlOption withContentType() { - return new SignUrlOption("CONTENT_TYPE", true); + return new SignUrlOption(Option.CONTENT_TYPE, true); } /** @@ -259,16 +259,17 @@ public static SignUrlOption withContentType() { * When used, users of the signed URL should include the blob's md5 with their request. */ public static SignUrlOption withMd5() { - return new SignUrlOption("MD5", true); + return new SignUrlOption(Option.MD5, true); } /** - * The private key to use for signing the URL. - * A private key is required for signing. If not provided an attempt will be made to get - * it from the service credentials. + * Service account credentials which are used for signing the URL. + * If not provided an attempt will be made to get it from the environment. + * + * @see Service account */ - public static SignUrlOption signWith(PrivateKey privateKey) { - return new SignUrlOption("PRIVATE_KEY", privateKey); + public static SignUrlOption serviceAccount(ServiceAccountAuthCredentials credentials) { + return new SignUrlOption(Option.SERVICE_ACCOUNT_CRED, credentials); } } @@ -278,7 +279,7 @@ class ComposeRequest implements Serializable { private final List sourceBlobs; private final Blob target; - private final List targetOptions; + private final List targetOptions; public static class SourceBlob implements Serializable { @@ -597,7 +598,8 @@ public static Builder builder() { * accessible blobs, but don't want to require users to explicitly log in. * * @param blob the blob associated with the signed url - * @param expirationTimeMillis the signed URL expiration (epoch time in milliseconds) + * @param expirationTimeInSeconds the signed URL expiration (using epoch time) + * @see Signed-URLs */ - URL signUrl(Blob blob, long expirationTimeMillis, SignUrlOption... options); + URL signUrl(Blob blob, long expirationTimeInSeconds, SignUrlOption... options); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 508092e78db5..340b473cb0f3 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -38,7 +38,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.common.io.BaseEncoding; import com.google.common.primitives.Ints; +import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials; import com.google.gcloud.BaseService; import com.google.gcloud.ExceptionHandler; import com.google.gcloud.ExceptionHandler.Interceptor; @@ -46,9 +48,15 @@ import com.google.gcloud.spi.StorageRpc.Tuple; import java.io.Serializable; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; import java.net.URL; -import java.security.PrivateKey; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.Signature; +import java.security.SignatureException; import java.util.Arrays; +import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -83,10 +91,8 @@ public RetryResult beforeEval(Exception exception) { StorageServiceImpl(StorageServiceOptions options) { super(options); storageRpc = options.storageRpc(); - // todo: replace nulls with Value.asNull (per toPb) // todo: configure timeouts - https://developers.google.com/api-client-library/java/google-api-java-client/errors // todo: provide rewrite - https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite - // todo: provide signed urls - https://cloud.google.com/storage/docs/access-control#Signed-URLs // todo: check if we need to expose https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/insert vs using bucket update/patch } @@ -432,12 +438,65 @@ public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) { @Override public URL signUrl(Blob blob, long expiration, SignUrlOption... options) { - Map optionMap = Maps.newHashMapWithExpectedSize(options.length); + EnumMap optionMap = Maps.newEnumMap(SignUrlOption.Option.class); for (SignUrlOption option : options) { - optionMap.put(option.name(), option.value()); + optionMap.put(option.option(), option.value()); + } + ServiceAccountAuthCredentials cred = + (ServiceAccountAuthCredentials) optionMap.get(SignUrlOption.Option.SERVICE_ACCOUNT_CRED); + if (cred == null) { + checkArgument(options().authCredentials() instanceof ServiceAccountAuthCredentials, + "Signing key was not provided and could not be derived"); + cred = (ServiceAccountAuthCredentials) this.options().authCredentials(); + } + // construct signature data - see https://cloud.google.com/storage/docs/access-control#Signed-URLs + StringBuilder stBuilder = new StringBuilder(); + if (optionMap.containsKey(SignUrlOption.Option.HTTP_METHOD)) { + stBuilder.append(optionMap.get(SignUrlOption.Option.HTTP_METHOD)); + } else { + stBuilder.append(HttpMethod.GET); + } + stBuilder.append('\n'); + if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.MD5) , false)) { + checkArgument(blob.md5() != null, "Blob is missing a value for md5"); + stBuilder.append(blob.md5()); + } + stBuilder.append('\n'); + if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.CONTENT_TYPE) , false)) { + checkArgument(blob.contentType() != null, "Blob is missing a value for content-type"); + stBuilder.append(blob.contentType()); + } + stBuilder.append('\n'); + stBuilder.append(expiration).append('\n').append('\n'); + StringBuilder path = new StringBuilder(); + if (!blob.bucket().startsWith("/")) { + path.append('/'); + } + path.append(blob.bucket()); + if (!blob.bucket().endsWith("/")) { + path.append('/'); + } + if (blob.name().startsWith("/")) { + path.setLength(stBuilder.length() - 1); + } + path.append(blob.name()); + stBuilder.append(path); + try { + Signature signer = Signature.getInstance("SHA256withRSA"); + signer.initSign(cred.privateKey()); + signer.update(stBuilder.toString().getBytes("UTF-8")); + String signature = BaseEncoding.base64Url().encode(signer.sign()); + // todo - use options().host() - after default is correct and value is past to RPC + stBuilder = new StringBuilder("https://storage.googleapis.com").append(path); + stBuilder.append("?GoogleAccessId=").append(cred.account()); + stBuilder.append("&Expires=").append(expiration); + stBuilder.append("&Signature=").append(signature); + return new URL(stBuilder.toString()); + } catch (MalformedURLException | NoSuchAlgorithmException | UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } catch (SignatureException | InvalidKeyException e) { + throw new IllegalArgumentException("Invalid service account private key"); } - PrivateKey key = (PrivateKey) optionMap.get(""); - return null; } private Map optionMap(Long generation, Long metaGeneration, From 81e5efe533cfff84460a0d17a5d1b0c661b49ba0 Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 28 May 2015 15:50:02 -0700 Subject: [PATCH 8/8] complete work on signURL --- gcloud-java-core/pom.xml | 4 +- gcloud-java-examples/pom.xml | 11 ++++ .../gcloud/examples/StorageExample.java | 66 +++++++++++++++++-- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../gcloud/storage/StorageServiceImpl.java | 10 +-- .../com/google/gcloud/storage/CorsTest.java | 2 +- pom.xml | 13 ++++ 7 files changed, 96 insertions(+), 12 deletions(-) diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml index 78baf824c080..fa2e1c18972f 100644 --- a/gcloud-java-core/pom.xml +++ b/gcloud-java-core/pom.xml @@ -28,13 +28,13 @@ com.google.http-client google-http-client - 1.19.0 + 1.20.0 compile com.google.oauth-client google-oauth-client - 1.19.0 + 1.20.0 compile diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml index 66d9fc9c93e3..1c0357d63635 100644 --- a/gcloud-java-examples/pom.xml +++ b/gcloud-java-examples/pom.xml @@ -21,4 +21,15 @@ ${project.version} + + + + org.codehaus.mojo + exec-maven-plugin + + false + + + + diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java index b0d44c292d2c..ccf2cd6f5b76 100644 --- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -16,6 +16,8 @@ package com.google.gcloud.examples; +import com.google.gcloud.AuthCredentials; +import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials; import com.google.gcloud.RetryParams; import com.google.gcloud.spi.StorageRpc.Tuple; import com.google.gcloud.storage.BatchRequest; @@ -27,6 +29,7 @@ import com.google.gcloud.storage.StorageService; import com.google.gcloud.storage.StorageService.ComposeRequest; import com.google.gcloud.storage.StorageService.CopyRequest; +import com.google.gcloud.storage.StorageService.SignUrlOption; import com.google.gcloud.storage.StorageServiceFactory; import com.google.gcloud.storage.StorageServiceOptions; @@ -40,7 +43,14 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; import java.util.Arrays; +import java.util.Calendar; import java.util.HashMap; import java.util.Map; @@ -58,7 +68,8 @@ * -Dexec.args="[] list []| info [ []]| * download [local_file]| upload []| * delete +| cp | - * compose + | update_metadata [key=value]*"} + * compose + | update_metadata [key=value]*| + * sign_url "} * * * @@ -75,7 +86,7 @@ private static abstract class StorageAction { abstract void run(StorageService storage, T request) throws Exception; - abstract T parse(String... args) throws IllegalArgumentException, IOException; + abstract T parse(String... args) throws Exception; protected String params() { return ""; @@ -424,7 +435,7 @@ public String params() { * * @see Objects: update */ - private static class UpdateMetadata extends StorageAction>> { + private static class UpdateMetadataAction extends StorageAction>> { @Override public void run(StorageService storage, Tuple> tuple) @@ -467,6 +478,52 @@ public String params() { } } + /** + * This class demonstrates how to sign a url. + * URL will be valid for 1 day. + * + * @see Signed URLs + */ + private static class SignUrlAction extends + StorageAction> { + + private static final char[] PASSWORD = "notasecret".toCharArray(); + + @Override + public void run(StorageService storage, Tuple tuple) + throws Exception { + run(storage, tuple.x(), tuple.y()); + } + + private void run(StorageService storage, ServiceAccountAuthCredentials cred, Blob blob) + throws IOException { + Calendar cal = Calendar.getInstance(); + cal.add(Calendar.DATE, 1); + long expiration = cal.getTimeInMillis() / 1000; + System.out.println("Signed URL: " + + storage.signUrl(blob, expiration, SignUrlOption.serviceAccount(cred))); + } + + @Override + Tuple parse(String... args) + throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException, + UnrecoverableKeyException { + if (args.length != 4) { + throw new IllegalArgumentException(); + } + KeyStore keystore = KeyStore.getInstance("PKCS12"); + keystore.load(Files.newInputStream(Paths.get(args[0])), PASSWORD); + PrivateKey privateKey = (PrivateKey) keystore.getKey("privatekey", PASSWORD); + ServiceAccountAuthCredentials cred = AuthCredentials.createFor(args[1], privateKey); + return Tuple.of(cred, Blob.of(args[2], args[3])); + } + + @Override + public String params() { + return " "; + } + } + static { ACTIONS.put("info", new InfoAction()); ACTIONS.put("delete", new DeleteAction()); @@ -475,7 +532,8 @@ public String params() { ACTIONS.put("download", new DownloadAction()); ACTIONS.put("cp", new CopyAction()); ACTIONS.put("compose", new ComposeAction()); - ACTIONS.put("update_metadata", new UpdateMetadata()); + ACTIONS.put("update_metadata", new UpdateMetadataAction()); + ACTIONS.put("sign_url", new SignUrlAction()); } public static void printUsage() { diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 12c74b811ea9..f63c57e3c784 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -81,8 +81,8 @@ public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; - // todo: validate what is returned by getRootURL and use that for host default (and use host()) storage = new Storage.Builder(transport, new JacksonFactory(), initializer) + .setRootUrl(options.host()) .setApplicationName("gcloud-java") .build(); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 340b473cb0f3..ecf7064a503c 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -29,6 +29,7 @@ import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.Function; @@ -51,6 +52,7 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; +import java.net.URLEncoder; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.Signature; @@ -467,7 +469,7 @@ public URL signUrl(Blob blob, long expiration, SignUrlOption... options) { stBuilder.append(blob.contentType()); } stBuilder.append('\n'); - stBuilder.append(expiration).append('\n').append('\n'); + stBuilder.append(expiration).append('\n'); StringBuilder path = new StringBuilder(); if (!blob.bucket().startsWith("/")) { path.append('/'); @@ -484,9 +486,9 @@ public URL signUrl(Blob blob, long expiration, SignUrlOption... options) { try { Signature signer = Signature.getInstance("SHA256withRSA"); signer.initSign(cred.privateKey()); - signer.update(stBuilder.toString().getBytes("UTF-8")); - String signature = BaseEncoding.base64Url().encode(signer.sign()); - // todo - use options().host() - after default is correct and value is past to RPC + signer.update(stBuilder.toString().getBytes(UTF_8)); + String signature = + URLEncoder.encode(BaseEncoding.base64().encode(signer.sign()), UTF_8.name()); stBuilder = new StringBuilder("https://storage.googleapis.com").append(path); stBuilder.append("?GoogleAccessId=").append(cred.account()); stBuilder.append("&Expires=").append(expiration); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CorsTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CorsTest.java index f29020c6380f..f978cb87f3d1 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CorsTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CorsTest.java @@ -38,7 +38,7 @@ public void testOrigin() { public void corsTest() { List origins = ImmutableList.of(Origin.any(), Origin.of("o")); List headers = ImmutableList.of("h1", "h2"); - List methods = ImmutableList.of(HttpMethod.ANY); + List methods = ImmutableList.of(HttpMethod.GET); Cors cors = Cors.builder() .maxAgeSeconds(100) .origins(origins) diff --git a/pom.xml b/pom.xml index e6fafbe5e7c2..8f56620542bc 100644 --- a/pom.xml +++ b/pom.xml @@ -85,6 +85,19 @@ + + + + org.codehaus.mojo + exec-maven-plugin + 1.3.2 + + true + java + + + + org.codehaus.mojo