diff --git a/gcloud-java-contrib/README.md b/gcloud-java-contrib/README.md index 23713f7450a3..b8bef6eb977e 100644 --- a/gcloud-java-contrib/README.md +++ b/gcloud-java-contrib/README.md @@ -3,38 +3,10 @@ Google Cloud Java Contributions Packages that provide higher-level abstraction/functionality for common gcloud-java use cases. -Quickstart ----------- -If you are using Maven, add this to your pom.xml file -```xml - - com.google.gcloud - gcloud-java-contrib - 0.1.3 - -``` -If you are using Gradle, add this to your dependencies -```Groovy -compile 'com.google.gcloud:gcloud-java-contrib:0.1.3' -``` -If you are using SBT, add this to your dependencies -```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java-contrib" % "0.1.3" -``` - -Java Versions -------------- - -Java 7 or above is required for using this client. - -Versioning ----------- - -This library follows [Semantic Versioning] (http://semver.org/). - -It is currently in major version zero (``0.y.z``), which means that anything -may change at any time and the public API should not be considered -stable. +Contents +-------- + + * [gcloud-java-nio](./gcloud-java-nio/): NIO Filesystem Provider for Google Cloud Storage. Contributing ------------ diff --git a/gcloud-java-contrib/gcloud-java-nio/pom.xml b/gcloud-java-contrib/gcloud-java-nio/pom.xml index 69d3c5157c96..3f06179f30b7 100644 --- a/gcloud-java-contrib/gcloud-java-nio/pom.xml +++ b/gcloud-java-contrib/gcloud-java-nio/pom.xml @@ -49,11 +49,6 @@ 1.1 provided - - com.google.appengine.tools - appengine-gcs-client - 0.5 - junit junit @@ -77,24 +72,6 @@ mockito-core 1.9.5 - - com.google.appengine - appengine-testing - 1.9.30 - test - - - com.google.appengine - appengine-api-stubs - 1.9.30 - test - - - com.google.appengine - appengine-local-endpoints - 1.9.30 - test - diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageConfiguration.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageConfiguration.java index e939edfb5a19..6659bd4d8857 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageConfiguration.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageConfiguration.java @@ -6,11 +6,13 @@ import java.util.Map; -/** Configuration class for {@link CloudStorageFileSystem#forBucket} */ +/** CloudStorageConfiguration is the configuration class for + * {@link CloudStorageFileSystem#forBucket}. */ @AutoValue public abstract class CloudStorageConfiguration { - /** Returns the path of the current working directory. Defaults to the root directory. */ + /** Returns the path of the current working directory. Defaults to the root directory. + */ public abstract String workingDirectory(); /** @@ -46,7 +48,8 @@ public static Builder builder() { return new Builder(); } - /** Builder for {@link CloudStorageConfiguration}. */ + /** Builder for {@link CloudStorageConfiguration}. + */ public static final class Builder { private String workingDirectory = UnixPath.ROOT; @@ -87,7 +90,8 @@ public Builder stripPrefixSlash(boolean value) { return this; } - /** Configures if paths with a trailing slash should be treated as fake directories. */ + /** Configures if paths with a trailing slash should be treated as fake directories. + */ public Builder usePseudoDirectories(boolean value) { usePseudoDirectories = value; return this; @@ -103,7 +107,9 @@ public Builder blockSize(int value) { return this; } - /** Creates a new instance, but does not destroy the builder. */ + + /** Creates a new instance, but does not destroy the builder. + */ public CloudStorageConfiguration build() { return new AutoValue_CloudStorageConfiguration( workingDirectory, diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeView.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeView.java index 5a4d4bc7c28c..181870560bfe 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeView.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeView.java @@ -2,8 +2,9 @@ import static com.google.common.base.Verify.verifyNotNull; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; import com.google.common.base.MoreObjects; +import com.google.gcloud.storage.BlobInfo; +import com.google.gcloud.storage.Storage; import java.io.IOException; import java.nio.file.NoSuchFileException; @@ -14,19 +15,22 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; -/** Metadata view for a Google Cloud Storage object. */ +/** Metadata view for a Google Cloud Storage object. + */ @Immutable public final class CloudStorageFileAttributeView implements BasicFileAttributeView { - private final CloudStorageFileSystemProvider provider; + //private final CloudStorageFileSystemProvider provider; + private final Storage storage; private final CloudStoragePath path; - CloudStorageFileAttributeView(CloudStorageFileSystemProvider provider, CloudStoragePath path) { - this.provider = verifyNotNull(provider); + CloudStorageFileAttributeView(Storage storage, CloudStoragePath path) { + this.storage = verifyNotNull(storage); this.path = verifyNotNull(path); } - /** Returns {@value CloudStorageFileSystem#GCS_VIEW} */ + /** Returns {@value CloudStorageFileSystem#GCS_VIEW}. + */ @Override public String name() { return CloudStorageFileSystem.GCS_VIEW; @@ -36,19 +40,18 @@ public String name() { public CloudStorageFileAttributes readAttributes() throws IOException { if (path.seemsLikeADirectory() && path.getFileSystem().config().usePseudoDirectories()) { - return CloudStoragePseudoDirectoryAttributes.SINGLETON_INSTANCE; + return new CloudStoragePseudoDirectoryAttributes(path); } - GcsFileMetadata metadata = provider.getGcsService().getMetadata(path.getGcsFilename()); - if (metadata == null) { + BlobInfo blobInfo = storage.get(path.getBlobId()); + if (blobInfo == null) { throw new NoSuchFileException(path.toUri().toString()); } - return new CloudStorageObjectAttributes(metadata); + + return new CloudStorageObjectAttributes(blobInfo); } /** * This feature is not supported, since Cloud Storage objects are immutable. - * - * @throws UnsupportedOperationException */ @Override public void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTime createTime) { @@ -59,19 +62,19 @@ public void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTim public boolean equals(@Nullable Object other) { return this == other || other instanceof CloudStorageFileAttributeView - && Objects.equals(provider, ((CloudStorageFileAttributeView) other).provider) + && Objects.equals(storage, ((CloudStorageFileAttributeView) other).storage) && Objects.equals(path, ((CloudStorageFileAttributeView) other).path); } @Override public int hashCode() { - return Objects.hash(provider, path); + return Objects.hash(storage, path); } @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("provider", provider) + .add("storage", storage) .add("path", path) .toString(); } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributes.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributes.java index 774a5499431d..a384e1be2c73 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributes.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributes.java @@ -2,8 +2,10 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; +import com.google.gcloud.storage.Acl; import java.nio.file.attribute.BasicFileAttributes; +import java.util.List; /** Interface for attributes on a cloud storage file or pseudo-directory. */ public interface CloudStorageFileAttributes extends BasicFileAttributes { @@ -27,7 +29,7 @@ public interface CloudStorageFileAttributes extends BasicFileAttributes { * * @see "https://developers.google.com/storage/docs/reference-headers#acl" */ - Optional acl(); + Optional> acl(); /** * Returns the {@code Cache-Control} HTTP header value, if set on this object. diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystem.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystem.java index b4318be63f31..8cec0845600c 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystem.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystem.java @@ -89,17 +89,20 @@ public CloudStorageFileSystemProvider provider() { return provider; } - /** Returns the Cloud Storage bucket name being served by this file system. */ + /** Returns the Cloud Storage bucket name being served by this file system. + */ public String bucket() { return bucket; } - /** Returns the configuration object for this filesystem instance. */ + /** Returns the configuration object for this filesystem instance. + */ public CloudStorageConfiguration config() { return config; } - /** Converts a cloud storage object name to a {@link Path} object. */ + /** Converts a cloud storage object name to a {@link Path} object. + */ @Override public CloudStoragePath getPath(String first, String... more) { checkArgument(!first.startsWith(URI_SCHEME + ":"), @@ -107,23 +110,27 @@ public CloudStoragePath getPath(String first, String... more) { return CloudStoragePath.getPath(this, first, more); } - /** Does nothing. */ + /** Does nothing. + */ @Override public void close() {} - /** Returns {@code true} */ + /** Returns {@code true}. + */ @Override public boolean isOpen() { return true; } - /** Returns {@code false} */ + /** Returns {@code false}. + */ @Override public boolean isReadOnly() { return false; } - /** Returns {@value UnixPath#SEPARATOR} */ + /** Returns {@value UnixPath#SEPARATOR}. + */ @Override public String getSeparator() { return "" + UnixPath.SEPARATOR; @@ -144,24 +151,26 @@ public Set supportedFileAttributeViews() { return SUPPORTED_VIEWS; } - /** @throws UnsupportedOperationException */ + /** Always throws {@link UnsupportedOperationException}. */ @Override public PathMatcher getPathMatcher(String syntaxAndPattern) { - // TODO(b/18997520): Implement me. + // TODO: Implement me. throw new UnsupportedOperationException(); } - /** @throws UnsupportedOperationException */ + /** Always throws {@link UnsupportedOperationException}. + */ @Override public UserPrincipalLookupService getUserPrincipalLookupService() { - // TODO(b/18997520): Implement me. + // TODO: Implement me. throw new UnsupportedOperationException(); } - /** @throws UnsupportedOperationException */ + /** Always throws {@link UnsupportedOperationException}. + */ @Override public WatchService newWatchService() throws IOException { - // TODO(b/18997520): Implement me. + // TODO: Implement me. throw new UnsupportedOperationException(); } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java index f1964b231ed1..240140cb071f 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java @@ -3,32 +3,29 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Strings.isNullOrEmpty; -import static com.google.common.base.Suppliers.memoize; import static com.google.gcloud.storage.contrib.nio.CloudStorageFileSystem.URI_SCHEME; -import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.buildFileOptions; import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.checkBucket; import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.checkNotNullArray; import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.checkPath; -import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.copyFileOptions; import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.stripPathFromUri; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsFileOptions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsInputChannel; -import com.google.appengine.tools.cloudstorage.GcsOutputChannel; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; import com.google.auto.service.AutoService; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; -import com.google.common.base.Supplier; +import com.google.common.base.Throwables; import com.google.common.primitives.Ints; +import com.google.gcloud.storage.Acl; +import com.google.gcloud.storage.BlobId; +import com.google.gcloud.storage.BlobInfo; +import com.google.gcloud.storage.CopyWriter; +import com.google.gcloud.storage.Storage; +import com.google.gcloud.storage.StorageException; +import com.google.gcloud.storage.StorageOptions; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; import java.nio.file.AccessMode; import java.nio.file.AtomicMoveNotSupportedException; @@ -48,7 +45,10 @@ import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.FileAttributeView; import java.nio.file.spi.FileSystemProvider; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -56,19 +56,23 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; -/** Google Cloud Storage {@link FileSystemProvider} */ +/** + * Google Cloud Storage {@link FileSystemProvider}. + */ @ThreadSafe @AutoService(FileSystemProvider.class) public final class CloudStorageFileSystemProvider extends FileSystemProvider { - private static final Supplier gcsServiceSupplier = - memoize(new Supplier() { - @Override - public GcsService get() { - return GcsServiceFactory.createGcsService(); - }}); + private final Storage storage; - private final GcsService gcsService; + // used only when we create a new instance of CloudStorageFileSystemProvider. + private static StorageOptions storageOptions; + + /** Those options are only used by the CloudStorageFileSystemProvider ctor. */ + @VisibleForTesting + public static void setGCloudOptions(StorageOptions newStorageOptions) { + storageOptions = newStorageOptions; + } /** * Default constructor which should only be called by Java SPI. @@ -77,15 +81,15 @@ public GcsService get() { * @see CloudStorageFileSystem#forBucket(String) */ public CloudStorageFileSystemProvider() { - this(gcsServiceSupplier.get()); + this(storageOptions); } - private CloudStorageFileSystemProvider(GcsService gcsService) { - this.gcsService = checkNotNull(gcsService); - } - - GcsService getGcsService() { - return gcsService; + private CloudStorageFileSystemProvider(@Nullable StorageOptions gcsStorageOptions) { + if (gcsStorageOptions == null) { + this.storage = StorageOptions.defaultInstance().service(); + } else { + this.storage = gcsStorageOptions.service(); + } } @Override @@ -127,7 +131,7 @@ public SeekableByteChannel newByteChannel( checkNotNull(path); checkNotNullArray(attrs); if (options.contains(StandardOpenOption.WRITE)) { - // TODO(b/18997618): Make our OpenOptions implement FileAttribute. Also remove buffer option. + // TODO: Make our OpenOptions implement FileAttribute. Also remove buffer option. return newWriteChannel(path, options); } else { return newReadChannel(path, options); @@ -165,14 +169,40 @@ private SeekableByteChannel newReadChannel( if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { throw new CloudStoragePseudoDirectoryException(cloudPath); } - return CloudStorageReadChannel.create(gcsService, cloudPath.getGcsFilename(), 0); + return CloudStorageReadChannel.create(storage, cloudPath.getBlobId(), 0); } private SeekableByteChannel newWriteChannel( Path path, Set options) throws IOException { - boolean wantCreateNew = false; + + CloudStoragePath cloudPath = checkPath(path); + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + throw new CloudStoragePseudoDirectoryException(cloudPath); + } + BlobId file = cloudPath.getBlobId(); + BlobInfo.Builder infoBuilder = BlobInfo.builder(file); + List writeOptions = new ArrayList<>(); + List acls = new ArrayList<>(); + + + HashMap metas = new HashMap<>(); for (OpenOption option : options) { - if (option instanceof StandardOpenOption) { + if (option instanceof OptionMimeType) { + infoBuilder.contentType(((OptionMimeType) option).mimeType()); + } else if (option instanceof OptionCacheControl) { + infoBuilder.cacheControl(((OptionCacheControl) option).cacheControl()); + } else if (option instanceof OptionContentDisposition) { + infoBuilder.contentDisposition(((OptionContentDisposition) option).contentDisposition()); + } else if (option instanceof OptionContentEncoding) { + infoBuilder.contentEncoding(((OptionContentEncoding) option).contentEncoding()); + } else if (option instanceof OptionUserMetadata) { + OptionUserMetadata opMeta = (OptionUserMetadata) option; + metas.put(opMeta.key(), opMeta.value()); + } else if (option instanceof OptionAcl) { + acls.add(((OptionAcl) option).acl()); + } else if (option instanceof OptionBlockSize) { + // TODO: figure out how to plumb in block size. + } else if (option instanceof StandardOpenOption) { switch ((StandardOpenOption) option) { case CREATE: case TRUNCATE_EXISTING: @@ -183,7 +213,7 @@ private SeekableByteChannel newWriteChannel( // Ignored by specification. break; case CREATE_NEW: - wantCreateNew = true; + writeOptions.add(Storage.BlobWriteOption.doesNotExist()); break; case READ: throw new IllegalArgumentException("READ+WRITE not supported yet"); @@ -195,24 +225,25 @@ private SeekableByteChannel newWriteChannel( throw new UnsupportedOperationException(option.toString()); } } else if (option instanceof CloudStorageOption) { - // These will be interpreted later. + // XXX: We need to interpret these later } else { throw new UnsupportedOperationException(option.toString()); } } - CloudStoragePath cloudPath = checkPath(path); - if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { - throw new CloudStoragePseudoDirectoryException(cloudPath); + + if (!metas.isEmpty()) { + infoBuilder.metadata(metas); } - if (wantCreateNew) { - // XXX: Java's documentation says this should be atomic. - if (gcsService.getMetadata(cloudPath.getGcsFilename()) != null) { - throw new FileAlreadyExistsException(cloudPath.toString()); - } + if (!acls.isEmpty()) { + infoBuilder.acl(acls); + } + + try { + return new CloudStorageWriteChannel(storage.writer(infoBuilder.build(), + writeOptions.toArray(new Storage.BlobWriteOption[0]))); + } catch (StorageException oops) { + throw asIOException(oops); } - GcsFilename file = cloudPath.getGcsFilename(); - GcsFileOptions fileOptions = buildFileOptions(new GcsFileOptions.Builder(), options.toArray()); - return new CloudStorageWriteChannel(gcsService.createOrReplace(file, fileOptions)); } @Override @@ -229,12 +260,12 @@ public InputStream newInputStream(Path path, OpenOption... options) throws IOExc } @Override - public final boolean deleteIfExists(Path path) throws IOException { + public boolean deleteIfExists(Path path) throws IOException { CloudStoragePath cloudPath = checkPath(path); if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { throw new CloudStoragePseudoDirectoryException(cloudPath); } - return gcsService.delete(cloudPath.getGcsFilename()); + return storage.delete(cloudPath.getBlobId()); } @Override @@ -261,6 +292,14 @@ public void move(Path source, Path target, CopyOption... options) throws IOExcep public void copy(Path source, Path target, CopyOption... options) throws IOException { boolean wantCopyAttributes = false; boolean wantReplaceExisting = false; + boolean setContentType = false; + boolean setCacheControl = false; + boolean setContentEncoding = false; + boolean setContentDisposition = false; + + CloudStoragePath toPath = checkPath(target); + BlobInfo.Builder tgtInfoBuilder = BlobInfo.builder(toPath.getBlobId()).contentType(""); + int blockSize = -1; for (CopyOption option : options) { if (option instanceof StandardCopyOption) { @@ -278,18 +317,34 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep } else if (option instanceof CloudStorageOption) { if (option instanceof OptionBlockSize) { blockSize = ((OptionBlockSize) option).size(); + } else if (option instanceof OptionMimeType) { + tgtInfoBuilder.contentType(((OptionMimeType) option).mimeType()); + setContentType = true; + } else if (option instanceof OptionCacheControl) { + tgtInfoBuilder.cacheControl(((OptionCacheControl) option).cacheControl()); + setCacheControl = true; + } else if (option instanceof OptionContentEncoding) { + tgtInfoBuilder.contentEncoding(((OptionContentEncoding) option).contentEncoding()); + setContentEncoding = true; + } else if (option instanceof OptionContentDisposition) { + tgtInfoBuilder.contentDisposition(((OptionContentDisposition) option) + .contentDisposition()); + setContentDisposition = true; + } else { + throw new UnsupportedOperationException(option.toString()); } - // The rest will be interpreted later. } else { throw new UnsupportedOperationException(option.toString()); } } + CloudStoragePath fromPath = checkPath(source); - CloudStoragePath toPath = checkPath(target); blockSize = blockSize != -1 ? blockSize : Ints.max(fromPath.getFileSystem().config().blockSize(), toPath.getFileSystem().config().blockSize()); + // TODO: actually use blockSize + if (fromPath.seemsLikeADirectory() && toPath.seemsLikeADirectory()) { if (fromPath.getFileSystem().config().usePseudoDirectories() && toPath.getFileSystem().config().usePseudoDirectories()) { @@ -307,41 +362,54 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep if (toPath.seemsLikeADirectoryAndUsePseudoDirectories()) { throw new CloudStoragePseudoDirectoryException(toPath); } - GcsFilename from = fromPath.getGcsFilename(); - GcsFilename to = toPath.getGcsFilename(); - GcsFileMetadata metadata = gcsService.getMetadata(from); - if (metadata == null) { - throw new NoSuchFileException(source.toString()); - } - if (fromPath.equals(toPath)) { - return; - } - if (!wantReplaceExisting && gcsService.getMetadata(to) != null) { - throw new FileAlreadyExistsException(target.toString()); - } - GcsFileOptions.Builder builder = wantCopyAttributes - ? copyFileOptions(metadata.getOptions()) - : new GcsFileOptions.Builder(); - GcsFileOptions fileOptions = buildFileOptions(builder, options); - try (GcsInputChannel input = gcsService.openReadChannel(from, 0); - GcsOutputChannel output = gcsService.createOrReplace(to, fileOptions)) { - ByteBuffer block = ByteBuffer.allocate(blockSize); - while (input.read(block) != -1) { - block.flip(); - while (block.hasRemaining()) { - output.write(block); + + try { + + + if (wantCopyAttributes) { + BlobInfo blobInfo = storage.get(fromPath.getBlobId()); + if (null == blobInfo) { + throw new NoSuchFileException(fromPath.toString()); + } + if (!setCacheControl) { + tgtInfoBuilder.cacheControl(blobInfo.cacheControl()); + } + if (!setContentType) { + tgtInfoBuilder.contentType(blobInfo.contentType()); + } + if (!setContentEncoding) { + tgtInfoBuilder.contentEncoding(blobInfo.contentEncoding()); } - block.clear(); + if (!setContentDisposition) { + tgtInfoBuilder.contentDisposition(blobInfo.contentDisposition()); + } + tgtInfoBuilder.acl(blobInfo.acl()); + tgtInfoBuilder.metadata(blobInfo.metadata()); } + + BlobInfo tgtInfo = tgtInfoBuilder.build(); + Storage.CopyRequest.Builder copyReqBuilder = Storage.CopyRequest.builder() + .source(fromPath.getBlobId()); + if (wantReplaceExisting) { + copyReqBuilder = copyReqBuilder.target(tgtInfo); + } else { + copyReqBuilder = copyReqBuilder.target(tgtInfo, Storage.BlobTargetOption.doesNotExist()); + } + CopyWriter copyWriter = storage.copy(copyReqBuilder.build()); + copyWriter.result(); + } catch (StorageException oops) { + throw asIOException(oops); } } + @Override public boolean isSameFile(Path path, Path path2) { return checkPath(path).equals(checkPath(path2)); } - /** Returns {@code false} */ + /** Returns {@code false}. + */ @Override public boolean isHidden(Path path) { checkPath(path); @@ -364,7 +432,8 @@ public void checkAccess(Path path, AccessMode... modes) throws IOException { if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { return; } - if (gcsService.getMetadata(cloudPath.getGcsFilename()) == null) { + if (storage.get(cloudPath.getBlobId(), Storage.BlobGetOption.fields(Storage.BlobField.ID)) + == null) { throw new NoSuchFileException(path.toString()); } } @@ -372,68 +441,71 @@ public void checkAccess(Path path, AccessMode... modes) throws IOException { @Override public A readAttributes( Path path, Class type, LinkOption... options) throws IOException { - CloudStoragePath cloudPath = checkPath(path); checkNotNull(type); checkNotNullArray(options); if (type != CloudStorageFileAttributes.class && type != BasicFileAttributes.class) { throw new UnsupportedOperationException(type.getSimpleName()); } + CloudStoragePath cloudPath = checkPath(path); if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { @SuppressWarnings("unchecked") - A result = (A) CloudStoragePseudoDirectoryAttributes.SINGLETON_INSTANCE; + A result = (A) new CloudStoragePseudoDirectoryAttributes(cloudPath); return result; } - GcsFileMetadata metadata = gcsService.getMetadata(cloudPath.getGcsFilename()); - if (metadata == null) { - throw new NoSuchFileException(path.toString()); + BlobInfo blobInfo = storage.get(cloudPath.getBlobId()); + // null size indicate a file that we haven't closed yet, so GCS treats it as not there yet. + if (null == blobInfo || blobInfo.size() == null) { + throw new NoSuchFileException( + cloudPath.getBlobId().bucket() + "/" + cloudPath.getBlobId().name()); } + CloudStorageObjectAttributes ret; + ret = new CloudStorageObjectAttributes(blobInfo); @SuppressWarnings("unchecked") - A result = (A) new CloudStorageObjectAttributes(metadata); + A result = (A) ret; return result; } + @Override + public Map readAttributes(Path path, String attributes, LinkOption... options) { + // Java 7 NIO defines at least eleven string attributes we'd want to support + // (eg. BasicFileAttributeView and PosixFileAttributeView), so rather than a partial + // implementation we rely on the other overload for now. + throw new UnsupportedOperationException(); + } + @Override public V getFileAttributeView( Path path, Class type, LinkOption... options) { - CloudStoragePath cloudPath = checkPath(path); checkNotNull(type); checkNotNullArray(options); if (type != CloudStorageFileAttributeView.class && type != BasicFileAttributeView.class) { throw new UnsupportedOperationException(type.getSimpleName()); } + CloudStoragePath cloudPath = checkPath(path); @SuppressWarnings("unchecked") - V result = (V) new CloudStorageFileAttributeView(this, cloudPath); + V result = (V) new CloudStorageFileAttributeView(storage, cloudPath); return result; } - /** Does nothing since GCS uses fake directories. */ + /** Does nothing since GCS uses fake directories. + */ @Override public void createDirectory(Path dir, FileAttribute... attrs) { checkPath(dir); checkNotNullArray(attrs); } - /** @throws UnsupportedOperationException */ - @Override - public DirectoryStream newDirectoryStream(Path dir, Filter filter) { - // TODO(b/18997618): Implement me. - throw new UnsupportedOperationException(); - } - - /** - * This feature is not supported. Please use {@link #readAttributes(Path, Class, LinkOption...)} - * - * @throws UnsupportedOperationException + /** Always @throws UnsupportedOperationException. */ @Override - public Map readAttributes(Path path, String attributes, LinkOption... options) { + public DirectoryStream newDirectoryStream(Path dir, Filter filter) { + // TODO: Implement me. throw new UnsupportedOperationException(); } /** * This feature is not supported, since Cloud Storage objects are immutable. - * - * @throws UnsupportedOperationException + * Always @throws UnsupportedOperationException. */ @Override public void setAttribute(Path path, String attribute, Object value, LinkOption... options) { @@ -442,8 +514,7 @@ public void setAttribute(Path path, String attribute, Object value, LinkOption.. /** * This feature is not supported. - * - * @throws UnsupportedOperationException + * Always @throws UnsupportedOperationException. */ @Override public FileStore getFileStore(Path path) { @@ -454,18 +525,42 @@ public FileStore getFileStore(Path path) { public boolean equals(@Nullable Object other) { return this == other || other instanceof CloudStorageFileSystemProvider - && Objects.equals(gcsService, ((CloudStorageFileSystemProvider) other).gcsService); + && Objects.equals(storage, ((CloudStorageFileSystemProvider) other).storage); } @Override public int hashCode() { - return Objects.hash(gcsService); + return Objects.hash(storage); } @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("gcsService", gcsService) + .add("storage", storage) .toString(); } + + + + private IOException asIOException(StorageException oops) { + if (oops.code() == 404) { + return new NoSuchFileException(oops.reason()); + } + // TODO: research if other codes should be translated to IOException. + + // RPC API can only throw StorageException, but CloudStorageFileSystemProvider + // can only throw IOException. Square peg, round hole. + Throwable cause = oops.getCause(); + try { + if (cause instanceof FileAlreadyExistsException) { + throw new FileAlreadyExistsException(((FileAlreadyExistsException) cause).getReason()); + } + // fallback + Throwables.propagateIfInstanceOf(oops.getCause(), IOException.class); + } catch (IOException okEx) { + return okEx; + } + return new IOException(oops.getMessage(), oops); + } + } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageObjectAttributes.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageObjectAttributes.java index b3265cd6276b..987f889a1850 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageObjectAttributes.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageObjectAttributes.java @@ -3,38 +3,44 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.gcloud.storage.contrib.nio.CloudStorageFileSystem.FILE_TIME_UNKNOWN; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; import com.google.common.base.MoreObjects; import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; +import com.google.gcloud.storage.Acl; +import com.google.gcloud.storage.BlobInfo; import java.nio.file.attribute.FileTime; +import java.util.List; import java.util.Objects; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; -/** Metadata for a Google Cloud Storage object. */ +/** + * Metadata for a Google Cloud Storage object. + */ @Immutable final class CloudStorageObjectAttributes implements CloudStorageFileAttributes { - private final GcsFileMetadata metadata; + @Nonnull + private final BlobInfo info; - CloudStorageObjectAttributes(GcsFileMetadata metadata) { - this.metadata = checkNotNull(metadata); + CloudStorageObjectAttributes(BlobInfo info) { + this.info = checkNotNull(info); } @Override public long size() { - return metadata.getLength(); + return info.size(); } @Override public FileTime creationTime() { - if (metadata.getLastModified() == null) { + if (info.updateTime() == null) { return FILE_TIME_UNKNOWN; } - return FileTime.fromMillis(metadata.getLastModified().getTime()); + return FileTime.fromMillis(info.updateTime()); } @Override @@ -42,16 +48,20 @@ public FileTime lastModifiedTime() { return creationTime(); } - /** Returns the HTTP etag hash for this object. */ + /** + * Returns the HTTP etag hash for this object. + */ @Override public Optional etag() { - return Optional.fromNullable(metadata.getEtag()); + return Optional.fromNullable(info.etag()); } - /** Returns the mime type (e.g. text/plain) if it was set for this object. */ + /** + * Returns the mime type (e.g. text/plain) if it was set for this object. + */ @Override public Optional mimeType() { - return Optional.fromNullable(metadata.getOptions().getMimeType()); + return Optional.fromNullable(info.contentType()); } /** @@ -60,8 +70,8 @@ public Optional mimeType() { * @see "https://developers.google.com/storage/docs/reference-headers#acl" */ @Override - public Optional acl() { - return Optional.fromNullable(metadata.getOptions().getAcl()); + public Optional> acl() { + return Optional.fromNullable(info.acl()); } /** @@ -71,7 +81,7 @@ public Optional acl() { */ @Override public Optional cacheControl() { - return Optional.fromNullable(metadata.getOptions().getCacheControl()); + return Optional.fromNullable(info.cacheControl()); } /** @@ -81,7 +91,7 @@ public Optional cacheControl() { */ @Override public Optional contentEncoding() { - return Optional.fromNullable(metadata.getOptions().getContentEncoding()); + return Optional.fromNullable(info.contentEncoding()); } /** @@ -91,7 +101,7 @@ public Optional contentEncoding() { */ @Override public Optional contentDisposition() { - return Optional.fromNullable(metadata.getOptions().getContentDisposition()); + return Optional.fromNullable(info.contentDisposition()); } /** @@ -101,7 +111,10 @@ public Optional contentDisposition() { */ @Override public ImmutableMap userMetadata() { - return ImmutableMap.copyOf(metadata.getOptions().getUserMetadata()); + if (null == info.metadata()) { + return ImmutableMap.of(); + } + return ImmutableMap.copyOf(info.metadata()); } @Override @@ -131,25 +144,25 @@ public FileTime lastAccessTime() { @Override public Object fileKey() { - return metadata.getFilename(); + return info.id(); } @Override public boolean equals(@Nullable Object other) { return this == other || other instanceof CloudStorageObjectAttributes - && Objects.equals(metadata, ((CloudStorageObjectAttributes) other).metadata); + && Objects.equals(info, ((CloudStorageObjectAttributes) other).info); } @Override public int hashCode() { - return Objects.hash(metadata); + return info.hashCode(); } @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("metadata", metadata) + .add("info", info) .toString(); } } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOption.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOption.java index 4142cef2bb6a..d7964628d61b 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOption.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOption.java @@ -7,11 +7,11 @@ public interface CloudStorageOption { /** Interface for GCS options that can be specified when opening files. */ - public interface Open extends CloudStorageOption, OpenOption {} + interface Open extends CloudStorageOption, OpenOption {} /** Interface for GCS options that can be specified when copying files. */ - public interface Copy extends CloudStorageOption, CopyOption {} + interface Copy extends CloudStorageOption, CopyOption {} /** Interface for GCS options that can be specified when opening or copying files. */ - public interface OpenCopy extends Open, Copy {} + interface OpenCopy extends Open, Copy {} } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptions.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptions.java index c66842d44c03..d852c0c098c3 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptions.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptions.java @@ -1,5 +1,7 @@ package com.google.gcloud.storage.contrib.nio; +import com.google.gcloud.storage.Acl; + /** Helper class for specifying options when opening and copying Cloud Storage files. */ public final class CloudStorageOptions { @@ -45,7 +47,7 @@ public static CloudStorageOption.OpenCopy withContentEncoding(String contentEnco * * @see "https://developers.google.com/storage/docs/reference-headers#acl" */ - public static CloudStorageOption.OpenCopy withAcl(String acl) { + public static CloudStorageOption.OpenCopy withAcl(Acl acl) { return OptionAcl.create(acl); } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePath.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePath.java index 7d8e88753ce8..1107dd2c5c7a 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePath.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePath.java @@ -6,8 +6,8 @@ import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.checkNotNullArray; import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.checkPath; -import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.common.collect.UnmodifiableIterator; +import com.google.gcloud.storage.BlobId; import java.io.File; import java.net.URI; @@ -27,7 +27,7 @@ import javax.annotation.concurrent.Immutable; /** - * Google Cloud Storage {@link Path} + * Google Cloud Storage {@link Path}. * * @see UnixPath */ @@ -51,14 +51,16 @@ static CloudStoragePath getPath( fileSystem, UnixPath.getPath(fileSystem.config().permitEmptyPathComponents(), path, more)); } - /** Returns the Cloud Storage bucket name being served by this file system. */ + /** Returns the Cloud Storage bucket name being served by this file system. + */ public String bucket() { return fileSystem.bucket(); } - /** Returns path converted to a {@link GcsFilename} so I/O can be performed. */ - GcsFilename getGcsFilename() { - return new GcsFilename(bucket(), toRealPath().path.toString()); + /** Returns path converted to a {@link BlobId} so I/O can be performed. + */ + BlobId getBlobId() { + return BlobId.of(bucket(), toRealPath().path.toString()); } boolean seemsLikeADirectory() { @@ -128,8 +130,6 @@ private UnixPath toRealPathInternal(boolean errorCheck) { /** * Returns path without extra slashes or {@code .} and {@code ..} and preserves trailing slash. - * - * @see java.nio.file.Path#normalize() */ @Override public CloudStoragePath normalize() { @@ -222,24 +222,25 @@ public boolean endsWith(String other) { return path.endsWith(getUnixPath(other)); } - /** @throws UnsupportedOperationException */ + /** Always @throws UnsupportedOperationException. + */ @Override public WatchKey register(WatchService watcher, Kind[] events, Modifier... modifiers) { - // TODO(b/18998105): Implement me. + // TODO: Implement me. throw new UnsupportedOperationException(); } - /** @throws UnsupportedOperationException */ + /** Always @throws UnsupportedOperationException. + */ @Override public WatchKey register(WatchService watcher, Kind... events) { - // TODO(b/18998105): Implement me. + // TODO: Implement me. throw new UnsupportedOperationException(); } /** * This operation is not supported, since GCS files aren't backed by the local file system. - * - * @throws UnsupportedOperationException + * Always @throws UnsupportedOperationException. */ @Override public File toFile() { diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java index fc381de3244b..b3874be99f54 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java @@ -4,14 +4,19 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; +import com.google.gcloud.storage.Acl; import java.nio.file.attribute.FileTime; +import java.util.List; /** Metadata for a cloud storage pseudo-directory. */ final class CloudStoragePseudoDirectoryAttributes implements CloudStorageFileAttributes { - static final CloudStoragePseudoDirectoryAttributes SINGLETON_INSTANCE = - new CloudStoragePseudoDirectoryAttributes(); + private final String id; + + CloudStoragePseudoDirectoryAttributes(CloudStoragePath path) { + this.id = path.toUri().toString(); + } @Override public boolean isDirectory() { @@ -35,7 +40,7 @@ public boolean isSymbolicLink() { @Override public Object fileKey() { - return null; + return id; } @Override @@ -69,7 +74,7 @@ public Optional mimeType() { } @Override - public Optional acl() { + public Optional> acl() { return Optional.absent(); } @@ -92,6 +97,4 @@ public Optional contentDisposition() { public ImmutableMap userMetadata() { return ImmutableMap.of(); } - - private CloudStoragePseudoDirectoryAttributes() {} } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannel.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannel.java index f4a29dde941c..47e31afd8f71 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannel.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannel.java @@ -2,10 +2,10 @@ import static com.google.common.base.Preconditions.checkArgument; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsInputChannel; -import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.gcloud.ReadChannel; +import com.google.gcloud.storage.BlobId; +import com.google.gcloud.storage.BlobInfo; +import com.google.gcloud.storage.Storage; import java.io.IOException; import java.nio.ByteBuffer; @@ -13,9 +13,9 @@ import java.nio.channels.NonWritableChannelException; import java.nio.channels.SeekableByteChannel; import java.nio.file.NoSuchFileException; - import javax.annotation.concurrent.ThreadSafe; + /** * Cloud Storage read channel. * @@ -25,28 +25,27 @@ final class CloudStorageReadChannel implements SeekableByteChannel { static CloudStorageReadChannel create( - GcsService gcsService, GcsFilename file, long position) throws IOException { + Storage gcsStorage, BlobId file, long position) throws IOException { // XXX: Reading size and opening file should be atomic. - long size = fetchSize(gcsService, file); - return new CloudStorageReadChannel(gcsService, file, position, size, - gcsService.openReadChannel(file, position)); + long size = fetchSize(gcsStorage, file); + ReadChannel channel = gcsStorage.reader(file); + if (position > 0) { + channel.seek((int) position); + } + return new CloudStorageReadChannel(position, size, channel); } - private final GcsService gcsService; - private final GcsFilename file; + private final ReadChannel channel; private long position; private long size; - private GcsInputChannel channel; - private CloudStorageReadChannel( - GcsService gcsService, GcsFilename file, long position, long size, GcsInputChannel channel) { - this.gcsService = gcsService; - this.file = file; + private CloudStorageReadChannel(long position, long size, ReadChannel channel) { this.position = position; this.size = size; this.channel = channel; } + @Override public boolean isOpen() { synchronized (this) { @@ -101,10 +100,8 @@ public SeekableByteChannel position(long newPosition) throws IOException { if (newPosition == position) { return this; } + channel.seek((int) newPosition); position = newPosition; - size = fetchSize(gcsService, file); - channel.close(); - channel = gcsService.openReadChannel(file, position); return this; } } @@ -125,13 +122,12 @@ private void checkOpen() throws ClosedChannelException { } } - private static long fetchSize(GcsService gcsService, GcsFilename file) throws IOException, - NoSuchFileException { - GcsFileMetadata metadata = gcsService.getMetadata(file); - if (metadata == null) { + private static long fetchSize(Storage gcsStorage, BlobId file) throws IOException { + BlobInfo blobInfo = gcsStorage.get(file); + if (blobInfo == null) { throw new NoSuchFileException( - String.format("gs://%s/%s", file.getBucketName(), file.getObjectName())); + String.format("gs://%s/%s", file.bucket(), file.name())); } - return metadata.getLength(); + return blobInfo.size(); } } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageUtil.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageUtil.java index 79f46af172ee..9a33d7801566 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageUtil.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageUtil.java @@ -2,23 +2,19 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Strings.isNullOrEmpty; - -import com.google.appengine.tools.cloudstorage.GcsFileOptions; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.ProviderMismatchException; -import java.util.Map; import java.util.regex.Pattern; final class CloudStorageUtil { - private static final Pattern BUCKET_PATTERN = Pattern.compile("[a-z0-9][-._a-z0-9.]+[a-z0-9]"); + private static final Pattern BUCKET_PATTERN = Pattern.compile("[a-z0-9][-._a-z0-9]+[a-z0-9]"); static void checkBucket(String bucket) { - // TODO(b/18998200): The true check is actually more complicated. Consider implementing it. + // TODO: The true check is actually more complicated. Consider implementing it. checkArgument(BUCKET_PATTERN.matcher(bucket).matches(), "" + "Invalid bucket name: '" + bucket + "'. " + "GCS bucket names must contain only lowercase letters, numbers, dashes (-), " @@ -34,50 +30,6 @@ static CloudStoragePath checkPath(Path path) { return (CloudStoragePath) path; } - static GcsFileOptions.Builder copyFileOptions(GcsFileOptions options) { - GcsFileOptions.Builder builder = new GcsFileOptions.Builder(); - if (!isNullOrEmpty(options.getAcl())) { - builder.acl(options.getAcl()); - } - if (!isNullOrEmpty(options.getCacheControl())) { - builder.cacheControl(options.getCacheControl()); - } - if (!isNullOrEmpty(options.getContentDisposition())) { - builder.contentDisposition(options.getContentDisposition()); - } - if (!isNullOrEmpty(options.getContentEncoding())) { - builder.contentEncoding(options.getContentEncoding()); - } - if (!isNullOrEmpty(options.getMimeType())) { - builder.mimeType(options.getMimeType()); - } - for (Map.Entry entry : options.getUserMetadata().entrySet()) { - builder.addUserMetadata(entry.getKey(), entry.getValue()); - } - return builder; - } - - @SafeVarargs - static GcsFileOptions buildFileOptions(GcsFileOptions.Builder builder, T... options) { - for (Object option : options) { - if (option instanceof OptionAcl) { - builder.acl(((OptionAcl) option).acl()); - } else if (option instanceof OptionCacheControl) { - builder.cacheControl(((OptionCacheControl) option).cacheControl()); - } else if (option instanceof OptionContentDisposition) { - builder.contentDisposition(((OptionContentDisposition) option).contentDisposition()); - } else if (option instanceof OptionContentEncoding) { - builder.contentEncoding(((OptionContentEncoding) option).contentEncoding()); - } else if (option instanceof OptionMimeType) { - builder.mimeType(((OptionMimeType) option).mimeType()); - } else if (option instanceof OptionUserMetadata) { - OptionUserMetadata metadata = (OptionUserMetadata) option; - builder.addUserMetadata(metadata.key(), metadata.value()); - } - } - return builder.build(); - } - static URI stripPathFromUri(URI uri) { try { return new URI( @@ -101,10 +53,5 @@ static void checkNotNullArray(T... values) { } } - static boolean getPropertyBoolean(String property, boolean defaultValue) { - String value = System.getProperty(property); - return value != null ? Boolean.valueOf(value) : defaultValue; - } - private CloudStorageUtil() {} } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannel.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannel.java index 2b5a004e1bc4..92c95fb688d5 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannel.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannel.java @@ -1,6 +1,6 @@ package com.google.gcloud.storage.contrib.nio; -import com.google.appengine.tools.cloudstorage.GcsOutputChannel; +import com.google.gcloud.WriteChannel; import java.io.IOException; import java.nio.ByteBuffer; @@ -20,11 +20,11 @@ @ThreadSafe final class CloudStorageWriteChannel implements SeekableByteChannel { - private GcsOutputChannel channel; + private final WriteChannel channel; private long position; private long size; - CloudStorageWriteChannel(GcsOutputChannel channel) { + CloudStorageWriteChannel(WriteChannel channel) { this.channel = channel; } @@ -83,8 +83,8 @@ public long size() throws IOException { @Override public SeekableByteChannel truncate(long newSize) throws IOException { - // TODO(b/18997913): Emulate this functionality by closing and rewriting old file up to newSize. - // Or maybe just swap out GcsService for the Apiary client. + // TODO: Emulate this functionality by closing and rewriting old file up to newSize. + // Or maybe just swap out GcsStorage for the API client. throw new UnsupportedOperationException(); } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionAcl.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionAcl.java index c0068ee1c61d..2224cdacebb7 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionAcl.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionAcl.java @@ -1,13 +1,14 @@ package com.google.gcloud.storage.contrib.nio; import com.google.auto.value.AutoValue; +import com.google.gcloud.storage.Acl; @AutoValue abstract class OptionAcl implements CloudStorageOption.OpenCopy { - static OptionAcl create(String acl) { + static OptionAcl create(Acl acl) { return new AutoValue_OptionAcl(acl); } - abstract String acl(); + abstract Acl acl(); } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionUserMetadata.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionUserMetadata.java index 9851d2ad5c4f..af8ea0306a93 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionUserMetadata.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionUserMetadata.java @@ -10,5 +10,6 @@ static OptionUserMetadata create(String key, String value) { } abstract String key(); + abstract String value(); } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/UnixPath.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/UnixPath.java index 7b80a6299be6..2536aa8d3d7c 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/UnixPath.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/UnixPath.java @@ -403,7 +403,7 @@ public UnixPath toAbsolutePath(UnixPath currentWorkingDirectory) { return isAbsolute() ? this : currentWorkingDirectory.resolve(this); } - /** Returns {@code toAbsolutePath(ROOT_PATH)} */ + /** Returns {@code toAbsolutePath(ROOT_PATH)}. */ public UnixPath toAbsolutePath() { return toAbsolutePath(ROOT_PATH); } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/AppEngineRule.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/AppEngineRule.java deleted file mode 100644 index 090ffa04ef77..000000000000 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/AppEngineRule.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.google.gcloud.storage.contrib.nio; - -import com.google.appengine.tools.development.testing.LocalDatastoreServiceTestConfig; -import com.google.appengine.tools.development.testing.LocalServiceTestHelper; - -import org.junit.rules.ExternalResource; - -import java.io.IOException; - -/** JUnit rule for App Engine testing environment. */ -public final class AppEngineRule extends ExternalResource { - - private LocalServiceTestHelper helper; - - @Override - protected void before() throws IOException { - helper = new LocalServiceTestHelper(new LocalDatastoreServiceTestConfig()); - helper.setUp(); - } - - @Override - protected void after() { - helper.tearDown(); - } -} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java index c376d2456dfc..7ad64388c3e0 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java @@ -6,6 +6,7 @@ import com.google.common.testing.EqualsTester; import com.google.common.testing.NullPointerTester; +import com.google.gcloud.storage.testing.LocalGcsHelper; import org.junit.Before; import org.junit.Rule; @@ -30,13 +31,11 @@ public class CloudStorageFileAttributeViewTest { @Rule public final ExpectedException thrown = ExpectedException.none(); - @Rule - public final AppEngineRule appEngineRule = new AppEngineRule(); - private Path path; @Before - public void before() throws Exception { + public void before() { + CloudStorageFileSystemProvider.setGCloudOptions(LocalGcsHelper.options()); path = Paths.get(URI.create("gs://red/water")); } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributesTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributesTest.java index 856a5dcb2c8c..810f67ceb1df 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributesTest.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributesTest.java @@ -11,9 +11,10 @@ import com.google.common.testing.EqualsTester; import com.google.common.testing.NullPointerTester; +import com.google.gcloud.storage.Acl; +import com.google.gcloud.storage.testing.LocalGcsHelper; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -29,14 +30,14 @@ public class CloudStorageFileAttributesTest { private static final byte[] HAPPY = "(✿◕ ‿◕ )ノ".getBytes(UTF_8); - @Rule - public final AppEngineRule appEngineRule = new AppEngineRule(); private Path path; private Path dir; + /** empty test storage and make sure we use it instead of the real GCS. Create a few paths. **/ @Before - public void before() throws Exception { + public void before() { + CloudStorageFileSystemProvider.setGCloudOptions(LocalGcsHelper.options()); path = Paths.get(URI.create("gs://bucket/randompath")); dir = Paths.get(URI.create("gs://bucket/randompath/")); } @@ -57,9 +58,10 @@ public void testMimeType() throws Exception { @Test public void testAcl() throws Exception { - Files.write(path, HAPPY, withAcl("potato")); + Acl acl = Acl.of(new Acl.User("serf@example.com"), Acl.Role.READER); + Files.write(path, HAPPY, withAcl(acl)); assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).acl().get()) - .isEqualTo("potato"); + .contains(acl); } @Test @@ -129,6 +131,25 @@ public void testEquals_equalsTester() throws Exception { new EqualsTester().addEqualityGroup(a1, a2).addEqualityGroup(b1, b2).testEquals(); } + @Test + public void testFilekey() throws Exception { + Files.write(path, HAPPY, withMimeType("text/plain")); + Path path2 = Paths.get(URI.create("gs://bucket/anotherrandompath")); + Files.write(path2, HAPPY, withMimeType("text/plain")); + + // diff files cannot have same filekey + CloudStorageFileAttributes a1 = Files.readAttributes(path, CloudStorageFileAttributes.class); + CloudStorageFileAttributes a2 = Files.readAttributes(path2, CloudStorageFileAttributes.class); + assertThat(a1.fileKey()).isNotEqualTo(a2.fileKey()); + + // same for directories + CloudStorageFileAttributes b1 = Files.readAttributes(dir, CloudStorageFileAttributes.class); + CloudStorageFileAttributes b2 = Files.readAttributes( + Paths.get(URI.create("gs://bucket/jacket/")), CloudStorageFileAttributes.class); + assertThat(a1.fileKey()).isNotEqualTo(b1.fileKey()); + assertThat(b1.fileKey()).isNotEqualTo(b2.fileKey()); + } + @Test public void testNullness() throws Exception { Files.write(path, HAPPY); diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java index 8dad56ff984e..9f07505acb70 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java @@ -3,7 +3,10 @@ import static com.google.common.truth.Truth.assertThat; import static com.google.gcloud.storage.contrib.nio.CloudStorageFileSystem.forBucket; import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withCacheControl; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withContentDisposition; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withContentEncoding; import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withMimeType; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withUserMetadata; import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; @@ -11,15 +14,12 @@ import static java.nio.file.StandardOpenOption.CREATE_NEW; import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import static java.nio.file.StandardOpenOption.WRITE; -import static org.junit.Assume.assumeTrue; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; import com.google.common.collect.ImmutableList; import com.google.common.testing.NullPointerTester; +import com.google.gcloud.storage.testing.LocalGcsHelper; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -48,6 +48,7 @@ /** Unit tests for {@link CloudStorageFileSystemProvider}. */ @RunWith(JUnit4.class) +@SuppressWarnings("resource") public class CloudStorageFileSystemProviderTest { private static final List FILE_CONTENTS = ImmutableList.of( @@ -65,8 +66,10 @@ public class CloudStorageFileSystemProviderTest { @Rule public final ExpectedException thrown = ExpectedException.none(); - @Rule - public final AppEngineRule appEngineRule = new AppEngineRule(); + @Before + public void before() { + CloudStorageFileSystemProvider.setGCloudOptions(LocalGcsHelper.options()); + } @Test public void testSize() throws Exception { @@ -249,15 +252,13 @@ public void testNewOutputStream_truncateExplicitly() throws Exception { public void testNewOutputStream_trailingSlash() throws Exception { Path path = Paths.get(URI.create("gs://bucket/wat/")); thrown.expect(CloudStoragePseudoDirectoryException.class); - try (OutputStream output = Files.newOutputStream(path)) { - } + Files.newOutputStream(path); } @Test public void testNewOutputStream_createNew() throws Exception { Path path = Paths.get(URI.create("gs://cry/wednesday")); - try (OutputStream output = Files.newOutputStream(path, CREATE_NEW)) { - } + Files.newOutputStream(path, CREATE_NEW); } @Test @@ -265,8 +266,7 @@ public void testNewOutputStream_createNew_alreadyExists() throws Exception { Path path = Paths.get(URI.create("gs://cry/wednesday")); Files.write(path, SINGULARITY.getBytes(UTF_8)); thrown.expect(FileAlreadyExistsException.class); - try (OutputStream output = Files.newOutputStream(path, CREATE_NEW)) { - } + Files.newOutputStream(path, CREATE_NEW); } @Test @@ -282,9 +282,7 @@ public void testWrite_objectNameWithExtraSlashes_canBeNormalized() throws Except Path path = fs.getPath("adipose//yep").normalize(); Files.write(path, FILE_CONTENTS, UTF_8); assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); - GcsService gcsService = GcsServiceFactory.createGcsService(); - assertThat(gcsService.getMetadata(new GcsFilename("greenbean", "adipose/yep"))).isNotNull(); - assertThat(Files.exists(path)).isTrue(); + assertThat(Files.exists(fs.getPath("adipose", "yep"))).isTrue(); } } @@ -294,8 +292,6 @@ public void testWrite_objectNameWithExtraSlashes_permitEmptyPathComponents() thr Path path = fs.getPath("adipose//yep"); Files.write(path, FILE_CONTENTS, UTF_8); assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); - GcsService gcsService = GcsServiceFactory.createGcsService(); - assertThat(gcsService.getMetadata(new GcsFilename("greenbean", "adipose//yep"))).isNotNull(); assertThat(Files.exists(path)).isTrue(); } } @@ -305,8 +301,6 @@ public void testWrite_absoluteObjectName_prefixSlashGetsRemoved() throws Excepti Path path = Paths.get(URI.create("gs://greenbean/adipose/yep")); Files.write(path, FILE_CONTENTS, UTF_8); assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); - GcsService gcsService = GcsServiceFactory.createGcsService(); - assertThat(gcsService.getMetadata(new GcsFilename("greenbean", "adipose/yep"))).isNotNull(); assertThat(Files.exists(path)).isTrue(); } @@ -320,8 +314,6 @@ public void testWrite_absoluteObjectName_disableStrip_slashGetsPreserved() throw Path path = fs.getPath("/adipose/yep"); Files.write(path, FILE_CONTENTS, UTF_8); assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); - GcsService gcsService = GcsServiceFactory.createGcsService(); - assertThat(gcsService.getMetadata(new GcsFilename("greenbean", "/adipose/yep"))).isNotNull(); assertThat(Files.exists(path)).isTrue(); } } @@ -333,6 +325,30 @@ public void testWrite() throws Exception { assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); } + @Test + public void testWriteOnClose() throws Exception { + Path path = Paths.get(URI.create("gs://greenbean/adipose")); + try (SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.WRITE)) { + // writing lots of contents to defeat channel-internal buffering. + for (int i = 0; i < 9999; i++) { + for (String s : FILE_CONTENTS) { + chan.write(ByteBuffer.wrap(s.getBytes(UTF_8))); + } + } + try { + Files.size(path); + // we shouldn't make it to this line. Not using thrown.expect because + // I still want to run a few lines after the exception. + assertThat(false).isTrue(); + } catch (NoSuchFileException nsf) { + // that's what we wanted, we're good. + } + } + // channel now closed, the file should be there and with the new contents. + assertThat(Files.exists(path)).isTrue(); + assertThat(Files.size(path)).isGreaterThan(100L); + } + @Test public void testWrite_trailingSlash() throws Exception { thrown.expect(CloudStoragePseudoDirectoryException.class); @@ -385,26 +401,20 @@ public void testDelete_trailingSlash_disablePseudoDirectories() throws Exception try (CloudStorageFileSystem fs = forBucket("pumpkin", usePseudoDirectories(false))) { Path path = fs.getPath("wat/"); Files.write(path, FILE_CONTENTS, UTF_8); - GcsService gcsService = GcsServiceFactory.createGcsService(); - assertThat(gcsService.getMetadata(new GcsFilename("pumpkin", "wat/"))).isNotNull(); + assertThat(Files.exists(path)); Files.delete(path); - assertThat(gcsService.getMetadata(new GcsFilename("pumpkin", "wat/"))).isNull(); + assertThat(!Files.exists(path)); } } @Test public void testDelete_notFound() throws Exception { - GcsService gcsService = GcsServiceFactory.createGcsService(); - assumeTrue(!gcsService.delete(new GcsFilename("loveh", "passionehu"))); // XXX: b/15832793 thrown.expect(NoSuchFileException.class); Files.delete(Paths.get(URI.create("gs://loveh/passionehu"))); } @Test public void testDeleteIfExists() throws Exception { - GcsService gcsService = GcsServiceFactory.createGcsService(); - assumeTrue(!gcsService.delete(new GcsFilename("loveh", "passionehu"))); // XXX: b/15832793 - assertThat(Files.deleteIfExists(Paths.get(URI.create("gs://love/passionz")))).isFalse(); Files.write(Paths.get(URI.create("gs://love/passionz")), "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); assertThat(Files.deleteIfExists(Paths.get(URI.create("gs://love/passionz")))).isTrue(); } @@ -528,12 +538,20 @@ public void testCopy_withCopyAttributes_preservesAttributes() throws Exception { Path target = Paths.get(URI.create("gs://greenbean/adipose")); Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), withMimeType("text/lolcat"), - withCacheControl("public; max-age=666")); + withCacheControl("public; max-age=666"), + withContentEncoding("foobar"), + withContentDisposition("my-content-disposition"), + withUserMetadata("answer", "42")); Files.copy(source, target, COPY_ATTRIBUTES); - GcsService gcsService = GcsServiceFactory.createGcsService(); - GcsFileMetadata metadata = gcsService.getMetadata(new GcsFilename("greenbean", "adipose")); - assertThat(metadata.getOptions().getMimeType()).isEqualTo("text/lolcat"); - assertThat(metadata.getOptions().getCacheControl()).isEqualTo("public; max-age=666"); + + CloudStorageFileAttributes attributes = + Files.readAttributes(target, CloudStorageFileAttributes.class); + assertThat(attributes.mimeType()).hasValue("text/lolcat"); + assertThat(attributes.cacheControl()).hasValue("public; max-age=666"); + assertThat(attributes.contentEncoding()).hasValue("foobar"); + assertThat(attributes.contentDisposition()).hasValue("my-content-disposition"); + assertThat(attributes.userMetadata().containsKey("answer")).isTrue(); + assertThat(attributes.userMetadata().get("answer")).isEqualTo("42"); } @Test @@ -542,27 +560,39 @@ public void testCopy_withoutOptions_doesntPreservesAttributes() throws Exception Path target = Paths.get(URI.create("gs://greenbean/adipose")); Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), withMimeType("text/lolcat"), - withCacheControl("public; max-age=666")); + withCacheControl("public; max-age=666"), + withUserMetadata("answer", "42")); Files.copy(source, target); - GcsService gcsService = GcsServiceFactory.createGcsService(); - GcsFileMetadata metadata = gcsService.getMetadata(new GcsFilename("greenbean", "adipose")); - assertThat(metadata.getOptions().getMimeType()).isNull(); - assertThat(metadata.getOptions().getCacheControl()).isNull(); + + CloudStorageFileAttributes attributes = + Files.readAttributes(target, CloudStorageFileAttributes.class); + String mimeType = attributes.mimeType().orNull(); + String cacheControl = attributes.cacheControl().orNull(); + assertThat(mimeType).isNotEqualTo("text/lolcat"); + assertThat(cacheControl).isNull(); + assertThat(attributes.userMetadata().containsKey("answer")).isFalse(); } @Test public void testCopy_overwriteAttributes() throws Exception { Path source = Paths.get(URI.create("gs://military/fashion.show")); - Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Path target1 = Paths.get(URI.create("gs://greenbean/adipose")); + Path target2 = Paths.get(URI.create("gs://greenbean/round")); Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), withMimeType("text/lolcat"), withCacheControl("public; max-age=666")); - Files.copy(source, target, COPY_ATTRIBUTES, + Files.copy(source, target1, COPY_ATTRIBUTES); + Files.copy(source, target2, COPY_ATTRIBUTES, withMimeType("text/palfun")); - GcsService gcsService = GcsServiceFactory.createGcsService(); - GcsFileMetadata metadata = gcsService.getMetadata(new GcsFilename("greenbean", "adipose")); - assertThat(metadata.getOptions().getMimeType()).isEqualTo("text/palfun"); - assertThat(metadata.getOptions().getCacheControl()).isEqualTo("public; max-age=666"); + + CloudStorageFileAttributes attributes = + Files.readAttributes(target1, CloudStorageFileAttributes.class); + assertThat(attributes.mimeType()).hasValue("text/lolcat"); + assertThat(attributes.cacheControl()).hasValue("public; max-age=666"); + + attributes = Files.readAttributes(target2, CloudStorageFileAttributes.class); + assertThat(attributes.mimeType()).hasValue("text/palfun"); + assertThat(attributes.cacheControl()).hasValue("public; max-age=666"); } @Test @@ -573,11 +603,22 @@ public void testNullness() throws Exception { .setDefault(Path.class, fs.getPath("and/one")) .setDefault(OpenOption.class, StandardOpenOption.CREATE) .setDefault(CopyOption.class, StandardCopyOption.COPY_ATTRIBUTES); - tester.testAllPublicStaticMethods(CloudStorageFileSystemProvider.class); + // can't do that, setGCloudOptions accepts a null argument. + // TODO(jart): Figure out how to re-enable this. + //tester.testAllPublicStaticMethods(CloudStorageFileSystemProvider.class); tester.testAllPublicInstanceMethods(new CloudStorageFileSystemProvider()); } } + @Test + public void testProviderEquals() throws Exception { + Path path1 = Paths.get(URI.create("gs://bucket/tuesday")); + Path path2 = Paths.get(URI.create("gs://blood/wednesday")); + Path path3 = Paths.get("tmp"); + assertThat(path1.getFileSystem().provider()).isEqualTo(path2.getFileSystem().provider()); + assertThat(path1.getFileSystem().provider()).isNotEqualTo(path3.getFileSystem().provider()); + } + private static CloudStorageConfiguration permitEmptyPathComponents(boolean value) { return CloudStorageConfiguration.builder() .permitEmptyPathComponents(value) diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java index 7cb750238b14..fbdfbca88350 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java @@ -5,8 +5,9 @@ import com.google.common.testing.EqualsTester; import com.google.common.testing.NullPointerTester; +import com.google.gcloud.storage.testing.LocalGcsHelper; -import org.junit.Rule; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -31,8 +32,11 @@ public class CloudStorageFileSystemTest { + "The Heart-ache, and the thousand Natural shocks\n" + "That Flesh is heir to? 'Tis a consummation\n"; - @Rule - public final AppEngineRule appEngineRule = new AppEngineRule(); + + @Before + public void before() { + CloudStorageFileSystemProvider.setGCloudOptions(LocalGcsHelper.options()); + } @Test public void testGetPath() throws Exception { diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptionsTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptionsTest.java index 0ed9dc5f7507..5620896761e0 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptionsTest.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptionsTest.java @@ -1,6 +1,5 @@ package com.google.gcloud.storage.contrib.nio; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.truth.Truth.assertThat; import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withAcl; import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withCacheControl; @@ -11,11 +10,11 @@ import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withoutCaching; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.common.testing.NullPointerTester; +import com.google.gcloud.storage.Acl; +import com.google.gcloud.storage.testing.LocalGcsHelper; -import org.junit.Rule; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -29,15 +28,18 @@ @RunWith(JUnit4.class) public class CloudStorageOptionsTest { - @Rule - public final AppEngineRule appEngineRule = new AppEngineRule(); + @Before + public void before() { + CloudStorageFileSystemProvider.setGCloudOptions(LocalGcsHelper.options()); + } @Test public void testWithoutCaching() throws Exception { Path path = Paths.get(URI.create("gs://bucket/path")); Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), withoutCaching()); - assertThat(getMetadata("bucket", "path").getOptions().getCacheControl()).isEqualTo("no-cache"); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).cacheControl().get()) + .isEqualTo("no-cache"); } @Test @@ -45,16 +47,18 @@ public void testCacheControl() throws Exception { Path path = Paths.get(URI.create("gs://bucket/path")); Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), withCacheControl("potato")); - assertThat(getMetadata("bucket", "path").getOptions().getCacheControl()).isEqualTo("potato"); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).cacheControl().get()) + .isEqualTo("potato"); } @Test public void testWithAcl() throws Exception { Path path = Paths.get(URI.create("gs://bucket/path")); + Acl acl = Acl.of(new Acl.User("king@example.com"), Acl.Role.OWNER); Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), - withAcl("mine empire of dirt")); - assertThat(getMetadata("bucket", "path").getOptions().getAcl()) - .isEqualTo("mine empire of dirt"); + withAcl(acl)); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).acl().get()) + .contains(acl); } @Test @@ -62,7 +66,8 @@ public void testWithContentDisposition() throws Exception { Path path = Paths.get(URI.create("gs://bucket/path")); Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), withContentDisposition("bubbly fun")); - assertThat(getMetadata("bucket", "path").getOptions().getContentDisposition()) + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class).contentDisposition().get()) .isEqualTo("bubbly fun"); } @@ -71,7 +76,8 @@ public void testWithContentEncoding() throws Exception { Path path = Paths.get(URI.create("gs://bucket/path")); Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), withContentEncoding("gzip")); - assertThat(getMetadata("bucket", "path").getOptions().getContentEncoding()).isEqualTo("gzip"); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).contentEncoding().get()) + .isEqualTo("gzip"); } @Test @@ -80,9 +86,14 @@ public void testWithUserMetadata() throws Exception { Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), withUserMetadata("nolo", "contendere"), withUserMetadata("eternal", "sadness")); - GcsFileMetadata metadata = getMetadata("bucket", "path"); - assertThat(metadata.getOptions().getUserMetadata().get("nolo")).isEqualTo("contendere"); - assertThat(metadata.getOptions().getUserMetadata().get("eternal")).isEqualTo("sadness"); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class) + .userMetadata().get("nolo")) + .isEqualTo("contendere"); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class) + .userMetadata().get("eternal")) + .isEqualTo("sadness"); } @Test @@ -90,11 +101,8 @@ public void testWithMimeType_string() throws Exception { Path path = Paths.get(URI.create("gs://bucket/path")); Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), withMimeType("text/plain")); - assertThat(getMetadata("bucket", "path").getOptions().getMimeType()).isEqualTo("text/plain"); - } - - private static GcsFileMetadata getMetadata(String bucket, String objectName) throws Exception { - return createGcsService().getMetadata(new GcsFilename(bucket, objectName)); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).mimeType().get()) + .isEqualTo("text/plain"); } @Test diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStoragePathTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStoragePathTest.java index aaf9e5b4fd16..617a0c0a9c4e 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStoragePathTest.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStoragePathTest.java @@ -6,7 +6,9 @@ import com.google.common.collect.Iterables; import com.google.common.testing.EqualsTester; import com.google.common.testing.NullPointerTester; +import com.google.gcloud.storage.testing.LocalGcsHelper; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -27,8 +29,10 @@ public class CloudStoragePathTest { @Rule public final ExpectedException thrown = ExpectedException.none(); - @Rule - public final AppEngineRule appEngineRule = new AppEngineRule(); + @Before + public void before() { + CloudStorageFileSystemProvider.setGCloudOptions(LocalGcsHelper.options()); + } @Test public void testCreate_neverRemoveExtraSlashes() { @@ -50,21 +54,21 @@ public void testCreate_preservesTrailingSlash() { public void testGetGcsFilename_empty_notAllowed() { try (CloudStorageFileSystem fs = forBucket("doodle")) { thrown.expect(IllegalArgumentException.class); - fs.getPath("").getGcsFilename(); + fs.getPath("").getBlobId(); } } @Test public void testGetGcsFilename_stripsPrefixSlash() { try (CloudStorageFileSystem fs = forBucket("doodle")) { - assertThat(fs.getPath("/hi").getGcsFilename().getObjectName()).isEqualTo("hi"); + assertThat(fs.getPath("/hi").getBlobId().name()).isEqualTo("hi"); } } @Test public void testGetGcsFilename_overrideStripPrefixSlash_doesntStripPrefixSlash() { try (CloudStorageFileSystem fs = forBucket("doodle", stripPrefixSlash(false))) { - assertThat(fs.getPath("/hi").getGcsFilename().getObjectName()).isEqualTo("/hi"); + assertThat(fs.getPath("/hi").getBlobId().name()).isEqualTo("/hi"); } } @@ -72,14 +76,14 @@ public void testGetGcsFilename_overrideStripPrefixSlash_doesntStripPrefixSlash() public void testGetGcsFilename_extraSlashes_throwsIae() { try (CloudStorageFileSystem fs = forBucket("doodle")) { thrown.expect(IllegalArgumentException.class); - fs.getPath("a//b").getGcsFilename(); + fs.getPath("a//b").getBlobId().name(); } } @Test public void testGetGcsFilename_overridepermitEmptyPathComponents() { try (CloudStorageFileSystem fs = forBucket("doodle", permitEmptyPathComponents(true))) { - assertThat(fs.getPath("a//b").getGcsFilename().getObjectName()).isEqualTo("a//b"); + assertThat(fs.getPath("a//b").getBlobId().name()).isEqualTo("a//b"); } } @@ -87,7 +91,7 @@ public void testGetGcsFilename_overridepermitEmptyPathComponents() { public void testGetGcsFilename_freaksOutOnExtraSlashesAndDotDirs() { try (CloudStorageFileSystem fs = forBucket("doodle")) { thrown.expect(IllegalArgumentException.class); - fs.getPath("a//b/..").getGcsFilename(); + fs.getPath("a//b/..").getBlobId().name(); } } @@ -212,7 +216,8 @@ public void testToRealPath_extraSlashes_throwsIae() { @Test public void testToRealPath_overridePermitEmptyPathComponents_extraSlashes_slashesRemain() { try (CloudStorageFileSystem fs = forBucket("doodle", permitEmptyPathComponents(true))) { - assertThat(fs.getPath("/life///b/./good/").toRealPath().toString()).isEqualTo("life///b/./good/"); + assertThat(fs.getPath("/life///b/./good/").toRealPath().toString()) + .isEqualTo("life///b/./good/"); } } @@ -396,19 +401,21 @@ public void testEndsWith() { } } - /** @see "http://stackoverflow.com/a/10068306" */ + /** @see "http://stackoverflow.com/a/10068306". + */ @Test public void testResolve_willWorkWithRecursiveCopy() throws Exception { try (FileSystem fsSource = FileSystems.getFileSystem(URI.create("gs://hello")); FileSystem fsTarget = FileSystems.getFileSystem(URI.create("gs://cat"))) { Path targetPath = fsTarget.getPath("/some/folder/"); - Path relativeSourcePath = fsSource.getPath("file.txt"); - assertThat((Object) targetPath.resolve(relativeSourcePath)) + Path relSrcPath = fsSource.getPath("file.txt"); + assertThat((Object) targetPath.resolve(relSrcPath)) .isEqualTo(fsTarget.getPath("/some/folder/file.txt")); } } - /** @see "http://stackoverflow.com/a/10068306" */ + /** @see "http://stackoverflow.com/a/10068306". + */ @Test public void testRelativize_willWorkWithRecursiveCopy() throws Exception { try (FileSystem fsSource = FileSystems.getFileSystem(URI.create("gs://hello")); diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java index c46ec0ac7981..4b355d076552 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java @@ -2,7 +2,6 @@ import static com.google.common.truth.Truth.assertThat; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -11,11 +10,10 @@ import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsFileOptions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsInputChannel; -import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.gcloud.ReadChannel; +import com.google.gcloud.storage.BlobId; +import com.google.gcloud.storage.BlobInfo; +import com.google.gcloud.storage.Storage; import org.junit.Before; import org.junit.Rule; @@ -35,21 +33,22 @@ public class CloudStorageReadChannelTest { @Rule public final ExpectedException thrown = ExpectedException.none(); - private final GcsService gcsService = mock(GcsService.class); - private final GcsInputChannel gcsChannel = mock(GcsInputChannel.class); - private final GcsFilename file = new GcsFilename("enya", "rocks"); - private final GcsFileOptions options = GcsFileOptions.getDefaultInstance(); - private final GcsFileMetadata metadata = new GcsFileMetadata(file, options, null, 42, null); private CloudStorageReadChannel chan; + private final Storage gcsStorage = mock(Storage.class); + private final BlobId file = BlobId.of("enya", "rocks"); + private final BlobInfo metadata = BlobInfo.builder(file).size(42L).build(); + private final ReadChannel gcsChannel = mock(ReadChannel.class); + + /** Set up the mocks. **/ @Before public void before() throws Exception { - when(gcsService.getMetadata(eq(file))).thenReturn(metadata); - when(gcsService.openReadChannel(eq(file), anyInt())).thenReturn(gcsChannel); + when(gcsStorage.get(file)).thenReturn(metadata); + when(gcsStorage.reader(eq(file))).thenReturn(gcsChannel); when(gcsChannel.isOpen()).thenReturn(true); - chan = CloudStorageReadChannel.create(gcsService, file, 0); - verify(gcsService).getMetadata(eq(file)); - verify(gcsService).openReadChannel(eq(file), eq(0L)); + chan = CloudStorageReadChannel.create(gcsStorage, file, 0); + verify(gcsStorage).get(eq(file)); + verify(gcsStorage).reader(eq(file)); } @Test @@ -61,7 +60,7 @@ public void testRead() throws Exception { assertThat(chan.position()).isEqualTo(1L); verify(gcsChannel).read(any(ByteBuffer.class)); verify(gcsChannel, times(3)).isOpen(); - verifyNoMoreInteractions(gcsService, gcsChannel); + verifyNoMoreInteractions(gcsStorage, gcsChannel); } @Test @@ -91,7 +90,7 @@ public void testIsOpen() throws Exception { assertThat(chan.isOpen()).isFalse(); verify(gcsChannel, times(2)).isOpen(); verify(gcsChannel).close(); - verifyNoMoreInteractions(gcsService, gcsChannel); + verifyNoMoreInteractions(gcsStorage, gcsChannel); } @Test @@ -99,7 +98,7 @@ public void testSize() throws Exception { assertThat(chan.size()).isEqualTo(42L); verify(gcsChannel).isOpen(); verifyZeroInteractions(gcsChannel); - verifyNoMoreInteractions(gcsService); + verifyNoMoreInteractions(gcsStorage); } @Test @@ -137,10 +136,9 @@ public void testSetPosition() throws Exception { chan.position(1L); assertThat(chan.position()).isEqualTo(1L); assertThat(chan.size()).isEqualTo(42L); - verify(gcsChannel).close(); + verify(gcsChannel).seek(1); verify(gcsChannel, times(5)).isOpen(); - verify(gcsService, times(2)).getMetadata(eq(file)); - verify(gcsService).openReadChannel(eq(file), eq(1L)); - verifyNoMoreInteractions(gcsService, gcsChannel); + verifyNoMoreInteractions(gcsStorage, gcsChannel); } + } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannelTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannelTest.java index 7b0e298b56c0..4a8d321e495f 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannelTest.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannelTest.java @@ -10,7 +10,7 @@ import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsOutputChannel; +import com.google.gcloud.WriteChannel; import org.junit.Before; import org.junit.Rule; @@ -30,11 +30,11 @@ public class CloudStorageWriteChannelTest { @Rule public final ExpectedException thrown = ExpectedException.none(); - private final GcsOutputChannel gcsChannel = mock(GcsOutputChannel.class); + private final WriteChannel gcsChannel = mock(WriteChannel.class); private CloudStorageWriteChannel chan = new CloudStorageWriteChannel(gcsChannel); @Before - public void before() throws Exception { + public void before() { when(gcsChannel.isOpen()).thenReturn(true); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java index b27d00d68a16..2524bfa2725c 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java @@ -228,7 +228,10 @@ Builder owner(Acl.Entity owner) { return this; } - Builder size(Long size) { + /** + * Sets the blob's size in bytes. + */ + public Builder size(Long size) { this.size = size; return this; } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/FakeStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/FakeStorageRpc.java new file mode 100644 index 000000000000..a04e8b73c1fd --- /dev/null +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/FakeStorageRpc.java @@ -0,0 +1,277 @@ +package com.google.gcloud.storage.testing; + +import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.StorageObject; +import com.google.gcloud.spi.StorageRpc; +import com.google.gcloud.storage.Storage; +import com.google.gcloud.storage.StorageException; + +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.nio.file.FileAlreadyExistsException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.concurrent.NotThreadSafe; + +/** + * A bare-bones in-memory implementation of Storage, meant for testing. + * See LocalGcsHelper. + * + * This class is NOT thread-safe. + */ +@NotThreadSafe +public class FakeStorageRpc implements StorageRpc { + + // fullname -> metadata + Map stuff = new HashMap<>(); + // fullname -> contents + Map contents = new HashMap<>(); + // fullname -> future contents that will be visible on close. + Map futureContents = new HashMap<>(); + + private final boolean throwIfOption; + + /** + * @param throwIfOption if true, we throw when given any option. + */ + public FakeStorageRpc(boolean throwIfOption) { + this.throwIfOption = throwIfOption; + } + + // remove all files + void reset() { + stuff = new HashMap<>(); + contents = new HashMap<>(); + } + + @Override + public Bucket create(Bucket bucket, Map options) throws StorageException { + throw new UnsupportedOperationException(); + } + + @Override + public StorageObject create(StorageObject object, InputStream content, Map options) + throws StorageException { + potentiallyThrow(options); + String key = fullname(object); + stuff.put(key, object); + try { + contents.put(key, com.google.common.io.ByteStreams.toByteArray(content)); + } catch (IOException e) { + throw new StorageException(e); + } + // TODO: crc, etc + return object; + } + + @Override + public Tuple> list(Map options) throws StorageException { + throw new UnsupportedOperationException(); + } + + @Override + public Tuple> list(String bucket, Map options) + throws StorageException { + potentiallyThrow(options); + return null; + } + + /** + * Returns the requested bucket or {@code null} if not found. + */ + @Override + public Bucket get(Bucket bucket, Map options) throws StorageException { + potentiallyThrow(options); + return null; + } + + /** + * Returns the requested storage object or {@code null} if not found. + */ + @Override + public StorageObject get(StorageObject object, Map options) throws StorageException { + // we allow the "ID" option because we need to, but then we give a whole answer anyways + // because the caller won't mind the extra fields. + if (throwIfOption && !options.isEmpty() && options.size()>1 + && options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) { + throw new UnsupportedOperationException(); + } + + String key = fullname(object); + if (stuff.containsKey(key)) { + StorageObject ret = stuff.get(key); + if (contents.containsKey(key)) { + ret.setSize(BigInteger.valueOf(contents.get(key).length)); + } + ret.setId(key); + return ret; + } + return null; + } + + @Override + public Bucket patch(Bucket bucket, Map options) throws StorageException { + potentiallyThrow(options); + return null; + } + + @Override + public StorageObject patch(StorageObject storageObject, Map options) + throws StorageException { + potentiallyThrow(options); + return null; + } + + @Override + public boolean delete(Bucket bucket, Map options) throws StorageException { + return false; + } + + @Override + public boolean delete(StorageObject object, Map options) throws StorageException { + String key = fullname(object); + contents.remove(key); + return null != stuff.remove(key); + } + + @Override + public BatchResponse batch(BatchRequest request) throws StorageException { + return null; + } + + @Override + public StorageObject compose(Iterable sources, StorageObject target, + Map targetOptions) throws StorageException { + return null; + } + + @Override + public byte[] load(StorageObject storageObject, Map options) throws StorageException { + String key = fullname(storageObject); + if (!contents.containsKey(key)) { + throw new StorageException(404, "File not found: " + key); + } + return contents.get(key); + } + + @Override + public Tuple read( + StorageObject from, Map options, long zposition, int zbytes) + throws StorageException { + potentiallyThrow(options); + String key = fullname(from); + if (!contents.containsKey(key)) { + throw new StorageException(404, "File not found: " + key); + } + long position = zposition; + int bytes = zbytes; + if (position < 0) { + position = 0; + } + byte[] full = contents.get(key); + if ((int) position + bytes > full.length) { + bytes = full.length - (int) position; + } + if (bytes <= 0) { + // special case: you're trying to read past the end + return Tuple.of("etag-goes-here", new byte[0]); + } + byte[] ret = new byte[bytes]; + System.arraycopy(full, (int) position, ret, 0, bytes); + return Tuple.of("etag-goes-here", ret); + } + + @Override + public String open(StorageObject object, Map options) throws StorageException { + String key = fullname(object); + boolean mustNotExist = false; + for (Option option : options.keySet()) { + if (option instanceof StorageRpc.Option) { + // this is a bit of a hack, since we don't implement generations. + if ((StorageRpc.Option) option == Option.IF_GENERATION_MATCH + && ((Long) options.get(option)).longValue() == 0L) { + mustNotExist = true; + } + } + } + if (mustNotExist && stuff.containsKey(key)) { + throw new StorageException(new FileAlreadyExistsException(key)); + } + stuff.put(key, object); + + return fullname(object); + } + + @Override + public void write(String uploadId, byte[] toWrite, int toWriteOffset, long destOffset, + int length, boolean last) throws StorageException { + // this may have a lot more allocations than ideal, but it'll work. + byte[] bytes; + if (futureContents.containsKey(uploadId)) { + bytes = futureContents.get(uploadId); + if (bytes.length < length + destOffset) { + bytes = new byte[(int) (length + destOffset)]; + } + } else { + bytes = new byte[(int) (length + destOffset)]; + } + System.arraycopy(toWrite, toWriteOffset, bytes, (int) destOffset, length); + // we want to mimic the GCS behavior that file contents are only visible on close. + if (last) { + contents.put(uploadId, bytes); + futureContents.remove(uploadId); + } else { + futureContents.put(uploadId, bytes); + } + } + + @Override + public RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException { + String sourceKey = fullname(rewriteRequest.source); + // a little hackish, just good enough for the tests to work. + if (!contents.containsKey(sourceKey)) { + throw new StorageException(404, "File not found: " + sourceKey); + } + + boolean mustNotExist = false; + for (Option option : rewriteRequest.targetOptions.keySet()) { + if (option instanceof StorageRpc.Option) { + // this is a bit of a hack, since we don't implement generations. + if ((StorageRpc.Option) option == Option.IF_GENERATION_MATCH + && ((Long) rewriteRequest.targetOptions.get(option)).longValue() == 0L) { + mustNotExist = true; + } + } + } + + String destKey = fullname(rewriteRequest.target); + if (mustNotExist && contents.containsKey(destKey)) { + throw new StorageException(new FileAlreadyExistsException(destKey)); + } + + stuff.put(destKey, rewriteRequest.target); + + byte[] data = contents.get(sourceKey); + contents.put(destKey, Arrays.copyOf(data, data.length)); + return new RewriteResponse(rewriteRequest, rewriteRequest.target, data.length, true, + "rewriteToken goes here", data.length); + } + + @Override + public RewriteResponse continueRewrite(RewriteResponse previousResponse) throws StorageException { + throw new UnsupportedOperationException(); + } + + private String fullname(StorageObject so) { + return (so.getBucket() + "/" + so.getName()); + } + + private void potentiallyThrow(Map options) throws UnsupportedOperationException { + if (throwIfOption && !options.isEmpty()) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/LocalGcsHelper.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/LocalGcsHelper.java new file mode 100644 index 000000000000..1c3a64452441 --- /dev/null +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/LocalGcsHelper.java @@ -0,0 +1,68 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage.testing; + +import com.google.gcloud.spi.ServiceRpcFactory; +import com.google.gcloud.spi.StorageRpc; +import com.google.gcloud.storage.StorageOptions; + +/** + * Utility to create an in-memory storage configuration for testing. Storage options can be + * obtained via the {@link #options()} method. Returned options will point to FakeStorageRpc. + */ +public class LocalGcsHelper { + + // used for testing. Will throw if you pass it an option. + private static final FakeStorageRpc instance = new FakeStorageRpc(true); + + /** + * Returns a {@link StorageOptions} that use the static FakeStorageRpc instance, + * and resets it first so you start from a clean slate. + * That instance will throw if you pass it any option. * + */ + public static StorageOptions options() { + instance.reset(); + return StorageOptions.builder() + .projectId("dummy-project-for-testing") + .serviceRpcFactory( + new ServiceRpcFactory() { + @Override + public StorageRpc create(StorageOptions options) { + return instance; + } + }) + .build(); + } + + /** + * Returns a {@link StorageOptions} that creates a new FakeStorageRpc instance + * with the given option. + */ + public static StorageOptions customOptions(final boolean throwIfOptions) { + return StorageOptions.builder() + .projectId("dummy-project-for-testing") + .serviceRpcFactory( + new ServiceRpcFactory() { + @Override + public StorageRpc create(StorageOptions options) { + return new FakeStorageRpc(throwIfOptions); + } + }) + .build(); + } + +} diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/package-info.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/package-info.java index 8afdd8a9660d..2ea1866f2966 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/package-info.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/package-info.java @@ -15,7 +15,9 @@ */ /** - * A testing helper for Google Cloud Storage. + * Two testing helpers for Google Cloud Storage. + * + * RemoteGcsHelper helps with testing on the actual cloud. * *

A simple usage example: * @@ -32,6 +34,21 @@ * RemoteGcsHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS); * } * + * LocalGcsHelper helps with testing on an in-memory filesystem (this is best for unit tests). + * Note that this filesystem isn't a complete implementation. In particular, it is not thread-safe. + * + *

A simple usage example: + * + * 

+ * CloudStorageFileSystemProvider.setGCloudOptions(LocalGcsHelper.options());
+ * Path path = Paths.get(URI.create("gs://bucket/wednesday"));
+ * thrown.expect(NoSuchFileException.class);
+ * Files.newByteChannel(path);
+ *
+ * + *

The first line ensures that the test uses the in-memory GCS instead of the real one + * (note that you have to set the cloud options before the first usage of the gs:// prefix). + * * @see * gcloud-java tools for testing */