Add a feature to list the buckets (googleapis#3915)

jean-philippe-martin · chingor13 · commit aa93a47a4157 · 2018-11-07T09:31:59.000-08:00
Also, update the README to reflect recent update to pseudoDirs.
diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/README.md b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/README.md
@@ -138,7 +138,6 @@ Limitations
 
 This library is usable, but not yet complete. The following features are not
 yet implemented:
- * Listing all the buckets
  * Resuming upload or download
  * Generations
  * File attributes
@@ -153,8 +152,9 @@ subset via a familiar interface.
 **NOTE:** Cloud Storage uses a flat namespace and therefore doesn't support real
 directories. So this library supports what's known as "pseudo-directories". Any
 path that includes a trailing slash, will be considered a directory. It will
-always be assumed to exist, without performing any I/O. This allows you to do
-path manipulation in the same manner as you would with the normal UNIX file
+always be assumed to exist, without performing any I/O. Paths without the trailing
+slash will result in an I/O operation to check a file is present in that "directory".
+This allows you to do path manipulation in the same manner as you would with the normal UNIX file
 system implementation. You can disable this feature with
 `CloudStorageConfiguration.usePseudoDirectories()`.
 
diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java
@@ -19,6 +19,10 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.google.api.gax.paging.Page;
+import com.google.cloud.storage.Bucket;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageException;
 import com.google.cloud.storage.StorageOptions;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
@@ -85,6 +89,27 @@ static CloudStorageConfiguration getDefaultCloudStorageConfiguration() {
     return userSpecifiedDefault;
   }
 
+  /**
+   * Lists the project's buckets. Pass "null" to use the default project.
+   *
+   * <p>Example of listing buckets, specifying the page size and a name prefix.
+   * <pre> {@code
+   * String prefix = "bucket_";
+   * Page<Bucket> buckets = CloudStorageFileSystem.listBuckets("my-project", BucketListOption.prefix(prefix));
+   * Iterator<Bucket> bucketIterator = buckets.iterateAll();
+   * while (bucketIterator.hasNext()) {
+   *   Bucket bucket = bucketIterator.next();
+   *   // do something with the bucket
+   * }
+   * }</pre>
+   *
+   * @throws StorageException upon failure
+   */
+  public static Page<Bucket> listBuckets(@Nullable String project, Storage.BucketListOption... options) {
+    CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider(null, StorageOptions.newBuilder().setProjectId(project).build());
+    return provider.listBuckets(options);
+  }
+
   /**
    * Returns Google Cloud Storage {@link FileSystem} object for {@code bucket}.
    *
diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java
@@ -26,6 +26,7 @@
 import com.google.cloud.storage.Blob;
 import com.google.cloud.storage.BlobId;
 import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Bucket;
 import com.google.cloud.storage.CopyWriter;
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.Storage.BlobGetOption;
@@ -34,7 +35,6 @@
 import com.google.cloud.storage.StorageOptions;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.MoreObjects;
-import com.google.common.base.Strings;
 import com.google.common.collect.AbstractIterator;
 import com.google.common.net.UrlEscapers;
 import com.google.common.primitives.Ints;
@@ -915,6 +915,35 @@ public CloudStorageFileSystemProvider withNoUserProject() {
     return new CloudStorageFileSystemProvider("", this.storageOptions);
   }
 
+  /**
+   * Returns the project that is assigned to this provider.
+   */
+  public String getProject() {
+    initStorage();
+    return storage.getOptions().getProjectId();
+  }
+
+  /**
+   * Lists the project's buckets. But use the one in CloudStorageFileSystem.
+   *
+   * <p>Example of listing buckets, specifying the page size and a name prefix.
+   * <pre> {@code
+   * String prefix = "bucket_";
+   * Page<Bucket> buckets = provider.listBuckets(BucketListOption.prefix(prefix));
+   * Iterator<Bucket> bucketIterator = buckets.iterateAll();
+   * while (bucketIterator.hasNext()) {
+   *   Bucket bucket = bucketIterator.next();
+   *   // do something with the bucket
+   * }
+   * }</pre>
+   *
+   * @throws StorageException upon failure
+   */
+  Page<Bucket> listBuckets(Storage.BucketListOption... options) {
+    initStorage();
+    return storage.list(options);
+  }
+
   private IOException asIoException(StorageException oops) {
     // RPC API can only throw StorageException, but CloudStorageFileSystemProvider
     // can only throw IOException. Square peg, round hole.
diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java
@@ -16,11 +16,13 @@
 
 package com.google.cloud.storage.contrib.nio.it;
 
+import static com.google.common.collect.ImmutableList.copyOf;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.client.http.HttpResponseException;
+import com.google.cloud.storage.Bucket;
 import com.google.cloud.storage.Storage.BlobTargetOption;
 import com.google.cloud.storage.StorageException;
 import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration;
@@ -298,6 +300,21 @@ private void assertIsRequesterPaysException(String message, StorageException sex
 
   // End of tests related to the "requester pays" feature
 
+  @Test
+  public void testListBuckets() throws IOException {
+    boolean bucketFound = false;
+    boolean rpBucketFound = false;
+    for (Bucket b : CloudStorageFileSystem.listBuckets(project).iterateAll()) {
+      bucketFound |= BUCKET.equals(b.getName());
+      rpBucketFound |= REQUESTER_PAYS_BUCKET.equals(b.getName());
+    }
+    assertWithMessage("listBucket should have found the test bucket")
+        .that(bucketFound).isTrue();
+    assertWithMessage("listBucket should have found the test requester-pays bucket")
+        .that(rpBucketFound).isTrue();
+  }
+
+
   @Test
   public void testFileExists() throws IOException {
     CloudStorageFileSystem testBucket = getTestBucket();
@@ -736,7 +753,7 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx
     }
 
     public ImmutableList<Path> getPaths() {
-      return ImmutableList.copyOf(paths);
+      return copyOf(paths);
     }
   }
 
