docs: Add in Transfer Manager chunked upload/download samples (#2518)

googleapis · May 1, 2024 · d1f6bcc · d1f6bcc
1 parent b7d673c
commit d1f6bcc
Show file tree

Hide file tree

Showing 3 changed files with 132 additions and 0 deletions.
diff --git a/...pets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java b/...pets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package com.example.storage.transfermanager;
+
+// [START storage_transfer_manager_download_chunks_concurrently]
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.transfermanager.DownloadResult;
+import com.google.cloud.storage.transfermanager.ParallelDownloadConfig;
+import com.google.cloud.storage.transfermanager.TransferManager;
+import com.google.cloud.storage.transfermanager.TransferManagerConfig;
+import java.nio.file.Path;
+import java.util.List;
+
+class AllowDivideAndConquerDownload {
+
+  public static void divideAndConquerDownloadAllowed(List<BlobInfo> blobs,
+      String bucketName, Path destinationDirectory) {
+    TransferManager transferManager = TransferManagerConfig.newBuilder()
+        .setAllowDivideAndConquerDownload(true)
+        .build()
+        .getService();
+    ParallelDownloadConfig parallelDownloadConfig = ParallelDownloadConfig.newBuilder()
+        .setBucketName(bucketName)
+        .setDownloadDirectory(destinationDirectory)
+        .build();
+    List<DownloadResult> results = transferManager
+        .downloadBlobs(blobs, parallelDownloadConfig)
+        .getDownloadResults();
+
+    for (DownloadResult result : results) {
+      System.out.println("Download of " + result.getInput().getName()
+          + " completed with status "
+          + result.getStatus());
+    }
+
+  }
+}
+// [END storage_transfer_manager_download_chunks_concurrently]
diff --git a/...ppets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java b/...ppets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package com.example.storage.transfermanager;
+
+// [START storage_transfer_manager_upload_chunks_concurrently]
+import com.google.cloud.storage.transfermanager.ParallelUploadConfig;
+import com.google.cloud.storage.transfermanager.TransferManager;
+import com.google.cloud.storage.transfermanager.TransferManagerConfig;
+import com.google.cloud.storage.transfermanager.UploadResult;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+
+class AllowParallelCompositeUpload {
+
+  public static void parallelCompositeUploadAllowed(String bucketName, List<Path> files)
+      throws IOException {
+    TransferManager transferManager = TransferManagerConfig
+        .newBuilder()
+        .setAllowParallelCompositeUpload(true)
+        .build()
+        .getService();
+    ParallelUploadConfig parallelUploadConfig =
+        ParallelUploadConfig.newBuilder().setBucketName(bucketName).build();
+    List<UploadResult> results =
+        transferManager.uploadFiles(files, parallelUploadConfig).getUploadResults();
+    for (UploadResult result : results) {
+      System.out.println(
+          "Upload for "
+              + result.getInput().getName()
+              + " completed with status "
+              + result.getStatus());
+    }
+  }
+}
+// [END storage_transfer_manager_upload_chunks_concurrently]
diff --git a/.../snippets/src/test/java/com/example/storage/transfermanager/ITTransferManagerSamples.java b/.../snippets/src/test/java/com/example/storage/transfermanager/ITTransferManagerSamples.java
@@ -25,11 +25,19 @@
 import com.google.cloud.storage.testing.RemoteStorageHelper;
 import com.google.cloud.testing.junit4.StdOutCaptureRule;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import java.io.File;
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.OpenOption;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
@@ -39,6 +47,7 @@ public class ITTransferManagerSamples {
   private static final String BUCKET = RemoteStorageHelper.generateBucketName();
   private static Storage storage;
   private static List<BlobInfo> blobs;
+  private static List<BlobInfo> bigBlob;
   private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT");
   @Rule public final StdOutCaptureRule stdOutCaptureRule = new StdOutCaptureRule();
   @Rule public final TemporaryFolder tmp = new TemporaryFolder();
@@ -112,4 +121,23 @@ public void downloadFiles() {
     assertThat(snippetOutput.contains("blob2")).isTrue();
     assertThat(snippetOutput.contains("blob3")).isTrue();
   }
+
+  @Test
+  public void uploadAllowPCU() throws IOException {
+    File tmpFile = tmpDirectory.newFile("fileDirUpload.txt");
+    AllowParallelCompositeUpload
+        .parallelCompositeUploadAllowed(BUCKET, Collections.singletonList(tmpFile.toPath()));
+    String snippetOutput = stdOutCaptureRule.getCapturedOutputAsUtf8String();
+    assertThat(snippetOutput.contains("fileDirUpload.txt")).isTrue();
+  }
+
+  @Test
+  public void downloadAllowDivideAndConquer() {
+    AllowDivideAndConquerDownload
+        .divideAndConquerDownloadAllowed(blobs, BUCKET, tmp.getRoot().toPath());
+    String snippetOutput = stdOutCaptureRule.getCapturedOutputAsUtf8String();
+    assertThat(snippetOutput.contains("blob1")).isTrue();
+    assertThat(snippetOutput.contains("blob2")).isTrue();
+    assertThat(snippetOutput.contains("blob3")).isTrue();
+  }
 }