mydomain
No ADS
No ADS

Java Awssdk S3 S3TransferManager download object

  1. Library
  2. Download a file
  3. Download multiple files
  4. Download with pause/resume feature
S3TransferManager is a high-level transfer utility built on top of S3Client. It provides a simple API to allow you to transfer files and directories between your application and Amazon S3.
S3TransferManager also allows you to monitor transfer progress in real-time as well as pause transfers to perform at a later time.
MyUtils.java (*)
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.SizeConstant;

public class MyUtils {

	public static S3TransferManager createS3TransferManager(Region region) {
		AwsCredentials credentials = AwsBasicCredentials.create("accessKeyId", "secretAccessKey");
		AwsCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(credentials);

		S3AsyncClient s3AsyncClient = S3AsyncClient.crtBuilder() //
				.credentialsProvider(credentialsProvider) //
				.region(region) //
				.targetThroughputInGbps(20.0) //
				.minimumPartSizeInBytes(10 * SizeConstant.MB) //
				.build();

		return S3TransferManager.builder().s3Client(s3AsyncClient).build();
	}
}
To create an S3TransferManager object, you need to create an AwsCredentialsProvider object, which provides credentials that allow you to interact with AWS. See the article below to create an AwsCredentialsProvider suitable for your purposes.

1. Library

<!-- https://mvnrepository.com/artifact/software.amazon.awssdk/s3 -->
<dependency>
	<groupId>software.amazon.awssdk</groupId>
	<artifactId>s3</artifactId>
	<version>2.21.10</version>
</dependency>

<!-- https://mvnrepository.com/artifact/software.amazon.awssdk/s3-transfer-manager -->
<dependency>
	<groupId>software.amazon.awssdk</groupId>
	<artifactId>s3-transfer-manager</artifactId>
	<version>2.21.10</version>
</dependency>

<!-- https://mvnrepository.com/artifact/software.amazon.awssdk.crt/aws-crt -->
<dependency>
	<groupId>software.amazon.awssdk.crt</groupId>
	<artifactId>aws-crt</artifactId>
	<version>0.28.0</version>
</dependency>

<!-- Log Library -->
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
<dependency>
	<groupId>org.slf4j</groupId>
	<artifactId>slf4j-api</artifactId>
	<version>2.0.9</version>
</dependency>

<!-- Log Library -->
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-simple -->
<dependency>
	<groupId>org.slf4j</groupId>
	<artifactId>slf4j-simple</artifactId>
	<version>2.0.9</version>
</dependency>

2. Download a file

A simple example using S3TransferManager to download a file from an S3 Bucket to your local computer.
DownloadOneFileExample1.java
package org.o7planning.java_14249_awssdk_s3;

import java.nio.file.Paths;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletionException;

import org.o7planning.java_14249_awssdk_s3.utils.MyUtils;

import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload;
import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest;
import software.amazon.awssdk.transfer.s3.model.FileDownload;
import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener;

public class DownloadOneFileExample1 {
	private static final Region MY_BUCKET_REGION = Region.EU_CENTRAL_1;
	private static final String BUCKET_NAME = "my-bucket-name";

	public static void downloadFile(S3TransferManager transferManager, String bucketName, //
			String key, String filePath) {
		DownloadFileRequest downloadFileRequest = DownloadFileRequest.builder()
				.getObjectRequest(req -> req.bucket(bucketName).key(key)) // bucketName & key
				.destination(Paths.get(filePath))
				// Attaching a LoggingTransferListener that will log the progress
				.addTransferListener(LoggingTransferListener.create()).build();

		FileDownload fileDownload = transferManager.downloadFile(downloadFileRequest);
		try {
			// Wait for the transfer to complete
			CompletedFileDownload completedFileDownload = fileDownload //
					.completionFuture() // CompletableFuture<CompletedFileDownload>
					.join(); // Wait until completed, and return value.
			//
			System.out.println("SuccessfullyCompleted");
			System.out.println(" --> " + completedFileDownload.toString());
		} catch (CancellationException e) {
			System.out.println("CancellationException");
			System.out.println(" --> (It could be by the user calling the pause method.)");
		} catch (CompletionException e) {
			System.out.println("Completed with error: " + e);
		}
	}

	public static void main(String[] args) {
		// There are several ways to create an S3TransferManager
		// @See my article 14231.
		S3TransferManager transferManager = MyUtils.createS3TransferManager(MY_BUCKET_REGION);
		//
		// Test download a video file 11.9MB
		//
		downloadFile(transferManager, //
				BUCKET_NAME, // bucketName
				"static/videos/sample-15s.mp4", // key
				"/Volumes/New/Test/download/sample-15s.mp4" // filePath to save.
		);
	}
}
Output:
[main] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - Transfer initiated...
[AwsEventLoop 12] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - |=================   | 87.9%
[Thread-1] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - |====================| 100.0%
SuccessfullyCompleted
 --> CompletedFileDownload(response=GetObjectResponse(AcceptRanges=bytes, LastModified=2023-11-02T18:54:31Z, ContentLength=11916526, ETag="3a1a600c79846a4af23e4eaeb62d4f77-2", ChecksumCRC32=owx0EQ==-2, ContentType=application/octet-stream, ServerSideEncryption=AES256, Metadata={}))
[sdk-async-response-0-0] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - Transfer complete!

3. Download multiple files

Simple example using S3TransferManager to download multiple files with the specified prefix.
DownloadMultiFilesExample1.java
package org.o7planning.java_14249_awssdk_s3;

import java.nio.file.Paths;

import org.o7planning.java_14249_awssdk_s3.utils.MyUtils;

import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload;
import software.amazon.awssdk.transfer.s3.model.DirectoryDownload;
import software.amazon.awssdk.transfer.s3.model.DownloadDirectoryRequest;
import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener;

public class DownloadMultiFilesExample1 {

	private static final Region MY_BUCKET_REGION = Region.EU_CENTRAL_1;
	private static final String BUCKET_NAME = "my-bucket-name";

	public static Integer downloadMultiFiles(S3TransferManager transferManager, //
			String bucketName, String keyPrefix, String destinationDirectory) {
		// Create DownloadDirectoryRequest
		DownloadDirectoryRequest downloadDirectoryRequest = DownloadDirectoryRequest.builder() //
				.destination(Paths.get(destinationDirectory))//
				.bucket(bucketName) //
				// Only download objects with prefix
				.listObjectsV2RequestTransformer(l -> l.prefix(keyPrefix)) //
				// Optional, add LoggingTransferListener for each file download.
				.downloadFileRequestTransformer(
						request -> request.addTransferListener(LoggingTransferListener.create()))
				.build();

		DirectoryDownload directoryDownload = transferManager.downloadDirectory(downloadDirectoryRequest);

		// Wait for the transfer to complete
		CompletedDirectoryDownload completedDirectoryDownload = directoryDownload //
				.completionFuture() // CompletableFuture<CompletedFileDownload>
				.join(); // Wait until completed, and return value.
		// Print out any failed downloads
		completedDirectoryDownload.failedTransfers().forEach(System.err::println);

		return completedDirectoryDownload.failedTransfers().size();
	}

	public static void main(String[] args) {
		// There are several ways to create an S3TransferManager
		// @See my article 14231.
		S3TransferManager transferManager = MyUtils.createS3TransferManager(MY_BUCKET_REGION);
		//
		// Test download all files with keyPrefix:
		//
		int failCount = downloadMultiFiles(transferManager, //
				BUCKET_NAME, // bucketName
				"static/images/", // keyPrefix
				"/Volumes/New/Test/download/" // destinationDirectory
		);
		System.out.println("failCount: " + failCount);
	}
}
No ADS
Output:
[sdk-async-response-0-0] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - Transfer initiated...
[sdk-async-response-0-0] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - Transfer initiated...
[AwsEventLoop 8] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - |====================| 100.0%
[sdk-async-response-0-1] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - Transfer complete!
[AwsEventLoop 6] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - |====================| 100.0%
[sdk-async-response-0-2] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - Transfer complete!
failCount: 0

4. Download with pause/resume feature

For example, download a file with S3TransferManager and the pause/resume feature.
ResumableFileDownloadThread.java
package org.o7planning.java_14249_awssdk_s3;

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletionException;

import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.CompletedFileDownload;
import software.amazon.awssdk.transfer.s3.model.FileDownload;
import software.amazon.awssdk.transfer.s3.model.ResumableFileDownload;
import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener;
import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest;

public class ResumableFileDownloadThread extends Thread {
	private S3TransferManager transferManager;
	private String bucketName;
	private String key;
	private String filePath;

	private FileDownload fileDownload;
	private boolean isPause;
	private boolean successfullyCompleted;

	public ResumableFileDownloadThread(S3TransferManager transferManager, //
			String bucketName, String key, String filePath) {
		super();
		this.transferManager = transferManager;
		this.bucketName = bucketName;
		this.key = key;
		this.filePath = filePath;
	}

	private String resumableStateFile(String key) {
		return key.replace('/', '_') + "_resumableFileDownload.json";
	}

	public void pauseDownload() {
		if (fileDownload == null || isPause || successfullyCompleted) {
			return;
		}
		System.out.println("--> User calls pauseDownload!");

		// Pause the download
		ResumableFileDownload resumableFileDownload = fileDownload.pause();
		isPause = true;
		// Optionally, persist the resumableFileDownload
		Path path = Paths.get(this.resumableStateFile(key));
		resumableFileDownload.serializeToFile(path);
	}

	public void resumeDownload() {
		if (fileDownload == null || !isPause || successfullyCompleted) {
			return;
		}
		isPause = false;
		System.out.println("--> User calls resumeDownload!");

		Path path = Paths.get(this.resumableStateFile(key));
		// Retrieve the resumableFileDownload from the file
		//
		// WARNING: Test with: Awssdk 2.21.10
		// The "TransferListener" is not restored. (Need to wait for newer version of
		// Awssdk)
		//
		ResumableFileDownload persistedResumableFileDownload = ResumableFileDownload.fromFile(path);

		// Resume the download
		this.fileDownload = transferManager.resumeDownloadFile(persistedResumableFileDownload);

		this.handleCompletionState(fileDownload);
	}

	@Override
	public void run() {
		DownloadFileRequest downloadFileRequest = DownloadFileRequest.builder()
				.getObjectRequest(req -> req.bucket(bucketName).key(key)) // bucketName & key
				.destination(Paths.get(filePath))
				// Attaching a LoggingTransferListener that will log the progress
				.addTransferListener(LoggingTransferListener.create())//
				.build();
		// Initiate the transfer
		this.fileDownload = transferManager.downloadFile(downloadFileRequest);

		this.handleCompletionState(fileDownload);
	}

	private void handleCompletionState(FileDownload fileDownload) {
		try {
			CompletedFileDownload completedFileDownload = fileDownload //
					.completionFuture() // CompletableFuture<CompletedDirectoryDownload>
					.join(); // Wait until completed, and return value.
			//
			successfullyCompleted = true;
			System.out.println("handleCompletionState: SuccessfullyCompleted");
			System.out.println(" --> " + completedFileDownload.toString());
		} catch (CancellationException e) {
			System.out.println("handleCompletionState: CancellationException");
			System.out.println(" --> (It could be by the user calling the pause method.)");
		} catch (CompletionException e) {
			System.out.println("handleCompletionState: Completed with error: " + e);
		}
	}
}
No ADS
Write a class that implements the TransferListener interface to listen for data transfer status.
MyTransferListener.java
package org.o7planning.java_14249_awssdk_s3;

import software.amazon.awssdk.transfer.s3.progress.TransferListener;
import software.amazon.awssdk.transfer.s3.progress.TransferListener.Context.BytesTransferred;
import software.amazon.awssdk.transfer.s3.progress.TransferListener.Context.TransferComplete;
import software.amazon.awssdk.transfer.s3.progress.TransferListener.Context.TransferFailed;
import software.amazon.awssdk.transfer.s3.progress.TransferListener.Context.TransferInitiated;
import software.amazon.awssdk.transfer.s3.progress.TransferProgressSnapshot;

public class MyTransferListener implements TransferListener {

	@Override
	public void bytesTransferred(BytesTransferred context) {
		TransferProgressSnapshot progress = context.progressSnapshot();
		System.out.println("MyTransferListener.bytesTransferred: " //
				+ (progress.ratioTransferred().getAsDouble() * 100) + " %");
	}

	@Override
	public void transferInitiated(TransferInitiated context) {
		TransferProgressSnapshot progress = context.progressSnapshot();
		System.out.println("MyTransferListener.transferInitiated: " //
				+ progress.ratioTransferred().getAsDouble());
	}

	@Override
	public void transferComplete(TransferComplete context) {
		TransferProgressSnapshot progress = context.progressSnapshot();
		System.out.println("MyTransferListener.bytesTransferred: " //
				+ progress.ratioTransferred().getAsDouble());
	}

	@Override
	public void transferFailed(TransferFailed context) {
		System.out.println("MyTransferListener.transferFailed (Or pause called): " //
				+ context.exception().toString());
	}
}
Write a class to simulate the user pausing the download and resuming the download later.
ResumableFileDownloadThreadTest.java
package org.o7planning.java_14249_awssdk_s3;

import java.time.Duration;

import org.o7planning.java_14249_awssdk_s3.utils.MyUtils;

import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.transfer.s3.S3TransferManager;

public class ResumableFileDownloadThreadTest {
	private static final Region MY_BUCKET_REGION = Region.EU_CENTRAL_1;
	private static final String BUCKET_NAME = "my-bucket-name";

	public static void main(String[] args) {
		S3TransferManager transferManager = MyUtils.createS3TransferManager(MY_BUCKET_REGION);

		// Test with a large File...
		ResumableFileDownloadThread downloadThread = new ResumableFileDownloadThread(transferManager, //
				BUCKET_NAME, // bucketName
				"static/videos/sample-16m.mp4", // key   (171MB)
				"/Volumes/New/Test/download/sample-16m.mp4" // filePath
		);
		//
		downloadThread.start();

		try {
			Thread.sleep(Duration.ofSeconds(60));
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		// The User calls pauseDownload method.
		downloadThread.pauseDownload();
		try {
			Thread.sleep(Duration.ofSeconds(3));
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		// The User calls resumeDownload method.
		downloadThread.resumeDownload();
	} 
}
Output:
[Thread-1] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - Transfer initiated...
[AwsEventLoop 9] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - |                    | 3.0%
[Thread-2] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - |=                   | 6.0%
[Thread-2] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - |==                  | 12.1%
[Thread-3] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - |===                 | 15.1%
[Thread-3] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - |====                | 21.2%
[AwsEventLoop 9] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - |=====               | 27.3%
[AwsEventLoop 9] INFO software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - |======              | 30.3%
--> User calls pauseDownload!
handleCompletionState: CancellationException
 --> (It could be by the user calling the pause method.)
[main] WARN software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener - Transfer failed.
java.util.concurrent.CancellationException
	at java.base/java.util.concurrent.CompletableFuture.cancel(CompletableFuture.java:2510)
	at software.amazon.awssdk.transfer.s3.internal.model.DefaultFileDownload.doPause(DefaultFileDownload.java:64)
	at software.amazon.awssdk.utils.Lazy.getValue(Lazy.java:50)
	at software.amazon.awssdk.transfer.s3.internal.model.DefaultFileDownload.pause(DefaultFileDownload.java:60)
	at org.o7planning.java_14249_awssdk_s3.ResumableFileDownloadThread.pauseDownload(ResumableFileDownloadThread.java:45)
	at org.o7planning.java_14249_awssdk_s3.ResumableFileDownloadThreadTest.main(ResumableFileDownloadThreadTest.java:32)
--> User calls resumeDownload!
handleCompletionState: SuccessfullyCompleted
 --> CompletedFileDownload(response=GetObjectResponse(AcceptRanges=bytes, LastModified=2023-11-03T05:03:30Z, ContentLength=120308024, ETag="8b7b7bb1f884ede76fd170c32eb00051-17", ContentRange=bytes 52428800-172736823/172736824, ContentType=application/octet-stream, ServerSideEncryption=AES256, Metadata={}))
No ADS
No ADS