s3 api接口的调用
最近公司使用s3作文件存储服务器，所以在程序中须要调用s3的api，目前程序中使用了python和java版本的s3的api，简单作下记录，方便之后使用。html
1、s3 api使用python版java
1.安装boto3的sdkpython
pip install boto3==1.4.7api
2.建立s3的api客户端浏览器
import boto3
from botocore.client import Config
import os
import sys
import threading服务器
#建立s3连接，若是s3服务器是第四代，则须要设置signature_version='s3v4'
s3_client = boto3.client('s3', endpoint_url='s3服务器地址',
aws_access_key_id='s3服务器的access_key_id',
aws_secret_access_key='s3服务器的secret_access_key',
region_name='s3服务器的时区，这里能够填写cn-north-1',
config=Config(signature_version='s3'))dom
3.获取s3中bucket列表ide
bucket_list = s3_client.list_buckets()测试
print bucket_listui
4.建立bucket
bucket = s3_client.create_bucket(Bucket='bucket的名称')
print bucket
5.获取bucket信息
bucket_info = s3_client.head_bucket(Bucket='bucket的名称')
print bucket_info
6.删除bucket
#bucket_delete = s3_client.delete_bucket(Bucket='bucket的名称')
7.上传文件到s3服务器
# s3_client.upload_file("上传的源文件地址", "bucket名称", "上传文件在s3上对应的key名称", ExtraArgs={'ACL': 'public-read'})
或者
s3_client.upload_file("上传的源文件地址", "bucket名称", "上传文件在s3上对应的key名称",
ExtraArgs={'ACL': 'public-read'},
Callback=UploadProgressPercentage("上传的源文件地址"))
Callback属性对应的类方法以下，该类方法在控制台中打印了上传文件的进度
class UploadProgressPercentage(object):
def __init__(self, filename):
self._filename = filename
self._size = float(os.path.getsize(filename))
self._seen_so_far = 0
self._lock = threading.Lock()
def __call__(self, bytes_amount):
# To simplify we'll assume this is hooked up
# to a single filename.
with self._lock:
self._seen_so_far += bytes_amount
percentage = (self._seen_so_far / self._size) * 100
sys.stdout.write(
"\r%s %s / %s (%.2f%%)" % (
self._filename, self._seen_so_far, self._size,
percentage))
sys.stdout.flush()
8.获取bucket下文件列表
object_list = s3_client.list_objects(Bucket='bucket名称')
print object_list
9.查看bucket下的某个文件信息
object_info = s3_client.get_object(Bucket='bucket名称', Key='文件对应的key名称')
print object_info
10.删除文件
object_delete = s3_client.delete_object(Bucket='bucket名称', Key='文件对应的key名称')
11.下载文件
s3_client.download_file("bucket名称", "文件对应的key名称", "文件下载到的地址")
或者
s3client.download_file("bucket名称", "文件对应的key名称", "文件下载到的地址",Callback=DownloadProgressPercentage("文件下载到的地址"))
Callback属性对应的类方法以下，该类方法在控制台中打印了下载文件的进度
class _DownloadProgressPercentage(object):
def __init__(self, filename):
self._filename = filename
self._seen_so_far = 0
self._lock = threading.Lock()
def __call__(self, bytes_amount):
# To simplify we'll assume this is hooked up
# to a single filename.
with self._lock:
self._seen_so_far += bytes_amount
sys.stdout.write(
"\r%s --> %s bytes transferred" % (
self._filename, self._seen_so_far))
sys.stdout.flush()
参考文档
boto3 api官方文档：http://boto3.readthedocs.io/en/latest/guide/quickstart.html
1.s3的client的api
2.s3的client的api上传下载示例
2、3 api使用java版
1.在pom.xml中增长依赖包
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.7.3</version>
</dependency>
2.java中使用s3的api的demo
package org.jenkinsci.plugins.s3_step;import java.io.BufferedInputStream;import java.io.BufferedOutputStream;import java.io.ByteArrayInputStream;import java.io.File;import java.io.FileOutputStream;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import java.net.URL;import java.security.SecureRandom;import java.util.ArrayList;import java.util.List;import com.amazonaws.ClientConfiguration;import com.amazonaws.auth.AWSCredentials;import com.amazonaws.auth.BasicAWSCredentials;import com.amazonaws.regions.Region;import com.amazonaws.regions.Regions;import com.amazonaws.services.s3.AmazonS3;import com.amazonaws.services.s3.AmazonS3Client;import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;import com.amazonaws.services.s3.model.Bucket;import com.amazonaws.services.s3.model.BucketPolicy;import com.amazonaws.services.s3.model.CannedAccessControlList;import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;import com.amazonaws.services.s3.model.CopyObjectResult;import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;import com.amazonaws.services.s3.model.GetBucketPolicyRequest;import com.amazonaws.services.s3.model.GetObjectRequest;import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;import com.amazonaws.services.s3.model.ListObjectsRequest;import com.amazonaws.services.s3.model.ListPartsRequest;import com.amazonaws.services.s3.model.MultipartUpload;import com.amazonaws.services.s3.model.MultipartUploadListing;import com.amazonaws.services.s3.model.ObjectListing;import com.amazonaws.services.s3.model.ObjectMetadata;import com.amazonaws.services.s3.model.PartETag;import com.amazonaws.services.s3.model.PartListing;import com.amazonaws.services.s3.model.PartSummary;import com.amazonaws.services.s3.model.PutObjectRequest;import com.amazonaws.services.s3.model.PutObjectResult;import com.amazonaws.services.s3.model.S3Object;import com.amazonaws.services.s3.model.S3ObjectInputStream;import com.amazonaws.services.s3.model.S3ObjectSummary;import com.amazonaws.services.s3.model.UploadPartRequest;import com.amazonaws.services.s3.model.UploadPartResult;public class Demo { AWSCredentials credentials = null; public AmazonS3 getS3Client(String access_key, String secret_key, String endpoint) { ClientConfiguration conf = null; if (conf == null) { conf = new ClientConfiguration(); credentials = new BasicAWSCredentials(access_key, secret_key); } AmazonS3 s3 = new AmazonS3Client(credentials, conf); s3.setRegion(Region.getRegion(Regions.CN_NORTH_1)); if (endpoint != null) s3.setEndpoint(endpoint); return s3; } // ============================ bucket ================================ public void testCreateBucket(AmazonS3 s3Client, String bucketName) { System.out.println("======================建立 Bucket=========================="); Bucket bucket = s3Client.createBucket(bucketName); System.out.println(bucket); System.out.println("建立 Bucket : " + bucket.getName()); System.out.println("======================建立 Bucket=========================="); } public void testDeleteBucket(AmazonS3 s3Client, String bucketName) { System.out.println("======================删除 Bucket=========================="); System.out.println("删除 Bucket : " + bucketName); s3Client.deleteBucket(bucketName); System.out.println("======================删除 Bucket=========================="); } public void testDoesBucketExist(AmazonS3 s3Client, String bucketName) { System.out.println("======================Bucket 是否存在=========================="); boolean bucketExist = s3Client.doesBucketExist(bucketName); System.out.println(String.format("%s : %s", bucketName, bucketExist)); System.out.println("======================Bucket 是否存在=========================="); } public void testListBuckets(AmazonS3 s3Client) { System.out.println("======================枚举 Buckets=========================="); List<Bucket> buckets = s3Client.listBuckets(); for (Bucket bucket : buckets) { System.out.println(String.format("%s - %s - %s", bucket.getName(), bucket.getOwner(), bucket.getCreationDate())); } System.out.println("======================枚举 Buckets=========================="); } public void testListObjects(AmazonS3 s3Client, String bucketName) { System.out.println("======================枚举 Objects=========================="); List<String> objectKeys = new ArrayList<String>(); String prefix = "/s3demo/test/";// for (int i = 0; i < 5; i++) {// objectKeys.add(prefix + i);// }// for (String objectKey : objectKeys) {// testPutObject(s3Client, bucketName, objectKey);// } ObjectListing objects = s3Client.listObjects(bucketName); List<S3ObjectSummary> s3Objects = objects.getObjectSummaries(); for (S3ObjectSummary s3os : s3Objects) { System.out.println(String.format("%s - %s - %s", s3os.getKey(), s3os.getETag(), s3os.getSize())); } // prefix ListObjectsRequest listObjReq = new ListObjectsRequest(); listObjReq.setBucketName(bucketName); listObjReq.setDelimiter("/"); listObjReq.setPrefix(prefix); ObjectListing preobjects = s3Client.listObjects(listObjReq); List<String> preobjs = preobjects.getCommonPrefixes(); for (String preobj : preobjs) { System.out.println(preobj + "\n"); } System.out.println("======================枚举 Objects=========================="); } // ============================ bucket ================================ // ============================ object ================================ public void testPutObject(AmazonS3 s3Client, String bucketName, String objectKey,String file_source_address) { System.out.println("======================上传 object=========================="); s3Client.putObject(new PutObjectRequest(bucketName, objectKey, new File(file_source_address)).withCannedAcl(CannedAccessControlList.PublicRead)); System.out.println("======================上传 object=========================="); } public void testGetObject(AmazonS3 s3Client, String bucketName,String objectKey) throws Exception { System.out.println("======================下载 object=========================="); try { S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName,objectKey)); //下载文件到指定地方 InputStream reader = new BufferedInputStream(s3Object.getObjectContent()); File file = new File("localFilename"); FileOutputStream fos = new FileOutputStream(file); OutputStream writer = new BufferedOutputStream(fos); int read = -1; while ( ( read = reader.read() ) != -1 ) { writer.write(read); } writer.flush(); writer.close(); reader.close(); //获取一个request，生成一个下载的url地址，能够在浏览器上下载 GeneratePresignedUrlRequest urlRequest = new GeneratePresignedUrlRequest(bucketName, objectKey); //生成公用的url URL url = s3Client.generatePresignedUrl(urlRequest); System.out.println("=========URL=================" + url + "============URL============="); if (url == null) { throw new Exception("can't get s3 file url!"); } System.out.println("URL=" + url.toString()); } catch (Exception e) { e.printStackTrace(); } System.out.println("======================下载 object=========================="); } public void testDeleteObject(AmazonS3 s3Client, String bucketName,String objectKey) { System.out.println("======================删除 object=========================="); s3Client.deleteObject(bucketName, objectKey); System.out.println("======================删除 object=========================="); } public void testCopyObject(AmazonS3 s3Client, String srcBucketName, String srcObjectKey, String dstBucketName, String dstObjectKey) { System.out.println("======================复制 object=========================="); CopyObjectResult cpres = s3Client.copyObject(srcBucketName, srcObjectKey, dstBucketName, dstObjectKey); String text = String.format("test copy objcet %s from %s to %s.%s", srcObjectKey, srcBucketName, dstBucketName, dstObjectKey); System.out.println(text); System.out.println("object etag : " + cpres.getETag()); System.out.println("======================复制 objects=========================="); } public void testMultiUpload(AmazonS3 s3Client, String bucketName, String objectKey) { System.out.println("======================分块上传=========================="); System.out.println("初始化分块上传"); InitiateMultipartUploadRequest initMultiUp = new InitiateMultipartUploadRequest(bucketName, objectKey); InitiateMultipartUploadResult initMultiUpRes = s3Client.initiateMultipartUpload(initMultiUp); String uploadId = initMultiUpRes.getUploadId(); System.out.println("分块上传 MultiUpload Id : " + uploadId); System.out.println("上传块"); List<PartETag> partETags = new ArrayList<PartETag>(); for (int i = 1; i <= 3; i++) { UploadPartRequest uploadPartReq = new UploadPartRequest(); uploadPartReq.setBucketName(bucketName); uploadPartReq.setKey(objectKey); uploadPartReq.setUploadId(uploadId); uploadPartReq.setPartNumber(i); byte[] part = new byte[1024 * 1024 * 6]; // 分块上传一个块必须大于等于5M if (i == 3) { uploadPartReq.setLastPart(true); part = new byte[1024 * 1024 * 3]; // 最后一个块能够小于5M } SecureRandom random = new SecureRandom(); random.nextBytes(part); System.out.println("块大小 = " + part.length); uploadPartReq.setPartSize(part.length); InputStream inputStream = new ByteArrayInputStream(part); uploadPartReq.setInputStream(inputStream); UploadPartResult uploadPartRes = s3Client.uploadPart(uploadPartReq); PartETag partETag = uploadPartRes.getPartETag(); partETags.add(partETag); System.out.println(String.format("上传块 %s - %s", partETag.getPartNumber(), partETag.getETag())); } CompleteMultipartUploadRequest comMultiReq = new CompleteMultipartUploadRequest( bucketName, objectKey, uploadId, partETags); CompleteMultipartUploadResult comMultires = s3Client.completeMultipartUpload(comMultiReq); System.out.println("完成上传 etag : " + comMultires.getETag()); System.out.println("测试终止上传"); String objectKey1 = "test_abort_multi_upload"; InitiateMultipartUploadRequest initMultiUp1 = new InitiateMultipartUploadRequest( bucketName, objectKey1); InitiateMultipartUploadResult initMultiUpRes1 = s3Client .initiateMultipartUpload(initMultiUp1); String uploadId1 = initMultiUpRes1.getUploadId(); System.out.println("终止上传 upload id : " + uploadId1); AbortMultipartUploadRequest abortMultiUp = new AbortMultipartUploadRequest(bucketName, objectKey1, uploadId1); s3Client.abortMultipartUpload(abortMultiUp); System.out.println("======================分块上传=========================="); } public void testListUploadIds(AmazonS3 s3Client, String bucketName) { System.out.println("======================枚举分块上传uploadId=========================="); String keyPrefix = "/demo/listupids"; for (int i = 1; i < 5; i++) { String objectKey = keyPrefix + i; InitiateMultipartUploadRequest initMultiUp = new InitiateMultipartUploadRequest(bucketName, objectKey); s3Client.initiateMultipartUpload(initMultiUp); } ListMultipartUploadsRequest listMultiUpReq = new ListMultipartUploadsRequest(bucketName); MultipartUploadListing multiUploads = s3Client.listMultipartUploads(listMultiUpReq); List<MultipartUpload> uploads = multiUploads.getMultipartUploads(); for (MultipartUpload up : uploads) { System.out.println(String.format("uploadId : %s - key : %s", up.getUploadId(), up.getKey())); } System.out.println("======================枚举分块上传uploadId=========================="); } public void testListMultiParts(AmazonS3 s3Client, String bucketName, String objectkey, String uploadId) { System.out.println("======================枚举分块上传part=========================="); ListPartsRequest listPartsReq = new ListPartsRequest(bucketName, objectkey, uploadId); PartListing parts = s3Client.listParts(listPartsReq); List<PartSummary> ps = parts.getParts(); for (PartSummary p : ps) { System.out.println(String.format("partNum : %s - partEtag : %s", p.getPartNumber(), p.getETag())); } System.out.println("======================枚举分块上传part=========================="); } public void testIsFileExists(AmazonS3 s3Client, String s3Path,String keyName){ System.out.println("======================判断文件是否已经存在=========================="); try{ boolean flag = false; S3Object s3Object = s3Client.getObject(s3Path, keyName); if(s3Object!=null){ flag = true; } if(flag){ System.out.println("File exists，filename:"+keyName); } }catch(Exception e){ e.printStackTrace(); } } // ============================ object ================================ public static void main(String[] args) throws IOException { String access_key = "s3的access key"; String secret_key = "s3的secret key"; String endpoint = "s3的地址"; Demo d = new Demo(); AmazonS3 s3Client = d.getS3Client(access_key, secret_key, endpoint); String bucketName = "demo"; String bucketName1 = "demo1"; //建立bucket// d.testCreateBucket(s3Client, bucketName); //查询bucket列表// d.testListBuckets(s3Client); //判断bucket是否存在// d.testDoesBucketExist(s3Client, bucketName); //删除bucket// d.testDeleteBucket(s3Client, bucketName2); //上传文件// String objectKey = "var/lib/pom.xml";// String objectKey1 = "var/lib/pom.xml";// String file_source_address = "E:/demo/test1/pom.xml";// d.testPutObject(s3Client, bucketName, objectKey,file_source_address); //下载文件// d.testGetObject(s3Client, bucketName, objectKey);// //复制文件// d.testCopyObject(s3Client, bucketName, objectKey, bucketName1, objectKey1); //bucket下面文件列表// d.testListObjects(s3Client, bucketName); //判断文件是否存在// d.testIsFileExists(s3Client, bucketName, objectKey); }}