大数据系列之:上传图片到cos、cos 安装python cos sdk 上传图片到cos 高级上传接口 分页列举桶内对象 cos桶之间复制 cos桶之间复制图片数据
安装python cos sdk
pip install -U cos-python-sdk-v5
上传图片到cos
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
import sys
import os
import logging
logging. basicConfig( level= logging. INFO, stream= sys. stdout)
secret_id = "*********************"
secret_key = "********************"
region = 'ap-shanghai'
token = None
scheme = 'https' config = CosConfig( Region= region, SecretId= secret_id, SecretKey= secret_key, Token= token, Scheme= scheme)
client = CosS3Client( config) folder_path = "" for root, dirs, files in os. walk( folder_path) : for file in files: if file . lower( ) . endswith( ( '.png' , '.jpg' , '.jpeg' , '.gif' ) ) : file_path = os. path. join( root, file ) print ( file_path) key = os. path. relpath( file_path, folder_path) key_path = "" + keyprint ( key_path) client. upload_file( Bucket= 'optics-data-1253431691' , LocalFilePath= file_path, Key= key_path, PartSize= 1 , MAXThread= 10 , EnableMD5= False ) print ( f'Uploaded { file_path} to COS' )
高级上传接口
根据文件大小自动选择简单上传或分块上传,分块上传具备断点续传功能。
response = client. upload_file( Bucket= 'optics-data-1253431691' , LocalFilePath= '' , Key= '/picture.jpg' , PartSize= 1 , MAXThread= 10 , EnableMD5= False
)
print ( response[ 'ETag' ] )
分页列举桶内对象
marker = ""
while True : response = client. list_objects( Bucket= 'optics-data-1253431691' , Prefix= '' , Marker= marker, MaxKeys= 10 ) if 'Contents' in response: for content in response[ 'Contents' ] : allPictures. append( content[ 'Key' ] ) if response[ 'IsTruncated' ] == 'false' : break marker = response[ "NextMarker" ]
cos桶之间复制
for source_path in allPictures: print ( source_path) picName = source_path. split( "/" ) [ - 1 ] response = client. copy( Bucket= 'optics-ai-data-1253431691' , Key= f"*/ { picName} " , CopySource= { 'Bucket' : 'optics-data-1253431691' , 'Key' : source_path, 'Region' : 'ap-shanghai' }
)
cos桶之间复制图片数据
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
import sys
import logging
logging. basicConfig( level= logging. INFO, stream= sys. stdout)
secret_id = "************"
secret_key = "***********"
region = 'ap-shanghai'
token = None
scheme = 'https' config = CosConfig( Region= region, SecretId= secret_id, SecretKey= secret_key, Token= token, Scheme= scheme)
client = CosS3Client( config) allPictures = [ ]
marker = ""
while True : response = client. list_objects( Bucket= 'optics-data-1253431691' , Prefix= '*/' , Marker= marker, MaxKeys= 10 ) if 'Contents' in response: for content in response[ 'Contents' ] : allPictures. append( content[ 'Key' ] ) if response[ 'IsTruncated' ] == 'false' : break marker = response[ "NextMarker" ] for source_path in allPictures: print ( source_path) picName = source_path. split( "/" ) [ - 1 ] response = client. copy( Bucket= 'optics-ai-data-1253431691' , Key= f"*/ { picName} " , CopySource= { 'Bucket' : 'optics-data-1253431691' , 'Key' : source_path, 'Region' : 'ap-shanghai' }
) print ( "成功把业务cos桶中图片同步到大数据cos桶中" )