upload a directory to s3 with boto

Solution 1:

The following function can be used to upload directory to s3 via boto.

    def uploadDirectory(path,bucketname):
        for root,dirs,files in os.walk(path):
            for file in files:
                s3C.upload_file(os.path.join(root,file),bucketname,file)

Provide a path to the directory and bucket name as the inputs. The files are placed directly into the bucket. Alter the last variable of the upload_file() function to place them in "directories".

Solution 2:

There is nothing in the boto library itself that would allow you to upload an entire directory. You could write your own code to traverse the directory using os.walk or similar and to upload each individual file using boto.

There is a command line utility in boto called s3put that could handle this or you could use the AWS CLI tool which has a lot of features that allow you to upload entire directories or even sync the S3 bucket with a local directory or vice-versa.

Solution 3:

I built the function based on the feedback from @JDPTET, however,

1. I needed to remove the common entire local path from getting uploaded to the bucket!
2. Not sure how many path separators I encounter - so I had to use os.path.normpath

    def upload_folder_to_s3(s3bucket, inputDir, s3Path):
        print("Uploading results to s3 initiated...")
        print("Local Source:",inputDir)
        os.system("ls -ltR " + inputDir)

        print("Dest  S3path:",s3Path)

        try:
            for path, subdirs, files in os.walk(inputDir):
                for file in files:
                    dest_path = path.replace(inputDir,"")
                    __s3file = os.path.normpath(s3Path + '/' + dest_path + '/' + file)
                    __local_file = os.path.join(path, file)
                    print("upload : ", __local_file, " to Target: ", __s3file, end="")
                    s3bucket.upload_file(__local_file, __s3file)
                    print(" ...Success")
        except Exception as e:
            print(" ... Failed!! Quitting Upload!!")
            print(e)
            raise e

    s3 = boto3.resource('s3', region_name='us-east-1')
    s3bucket = s3.Bucket("<<s3bucket_name>>")
    upload_folder_to_s3(s3bucket, "<<Local Folder>>", "<<s3 Path>>")

Solution 4:

This is the code I used which recursively upload files from the specified folder to the specified s3 path. Just add S3 credential and bucket details in the script:

https://gist.github.com/hari116/4ab5ebd885b63e699c4662cd8382c314/

#!/usr/bin/python
"""Usage: Add bucket name and credentials
          script.py <source folder> <s3 destination folder >"""

import os
from sys import argv
import boto3
from botocore.exceptions import NoCredentialsError

ACCESS_KEY = ''
SECRET_KEY = ''
host = ''
bucket_name = ''

local_folder, s3_folder = argv[1:3]
walks = os.walk(local_folder)
# Function to upload to s3
def upload_to_aws(bucket, local_file, s3_file):
    """local_file, s3_file can be paths"""
    s3 = boto3.client('s3', aws_access_key_id=ACCESS_KEY,
                      aws_secret_access_key=SECRET_KEY)
    print('  Uploading ' +local_file + ' as ' + bucket + '/' +s3_file)
    try:
        s3.upload_file(local_file, bucket, s3_file)
        print('  '+s3_file + ": Upload Successful")
        print('  ---------')
        return True
    except NoCredentialsError:
        print("Credentials not available")
        return False

"""For file names"""
for source, dirs, files in walks:
    print('Directory: ' + source)
    for filename in files:
        # construct the full local path
        local_file = os.path.join(source, filename)
        # construct the full Dropbox path
        relative_path = os.path.relpath(local_file, local_folder)
        s3_file = os.path.join(s3_folder, relative_path)
        # Invoke upload function
        upload_to_aws(bucket_name, local_file, s3_file)