Skip to content

Getting Started

Azure

Manager

Get container SAS URL

1
2
3
4
5
6
7
8
from object_storage.factories import get_manager

connection_string = 'DefaultEndpointsProtocol=https;AccountName=pilot;AccountKey=any;EndpointSuffix=core.windows.net'
azr_manager = get_manager('azure', connection_string)

account_sas = await azr_manager.get_container_sas('test')
print(blob_sas)
> 'https://pilot.blob.core.windows.net/test/file.txt?sp=rw&st=2023-04-28T15:15:14Z&se=2023-04-28T23:15:14Z&spr=https&sv=2021-12-02&sr=b&sig=account_signature'

Get file SAS URL

1
2
3
4
blob_sas = await azr_manager.get_blob_sas('test', 'small.txt')

print(blob_sas)
> 'https://pilot.blob.core.windows.net/test/file.txt?sp=rw&st=2023-04-28T15:15:14Z&se=2023-04-28T23:15:14Z&spr=https&sv=2021-12-02&sr=b&sig=blob_signature'

Create Container

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
container = await azr_manager.create_container('test')

print(container)
> {
    'etag': '"0x1B4CA41DA6414F0"',
    'last_modified': datetime.datetime(2023, 6, 7, 14, 24, 58, tzinfo=datetime.timezone.utc),
    'client_request_id': '14e46bc6-053f-11ee-bc91-0efc7f47dc62',
    'request_id': '6d6b659f-467e-45f5-a29f-181da356d6c6',
    'version': '2022-11-02',
    'date': datetime.datetime(2023, 6, 7, 14, 24, 58, tzinfo=datetime.timezone.utc)
}

Delete Container

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
container = await azr_manager.delete_container('test')

print(container)
> {
    'etag': '"0x1B4CA41DA6414F0"',
    'last_modified': datetime.datetime(2023, 6, 7, 14, 24, 58, tzinfo=datetime.timezone.utc),
    'client_request_id': '14e46bc6-053f-11ee-bc91-0efc7f47dc62',
    'request_id': '6d6b659f-467e-45f5-a29f-181da356d6c6',
    'version': '2022-11-02',
    'date': datetime.datetime(2023, 6, 7, 14, 24, 58, tzinfo=datetime.timezone.utc)
}

Info

In case you try to delete an container that does not exist, the method will raise the ResourceNotFoundError

List files in a specific container SAS URL

1
2
3
blobs_list = await azr_manager.list_objects('test')
print(blobs_list)
> [<class 'azure.storage.blob._models.BlobProperties'>, ...]

Check if container exists

1
2
3
4
5
6
connection_string = 'DefaultEndpointsProtocol=https;AccountName=pilot;AccountKey=any;EndpointSuffix=core.windows.net'
azr_manager = get_manager('azure', connection_string)

is_exists = await azr_manager.is_container_exists('test')
print(is_exists)
> True

File Client

Upload File

Uploading a file with default settings
1
2
3
4
5
6
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Upload a file to the blob using default chunk size and no progress callback
result = await azure_blob_client.upload_file(file_path='path/to/file.txt')
print(result)
Uploading a file with custom chunk size and progress tracking
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Define a progress callback function
async def progress_callback(file_name: str, uploaded_bytes: int, total_bytes: int):
    print(f"Uploading {file_name}: {uploaded_bytes}/{total_bytes} bytes")

# Upload a file to the blob with a custom chunk size and progress tracking
result = await azure_blob_client.upload_file(
    file_path='path/to/large_file.txt', chunk_size=8 * 1024 * 1024, progress_callback=progress_callback
)
print(result)
Uploading a file from byte stream
1
2
3
4
5
6
7
8
9
# Create container client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# open file in byte stream
with open('path/to/file.txt', 'rb') as file:
    file_bytes = BytesIO(file.read())
    # Upload the 'key' file to the blob using default chunk size and no progress callback
    result = await azure_blob_client.upload_file_from_bytes(file_bytes)
    print(result)

Resume Upload

Resuming an interrupted file upload with default settings
1
2
3
4
5
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Resume an interrupted file upload to the container using default chunk size and no progress callback
await azure_blob_client.resume_upload(file_path='path/to/file.txt')
Resuming an interrupted file upload with custom chunk size
1
2
3
4
5
6
7
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Resume an interrupted file upload to the blob with a custom chunk size
await azure_blob_client.resume_upload(
    file_path='path/to/large_file.txt', chunk_size=8 * 1024 * 1024
)
Resuming an interrupted file upload with progress tracking
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Define a progress callback function
async def progress_callback(file_name: str, uploaded_bytes: int, total_bytes: int):
    print(f"Resuming upload of {file_name}: {uploaded_bytes}/{total_bytes} bytes")

# Resume an interrupted file upload to the blob with progress tracking
await azure_blob_client.resume_upload(
    file_path='path/to/file.txt', progress_callback=progress_callback
)

Download File to Bytes

Downloading a file to bytes with default settings
1
2
3
4
5
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Download a file from the blob to bytes using default chunk size and no progress callback
file_bytes = await azure_blob_client.download_file_to_bytes()
Downloading a file to bytes with custom chunk size
1
2
3
4
5
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Download a file from the blob to bytes with a custom chunk size
file_bytes = await azure_blob_client.download_file_to_bytes(chunk_size=8 * 1024 * 1024)
Downloading a file to bytes with progress tracking
1
2
3
4
5
6
7
8
9
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Define a progress callback function
async def progress_callback(file_name: str, downloaded_bytes: int, total_bytes: int):
    print(f"Downloading {file_name}: {downloaded_bytes}/{total_bytes} bytes")

# Download a file from the blob to bytes with progress tracking
file_bytes = await azure_blob_client.download_file_to_bytes(progress_callback=progress_callback)

Download File

Downloading a file to a specified path with default settings
1
2
3
4
5
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Download a file from the blob to a specified path using default chunk size and no progress callback
await azure_blob_client.download_file(file_path="/path/to/save/file.txt")
Downloading a file to a specified path with custom chunk size
1
2
3
4
5
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Download a file from the blob to a specified path with a custom chunk size
await azure_blob_client.download_file(file_path="/path/to/save/file.txt", chunk_size=8 * 1024 * 1024)
Downloading a file to a specified path with progress tracking
1
2
3
4
5
6
7
8
9
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Define a progress callback function
async def progress_callback(file_name: str, downloaded_bytes: int, total_bytes: int):
    print(f"Downloading {file_name}: {downloaded_bytes}/{total_bytes} bytes")

# Download a file from the blob to a specified path with progress tracking
await azure_blob_client.download_file(file_path="/path/to/save/file.txt", progress_callback=progress_callback)

Copy File From a Specific URL

Copying a file from a URL to the Blob
1
2
3
4
5
6
7
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Copy a file from a URL to the container
source_url = "https://example.com/file.txt"
copy_result = await azure_blob_client.copy_file_from_url(source_url=source_url)
print(f"File copied successfully. Copy result: {copy_result}")
Copying a large file from a URL with progress tracking
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Copy a large file from a URL to the container with progress tracking
source_url = "https://example.com/large_file.zip"

def progress_callback(file_name: str, current_bytes: int, total_bytes: int):
    progress_percentage = (current_bytes / total_bytes) * 100
    print(f"Copying {file_name}: {progress_percentage:.2f}% complete")

copy_result = await azure_blob_client.copy_file_from_url(source_url=source_url)
print(f"File copied successfully. Copy result: {copy_result}")

Danger

When you move the file the original file remains in the container, so if it's a definitive copy you have to call .delete_file

Delete File

1
2
3
4
5
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Delete a file from a container using an AzureBlobClient
await azure_blob_client.delete_file()

Info

The settings to define if the deletion is permanent or not has to be configure in your Azure service. For more information you can check check Azure SDK soft deletion

Get File URL

1
2
3
4
5
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Get Blob URL
await azure_blob_client.get_file_url()

Get File Properties

1
2
3
4
5
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Get Blob properties
await azure_blob_client.get_file_properties()

Commit File

1
2
3
4
5
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Get `key` blob URL in the client Container
await azure_container_client.commit_file()

Info

It's important to know that this will only commit the chunks previous uploaded with uncommitted status.

Get File Chunks

1
2
3
4
5
6
7
8
9
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Get `key` blob URL in the client Container
committed_block_list, uncommitted_block_list = await azure_container_client.get_file_chunks()

# print the uncommitted_block_list block list
for block in uncommitted_block_list:
    print(block.id)

Check if blob exists

1
2
3
4
5
6
7
8
9
# Create blob client instance
azure_blob_client = AzureBlobClient(blob_sas_url='blob_sas_url')

# Checks if blobs exists
is_exists = await await azure_container_client.is_exists()

# `True` when it exists and `False` when it doesn't
print(is_exists)
> True

Container Client

Delete Container

1
2
3
4
5
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Delete container from the Storage.
await azure_container_client.delete()

Info

In case you try to delete an container that does not exist, the method will raise the ResourceNotFoundError

Upload File

Uploading the 'key' file with default settings to the client Container
1
2
3
4
5
6
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Upload the 'key' file to the blob using default chunk size and no progress callback
result = await azure_container_client.upload_file(key='file.txt', file_path='path/to/file.txt')
print(result)
Uploading the 'key' file with custom chunk size and progress tracking to the client Container
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Define a progress callback function
async def progress_callback(file_name: str, uploaded_bytes: int, total_bytes: int):
    print(f"Uploading {file_name}: {uploaded_bytes}/{total_bytes} bytes")

# Upload the 'key' file to the blob with a custom chunk size and progress tracking
result = await azure_container_client.upload_file(
    key='file.txt', file_path='path/to/large_file.txt', chunk_size=8 * 1024 * 1024, progress_callback=progress_callback
)
print(result)
Uploading the 'key' file from byte stream
1
2
3
4
5
6
7
8
9
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# open file in byte stream
with open('path/to/file.txt', 'rb') as file:
    file_bytes = BytesIO(file.read())
    # Upload the 'key' file to the blob using default chunk size and no progress callback
    result = await azure_container_client.upload_file_from_bytes('file.txt', file_bytes)
    print(result)

Resume Upload

Resuming an interrupted file upload with default settings to the client Container
1
2
3
4
5
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Resume an interrupted file upload to the container using default chunk size and no progress callback
await azure_container_client.resume_upload(key='file.txt', file_path='path/to/file.txt')
Resuming an interrupted file upload with custom chunk size to the client Container
1
2
3
4
5
6
7
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Resume an interrupted file upload to the blob with a custom chunk size
await azure_container_client.resume_upload(
    key='file.txt', file_path='path/to/large_file.txt', chunk_size=8 * 1024 * 1024
)
Resuming an interrupted file upload with progress tracking to the client Container
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Define a progress callback function
async def progress_callback(file_name: str, uploaded_bytes: int, total_bytes: int):
    print(f"Resuming upload of {file_name}: {uploaded_bytes}/{total_bytes} bytes")

# Resume an interrupted file upload to the blob with progress tracking
await azure_container_client.resume_upload(
    key='file.txt', file_path='path/to/file.txt', progress_callback=progress_callback
)

Download File to Bytes

Downloading the 'key' file to bytes with default settings from the client Container
1
2
3
4
5
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Download the 'key' file from the blob to bytes using default chunk size and no progress callback
file_bytes = await azure_container_client.download_file_to_bytes(key='file.txt')
Downloading the 'key' file to bytes with custom chunk size from the client Container
1
2
3
4
5
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Download the 'key' file from the blob to bytes with a custom chunk size
file_bytes = await azure_container_client.download_file_to_bytes(key='file.txt', chunk_size=8 * 1024 * 1024)
Downloading the 'key' file to bytes with progress tracking from the client Container
1
2
3
4
5
6
7
8
9
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Define a progress callback function
async def progress_callback(file_name: str, downloaded_bytes: int, total_bytes: int):
    print(f"Downloading {file_name}: {downloaded_bytes}/{total_bytes} bytes")

# Download the 'key' file from the blob to bytes with progress tracking
file_bytes = await azure_container_client.download_file_to_bytes(key='file.txt', progress_callback=progress_callback)

Download File

Downloading the 'key' file to a specified path with default settings from the client Container
1
2
3
4
5
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Download the 'key' file from the blob to a specified path using default chunk size and no progress callback
await azure_container_client.download_file(key='file.txt', file_path="/path/to/save/file.txt")
Downloading the 'key' file to a specified path with custom chunk size from the client Container
1
2
3
4
5
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Download the 'key' file from the blob to a specified path with a custom chunk size
await azure_container_client.download_file(key='file.txt', file_path="/path/to/save/file.txt", chunk_size=8 * 1024 * 1024)
Downloading the 'key' file to a specified path with progress tracking from the client Container
1
2
3
4
5
6
7
8
9
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Define a progress callback function
async def progress_callback(file_name: str, downloaded_bytes: int, total_bytes: int):
    print(f"Downloading {file_name}: {downloaded_bytes}/{total_bytes} bytes")

# Download the 'key' file from the blob to a specified path with progress tracking
await azure_container_client.download_file(key='file.txt', file_path="/path/to/save/file.txt", progress_callback=progress_callback)

Copy File From a Specific URL

Copying the 'key' file from a URL to the Blob to the client Container
1
2
3
4
5
6
7
8
9
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Copy the 'key' file from a URL to the container
source_url = "https://example.com/file.txt"

# The copied file will have have the name of the value passed in the `key`param
copy_result = await azure_container_client.copy_file_from_url(key='file.txt', source_url=source_url)
print(f"File copied successfully. Copy result: {copy_result}")
Copying a large file from a URL with progress tracking to the client Container
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Copy a large file from a URL to the container with progress tracking
source_url = "https://example.com/large_file.zip"

def progress_callback(file_name: str, current_bytes: int, total_bytes: int):
    progress_percentage = (current_bytes / total_bytes) * 100
    print(f"Copying {file_name}: {progress_percentage:.2f}% complete")

copy_result = await azure_container_client.copy_file_from_url(key='file.txt', source_url=source_url)
print(f"File copied successfully. Copy result: {copy_result}")
Copying the 'key' file from origin Container to destination Container
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
# Create origin and destination container clients instance
origin_azure_container_client = AzureContainerClient(container_sas_url='origin_container_sas_url')
destination_azure_container_client = AzureContainerClient(container_sas_url='destination_container_sas_url')

# Get Blob url of the file to be copied by `key`.
source_url = await origin_azure_container_client.get_file_url(key='file_orgin.txt')

# Copy source_url from origin container to `key` file in the destination container.
copy_result = await destination_azure_container_client.copy_file_from_url(key='destination_file.txt', source_url=source_url)
print(f"File copied successfully. Copy result: {copy_result}")

Danger

When you move the file the original file remains in the container, so if it's a definitive copy you have to call .delete_file

Delete File

1
2
3
4
5
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Delete the 'key' file from a the client Container using an AzureContainerClient
await azure_container_client.delete_file(key='file.txt')

Info

The settings to define if the deletion is permanent or not has to be configure in your Azure service. For more information you can check check Azure SDK soft deletion

Get File URL

1
2
3
4
5
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Get `key` blob URL in the client Container
await azure_container_client.get_file_url(key='file.txt')

Commit File

1
2
3
4
5
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Get `key` blob URL in the client Container
await azure_container_client.commit_file(key='file.txt')

Info

It's important to know that this will only commit the chunks previous uploaded with uncommitted status.

Get File Chunks

1
2
3
4
5
6
7
8
9
# Create container client instance
azure_container_client = AzureContainerClient(container_sas_url='container_sas_url')

# Get `key` blob URL in the client Container
committed_block_list, uncommitted_block_list = await azure_container_client.get_file_chunks(key='file.txt')

# print the uncommitted_block_list block list
for block in uncommitted_block_list:
    print(block.id)

Check if container exists

1
2
3
4
5
6
7
8
9
# Create blob client instance
azure_blob_client = AzureContainerClient(blob_sas_url='container_sas_url')

# Checks if container exists
is_exists = await await azure_container_client.is_exists()

# `True` when it exists and `False` when it doesn't
print(is_exists)
> True

Check if file with specific exists in container

File with 'key' in container
1
2
3
4
5
6
7
8
9
# Create blob client instance
azure_blob_client = AzureContainerClient(blob_sas_url='container_sas_url')

# Checks if container exists
is_exists = await await azure_container_client.is_file_exists(key='blob_key)

# `True` when it exists and `False` when it doesn't
print(is_exists)
> True

Restore a blob

restore a blob
1
2
3
4
5
# Create blob client instance
azure_container_client = AzureContainerClient(blob_sas_url='container_sas_url')

# restore a blob from delete state
azure_container_client.restore_blob(key='blob_key')

Batch restore blob

batch restore blob
1
2
3
4
5
# Create blob client instance
azure_container_client = AzureContainerClient(blob_sas_url='container_sas_url')

# batch restore blobs from delete state
azure_container_client.batch_restore_blobs(keys=['blob_key1', 'blob_key2'])