Skip to content

Instantly share code, notes, and snippets.

@ateska

ateska/streaming_tar.py

Last active December 28, 2024 13:44
Show Gist options
  • Save ateska/a9a53eddc70f6431067a77bacf1b33c2 to your computer and use it in GitHub Desktop.
Save ateska/a9a53eddc70f6431067a77bacf1b33c2 to your computer and use it in GitHub Desktop.
Download ZIP
Streaming asynchronous non-blocking tar using Python and asyncio
Raw
streaming_tar.py
import os.path
import aiohttp.web
async def get_tar(request):
'''
AIOHTTP server handler for GET request that wants to download files from a `directory` using TAR.
'''
directory = <specify the directory>
response = aiohttp.web.StreamResponse(
status=200,
reason='OK',
headers={
'Content-Type': 'application/x-tar',
'Content-Disposition': 'attachment; filename="archive.tar"',
},
)
await response.prepare(request)
# Iterate over directory and stream found files
for root, dirs, files in os.walk(directory):
for file in files:
file_path = os.path.join(root, file)
await stream_tar(response, file_path)
# Write two 512-byte blocks of zeros at the end of the archive
await response.write(b'\x00' * 1024)
await response.write_eof()
return response
async def stream_tar(response, file_path):
# Create a TarInfo object for each file and stream it with the file's content
file_name = os.path.basename(file_path)
file_size = os.path.getsize(file_path)
# Create a TarInfo object
info = tarfile.TarInfo(name=file_name)
info.size = file_size
info.mtime = os.path.getmtime(file_path)
info.mode = 0o640
info.type = tarfile.REGTYPE
# Write the tar header
header_bytes = info.tobuf(format=tarfile.USTAR_FORMAT)
await response.write(header_bytes)
# Stream the file content
with open(file_path, 'rb') as f:
while True:
chunk = f.read(8192)
if not chunk:
break
await response.write(chunk)
# Pad to full 512-byte blocks if needed
if file_size % 512 != 0:
await response.write(b'\x00' * (512 - (file_size % 512)))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment