Skip to content

Instantly share code, notes, and snippets.

@ateska

ateska/streaming_tar.py

Last active December 28, 2024 13:44
Show Gist options
  • Save ateska/a9a53eddc70f6431067a77bacf1b33c2 to your computer and use it in GitHub Desktop.
Save ateska/a9a53eddc70f6431067a77bacf1b33c2 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. ateska revised this gist Nov 5, 2023. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion streaming_tar.py
    View file
    Original file line number Diff line number Diff line change
    @@ -17,6 +17,7 @@ async def get_tar(request):
    )
    await response.prepare(request)

    # Iterate over directory and stream found files
    for root, dirs, files in os.walk(directory):
    for file in files:
    file_path = os.path.join(root, file)
    @@ -29,7 +30,6 @@ async def get_tar(request):
    return response



    async def stream_tar(response, file_path):
    # Create a TarInfo object for each file and stream it with the file's content
    file_name = os.path.basename(file_path)
  2. ateska revised this gist Nov 5, 2023. 1 changed file with 4 additions and 2 deletions.
    6 changes: 4 additions & 2 deletions streaming_tar.py
    View file
    Original file line number Diff line number Diff line change
    @@ -2,8 +2,10 @@
    import aiohttp.web

    async def get_tar(request):

    directory = <get the directory>
    '''
    AIOHTTP server handler for GET request that wants to download files from a `directory` using TAR.
    '''
    directory = <specify the directory>

    response = aiohttp.web.StreamResponse(
    status=200,
  3. ateska created this gist Nov 5, 2023.
    57 changes: 57 additions & 0 deletions streaming_tar.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,57 @@
    import os.path
    import aiohttp.web

    async def get_tar(request):

    directory = <get the directory>

    response = aiohttp.web.StreamResponse(
    status=200,
    reason='OK',
    headers={
    'Content-Type': 'application/x-tar',
    'Content-Disposition': 'attachment; filename="archive.tar"',
    },
    )
    await response.prepare(request)

    for root, dirs, files in os.walk(directory):
    for file in files:
    file_path = os.path.join(root, file)
    await stream_tar(response, file_path)

    # Write two 512-byte blocks of zeros at the end of the archive
    await response.write(b'\x00' * 1024)
    await response.write_eof()

    return response



    async def stream_tar(response, file_path):
    # Create a TarInfo object for each file and stream it with the file's content
    file_name = os.path.basename(file_path)
    file_size = os.path.getsize(file_path)

    # Create a TarInfo object
    info = tarfile.TarInfo(name=file_name)
    info.size = file_size
    info.mtime = os.path.getmtime(file_path)
    info.mode = 0o640
    info.type = tarfile.REGTYPE

    # Write the tar header
    header_bytes = info.tobuf(format=tarfile.USTAR_FORMAT)
    await response.write(header_bytes)

    # Stream the file content
    with open(file_path, 'rb') as f:
    while True:
    chunk = f.read(8192)
    if not chunk:
    break
    await response.write(chunk)

    # Pad to full 512-byte blocks if needed
    if file_size % 512 != 0:
    await response.write(b'\x00' * (512 - (file_size % 512)))