File size: 1,971 Bytes
ec0c8fa |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
from pathlib import Path
from typing import *
import requests
from tqdm import tqdm
__all__ = ["download_file", "download_bytes"]
def download_file(url: str, filepath: Union[str, Path], headers: dict = None, resume: bool = True) -> None:
# Ensure headers is a dict if not provided
headers = headers or {}
# Initialize local variables
file_path = Path(filepath)
downloaded_bytes = 0
# Check if we should resume the download
if resume and file_path.exists():
downloaded_bytes = file_path.stat().st_size
headers['Range'] = f"bytes={downloaded_bytes}-"
# Make a GET request to fetch the file
with requests.get(url, stream=True, headers=headers) as response:
response.raise_for_status() # This will raise an HTTPError if the status is 4xx/5xx
# Calculate the total size to download
total_size = downloaded_bytes + int(response.headers.get('content-length', 0))
# Display a progress bar while downloading
with (
tqdm(desc=f"Downloading {file_path.name}", total=total_size, unit='B', unit_scale=True, leave=False) as pbar,
open(file_path, 'ab') as file,
):
# Set the initial position of the progress bar
pbar.update(downloaded_bytes)
# Write the content to the file in chunks
for chunk in response.iter_content(chunk_size=4096):
file.write(chunk)
pbar.update(len(chunk))
def download_bytes(url: str, headers: dict = None) -> bytes:
# Ensure headers is a dict if not provided
headers = headers or {}
# Make a GET request to fetch the file
with requests.get(url, stream=True, headers=headers) as response:
response.raise_for_status() # This will raise an HTTPError if the status is 4xx/5xx
# Read the content of the response
return response.content
|