linux-iso/download_isos.py

#!/usr/bin/env python3

import argparse
import concurrent.futures
import os
import sys
import time
from pathlib import Path
from urllib.parse import urlparse

import requests
from tqdm import tqdm


class ISODownloader:
    def __init__(self, output_dir, max_workers=5, retry_attempts=3, chunk_size=8192):
        self.output_dir = Path(output_dir)
        self.max_workers = max_workers
        self.retry_attempts = retry_attempts
        self.chunk_size = chunk_size
        self.session = requests.Session()
        # Configure session with longer timeouts
        self.session.mount(
            "https://",
            requests.adapters.HTTPAdapter(
                max_retries=3, pool_connections=max_workers, pool_maxsize=max_workers
            ),
        )
        self.session.mount(
            "http://",
            requests.adapters.HTTPAdapter(
                max_retries=3, pool_connections=max_workers, pool_maxsize=max_workers
            ),
        )

    def download_file(self, url):
        """Download a single file with retry capability and progress tracking"""
        file_name = os.path.basename(urlparse(url).path)
        file_path = self.output_dir / file_name

        # Create output directory if it doesn't exist
        self.output_dir.mkdir(parents=True, exist_ok=True)

        # Check if file already exists and get its size
        file_size = file_path.stat().st_size if file_path.exists() else 0

        for attempt in range(1, self.retry_attempts + 1):
            try:
                # Setup the request with resume capability if file exists
                headers = {}
                if file_size > 0:
                    headers["Range"] = f"bytes={file_size}-"

                # Make the request with a timeout
                response = self.session.get(
                    url, headers=headers, stream=True, timeout=30, allow_redirects=True
                )

                if (
                    response.status_code == 416
                ):  # Range not satisfiable, file may be complete
                    print(f"File {file_name} appears to be complete, skipping.")
                    return True

                # Handle non-successful status codes
                response.raise_for_status()

                # Get total file size
                total_size = int(response.headers.get("content-length", 0))
                if file_size > 0 and response.status_code == 206:  # Partial content
                    total_size += file_size
                else:
                    # If not a resume, start from scratch
                    file_size = 0

                # Setup progress bar
                desc = f"[Attempt {attempt}] {file_name}"
                with tqdm(
                    total=total_size,
                    initial=file_size,
                    unit="B",
                    unit_scale=True,
                    desc=desc,
                    ncols=100,
                ) as pbar:
                    # Write to file
                    mode = "ab" if file_size > 0 else "wb"
                    with open(file_path, mode) as f:
                        for chunk in response.iter_content(chunk_size=self.chunk_size):
                            if chunk:  # filter out keep-alive chunks
                                f.write(chunk)
                                pbar.update(len(chunk))

                return True

            except (OSError, requests.exceptions.RequestException) as e:
                if attempt < self.retry_attempts:
                    wait_time = 2**attempt  # Exponential backoff
                    print(f"Error downloading {file_name}: {e}")
                    print(
                        f"Retrying in {wait_time} seconds... (Attempt {attempt}/{self.retry_attempts})"
                    )
                    time.sleep(wait_time)
                else:
                    print(
                        f"Failed to download {file_name} after {self.retry_attempts} attempts: {e}"
                    )
                    return False

    def download_all(self, urls):
        """Download all files in parallel"""
        results = []
        successful = 0
        failed = 0

        print(f"Starting downloads with {self.max_workers} parallel workers")

        with concurrent.futures.ThreadPoolExecutor(
            max_workers=self.max_workers
        ) as executor:
            # Submit all download tasks
            future_to_url = {
                executor.submit(self.download_file, url): url for url in urls
            }

            # Process results as they complete
            for future in concurrent.futures.as_completed(future_to_url):
                url = future_to_url[future]
                file_name = os.path.basename(urlparse(url).path)

                try:
                    if future.result():
                        successful += 1
                    else:
                        failed += 1
                        results.append(f"Failed: {file_name}")
                except Exception as e:
                    failed += 1
                    results.append(f"Error with {file_name}: {str(e)}")

        print("\nDownload Summary:")
        print(f"  Successful: {successful}")
        print(f"  Failed: {failed}")

        if failed > 0:
            print("\nFailed Downloads:")
            for result in results:
                print(f"  - {result}")

        return successful, failed, results


def main():
    parser = argparse.ArgumentParser(description="Download ISO files in parallel.")
    parser.add_argument(
        "--urls-file",
        default="urls.txt",
        help="File containing URLs to download (default: urls.txt)",
    )
    parser.add_argument(
        "--output-dir",
        default="./isos",
        help="Directory to save downloaded files (default: ./isos)",
    )
    parser.add_argument(
        "--max-workers",
        type=int,
        default=5,
        help="Maximum number of parallel downloads (default: 5)",
    )
    parser.add_argument(
        "--retries",
        type=int,
        default=5,
        help="Number of retry attempts per file (default: 5)",
    )
    parser.add_argument(
        "--chunk-size",
        type=int,
        default=8192,
        help="Chunk size for downloading in bytes (default: 8192)",
    )

    args = parser.parse_args()

    # Read URLs from file
    try:
        with open(args.urls_file) as f:
            urls = [line.strip() for line in f if line.strip()]
    except OSError as e:
        print(f"Error reading URLs file: {e}")
        sys.exit(1)

    if not urls:
        print("No URLs found in the file.")
        sys.exit(1)

    print(f"Found {len(urls)} URLs to download.")

    # Initialize downloader
    downloader = ISODownloader(
        args.output_dir,
        max_workers=args.max_workers,
        retry_attempts=args.retries,
        chunk_size=args.chunk_size,
    )

    # Start the download
    start_time = time.time()
    successful, failed, results = downloader.download_all(urls)
    elapsed_time = time.time() - start_time

    # Print summary
    print(f"\nDownload process completed in {elapsed_time:.2f} seconds.")
    print(f"Files saved to: {os.path.abspath(args.output_dir)}")

    if failed > 0:
        sys.exit(1)


if __name__ == "__main__":
    main()