Repository navigation
Expand file tree
/
Copy pathdocker_pull.py
More file actions
executable file
·2061 lines (1774 loc) · 76.9 KB
/
Copy pathdocker_pull.py
File metadata and controls
executable file
·2061 lines (1774 loc) · 76.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
Docker Image Puller - Pure Python CLI for pulling and saving Docker images
A pure Python tool for downloading container images from Registry v2 endpoints
without requiring Docker itself. Perfect for air-gapped environments, corporate
networks with proxies, and transferring images between systems.
Features:
- No Docker installation required (pure Python, standard library only)
- Docker Registry v2 and OCI endpoints, including private registries
- Full corporate proxy support with authentication
- Multi-architecture support (amd64, arm64, etc.)
- SSL/TLS configuration options
- Real-time download progress tracking
- Outputs Docker-compatible tar files
Usage:
python docker_pull.py <image:tag> [options]
Examples:
python docker_pull.py ubuntu:latest
python docker_pull.py ghcr.io/nextcloud-releases/all-in-one:latest
python docker_pull.py nginx:alpine --output my-nginx.tar
python docker_pull.py --arch arm64 alpine:latest
python docker_pull.py ubuntu:20.04 --proxy http://proxy:8080 --proxy-auth user:pass
Requirements:
- Python 3.11 or later
- Internet connection (or corporate network with proxy)
Copyright (c) 2025 ZacharyArthur
Licensed under the MIT License - see LICENSE file for details
"""
import argparse
import base64
import gzip
import hashlib
import ipaddress
import json
import logging
import os
import re
import shutil
import signal
import ssl
import sys
import tarfile
import tempfile
import threading
import time
import urllib.request
import zlib
from urllib.error import HTTPError, URLError
from urllib.parse import quote, urlencode, urlparse
from urllib.request import (
HTTPRedirectHandler,
ProxyHandler,
Request,
build_opener,
)
# Module-level logger
logger = logging.getLogger(__name__)
DOCKER_HUB_ALIASES = {"docker.io", "index.docker.io", "registry-1.docker.io"}
SUPPORTED_DIGEST_ALGORITHMS = {"sha256", "sha512"}
ARCHITECTURE_ALIASES = {
"aarch64": "arm64",
"i386": "386",
"x86-64": "amd64",
"x86_64": "amd64",
}
TAG_PATTERN = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}")
DIGEST_PATTERN = re.compile(r"([a-z0-9]+(?:[+._-][a-z0-9]+)*):([0-9a-fA-F]+)")
REPOSITORY_COMPONENT = r"[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*"
REPOSITORY_PATTERN = re.compile(rf"{REPOSITORY_COMPONENT}(?:/{REPOSITORY_COMPONENT})*")
REGISTRY_LABEL = r"(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]*[A-Za-z0-9])"
REGISTRY_HOST_PATTERN = re.compile(rf"{REGISTRY_LABEL}(?:\.{REGISTRY_LABEL})*")
MAX_MANIFEST_SIZE = 4 * 1024 * 1024
class RegistryError(Exception):
"""A registry response or image prevents a safe, complete pull."""
def validate_digest(digest):
"""Return a validated digest algorithm and lowercase encoded value."""
if not isinstance(digest, str):
raise RegistryError(f"Invalid descriptor digest: {digest!r}")
match = DIGEST_PATTERN.fullmatch(digest or "")
if not match:
raise RegistryError(f"Invalid descriptor digest: {digest!r}")
algorithm, encoded = match.groups()
if algorithm not in SUPPORTED_DIGEST_ALGORITHMS:
raise RegistryError(f"Unsupported digest algorithm: {algorithm}")
hasher = hashlib.new(algorithm)
if len(encoded) != hasher.digest_size * 2:
raise RegistryError(f"Invalid {algorithm} digest length")
return algorithm, encoded.lower()
def verify_digest(data, digest, subject="Content"):
"""Verify bytes against a validated descriptor digest."""
algorithm, expected = validate_digest(digest)
if hashlib.new(algorithm, data).hexdigest().lower() != expected:
raise RegistryError(f"{subject} digest mismatch for {digest}")
def normalize_architecture(architecture, variant=None):
"""Return a canonical architecture name for platform comparisons."""
architecture = architecture.lower() if isinstance(architecture, str) else ""
variant = variant.lower() if isinstance(variant, str) else ""
if architecture == "arm" and variant == "v8":
return "arm64"
return ARCHITECTURE_ALIASES.get(architecture, architecture)
def normalize_variant(architecture, variant=None):
"""Return the canonical variant used for platform comparisons."""
variant = variant.lower() if isinstance(variant, str) else ""
if normalize_architecture(architecture, variant) == "arm64" and variant in (
"",
"v8",
):
return ""
return variant
def get_auth_challenges(headers):
"""Return every WWW-Authenticate field from an HTTP header mapping."""
values = (
headers.get_all("WWW-Authenticate") if hasattr(headers, "get_all") else None
)
if values:
return values
value = headers.get("WWW-Authenticate")
return [value] if value else []
def parse_auth_challenges(challenges):
"""Parse supported Bearer and Basic challenges, preferring Bearer."""
if isinstance(challenges, str):
challenges = [challenges]
parsed = []
scheme_pattern = re.compile(r"(bearer|basic)\s+(.+)", re.IGNORECASE)
for header in challenges or []:
current_scheme = None
parameter_items = []
for item in urllib.request.parse_http_list(header):
match = scheme_pattern.fullmatch(item)
if match:
if current_scheme:
try:
values = urllib.request.parse_keqv_list(parameter_items)
except (IndexError, ValueError) as error:
raise ValueError(
"Malformed authentication challenge"
) from error
parsed.append(
(
current_scheme,
{key.lower(): value for key, value in values.items()},
)
)
current_scheme = match.group(1).lower()
parameter_items = [match.group(2)]
elif current_scheme:
parameter_items.append(item)
if current_scheme:
try:
values = urllib.request.parse_keqv_list(parameter_items)
except (IndexError, ValueError) as error:
raise ValueError("Malformed authentication challenge") from error
parsed.append(
(current_scheme, {key.lower(): value for key, value in values.items()})
)
return sorted(parsed, key=lambda item: item[0] != "bearer")
class OriginSafeRedirectHandler(HTTPRedirectHandler):
"""Keep credentials on same-origin redirects and strip them otherwise."""
@staticmethod
def _origin(url):
parsed = urlparse(url)
default_port = 443 if parsed.scheme.lower() == "https" else 80
return (
parsed.scheme.lower(),
(parsed.hostname or "").lower(),
parsed.port or default_port,
)
@staticmethod
def _drop_header(request, name):
"""Remove a header case-insensitively from both urllib header stores."""
lowered = name.lower()
for store in (request.headers, request.unredirected_hdrs):
for key in list(store):
if key.lower() == lowered:
del store[key]
def redirect_request(self, req, fp, code, msg, headers, newurl):
old_scheme = urlparse(req.full_url).scheme.lower()
new_scheme = urlparse(newurl).scheme.lower()
if new_scheme not in ("http", "https"):
raise HTTPError(
newurl, code, "Refusing redirect to unsupported scheme", headers, fp
)
if old_scheme == "https" and new_scheme != "https":
raise HTTPError(
newurl, code, "Refusing HTTPS downgrade redirect", headers, fp
)
try:
old_origin = self._origin(req.full_url)
new_origin = self._origin(newurl)
except ValueError as error:
raise HTTPError(
newurl,
code,
f"Refusing redirect with invalid URL: {error}",
headers,
fp,
) from error
redirected = super().redirect_request(req, fp, code, msg, headers, newurl)
if redirected:
self._drop_header(redirected, "Proxy-Authorization")
if redirected and old_origin != new_origin:
self._drop_header(redirected, "Authorization")
return redirected
# =============================================================================
# UTILITY FUNCTIONS
# =============================================================================
def parse_image_spec(image_spec, plain_http=False):
"""Return registry URL, repository, manifest reference, load name, and tag."""
if not image_spec or "://" in image_spec:
raise ValueError("Image must be in [registry/]repository[:tag] format")
if image_spec.count("@") > 1:
raise ValueError("Image must contain at most one digest reference")
image_with_tag, separator, digest = image_spec.rpartition("@")
if separator:
if "/" in digest or ":" not in digest:
raise ValueError(
"Image must be in [registry/]repository[:tag][@digest] format"
)
algorithm, encoded = validate_digest(digest)
digest = f"{algorithm}:{encoded}"
else:
image_with_tag = image_spec
digest = None
last_slash = image_with_tag.rfind("/")
last_colon = image_with_tag.rfind(":")
explicit_tag = last_colon > last_slash
if explicit_tag:
image_name = image_with_tag[:last_colon]
tag = image_with_tag[last_colon + 1 :]
else:
image_name, tag = image_with_tag, "latest"
if not image_name or not tag:
raise ValueError("Image name and tag must not be empty")
if not TAG_PATTERN.fullmatch(tag):
raise ValueError(f"Invalid image tag: {tag!r}")
first, separator, remainder = image_name.partition("/")
has_registry = bool(separator) and (
"." in first or ":" in first or first.lower() == "localhost"
)
registry = first if has_registry else "docker.io"
repository = remainder if has_registry else image_name
if has_registry:
if any(character.isspace() for character in registry):
raise ValueError(f"Invalid registry host: {registry!r}")
try:
parsed_registry = urlparse(f"//{registry}")
if registry.endswith(":") or parsed_registry.port == 0:
raise ValueError("port must be between 1 and 65535")
except ValueError as error:
raise ValueError(f"Invalid registry host {registry!r}: {error}") from error
registry_hostname = parsed_registry.hostname
if (
not registry_hostname
or parsed_registry.username is not None
or parsed_registry.password is not None
or parsed_registry.path
or parsed_registry.query
or parsed_registry.fragment
):
raise ValueError(f"Invalid registry host: {registry!r}")
if ":" in registry_hostname:
try:
ipaddress.IPv6Address(registry_hostname)
except ValueError as error:
raise ValueError(f"Invalid registry host: {registry!r}") from error
elif (
len(registry_hostname) > 253
or not REGISTRY_HOST_PATTERN.fullmatch(registry_hostname)
or any(len(label) > 63 for label in registry_hostname.split("."))
):
raise ValueError(f"Invalid registry host: {registry!r}")
if not repository:
raise ValueError("Image repository must not be empty")
load_name = f"{registry}/{repository}" if has_registry else repository
if registry.lower() in DOCKER_HUB_ALIASES and "/" not in repository:
repository = f"library/{repository}"
if repository.lower() != repository:
raise ValueError(f"Image repository must be lowercase: {repository!r}")
if len(repository) > 255 or not REPOSITORY_PATTERN.fullmatch(repository):
raise ValueError(f"Invalid image repository: {repository!r}")
registry_host = (
"registry-1.docker.io" if registry.lower() in DOCKER_HUB_ALIASES else registry
)
scheme = "http" if plain_http else "https"
reference = digest or tag
archive_tag = tag if not digest or explicit_tag else None
return (
f"{scheme}://{registry_host}",
repository,
reference,
load_name,
archive_tag,
)
def setup_logging(level=logging.INFO):
"""Configure logging for the application."""
logging.basicConfig(
level=level,
format="%(message)s", # Clean format for CLI tool
handlers=[logging.StreamHandler(sys.stdout)],
)
logger.setLevel(level)
def format_bytes(size):
"""Format bytes into a human-readable string."""
for unit in ["B", "KB", "MB", "GB"]:
if size < 1024.0:
return f"{int(size)} {unit}" if unit == "B" else f"{size:.1f} {unit}"
size /= 1024.0
return f"{size:.1f} TB"
def read_limited(response, limit, subject):
"""Read at most limit bytes from a small registry response."""
body = response.read(limit + 1)
if len(body) > limit:
raise RegistryError(f"{subject} exceeds the {format_bytes(limit)} limit")
return body
def log_http_error_body(error):
"""Log a bounded, escaped registry error response."""
try:
body = error.read(4096).decode("utf-8", "replace")
except (AttributeError, OSError):
return
if body:
logger.error("Error details: %r", body[:500])
def apply_default_file_mode(path):
"""Restore normal umask-derived permissions after mkstemp on POSIX."""
if os.name == "posix":
current_umask = os.umask(0)
os.umask(current_umask)
os.chmod(path, 0o666 & ~current_umask)
# =============================================================================
# CONFIGURATION CLASSES
# =============================================================================
class Config:
"""Configuration management for Docker Image Puller"""
def __init__(self, proxy_config=None, timeout_config=None):
# Proxy configuration
self.proxy_config = self._validate_proxy_config(dict(proxy_config or {}))
# Timeout configuration
timeout_config = timeout_config or {}
self.request_timeout = timeout_config.get("request_timeout", 30)
self.download_timeout = timeout_config.get("download_timeout", 300)
self.chunk_timeout = timeout_config.get("chunk_timeout", 60)
# Validate configuration
self._validate_config()
def _validate_proxy_config(self, proxy_config):
"""Validate and normalize proxy configuration.
Merges provided proxy settings with environment variables.
Environment variables are used as fallback when not explicitly provided.
Args:
proxy_config (dict): User-provided proxy configuration
Returns:
dict: Validated and normalized proxy configuration
"""
# Merge environment proxy settings
if not proxy_config.get("http_proxy"):
proxy_config["http_proxy"] = os.environ.get("HTTP_PROXY") or os.environ.get(
"http_proxy"
)
if not proxy_config.get("https_proxy"):
proxy_config["https_proxy"] = os.environ.get(
"HTTPS_PROXY"
) or os.environ.get("https_proxy")
if not proxy_config.get("no_proxy"):
proxy_config["no_proxy"] = os.environ.get("NO_PROXY") or os.environ.get(
"no_proxy"
)
for name in ("http_proxy", "https_proxy"):
url = proxy_config.get(name)
if not url:
continue
if not isinstance(url, str):
raise ValueError(f"{name} must be a URL")
try:
parsed = urlparse(url)
if parsed.netloc.endswith(":") or parsed.port == 0:
raise ValueError("port must be between 1 and 65535")
hostname = parsed.hostname
except ValueError as error:
raise ValueError(f"Invalid {name} URL: {error}") from error
if parsed.scheme not in ("http", "https") or not hostname:
raise ValueError(
f"Invalid {name} URL: expected HTTP(S) with a hostname"
)
return proxy_config
def _validate_config(self):
"""Validate configuration values"""
for name in ("request_timeout", "download_timeout"):
value = getattr(self, name)
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or value <= 0
):
raise ValueError(f"{name} must be a positive number")
if (
not isinstance(self.chunk_timeout, int)
or isinstance(self.chunk_timeout, bool)
or self.chunk_timeout <= 0
):
raise ValueError("chunk_timeout must be a positive integer")
def has_proxy(self):
"""Check if proxy configuration is present"""
return bool(
self.proxy_config.get("http_proxy") or self.proxy_config.get("https_proxy")
)
# =============================================================================
# SUPPORT CLASSES
# =============================================================================
class ProxyManager:
"""Handles proxy configuration and URL sanitization"""
def __init__(self, config):
self.config = config
self.opener = None
self.direct_opener = None
self.setup_proxy()
def _build_opener(self, proxy_handlers):
handlers = [ProxyHandler(proxy_handlers), OriginSafeRedirectHandler()]
if self.config.proxy_config.get("insecure"):
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
handlers.append(urllib.request.HTTPSHandler(context=context))
return build_opener(*handlers)
def setup_proxy(self):
"""Build per-instance proxied and direct urllib openers."""
if self.config.proxy_config.get("insecure"):
logger.warning("SSL certificate verification is disabled")
if not self.config.has_proxy():
self._setup_no_proxy()
return
proxy_handlers = {}
logger.info("Using proxy configuration:")
http_proxy = self.config.proxy_config.get("http_proxy")
https_proxy = self.config.proxy_config.get("https_proxy")
if http_proxy:
if self.config.proxy_config.get("proxy_auth"):
http_proxy = self._add_proxy_auth(
http_proxy, self.config.proxy_config["proxy_auth"]
)
proxy_handlers["http"] = http_proxy
logger.info(f" HTTP Proxy: {self.sanitize_proxy_url(http_proxy)}")
if https_proxy:
if self.config.proxy_config.get("proxy_auth"):
https_proxy = self._add_proxy_auth(
https_proxy, self.config.proxy_config["proxy_auth"]
)
proxy_handlers["https"] = https_proxy
logger.info(f" HTTPS Proxy: {self.sanitize_proxy_url(https_proxy)}")
if self.config.proxy_config.get("no_proxy"):
logger.info(f" No Proxy: {self.config.proxy_config.get('no_proxy')}")
self.opener = self._build_opener(proxy_handlers)
self.direct_opener = self._build_opener({})
def _setup_no_proxy(self):
"""Build an opener that explicitly ignores environment proxies."""
self.opener = self._build_opener({})
self.direct_opener = self.opener
def open(self, request, timeout, bypass_proxy=False):
"""Open a request through this puller's configured network path."""
opener = self.direct_opener if bypass_proxy else self.opener
if opener is None:
raise RuntimeError("Proxy manager is not initialized")
return opener.open(request, timeout=timeout)
def _add_proxy_auth(self, proxy_url, auth_string):
"""Add authentication credentials to proxy URL.
Args:
proxy_url (str): Base proxy URL
auth_string (str): Authentication in format "username:password"
Returns:
str: Proxy URL with embedded authentication credentials
"""
parsed = urlparse(proxy_url)
if parsed.username is not None:
return proxy_url
if ":" in auth_string:
username, password = auth_string.split(":", 1)
else:
logger.warning("Proxy authentication must be in format 'username:password'")
return proxy_url
credentials = f"{quote(username, safe='')}:{quote(password, safe='')}"
return parsed._replace(netloc=f"{credentials}@{parsed.netloc}").geturl()
def sanitize_proxy_url(self, url):
"""Remove credentials from proxy URL for display"""
if not url:
return url
try:
parsed = urlparse(url)
if parsed.username or parsed.password:
netloc = parsed.netloc.rpartition("@")[2]
return parsed._replace(netloc=netloc).geturl()
except (ValueError, TypeError, AttributeError):
return self._mask_credentials_fallback(url)
return url
def _mask_credentials_fallback(self, text):
"""Fallback credential masking for malformed URLs or general text"""
if not text:
return text
patterns = [
r"://[^:/@]+:[^@]+@", # ://user:pass@
r"://[^:/@]+@", # ://user@
]
result = text
for pattern in patterns:
result = re.sub(pattern, "://***:***@", result)
return result
def sanitize_debug_output(self, text):
"""Sanitize any text that might contain credentials for debug output"""
if not text:
return text
if isinstance(text, dict):
return {
key: (
"***"
if key.lower() in ("authorization", "proxy-authorization")
else value
)
for key, value in text.items()
}
return self._mask_credentials_fallback(str(text))
def should_bypass_proxy(self, host):
"""Check if a hostname or host:port should bypass the proxy."""
no_proxy = self.config.proxy_config.get("no_proxy")
return bool(
no_proxy
and urllib.request.proxy_bypass_environment( # pyright: ignore[reportAttributeAccessIssue]
host, {"no": no_proxy}
)
)
class ProgressReporter:
"""Terminal-safe progress reporting with speed and ETA calculation."""
def __init__(self, total_size=None, description="Download", show_speed=True):
"""Initialize progress reporter.
Args:
total_size (int, optional): Total expected size in bytes
description (str): Description to show before progress bar
show_speed (bool): Whether to show download speed and ETA
"""
self.total_size = total_size
self.downloaded = 0
self.description = description
self.show_speed = show_speed
self.start_time = time.time()
self.last_update = self.start_time
# Terminal width detection
self.terminal_width = self._get_terminal_width()
def _get_terminal_width(self):
"""Get terminal width, default to 80 if detection fails"""
try:
return shutil.get_terminal_size().columns
except (OSError, AttributeError):
return 80
def update(self, bytes_downloaded):
"""Update progress with new bytes downloaded.
Args:
bytes_downloaded (int): Additional bytes downloaded since last update
"""
if bytes_downloaded <= 0:
return
self.downloaded += bytes_downloaded
current_time = time.time()
# Throttle display updates
if current_time - self.last_update < 0.1 and self.downloaded < (
self.total_size or float("inf")
):
return
self._display_progress()
self.last_update = current_time
def _display_progress(self):
"""Display current progress bar and stats"""
if not logger.isEnabledFor(logging.INFO):
return
# Calculate progress percentage
if self.total_size:
progress_pct = min(100.0, (self.downloaded / self.total_size) * 100)
else:
progress_pct = 0
# Format downloaded amount
downloaded_str = format_bytes(self.downloaded)
# Build progress bar
if self.total_size:
total_str = format_bytes(self.total_size)
size_info = f"{downloaded_str}/{total_str}"
progress_bar = self._build_progress_bar(progress_pct)
else:
size_info = downloaded_str
progress_bar = f"[{'#' * 10}] ???%"
# Calculate speed and ETA
speed_info = ""
if self.show_speed:
elapsed = time.time() - self.start_time
if elapsed > 0:
speed = self.downloaded / elapsed
speed_str = f"{format_bytes(speed)}/s"
if self.total_size and speed > 0:
remaining = (self.total_size - self.downloaded) / speed
eta_str = self._format_duration(remaining)
speed_info = f" | {speed_str} | ETA: {eta_str}"
else:
speed_info = f" | {speed_str}"
# Build complete progress line
progress_line = f" {self.description}: {progress_bar} {size_info}{speed_info}"
# Truncate to terminal width if needed
if len(progress_line) > self.terminal_width:
available = (
self.terminal_width
- len(f" {self.description}: ")
- len(size_info)
- len(speed_info)
- 3
)
if available > 10: # Minimum width required
progress_bar = self._build_progress_bar(progress_pct, available)
progress_line = (
f" {self.description}: {progress_bar} {size_info}{speed_info}"
)
else:
# Minimal display for narrow terminals
progress_line = f" {downloaded_str} {int(progress_pct)}%"
# Print with carriage return for overwrite
print(f"\r{progress_line}", end="", flush=True)
def _build_progress_bar(self, progress_pct, width=30):
"""Build a terminal-safe progress bar.
Args:
progress_pct (float): Progress percentage (0-100)
width (int): Width of progress bar in characters
Returns:
str: Formatted progress bar
"""
filled_width = int(width * progress_pct / 100)
return (
f"[{'#' * filled_width}{'-' * (width - filled_width)}] {progress_pct:5.1f}%"
)
def _format_duration(self, seconds):
"""Format duration into human readable string.
Args:
seconds (float): Duration in seconds
Returns:
str: Formatted duration string
"""
if seconds < 60:
return f"{int(seconds)}s"
elif seconds < 3600:
minutes = int(seconds / 60)
secs = int(seconds % 60)
return f"{minutes}m{secs:02d}s"
else:
hours = int(seconds / 3600)
minutes = int((seconds % 3600) / 60)
return f"{hours}h{minutes:02d}m"
def finish(self):
"""Complete the progress display with a newline"""
if self.downloaded > 0 and logger.isEnabledFor(logging.INFO):
print() # New line to finish progress display
# =============================================================================
# CORE FUNCTIONALITY
# =============================================================================
class DockerImagePuller:
def __init__(
self,
auth_token=None,
proxy_config=None,
timeout_config=None,
username=None,
password=None,
):
if (username is None) != (password is None):
raise ValueError("username and password must be provided together")
config = Config(proxy_config, timeout_config)
self.registry_url = "https://registry-1.docker.io"
self.auth_token = auth_token
self.username = username
self.password = password
self.authorization = f"Bearer {auth_token}" if auth_token else None
self.request_timeout = config.request_timeout
self.download_timeout = config.download_timeout
self.chunk_timeout = config.chunk_timeout
self.proxy_manager = ProxyManager(config)
def _stream_download(self, response, digest, expected_size=None, destination=None):
"""Download, validate, and optionally retain a blob on disk.
Args:
response: HTTP response object to stream from
digest (str): Blob digest for identification in error messages
expected_size (int, optional): Expected download size in bytes
destination (str, optional): Final path for a disk-backed blob
Returns:
bytes or str: Validated data or destination path; None on failure
"""
# Setup timeout handling for stuck downloads
def timeout_handler(_signum, _frame):
raise TimeoutError(f"Download chunk timeout after {self.chunk_timeout}s")
algorithm, expected_digest = validate_digest(digest)
temp_data = (
open(destination + ".part", "w+b") # noqa: SIM115
if destination
else tempfile.NamedTemporaryFile(delete=False) # noqa: SIM115
)
total_size = 0
chunk_size = 65536 # 64KB chunks
hasher = hashlib.new(algorithm)
alarm_signal = getattr(signal, "SIGALRM", None)
set_alarm = getattr(signal, "alarm", None)
if threading.current_thread() is not threading.main_thread():
alarm_signal = set_alarm = None
previous_alarm_handler = None
last_fallback_update = 0.0
# Initialize progress reporter for files > 1MB
progress_reporter = None
if expected_size and expected_size > 1024 * 1024: # Over 1MB
blob_name = digest[:12] + "..." if len(digest) > 12 else digest
progress_reporter = ProgressReporter(expected_size, f"Layer {blob_name}")
try:
if alarm_signal is not None and set_alarm is not None:
previous_alarm_handler = signal.signal(alarm_signal, timeout_handler)
while True:
if set_alarm is not None:
set_alarm(self.chunk_timeout)
try:
chunk = response.read(chunk_size)
if not chunk:
break
chunk_len = len(chunk)
if (
expected_size is not None
and total_size + chunk_len > expected_size
):
raise RegistryError(
f"Blob {digest} exceeds its descriptor size of "
f"{expected_size} bytes"
)
temp_data.write(chunk)
hasher.update(chunk)
total_size += chunk_len
# Update progress reporter
if progress_reporter:
progress_reporter.update(chunk_len)
elif total_size > 1024 * 1024 and logger.isEnabledFor(logging.INFO):
current_time = time.monotonic()
if current_time - last_fallback_update >= 0.1:
mb_downloaded = total_size / (1024 * 1024)
print(f" Downloaded {mb_downloaded:.1f} MB...", end="\r")
last_fallback_update = current_time
except TimeoutError as error:
raise TimeoutError("Download chunk timeout") from error
finally:
if set_alarm is not None:
set_alarm(0) # Cancel timeout
# Finish progress reporting
if progress_reporter:
progress_reporter.finish()
elif total_size > 1024 * 1024 and logger.isEnabledFor(logging.INFO):
print() # New line after fallback progress
temp_data.close()
if expected_size is not None and total_size != expected_size:
logger.error(
"Blob %s size mismatch: expected %s bytes, received %s",
digest,
expected_size,
total_size,
)
return None
if hasher.hexdigest().lower() != expected_digest:
logger.error(f"Blob digest verification failed for {digest}")
return None
if destination:
os.replace(temp_data.name, destination)
return destination
with open(temp_data.name, "rb") as f:
return f.read()
except TimeoutError as e:
if progress_reporter:
progress_reporter.finish()
logger.error(f"Download timeout for blob {digest}: {e}")
return None
except Exception as e: # noqa: BLE001
if progress_reporter:
progress_reporter.finish()
logger.error(f"Streaming error for blob {digest}: {e}")
return None
finally:
# Cleanup
if alarm_signal is not None and set_alarm is not None:
set_alarm(0)
if previous_alarm_handler is not None:
signal.signal(alarm_signal, previous_alarm_handler)
temp_data.close()
try:
os.unlink(temp_data.name)
except OSError:
pass
def make_request(self, url, headers=None):
"""Make HTTP request with proxy handling"""
req_headers = headers or {}
logger.debug(f"Request URL: {self.proxy_manager.sanitize_debug_output(url)}")
logger.debug(
f"Headers: {self.proxy_manager.sanitize_debug_output(req_headers)}"
)
parsed_url = urlparse(url)
bypass_proxy = self.proxy_manager.should_bypass_proxy(parsed_url.netloc)
if bypass_proxy:
logger.debug(f"Bypassing proxy for {parsed_url.hostname}")
request = Request(url, headers=req_headers)
response = self.proxy_manager.open(
request, self.request_timeout, bypass_proxy=bypass_proxy
)
logger.debug(f"Response code: {response.code}")
return response
def get_auth_header(self, image_name, challenge=None):
"""Negotiate the registry's advertised authentication scheme."""
if self.auth_token:
return self.authorization
try:
challenge_has_repository_scope = challenge is not None
if challenge is None:
try:
with self.make_request(f"{self.registry_url}/v2/"):
return None
except HTTPError as error:
if error.code != 401:
return None
challenge = get_auth_challenges(error.headers)
if not challenge:
raise RegistryError(
"Registry did not provide an authentication challenge"
)
parsed_challenges = parse_auth_challenges(challenge)
if not parsed_challenges:
raise RegistryError(
"Registry did not provide a supported authentication challenge"
)
scheme, values = parsed_challenges[0]
credentials = None
if self.username is not None:
encoded = base64.b64encode(
f"{self.username}:{self.password}".encode()
).decode()
credentials = f"Basic {encoded}"
if scheme == "basic":
if not credentials:
raise RegistryError("Registry requires --username and --password")
self.authorization = credentials
return credentials