diff --git a/tools/docker_appender_.py b/tools/docker_appender_.py index 62a38b77c..8de6553ca 100755 --- a/tools/docker_appender_.py +++ b/tools/docker_appender_.py @@ -27,6 +27,7 @@ from containerregistry.client.v2_2 import docker_session from containerregistry.tools import logging_setup from containerregistry.tools import patched +from containerregistry.transport import transport from containerregistry.transport import transport_pool import httplib2 @@ -44,6 +45,9 @@ parser.add_argument( '--dst-image', action='store', help='The name of the new image.') +parser.add_argument( + '--cacert', help='The CA certificate to use.') + _THREADS = 8 @@ -56,7 +60,10 @@ def main(): raise Exception('--src-image, --dst-image and --tarball are required ' 'arguments.') - transport = transport_pool.Http(httplib2.Http, size=_THREADS) + transport_factory = transport.Factory() + if args.cacert is not None: + transport_factory = transport_factory.WithCaCert(args.cacert) + transports_pool = transport_pool.Http(transport_factory.Build, size=_THREADS) # This library can support push-by-digest, but the likelihood of a user # correctly providing us with the digest without using this library diff --git a/tools/docker_puller_.py b/tools/docker_puller_.py index 426dcfe75..ba6201457 100755 --- a/tools/docker_puller_.py +++ b/tools/docker_puller_.py @@ -31,10 +31,9 @@ from containerregistry.tools import logging_setup from containerregistry.tools import patched from containerregistry.transport import retry +from containerregistry.transport import transport from containerregistry.transport import transport_pool -import httplib2 - parser = argparse.ArgumentParser( description='Pull images from a Docker Registry.') @@ -48,12 +47,17 @@ parser.add_argument( '--tarball', action='store', help='Where to save the image tarball.') +parser.add_argument( + '--cacert', help='The CA certificate to use.') + _DEFAULT_TAG = 'i-was-a-digest' _PROCESSOR_ARCHITECTURE = 'amd64' _OPERATING_SYSTEM = 'linux' +_THREADS = 8 + # Today save.tarball expects a tag, which is emitted into one or more files # in the resulting tarball. If we don't translate the digest into a tag then @@ -83,9 +87,11 @@ def main(): logging.fatal('--name and --tarball are required arguments.') sys.exit(1) - retry_factory = retry.Factory() - retry_factory = retry_factory.WithSourceTransportCallable(httplib2.Http) - transport = transport_pool.Http(retry_factory.Build, size=8) + transport_factory = transport.Factory() + if args.cacert is not None: + transport_factory = transport_factory.WithCaCert(args.cacert) + retry_factory = retry.Factory().WithSourceTransportFactory(transport_factory) + transports_pool = transport_pool.Http(retry_factory.Build, size=_THREADS) if '@' in args.name: name = docker_name.Digest(args.name) @@ -113,7 +119,7 @@ def main(): try: with tarfile.open(name=args.tarball, mode='w:') as tar: logging.info('Pulling manifest list from %r ...', name) - with image_list.FromRegistry(name, creds, transport) as img_list: + with image_list.FromRegistry(name, creds, transports_pool) as img_list: if img_list.exists(): platform = image_list.Platform({ 'architecture': _PROCESSOR_ARCHITECTURE, @@ -126,13 +132,13 @@ def main(): # pytype: enable=wrong-arg-types logging.info('Pulling v2.2 image from %r ...', name) - with v2_2_image.FromRegistry(name, creds, transport, accept) as v2_2_img: + with v2_2_image.FromRegistry(name, creds, transports_pool, accept) as v2_2_img: if v2_2_img.exists(): save.tarball(_make_tag_if_digest(name), v2_2_img, tar) return logging.info('Pulling v2 image from %r ...', name) - with v2_image.FromRegistry(name, creds, transport) as v2_img: + with v2_image.FromRegistry(name, creds, transports_pool) as v2_img: with v2_compat.V22FromV2(v2_img) as v2_2_img: save.tarball(_make_tag_if_digest(name), v2_2_img, tar) return diff --git a/tools/docker_pusher_.py b/tools/docker_pusher_.py index 1ecb29210..5d7d2201f 100755 --- a/tools/docker_pusher_.py +++ b/tools/docker_pusher_.py @@ -29,6 +29,7 @@ from containerregistry.tools import logging_setup from containerregistry.tools import patched from containerregistry.transport import retry +from containerregistry.transport import transport from containerregistry.transport import transport_pool import httplib2 @@ -53,6 +54,9 @@ parser.add_argument( '--oci', action='store_true', help='Push the image with an OCI Manifest.') +parser.add_argument( + '--cacert', help='The CA certificate to use.') + _THREADS = 8 @@ -83,9 +87,11 @@ def main(): logging.fatal('--name and --tarball are required arguments.') sys.exit(1) - retry_factory = retry.Factory() - retry_factory = retry_factory.WithSourceTransportCallable(httplib2.Http) - transport = transport_pool.Http(retry_factory.Build, size=_THREADS) + transport_factory = transport.Factory() + if args.cacert is not None: + transport_factory = transport_factory.WithCaCert(args.cacert) + retry_factory = retry.Factory().WithSourceTransportFactory(transport_factory) + transports_pool = transport_pool.Http(retry_factory.Build, size=_THREADS) # This library can support push-by-digest, but the likelihood of a user # correctly providing us with the digest without using this library @@ -105,7 +111,7 @@ def main(): try: with docker_session.Push( - name, creds, transport, threads=_THREADS) as session: + name, creds, transports_pool, threads=_THREADS) as session: logging.info('Starting upload ...') if args.oci: with oci_compat.OCIFromV22(v2_2_img) as oci_img: diff --git a/tools/fast_puller_.py b/tools/fast_puller_.py index 4e3910c79..fb5f002c2 100755 --- a/tools/fast_puller_.py +++ b/tools/fast_puller_.py @@ -33,10 +33,9 @@ from containerregistry.tools import logging_setup from containerregistry.tools import patched from containerregistry.transport import retry +from containerregistry.transport import transport from containerregistry.transport import transport_pool -import httplib2 - parser = argparse.ArgumentParser( description='Pull images from a Docker Registry, faaaaast.') @@ -50,6 +49,9 @@ parser.add_argument( '--directory', action='store', help='Where to save the image\'s files.') +parser.add_argument( + '--cacert', help='The CA certificate to use.') + _THREADS = 8 _PROCESSOR_ARCHITECTURE = 'amd64' @@ -65,9 +67,11 @@ def main(): if not args.name or not args.directory: logging.fatal('--name and --directory are required arguments.') - retry_factory = retry.Factory() - retry_factory = retry_factory.WithSourceTransportCallable(httplib2.Http) - transport = transport_pool.Http(retry_factory.Build, size=_THREADS) + transport_factory = transport.Factory() + if args.cacert is not None: + transport_factory = transport_factory.WithCaCert(args.cacert) + retry_factory = retry.Factory().WithSourceTransportFactory(transport_factory) + transports_pool = transport_pool.Http(retry_factory.Build, size=_THREADS) if '@' in args.name: name = docker_name.Digest(args.name) @@ -94,7 +98,7 @@ def main(): try: logging.info('Pulling manifest list from %r ...', name) - with image_list.FromRegistry(name, creds, transport) as img_list: + with image_list.FromRegistry(name, creds, transports_pool) as img_list: if img_list.exists(): platform = image_list.Platform({ 'architecture': _PROCESSOR_ARCHITECTURE, @@ -107,13 +111,13 @@ def main(): # pytype: enable=wrong-arg-types logging.info('Pulling v2.2 image from %r ...', name) - with v2_2_image.FromRegistry(name, creds, transport, accept) as v2_2_img: + with v2_2_image.FromRegistry(name, creds, transports_pool, accept) as v2_2_img: if v2_2_img.exists(): save.fast(v2_2_img, args.directory, threads=_THREADS) return logging.info('Pulling v2 image from %r ...', name) - with v2_image.FromRegistry(name, creds, transport) as v2_img: + with v2_image.FromRegistry(name, creds, transports_pool) as v2_img: with v2_compat.V22FromV2(v2_img) as v2_2_img: save.fast(v2_2_img, args.directory, threads=_THREADS) return diff --git a/tools/fast_pusher_.py b/tools/fast_pusher_.py index 41497c056..fb5182f3f 100755 --- a/tools/fast_pusher_.py +++ b/tools/fast_pusher_.py @@ -34,9 +34,9 @@ from containerregistry.tools import logging_setup from containerregistry.tools import patched from containerregistry.transport import retry +from containerregistry.transport import transport from containerregistry.transport import transport_pool -import httplib2 from six.moves import zip # pylint: disable=redefined-builtin @@ -73,6 +73,9 @@ parser.add_argument( '--oci', action='store_true', help='Push the image with an OCI Manifest.') +parser.add_argument( + '--cacert', help='The CA certificate to use.') + _THREADS = 8 @@ -137,9 +140,11 @@ def main(): logging.fatal('--digest and --layer must have matching lengths.') sys.exit(1) - retry_factory = retry.Factory() - retry_factory = retry_factory.WithSourceTransportCallable(httplib2.Http) - transport = transport_pool.Http(retry_factory.Build, size=_THREADS) + transport_factory = transport.Factory() + if args.cacert is not None: + transport_factory = transport_factory.WithCaCert(args.cacert) + retry_factory = retry.Factory().WithSourceTransportFactory(transport_factory) + transports_pool = transport_pool.Http(retry_factory.Build, size=_THREADS) logging.info('Loading v2.2 image from disk ...') with v2_2_image.FromDisk( @@ -157,7 +162,7 @@ def main(): try: with docker_session.Push( - name, creds, transport, threads=_THREADS) as session: + name, creds, transports_pool, threads=_THREADS) as session: logging.info('Starting upload ...') if args.oci: with oci_compat.OCIFromV22(v2_2_img) as oci_img: diff --git a/transport/__init__.py b/transport/__init__.py index d28cba0cf..61240c847 100755 --- a/transport/__init__.py +++ b/transport/__init__.py @@ -23,6 +23,8 @@ from containerregistry.transport import retry_ setattr(x, 'retry', retry_) +from containerregistry.transport import transport_ +setattr(x, 'transport', transport_) from containerregistry.transport import transport_pool_ setattr(x, 'transport_pool', transport_pool_) diff --git a/transport/retry_.py b/transport/retry_.py index 2224e2b56..7ac122619 100755 --- a/transport/retry_.py +++ b/transport/retry_.py @@ -20,10 +20,8 @@ from containerregistry.transport import nested -import httplib2 import six.moves.http_client -DEFAULT_SOURCE_TRANSPORT_CALLABLE = httplib2.Http DEFAULT_MAX_RETRIES = 2 DEFAULT_BACKOFF_FACTOR = 0.5 RETRYABLE_EXCEPTION_TYPES = [ @@ -45,10 +43,10 @@ class Factory(object): def __init__(self): self.kwargs = {} - self.source_transport_callable = DEFAULT_SOURCE_TRANSPORT_CALLABLE + self.source_transport_factory = None - def WithSourceTransportCallable(self, source_transport_callable): - self.source_transport_callable = source_transport_callable + def WithSourceTransportFactory(self, source_transport_factory): + self.source_transport_factory = source_transport_factory return self def WithMaxRetries(self, max_retries): @@ -66,7 +64,7 @@ def WithShouldRetryFunction(self, should_retry_fn): def Build(self): """Returns a RetryTransport constructed with the given values. """ - return RetryTransport(self.source_transport_callable(), **self.kwargs) + return RetryTransport(self.source_transport_factory.Build(), **self.kwargs) class RetryTransport(nested.NestedTransport): diff --git a/transport/transport_.py b/transport/transport_.py new file mode 100644 index 000000000..e9eebdf0c --- /dev/null +++ b/transport/transport_.py @@ -0,0 +1,21 @@ +import httplib2 +import logging + +DEFAULT_SOURCE_TRANSPORT_CALLABLE = httplib2.Http + +class Factory(object): + """A factory for creating httplib2.Http client instance.""" + + def __init__(self, http_callable_transport = DEFAULT_SOURCE_TRANSPORT_CALLABLE): + self.kwargs = {} + self.http_callable_transport = http_callable_transport + + def WithCaCert(self, ca_certs): + self.kwargs['ca_certs'] = ca_certs + logging.info('Adding CA certificates of %s', ca_certs) + return self + + def Build(self): + """Returns a httplib2.Http client constructed with the given values. + """ + return self.http_callable_transport(**self.kwargs) \ No newline at end of file