diff --git a/.gitmodules b/.gitmodules index 152e601a3..82e0d0df6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -57,7 +57,7 @@ url = https://github.com/gsutil-mirrors/boto.git [submodule "gslib/vendored/oauth2client"] path = gslib/vendored/oauth2client - url = https://github.com/gsutil-mirrors/oauth2client.git + url = https://github.com/gurusai-voleti/oauth2client.git [submodule "third_party/google-auth-library-python"] path = third_party/google-auth-library-python url = https://github.com/googleapis/google-auth-library-python diff --git a/gslib/commands/config.py b/gslib/commands/config.py index a828dd48d..3ab04e676 100644 --- a/gslib/commands/config.py +++ b/gslib/commands/config.py @@ -772,7 +772,7 @@ def _WriteBotoConfigFile(self, if not HAS_CRYPTO: raise CommandException( 'Service account authentication via a .p12 file requires ' - 'either\nPyOpenSSL or PyCrypto 2.6 or later. Please install ' + 'either\ncryptography. Please install ' 'either of these\nto proceed, use a JSON-format key file, or ' 'configure a different type of credentials.') diff --git a/gslib/commands/signurl.py b/gslib/commands/signurl.py index 2df8b5e3b..6c293cd16 100644 --- a/gslib/commands/signurl.py +++ b/gslib/commands/signurl.py @@ -51,22 +51,9 @@ from gslib.utils.shim_util import GcloudStorageMap, GcloudStorageFlag from gslib.utils.signurl_helper import CreatePayload, GetFinalUrl, to_bytes -try: - # Check for openssl. - # pylint: disable=C6204 - from OpenSSL.crypto import FILETYPE_PEM - from OpenSSL.crypto import load_privatekey - from OpenSSL.crypto import sign - from OpenSSL.crypto import PKey - HAVE_OPENSSL = True -except ImportError: - load_privatekey = None - sign = None - HAVE_OPENSSL = False - FILETYPE_PEM = None - try: from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import padding from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.hazmat.primitives.serialization import pkcs12 @@ -76,6 +63,7 @@ HAVE_CRYPTO = False _CRYPTO_IMPORT_ERROR = "pyca/cryptography is not available. Either install it, or please consider using the .json keyfile" +HAVE_OPENSSL = HAVE_CRYPTO _AUTO_DETECT_REGION = 'auto' _MAX_EXPIRATION_TIME = timedelta(days=7) _MAX_EXPIRATION_TIME_WITH_MINUS_U = timedelta(hours=12) @@ -307,12 +295,6 @@ def _GenSignedUrl(key, signed_headers=signed_headers, string_to_sign_debug=string_to_sign_debug) else: - if six.PY2: - digest = b'RSA-SHA256' - else: - # Your IDE may complain about this due to a bad docstring in pyOpenSsl: - # https://github.com/pyca/pyopenssl/issues/741 - digest = 'RSA-SHA256' string_to_sign, canonical_query_string = CreatePayload( client_id=client_id, method=method, @@ -324,12 +306,10 @@ def _GenSignedUrl(key, signed_headers=signed_headers, billing_project=billing_project, string_to_sign_debug=string_to_sign_debug) - if isinstance(key, PKey): - raw_signature = sign(key, string_to_sign, digest) - else: - if not HAVE_CRYPTO: - raise CommandException(_CRYPTO_IMPORT_ERROR) - raw_signature = key.sign(to_bytes(string_to_sign), padding.PKCS1v15(), hashes.SHA256()) + if not HAVE_CRYPTO: + raise CommandException(_CRYPTO_IMPORT_ERROR) + string_to_sign = string_to_sign.replace('\r\n', '\n') + raw_signature = key.sign(to_bytes(string_to_sign), padding.PKCS1v15(), hashes.SHA256()) final_url = GetFinalUrl(raw_signature, gs_host, gcs_path, canonical_query_string) return final_url @@ -353,8 +333,7 @@ def _ReadJSONKeystore(ks_contents, passwd=None): Assumes this keystore was downloaded from the Cloud Platform Console. By default, JSON keystore private keys from the Cloud Platform Console - aren't encrypted so the passwd is optional as load_privatekey will - prompt for the PEM passphrase if the key is encrypted. + aren't encrypted so the passwd is optional. Arguments: ks_contents: JSON formatted string representing the keystore contents. Must @@ -378,10 +357,13 @@ def _ReadJSONKeystore(ks_contents, passwd=None): raise ValueError('JSON keystore doesn\'t contain required fields') client_email = ks['client_email'] - if passwd: - key = load_privatekey(FILETYPE_PEM, ks['private_key'], passwd) - else: - key = load_privatekey(FILETYPE_PEM, ks['private_key']) + if not HAVE_CRYPTO: + raise CommandException(_CRYPTO_IMPORT_ERROR) + + key = serialization.load_pem_private_key( + ks['private_key'].encode('utf-8'), + password=passwd.encode('utf-8') if passwd else None + ) return key, client_email @@ -617,8 +599,8 @@ def RunCommand(self): """Command entry point for signurl command.""" if not HAVE_OPENSSL: raise CommandException( - 'The signurl command requires the pyopenssl library (try pip ' - 'install pyopenssl or easy_install pyopenssl)') + 'The signurl command requires the cryptography library (try pip ' + 'install cryptography)') method, delta, content_type, passwd, region, use_service_account, billing_project = ( self._ParseAndCheckSubOpts()) @@ -631,7 +613,7 @@ def RunCommand(self): try: key, client_email = _ReadJSONKeystore( open(self.args[0], 'rb').read(), passwd) - except ValueError: + except: # Ignore and try parsing as a pkcs12. if not passwd: passwd = getpass.getpass('Keystore password:') diff --git a/gslib/tests/test_gcs_json_credentials.py b/gslib/tests/test_gcs_json_credentials.py index 617be5e9e..499ec6261 100644 --- a/gslib/tests/test_gcs_json_credentials.py +++ b/gslib/tests/test_gcs_json_credentials.py @@ -38,12 +38,6 @@ add_move(MovedModule("mock", "mock", "unittest.mock")) from six.moves import mock -try: - from OpenSSL.crypto import load_pkcs12 - HAS_OPENSSL = True -except ImportError: - HAS_OPENSSL = False - try: import cryptography HAS_CRYPTO = True diff --git a/gslib/tests/test_signurl.py b/gslib/tests/test_signurl.py index ea0ac6c86..c550df41f 100644 --- a/gslib/tests/test_signurl.py +++ b/gslib/tests/test_signurl.py @@ -189,9 +189,15 @@ def testSignUrlWithURLEncodeRequiredChars(self): lines = lines[1:] for obj, line, partial_url in zip(objs, lines, expected_partial_urls): - self.assertIn(obj, line) - self.assertIn(partial_url, line) - self.assertIn('x-goog-credential='+TEST_EMAIL, line) + # If we are on Windows, the output might be escaped to avoid crashes + try: + self.assertIn(obj, line) + except AssertionError: + # Escape the object string to match the 'backslashreplace' output + escaped_obj = obj.encode('ascii', 'backslashreplace').decode('ascii') + self.assertIn(escaped_obj, line) + self.assertIn(partial_url, line) + self.assertIn('x-goog-credential='+TEST_EMAIL, line) self.assertIn('%2Fus%2F', stdout) def testSignUrlWithWildcard(self): @@ -239,8 +245,9 @@ def testSignUrlOfNonObjectUrl(self): """Tests the signurl output of a non-existent file.""" self.RunGsUtil(['signurl', self._GetJSONKsFile(), 'gs://'], expected_status=1) - self.RunGsUtil(['signurl', 'file://tmp/abc', 'gs://bucket'], - expected_status=1) + self.RunGsUtil(['signurl', 'file:///tmp/abc', 'gs://bucket'], + expected_status=1, + return_stderr=True) @unittest.skipUnless(HAVE_OPENSSL, 'signurl requires pyopenssl.') diff --git a/gslib/utils/boto_util.py b/gslib/utils/boto_util.py index 3d00b30c9..919763e05 100644 --- a/gslib/utils/boto_util.py +++ b/gslib/utils/boto_util.py @@ -114,7 +114,7 @@ def ConfigureNoOpAuthIfNeeded(): raise CommandException('\n'.join( textwrap.wrap( 'Your gsutil is configured with an OAuth2 service account, but ' - 'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. ' + 'you do not have cryptography installed. ' 'Service account authentication requires one of these libraries; ' 'please reactivate your service account via the gcloud auth ' 'command and ensure any gcloud packages necessary for ' @@ -123,7 +123,7 @@ def ConfigureNoOpAuthIfNeeded(): raise CommandException('\n'.join( textwrap.wrap( 'Your gsutil is configured with an OAuth2 service account, but ' - 'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. ' + 'you do not have cryptography installed. ' 'Service account authentication requires one of these libraries; ' 'please install either of them to proceed, or configure a ' 'different type of credentials with "gsutil config".'))) diff --git a/gslib/utils/signurl_helper.py b/gslib/utils/signurl_helper.py index b0c6a9d22..d95fa78c8 100644 --- a/gslib/utils/signurl_helper.py +++ b/gslib/utils/signurl_helper.py @@ -135,7 +135,7 @@ def CreatePayload(client_id, def GetFinalUrl(raw_signature, host, path, canonical_query_string): """Get the final signed url.""" - signature = base64.b16encode(raw_signature).lower().decode() + signature = base64.b16encode(raw_signature).lower().decode(UTF8) return _SIGNED_URL_FORMAT.format(host=host, path=path, sig=signature, diff --git a/gslib/vendored/oauth2client b/gslib/vendored/oauth2client index 30b5394b0..d331bc033 160000 --- a/gslib/vendored/oauth2client +++ b/gslib/vendored/oauth2client @@ -1 +1 @@ -Subproject commit 30b5394b0295117b68ac54a57d96160757ce01d0 +Subproject commit d331bc033048174650dd4731b6edc5fe3e94cd55 diff --git a/setup.py b/setup.py index d6374dbba..7b065fc2c 100755 --- a/setup.py +++ b/setup.py @@ -47,7 +47,8 @@ # 3.0.5 is the last version that supports Python 3.3 or lower. 'mock>=2.0.0, <=3.0.5; python_version < "3.3"', 'monotonic>=1.4', - 'pyOpenSSL>=0.13, <=24.2.1', + 'pyOpenSSL>=26.0.0', + 'cryptography>=46.0.0', 'retry_decorator>=1.0.0', 'six>=1.17.0', # aiohttp is the extra dependency that contains requests lib.