Skip to content

Add SHA256 fallback if MD5 isn't available on the system #9534

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 9, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changes/next-release/enhancement-cloudformation-37336.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"type": "enhancement",
"category": "``cloudformation``",
"description": "High-level Cloudformation commands such as ``deploy`` and ``package`` will now fallback to SHA256 for checksumming if MD5 isn't present on the host"
}
24 changes: 17 additions & 7 deletions awscli/customizations/s3uploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

import hashlib
import logging
import threading
import os
import sys

import botocore
import botocore.exceptions
from botocore.compat import get_md5
from s3transfer.manager import TransferManager
from s3transfer.subscribers import BaseSubscriber

Expand All @@ -27,6 +27,16 @@
LOG = logging.getLogger(__name__)


def _get_checksum():
hashlib_params = {"usedforsecurity": False}
try:
checksum = get_md5(**hashlib_params)
except botocore.exceptions.MD5UnavailableError:
import hashlib
checksum = hashlib.sha256(**hashlib_params)
return checksum


class NoSuchBucketError(Exception):
def __init__(self, **kwargs):
msg = self.fmt.format(**kwargs)
Expand Down Expand Up @@ -128,7 +138,7 @@ def upload(self, file_name, remote_path):

def upload_with_dedup(self, file_name, extension=None):
"""
Makes and returns name of the S3 object based on the file's MD5 sum
Makes and returns name of the S3 object based on the file's checksum

:param file_name: file to upload
:param extension: String of file extension to append to the object
Expand All @@ -140,8 +150,8 @@ def upload_with_dedup(self, file_name, extension=None):
# and re-upload only if necessary. So the template points to same file
# in multiple places, this will upload only once

filemd5 = self.file_checksum(file_name)
remote_path = filemd5
file_checksum = self.file_checksum(file_name)
remote_path = file_checksum
if extension:
remote_path = remote_path + "." + extension

Expand Down Expand Up @@ -172,7 +182,7 @@ def make_url(self, obj_path):
def file_checksum(self, file_name):

with open(file_name, "rb") as file_handle:
md5 = hashlib.md5()
checksum = _get_checksum()
# Read file in chunks of 4096 bytes
block_size = 4096

Expand All @@ -182,13 +192,13 @@ def file_checksum(self, file_name):

buf = file_handle.read(block_size)
while len(buf) > 0:
md5.update(buf)
checksum.update(buf)
buf = file_handle.read(block_size)

# Restore file cursor's position
file_handle.seek(curpos)

return md5.hexdigest()
return checksum.hexdigest()

def to_path_style_s3_url(self, key, version=None):
"""
Expand Down
8 changes: 4 additions & 4 deletions awscli/examples/cloudformation/_package_description.rst
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ current working directory. The exception is ``AWS::ApiGateway::RestApi``;
if you don't specify a ``BodyS3Location``, this command will not upload an artifact to S3.

Before the command uploads artifacts, it checks if the artifacts are already
present in the S3 bucket to prevent unnecessary uploads. The command uses MD5
checksums to compare files. If the values match, the command doesn't upload the
artifacts. Use the ``--force-upload flag`` to skip this check and always upload the
artifacts.
present in the S3 bucket to prevent unnecessary uploads. If the values match, the
command doesn't upload the artifacts. Use the ``--force-upload flag`` to skip this
check and always upload the artifacts. The command uses MD5 checksums to compare
files by default. If MD5 is not available in the environment, a SHA256 checksum is used.

21 changes: 21 additions & 0 deletions tests/unit/customizations/test_s3uploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,27 @@ def test_file_checksum(self):
finally:
shutil.rmtree(tempdir)

@mock.patch("awscli.customizations.s3uploader.get_md5")
def test_file_checksum_fips_fallback(self, get_md5_mock):
num_chars = 4096*5
data = ''.join(random.choice(string.ascii_uppercase)
for _ in range(num_chars)).encode('utf-8')
checksum = hashlib.sha256(usedforsecurity=False)
checksum.update(data)
expected_checksum = checksum.hexdigest()

tempdir = tempfile.mkdtemp()
get_md5_mock.side_effect = botocore.exceptions.MD5UnavailableError()
try:
filename = os.path.join(tempdir, 'tempfile')
with open(filename, 'wb') as f:
f.write(data)

actual_checksum = self.s3uploader.file_checksum(filename)
self.assertEqual(expected_checksum, actual_checksum)
finally:
shutil.rmtree(tempdir)

def test_make_url(self):
path = "Hello/how/are/you"
expected = "s3://{0}/{1}".format(self.bucket_name, path)
Expand Down