diff --git a/README.md b/README.md index ba88d14..f6d04ea 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,38 @@ # s3-bash4 -_s3-bash4_ is a small collection of _Bash_ scripts to do simple interaction with _Amazon S3_ using [_AWS Signature Version 4_](http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html). The advantage of using _s3-bash4_ is that it's extremly lightweight and easy to use. No need to setup _Python_, _Java_, _Ruby_ and co. +_s3-bash4_ is a small collection of _Bash_ scripts to do simple interaction with _Amazon S3_ using [_AWS Signature Version 4_](http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html). The advantage of using _s3-bash4_ is that it's extremely lightweight and easy to use. No need to setup _Python_, _Java_, _Ruby_ and co. This is inspired by the discontinued [_s3-bash_](https://github.com/cosmin/s3-bash) from _cosmin_. I was in need of a _Bash_ version that supports the newer _AWS Signature Version 4_. -### Usage - # Get file from bucket: - s3-get -k {accesskey} -s /{path}/{secretid} -r {region} /{bucketname}/{filename} > {filename} +## Usage - # Upload file to bucket: - s3-put -k {accesskey} -s /{path}/{secretid} -r {region} -T /{path}/{filename} /{bucketname}/{filename} +```sh + # Get file from bucket: + s3-get -k {accesskey} -s /{path}/{secretid} -r {region} [-d {customdomain}] /{bucketname}/{filename} > {filename} - # Delete from bucket: - s3-delete -k {accesskey} -s /{path}/{secretid} -r {region} /{bucketname}/{filename} + # Upload file to bucket: + s3-put -k {accesskey} -s /{path}/{secretid} -r {region} [-d {customdomain}] -T /{path}/{filename} /{bucketname}/{filename} -### Environment Variables + # Delete from bucket: + s3-delete -k {accesskey} -s /{path}/{secretid} -r {region} [-d {customdomain}] /{bucketname}/{filename} +``` + +## Environment Variables Variable | Description ---------------------- | ----------------------------------------------------------- `AWS_ACCESS_KEY_ID` | Default _AWS Access Key ID_ `AWS_SECRET_ACCESS_KEY`| Default _AWS Secret Access Key_ `AWS_DEFAULT_REGION` | Default _AWS S3 Region_ +`S3_DEFAULT_DOMAIN` | Default _S3 custom domain_ `AWS_SECURITY_TOKEN` | Default _AWS Security Token for temporary credentials_ -### Requirements - - _OpenSSL_ - - _cUrl_ +## Requirements + +- _GNU Bash_ +- _OpenSSL_ +- _cUrl_ + +## License -### License -_Apache License Version 2.0_ +This work is released under the _Apache License Version 2.0_ diff --git a/STYLE_GUIDE b/STYLE_GUIDE index 9180961..a10a5c3 100644 --- a/STYLE_GUIDE +++ b/STYLE_GUIDE @@ -1 +1 @@ -This project follows the following style guide: https://google-styleguide.googlecode.com/svn/trunk/shell.xml +This project follows the following style guide: https://google.github.io/styleguide/shellguide.html diff --git a/bin/s3-delete b/bin/s3-delete index 16e80f8..41dc3da 100755 --- a/bin/s3-delete +++ b/bin/s3-delete @@ -2,6 +2,7 @@ # # Delete a file from S3 # (c) 2015 Chi Vinh Le +# (c) 2024 Orange SA — author: benoit.bailleux@orange.com # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy of @@ -17,12 +18,16 @@ set -euo pipefail -readonly PROJECT_PATH=$( cd $(dirname $0) ; cd ../; pwd -P ) -readonly SCRIPT_NAME="$(basename $0)" +PROJECT_PATH=$( cd "$(dirname "$0")" ; cd ../; pwd -P ) +readonly PROJECT_PATH +SCRIPT_NAME="$(basename "$0")" +readonly SCRIPT_NAME +# shellcheck disable=SC2034 # rule "METHOD appears unused" readonly METHOD="DELETE" # Includes -source ${PROJECT_PATH}/lib/s3-common.sh +# shellcheck source=../lib/s3-common.sh +source "${PROJECT_PATH}/lib/s3-common.sh" ## # Print help and exit @@ -33,21 +38,22 @@ source ${PROJECT_PATH}/lib/s3-common.sh ## printUsageAndExitWith() { printf "Usage:\n" - printf " ${SCRIPT_NAME} [-vi] [-k key] [-s file] [-r region] resource_path\n" - printf " ${SCRIPT_NAME} -h\n" + printf " %s [-vi] [-k key] [-s file] [-r region] [-d domain] resource_path\n" "${SCRIPT_NAME}" + printf " %s -h\n" "${SCRIPT_NAME}" printf "Example:\n" - printf " ${SCRIPT_NAME} -k key -s secret -r eu-central-1 /bucket/file.ext\n" + printf " %s -k key -s secret -r eu-central-1 /bucket/file.ext\n" "${SCRIPT_NAME}" printf "Options:\n" printf " --debug\tEnable debugging mode\n" printf " -h,--help\tPrint this help\n" printf " -i,--insecure\tUse http instead of https\n" printf " -k,--key\tAWS Access Key ID. Default to environment variable AWS_ACCESS_KEY_ID\n" printf " -r,--region\tAWS S3 Region. Default to environment variable AWS_DEFAULT_REGION\n" + printf " -d,--domain\tS3 custom network domain. Default to environment variable S3_DEFAULT_DOMAIN or 'amazonaws.com'\n" printf " -s,--secret\tFile containing AWS Secret Access Key. If not set, secret will be environment variable AWS_SECRET_ACCESS_KEY\n" printf " -t,--token\tSecurity token for temporary credentials. If not set, token will be environment variable AWS_SECURITY_TOKEN\n" printf " -v,--verbose\tVerbose output\n" printf " --version\tShow version\n" - exit $1 + exit "$1" } ## @@ -58,6 +64,7 @@ printUsageAndExitWith() { # AWS_ACCESS_KEY_ID string # AWS_SECRET_ACCESS_KEY string # AWS_REGION string +# S3_DEFAULT_DOMAIN string # AWS_SECURITY_TOKEN string # RESOURCE_PATH string # VERBOSE bool @@ -67,6 +74,7 @@ printUsageAndExitWith() { parseCommandLine() { # Init globals AWS_REGION=${AWS_DEFAULT_REGION:-""} + S3_DOMAIN=${S3_DEFAULT_DOMAIN:-""} AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-""} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-""} AWS_SECURITY_TOKEN=${AWS_SECURITY_TOKEN:-""} @@ -77,7 +85,7 @@ parseCommandLine() { # Parse options local remaining= local secretKeyFile= - while [[ $# > 0 ]]; do + while [[ $# -gt 0 ]]; do local key="$1" case ${key} in --version) showVersionAndExit;; @@ -85,29 +93,30 @@ parseCommandLine() { -h|--help) printUsageAndExitWith 0;; -v|--verbose) VERBOSE=true;; -i|--insecure) INSECURE=true;; - -r|--region) assertArgument $@; AWS_REGION=$2; shift;; - -k|--key) assertArgument $@; AWS_ACCESS_KEY_ID=$2; shift;; - -s|--secret) assertArgument $@; secretKeyFile=$2; shift;; - -t|--token) assertArgument $@; AWS_SECURITY_TOKEN=$2; shift;; + -r|--region) assertArgument "$@"; AWS_REGION=$2; shift;; + -d|--domain) assertArgument "$@"; S3_DOMAIN=$2; shift;; + -k|--key) assertArgument "$@"; AWS_ACCESS_KEY_ID=$2; shift;; + -s|--secret) assertArgument "$@"; secretKeyFile=$2; shift;; + -t|--token) assertArgument "$@"; AWS_SECURITY_TOKEN=$2; shift;; -*) err "Unknown option $1" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE};; + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}";; *) remaining="${remaining} \"${key}\"";; esac shift done # Set the non-parameters back into the positional parameters ($1 $2 ..) - eval set -- ${remaining} + eval set -- "${remaining}" # Read secret file if set - if ! [[ -z "${secretKeyFile}" ]]; then + if [[ -n "${secretKeyFile}" ]]; then AWS_SECRET_ACCESS_KEY=$(processAWSSecretFile "${secretKeyFile}") fi # Parse arguments - if [[ $# != 1 ]]; then + if [[ $# -ne 1 ]]; then err "You need to specify the resource path to download e.g. /bucket/file.ext" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi assertResourcePath "$1" @@ -115,19 +124,20 @@ parseCommandLine() { if [[ -z "${AWS_REGION}" ]]; then err "AWS Region not specified" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then err "AWS Access Key ID not specified" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi if [[ -z "${AWS_SECRET_ACCESS_KEY}" ]]; then err "AWS Secret Access Key not specified" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi # Freeze globals readonly AWS_REGION + readonly S3_DOMAIN readonly AWS_ACCESS_KEY_ID readonly AWS_SECRET_ACCESS_KEY readonly RESOURCE_PATH @@ -141,8 +151,8 @@ parseCommandLine() { ## main() { checkEnvironment - parseCommandLine $@ + parseCommandLine "$@" performRequest } -main $@ +main "$@" diff --git a/bin/s3-get b/bin/s3-get index 79d6471..6efe7e7 100755 --- a/bin/s3-get +++ b/bin/s3-get @@ -2,6 +2,7 @@ # # Download a file from S3 # (c) 2015 Chi Vinh Le +# (c) 2024 Orange SA — author: benoit.bailleux@orange.com # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy of @@ -17,12 +18,16 @@ set -euo pipefail -readonly PROJECT_PATH=$( cd $(dirname $0) ; cd ../; pwd -P ) -readonly SCRIPT_NAME="$(basename $0)" +PROJECT_PATH=$( cd "$(dirname "$0")" ; cd ../; pwd -P ) +readonly PROJECT_PATH +SCRIPT_NAME="$(basename "$0")" +readonly SCRIPT_NAME +# shellcheck disable=SC2034 # rule "METHOD appears unused" readonly METHOD="GET" # Includes -source ${PROJECT_PATH}/lib/s3-common.sh +# shellcheck source=../lib/s3-common.sh +source "${PROJECT_PATH}/lib/s3-common.sh" ## # Print help and exit @@ -33,21 +38,22 @@ source ${PROJECT_PATH}/lib/s3-common.sh ## printUsageAndExitWith() { printf "Usage:\n" - printf " ${SCRIPT_NAME} [-vi] [-k key] [-s file] [-r region] resource_path\n" - printf " ${SCRIPT_NAME} -h\n" + printf " %s [-vi] [-k key] [-s file] [-r region] [-d domain] resource_path\n" "${SCRIPT_NAME}" + printf " %s -h\n" "${SCRIPT_NAME}" printf "Example:\n" - printf " ${SCRIPT_NAME} -k key -s secret -r eu-central-1 /bucket/file.ext\n" + printf " %s -k key -s secret -r eu-central-1 /bucket/file.ext\n" "${SCRIPT_NAME}" printf "Options:\n" printf " --debug\tEnable debugging mode\n" printf " -h,--help\tPrint this help\n" printf " -i,--insecure\tUse http instead of https\n" printf " -k,--key\tAWS Access Key ID. Default to environment variable AWS_ACCESS_KEY_ID\n" printf " -r,--region\tAWS S3 Region. Default to environment variable AWS_DEFAULT_REGION\n" + printf " -d,--domain\tS3 custom network domain. Default to environment variable S3_DEFAULT_DOMAIN or 'amazonaws.com'\n" printf " -s,--secret\tFile containing AWS Secret Access Key. If not set, secret will be environment variable AWS_SECRET_ACCESS_KEY\n" printf " -t,--token\tSecurity token for temporary credentials. If not set, token will be environment variable AWS_SECURITY_TOKEN\n" printf " -v,--verbose\tVerbose output\n" printf " --version\tShow version\n" - exit $1 + exit "$1" } ## @@ -58,6 +64,7 @@ printUsageAndExitWith() { # AWS_ACCESS_KEY_ID string # AWS_SECRET_ACCESS_KEY string # AWS_REGION string +# S3_DEFAULT_DOMAIN string # AWS_SECURITY_TOKEN string # RESOURCE_PATH string # VERBOSE bool @@ -67,6 +74,7 @@ printUsageAndExitWith() { parseCommandLine() { # Init globals AWS_REGION=${AWS_DEFAULT_REGION:-""} + S3_DOMAIN=${S3_DEFAULT_DOMAIN:-""} AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-""} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-""} AWS_SECURITY_TOKEN=${AWS_SECURITY_TOKEN:-""} @@ -77,7 +85,7 @@ parseCommandLine() { # Parse options local remaining= local secretKeyFile= - while [[ $# > 0 ]]; do + while [[ $# -gt 0 ]]; do local key="$1" case ${key} in --version) showVersionAndExit;; @@ -85,29 +93,30 @@ parseCommandLine() { -h|--help) printUsageAndExitWith 0;; -v|--verbose) VERBOSE=true;; -i|--insecure) INSECURE=true;; - -r|--region) assertArgument $@; AWS_REGION=$2; shift;; - -k|--key) assertArgument $@; AWS_ACCESS_KEY_ID=$2; shift;; - -s|--secret) assertArgument $@; secretKeyFile=$2; shift;; - -t|--token) assertArgument $@; AWS_SECURITY_TOKEN=$2; shift;; + -r|--region) assertArgument "$@"; AWS_REGION=$2; shift;; + -d|--domain) assertArgument "$@"; S3_DOMAIN=$2; shift;; + -k|--key) assertArgument "$@"; AWS_ACCESS_KEY_ID=$2; shift;; + -s|--secret) assertArgument "$@"; secretKeyFile=$2; shift;; + -t|--token) assertArgument "$@"; AWS_SECURITY_TOKEN=$2; shift;; -*) err "Unknown option $1" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE};; + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}";; *) remaining="${remaining} \"${key}\"";; esac shift done # Set the non-parameters back into the positional parameters ($1 $2 ..) - eval set -- ${remaining} + eval set -- "${remaining}" # Read secret file if set - if ! [[ -z "${secretKeyFile}" ]]; then + if [[ -n "${secretKeyFile}" ]]; then AWS_SECRET_ACCESS_KEY=$(processAWSSecretFile "${secretKeyFile}") fi # Parse arguments - if [[ $# != 1 ]]; then + if [[ $# -ne 1 ]]; then err "You need to specify the resource path to download e.g. /bucket/file.ext" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi assertResourcePath "$1" @@ -115,19 +124,20 @@ parseCommandLine() { if [[ -z "${AWS_REGION}" ]]; then err "AWS Region not specified" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then err "AWS Access Key ID not specified" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi if [[ -z "${AWS_SECRET_ACCESS_KEY}" ]]; then err "AWS Secret Access Key not specified" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi # Freeze globals readonly AWS_REGION + readonly S3_DOMAIN readonly AWS_ACCESS_KEY_ID readonly AWS_SECRET_ACCESS_KEY readonly RESOURCE_PATH @@ -141,8 +151,8 @@ parseCommandLine() { ## main() { checkEnvironment - parseCommandLine $@ + parseCommandLine "$@" performRequest } -main $@ +main "$@" diff --git a/bin/s3-put b/bin/s3-put index 40397e6..d910256 100755 --- a/bin/s3-put +++ b/bin/s3-put @@ -2,6 +2,7 @@ # # Upload a file to S3 # (c) 2015 Chi Vinh Le +# (c) 2024 Orange SA — author: benoit.bailleux@orange.com # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy of @@ -17,12 +18,16 @@ set -euo pipefail -readonly PROJECT_PATH=$( cd $(dirname $0) ; cd ../; pwd -P ) -readonly SCRIPT_NAME="$(basename $0)" +PROJECT_PATH=$( cd "$(dirname "$0")" ; cd ../; pwd -P ) +readonly PROJECT_PATH +SCRIPT_NAME="$(basename "$0")" +readonly SCRIPT_NAME +# shellcheck disable=SC2034 # rule "METHOD appears unused" readonly METHOD="PUT" # Includes -source ${PROJECT_PATH}/lib/s3-common.sh +# shellcheck source=../lib/s3-common.sh +source "${PROJECT_PATH}/lib/s3-common.sh" ## # Print help and exit @@ -33,25 +38,25 @@ source ${PROJECT_PATH}/lib/s3-common.sh ## printUsageAndExitWith() { printf "Usage:\n" - printf " ${SCRIPT_NAME} [--debug] [-vip] [-k key] [-r region] [-s file] [-c content_type] -T file_to_upload resource_path\n" - printf " ${SCRIPT_NAME} -h\n" + printf " %s [--debug] [-vip] [-k key] [-r region] [-d domain] [-s file] [-c content_type] -T file_to_upload resource_path\n" "${SCRIPT_NAME}" + printf " %s -h\n" "${SCRIPT_NAME}" printf "Example:\n" - printf " ${SCRIPT_NAME} -k key -s secret -r eu-central-1 -T file.ext -c text/plain /bucket/file.ext\n" + printf " %s -k key -s secret -r eu-central-1 -T file.ext -c text/plain /bucket/file.ext\n" "${SCRIPT_NAME}" printf "Options:\n" - printf " -c,--content-type\tMIME content type\n" printf " --debug\tEnable debugging mode\n" + printf " -c,--content-type\tMIME content type\n" printf " -h,--help\tPrint this help\n" printf " -i,--insecure\tUse http instead of https\n" printf " -k,--key\tAWS Access Key ID. Default to environment variable AWS_ACCESS_KEY_ID\n" printf " -p,--public\tGrant public read on uploaded file\n" printf " -r,--region\tAWS S3 Region. Default to environment variable AWS_DEFAULT_REGION\n" + printf " -d,--domain\tS3 custom network domain. Default to environment variable S3_DEFAULT_DOMAIN or 'amazonaws.com'\n" printf " -s,--secret\tFile containing AWS Secret Access Key. If not set, secret will be environment variable AWS_SECRET_ACCESS_KEY\n" printf " -t,--token\tSecurity token for temporary credentials. If not set, token will be environment variable AWS_SECURITY_TOKEN\n" printf " -T,--upload-file\tPath to file to upload\n" printf " -v,--verbose\tVerbose output\n" printf " --version\tShow version\n" - - exit $1 + exit "$1" } ## @@ -62,6 +67,7 @@ printUsageAndExitWith() { # AWS_ACCESS_KEY_ID string # AWS_SECRET_ACCESS_KEY string # AWS_REGION string +# S3_DEFAULT_DOMAIN string # AWS_SECURITY_TOKEN string # RESOURCE_PATH string # FILE_TO_UPLOAD string @@ -75,6 +81,7 @@ printUsageAndExitWith() { parseCommandLine() { # Init globals AWS_REGION=${AWS_DEFAULT_REGION:-""} + S3_DOMAIN=${S3_DEFAULT_DOMAIN:-""} AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-""} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-""} AWS_SECURITY_TOKEN=${AWS_SECURITY_TOKEN:-""} @@ -88,7 +95,7 @@ parseCommandLine() { # Parse options local remaining= local secretKeyFile= - while [[ $# > 0 ]]; do + while [[ $# -gt 0 ]]; do local key="$1" case ${key} in --version) showVersionAndExit;; @@ -97,57 +104,59 @@ parseCommandLine() { -v|--verbose) VERBOSE=true;; -i|--insecure) INSECURE=true;; -p|--publish) PUBLISH=true;; - -c|--content-type) assertArgument $@; CONTENT_TYPE=$2; shift;; - -T|--upload-file) assertArgument $@; FILE_TO_UPLOAD=$2; shift;; - -r|--region) assertArgument $@; AWS_REGION=$2; shift;; - -k|--key) assertArgument $@; AWS_ACCESS_KEY_ID=$2; shift;; - -s|--secret) assertArgument $@; secretKeyFile=$2; shift;; - -t|--token) assertArgument $@; AWS_SECURITY_TOKEN=$2; shift;; + -c|--content-type) assertArgument "$@"; CONTENT_TYPE=$2; shift;; + -T|--upload-file) assertArgument "$@"; FILE_TO_UPLOAD=$2; shift;; + -r|--region) assertArgument "$@"; AWS_REGION=$2; shift;; + -d|--domain) assertArgument "$@"; S3_DOMAIN=$2; shift;; + -k|--key) assertArgument "$@"; AWS_ACCESS_KEY_ID=$2; shift;; + -s|--secret) assertArgument "$@"; secretKeyFile=$2; shift;; + -t|--token) assertArgument "$@"; AWS_SECURITY_TOKEN=$2; shift;; -*) err "Unknown option $1" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE};; + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}";; *) remaining="${remaining} \"${key}\"";; esac shift done # Set the non-parameters back into the positional parameters ($1 $2 ..) - eval set -- ${remaining} + eval set -- "${remaining}" # Read secret file if set - if ! [[ -z "${secretKeyFile}" ]]; then + if [[ -n "${secretKeyFile}" ]]; then AWS_SECRET_ACCESS_KEY=$(processAWSSecretFile "${secretKeyFile}") fi # Parse arguments - if [[ $# != 1 ]]; then + if [[ $# -ne 1 ]]; then err "You need to specify the resource path to upload to e.g. /bucket/file.ext" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi - assertResourcePath $1 + assertResourcePath "$1" RESOURCE_PATH="$1" if [[ -z "${FILE_TO_UPLOAD}" ]]; then err "You need to specify the file to upload using -T" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi assertFileExists "${FILE_TO_UPLOAD}" if [[ -z "${AWS_REGION}" ]]; then err "AWS Region not specified" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then err "AWS Access Key ID not specified" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi if [[ -z "${AWS_SECRET_ACCESS_KEY}" ]]; then err "AWS Secret Access Key not specified" - printUsageAndExitWith ${INVALID_USAGE_EXIT_CODE} + printUsageAndExitWith "${INVALID_USAGE_EXIT_CODE}" fi # Freeze globals readonly AWS_REGION + readonly S3_DOMAIN readonly AWS_ACCESS_KEY_ID readonly AWS_SECRET_ACCESS_KEY readonly RESOURCE_PATH @@ -163,8 +172,8 @@ parseCommandLine() { ## main() { checkEnvironment - parseCommandLine $@ + parseCommandLine "$@" performRequest } -main $@ +main "$@" diff --git a/lib/s3-common.sh b/lib/s3-common.sh index 7c8d8f8..b8e47fd 100644 --- a/lib/s3-common.sh +++ b/lib/s3-common.sh @@ -2,9 +2,22 @@ # # Common functions for s3-bash4 commands # (c) 2015 Chi Vinh Le +# (c) 2024 Orange SA — author: benoit.bailleux@orange.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. # Constants -readonly VERSION="0.0.1" +readonly VERSION="0.0.2" # Exit codes readonly INVALID_USAGE_EXIT_CODE=1 @@ -17,7 +30,7 @@ readonly INVALID_ENVIRONMENT_EXIT_CODE=3 # $1 string to output ## err() { - echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')] Error: $@" >&2 + echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')] Error:" "$@" >&2 } @@ -25,7 +38,7 @@ err() { # Display version and exit ## showVersionAndExit() { - printf "$VERSION\n" + printf "%s\n" "$VERSION" exit } @@ -33,7 +46,7 @@ showVersionAndExit() { # Helper for parsing the command line. ## assertArgument() { - if [[ $# -lt 2 ]]; then + if [[ $# -lt 2 || $2 =~ ^--?[a-z]+ ]]; then # One following string that is not another option err "Option $1 needs an argument." exit $INVALID_USAGE_EXIT_CODE fi @@ -58,7 +71,7 @@ assertResourcePath() { ## assertFileExists() { if [[ ! -f $1 ]]; then - err "$1 file doesn't exists" + err "$1 file does not exist" exit $INVALID_USER_DATA_EXIT_CODE fi } @@ -70,7 +83,7 @@ checkEnvironment() { programs=(openssl curl printf echo sed awk od date pwd dirname) for program in "${programs[@]}"; do - if [ ! -x "$(which $program)" ]; then + if [ ! -x "$(which "$program")" ]; then err "$program is required to run" exit $INVALID_ENVIRONMENT_EXIT_CODE fi @@ -102,19 +115,21 @@ processAWSSecretFile() { fi # limit file size to max 41 characters. 40 + potential null terminating character. - local fileSize="$(ls -l "$1" | awk '{ print $5 }')" + local fileSize + # shellcheck disable=SC2012 # Rule "Use find instead of ls to better handle non-alphanumeric filenames" + fileSize="$(ls -l "$1" | awk '{ print $5 }')" if [[ $fileSize -gt 41 ]]; then - err $errStr + err "$errStr" exit $INVALID_USER_DATA_EXIT_CODE fi - secret=$(<$1) + secret=$(<"$1") # exact string size should be 40. - if [[ ${#secret} != 40 ]]; then - err $errStr + if [[ ${#secret} -ne 40 ]]; then + err "$errStr" exit $INVALID_USER_DATA_EXIT_CODE fi - echo $secret + echo "$secret" } ## @@ -125,7 +140,7 @@ processAWSSecretFile() { # string hex ## hex256() { - printf "$1" | od -A n -t x1 | sed ':a;N;$!ba;s/[\n ]//g' + echo -n "$1" | od -A n -t x1 | tr -d "\n " } ## @@ -136,7 +151,9 @@ hex256() { # string hash ## sha256Hash() { - local output=$(printf "$1" | $SHACMD) + local output + # shellcheck disable=SC2059 # Rule "Don't use variables in the printf format string" + output=$(printf "$1" | $SHACMD) echo "${output%% *}" } @@ -148,7 +165,8 @@ sha256Hash() { # string hash ## sha256HashFile() { - local output=$($SHACMD $1) + local output + output=$($SHACMD "$1") echo "${output%% *}" } @@ -161,7 +179,8 @@ sha256HashFile() { # string signature ## hmac_sha256() { - printf "$2" | openssl dgst -binary -hex -sha256 -mac HMAC -macopt hexkey:$1 \ + # shellcheck disable=SC2059 # Rule "Don't use variables in the printf format string" + printf "$2" | openssl dgst -binary -hex -sha256 -mac HMAC -macopt hexkey:"$1" \ | sed 's/^.* //' } @@ -177,25 +196,35 @@ hmac_sha256() { # signature ## sign() { - local kSigning=$(hmac_sha256 $(hmac_sha256 $(hmac_sha256 \ - $(hmac_sha256 $(hex256 "AWS4$1") $2) $3) $4) "aws4_request") + local kSigning + kSigning=$(hmac_sha256 "$(hmac_sha256 "$(hmac_sha256 \ + "$(hmac_sha256 "$(hex256 "AWS4$1")" "$2")" "$3")" "$4")" "aws4_request") hmac_sha256 "${kSigning}" "$5" } ## # Get endpoint of specified region +# Uses the following Globals: +# S3_DEFAULT_DOMAIN string +# S3_DOMAIN string # Arguments: # $1 region # Returns: # amazon andpoint ## convS3RegionToEndpoint() { + local domain + if [[ -n "${S3_DOMAIN}" ]]; then + domain="${S3_DOMAIN}" + else + domain=${S3_DEFAULT_DOMAIN:-"amazonaws.com"} + fi case "$1" in - us-east-1) echo "s3.amazonaws.com" + us-east-1) echo "s3.${domain}" ;; - *) echo s3-${1}.amazonaws.com + *) echo "s3-${1}.${domain}" ;; - esac + esac } ## @@ -214,19 +243,24 @@ convS3RegionToEndpoint() { # INSECURE bool ## performRequest() { - local timestamp=$(date -u "+%Y-%m-%d %H:%M:%S") - local isoTimestamp=$(date -ud "${timestamp}" "+%Y%m%dT%H%M%SZ") - local dateScope=$(date -ud "${timestamp}" "+%Y%m%d") - local host=$(convS3RegionToEndpoint "${AWS_REGION}") + local timestamp + timestamp=$(date -u "+%Y-%m-%d %H:%M:%S") + local isoTimestamp + isoTimestamp=$(date -ud "${timestamp}" "+%Y%m%dT%H%M%SZ") + local dateScope + dateScope=$(date -ud "${timestamp}" "+%Y%m%d") + local host + host=$(convS3RegionToEndpoint "${AWS_REGION}") # Generate payload hash - if [[ $METHOD == "PUT" ]]; then - local payloadHash=$(sha256HashFile $FILE_TO_UPLOAD) + local payloadHash + if [[ ${METHOD} == "PUT" ]]; then + payloadHash=$(sha256HashFile "$FILE_TO_UPLOAD") else - local payloadHash=$(sha256Hash "") + payloadHash=$(sha256Hash "") fi - local cmd=("curl") + local cmd=("curl" "-sS") # Silent, but show errors local headers= local headerList= @@ -243,7 +277,7 @@ performRequest() { fi cmd+=("-X" "${METHOD}") - if [[ ${METHOD} == "PUT" && ! -z "${CONTENT_TYPE}" ]]; then + if [[ ${METHOD} == "PUT" && -n "${CONTENT_TYPE}" ]]; then cmd+=("-H" "Content-Type: ${CONTENT_TYPE}") headers+="content-type:${CONTENT_TYPE}\n" headerList+="content-type;" @@ -283,7 +317,8 @@ ${headerList} ${payloadHash}" # Generated request hash - local hashedRequest=$(sha256Hash "${canonicalRequest}") + local hashedRequest + hashedRequest=$(sha256Hash "${canonicalRequest}") # Generate signing data local stringToSign="AWS4-HMAC-SHA256 @@ -292,7 +327,8 @@ ${dateScope}/${AWS_REGION}/s3/aws4_request ${hashedRequest}" # Sign data - local signature=$(sign "${AWS_SECRET_ACCESS_KEY}" "${dateScope}" "${AWS_REGION}" \ + local signature + signature=$(sign "${AWS_SECRET_ACCESS_KEY}" "${dateScope}" "${AWS_REGION}" \ "s3" "${stringToSign}") local authorizationHeader="AWS4-HMAC-SHA256 Credential=${AWS_ACCESS_KEY_ID}/${dateScope}/${AWS_REGION}/s3/aws4_request, SignedHeaders=${headerList}, Signature=${signature}" diff --git a/test/run.sh b/test/run.sh index c9465a4..5ee4989 100755 --- a/test/run.sh +++ b/test/run.sh @@ -2,6 +2,7 @@ # # Run tests for s3-bash4 commands # (c) 2015 Chi Vinh Le +# (c) 2024 Orange SA — author: benoit.bailleux@orange.com # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy of @@ -17,11 +18,14 @@ set -euo pipefail -readonly PROJECT_PATH=$(dirname $(pwd)) -readonly SCRIPT_NAME="$(basename $0)" +PROJECT_PATH=$(dirname "$(pwd)") +readonly PROJECT_PATH +SCRIPT_NAME="$(basename "$0")" +readonly SCRIPT_NAME # Includes -source ${PROJECT_PATH}/lib/s3-common.sh +# shellcheck source=../lib/s3-common.sh +source "${PROJECT_PATH}/lib/s3-common.sh" ## # Print help and exit @@ -32,17 +36,18 @@ source ${PROJECT_PATH}/lib/s3-common.sh ## printUsageAndExitWith() { printf "Usage:\n" - printf " $SCRIPT_NAME [-k key] [-s file] [-r region] resource_path\n" - printf " $SCRIPT_NAME -h\n" + printf " %s [-k key] [-s file] [-r region] [-d domain] resource_path\n" "${SCRIPT_NAME}" + printf " %s -h\n" "${SCRIPT_NAME}" printf "Example:\n" - printf " $SCRIPT_NAME -k key -s secret -r eu-central-1 /bucket/file.ext\n" + printf " %s -k key -s secret -r eu-central-1 /bucket/file.ext\n" "${SCRIPT_NAME}" printf "Options:\n" printf " -h,--help\tPrint this help\n" printf " -k,--key\tAWS Access Key ID. Default to environment variable AWS_ACCESS_KEY_ID\n" printf " -r,--region\tAWS S3 Region. Default to environment variable AWS_DEFAULT_REGION\n" + printf " -d,--domain\tS3 custom network domain. Default to environment variable S3_DEFAULT_DOMAIN\n" printf " -s,--secret\tFile containing AWS Secret Access Key. If not set, secret will be environment variable AWS_SECRET_ACCESS_KEY\n" printf " --version\tShow version\n" - exit $1 + exit "$1" } ## @@ -58,38 +63,40 @@ printUsageAndExitWith() { parseCommandLine() { # Init globals AWS_REGION=${AWS_DEFAULT_REGION:-""} + S3_DOMAIN=${S3_DEFAULT_DOMAIN:-""} AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-""} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-""} # Parse options local remaining= local secretKeyFile= - while [[ $# > 0 ]]; do + while [[ $# -gt 0 ]]; do local key="$1" case $key in -h|--help) printUsageAndExitWith 0;; - -r|--region) assertArgument $@; AWS_REGION=$2; shift;; - -k|--key) assertArgument $@; AWS_ACCESS_KEY_ID=$2; shift;; - -s|--secret) assertArgument $@; secretKeyFile=$2; shift;; + -r|--region) assertArgument "$@"; AWS_REGION=$2; shift;; + -d|--domain) assertArgument "$@"; S3_DOMAIN=$2; shift;; + -k|--key) assertArgument "$@"; AWS_ACCESS_KEY_ID=$2; shift;; + -s|--secret) assertArgument "$@"; secretKeyFile=$2; shift;; -*) err "Unknown option $1" - printUsageAndExitWith $INVALID_USAGE_EXIT_CODE;; + printUsageAndExitWith "$INVALID_USAGE_EXIT_CODE";; *) remaining="$remaining \"$key\"";; esac shift done # Set the non-parameters back into the positional parameters ($1 $2 ..) - eval set -- $remaining + eval set -- "$remaining" # Read secret file if set - if ! [[ -z "$secretKeyFile" ]]; then + if [[ -n "$secretKeyFile" ]]; then AWS_SECRET_ACCESS_KEY=$(processAWSSecretFile "$secretKeyFile") fi # Parse arguments - if [[ $# != 1 ]]; then + if [[ $# -ne 1 ]]; then err "You need to specify the resource path to download e.g. /bucket/file.ext" - printUsageAndExitWith $INVALID_USAGE_EXIT_CODE + printUsageAndExitWith "$INVALID_USAGE_EXIT_CODE" fi assertResourcePath "$1" @@ -97,6 +104,7 @@ parseCommandLine() { # Freeze globals readonly AWS_REGION + readonly S3_DOMAIN readonly AWS_ACCESS_KEY_ID readonly AWS_SECRET_ACCESS_KEY readonly RESOURCE_PATH @@ -106,13 +114,14 @@ parseCommandLine() { # Main routine ## main() { - parseCommandLine $@ + parseCommandLine "$@" local get="${PROJECT_PATH}/bin/s3-get" local put="${PROJECT_PATH}/bin/s3-put" local delete="${PROJECT_PATH}/bin/s3-delete" local testfile="${PROJECT_PATH}/test/testfile" export AWS_DEFAULT_REGION=${AWS_REGION} + export S3_DEFAULT_DOMAIN=${S3_DOMAIN} export AWS_ACCESS_KEY_ID export AWS_SECRET_ACCESS_KEY @@ -126,4 +135,4 @@ main() { "${delete}" "${RESOURCE_PATH}" } -main $@ +main "$@"