diff --git a/app/config.py b/app/config.py index f03176f27f..9c630e0d0d 100644 --- a/app/config.py +++ b/app/config.py @@ -254,6 +254,7 @@ class Config(object): AWS_REGION = os.getenv("AWS_REGION", "us-east-1") AWS_ROUTE53_ZONE = os.getenv("AWS_ROUTE53_ZONE", "Z2OW036USASMAK") AWS_SES_REGION = os.getenv("AWS_SES_REGION", "us-east-1") + AWS_SES_SMTP = os.getenv("AWS_SES_SMTP", "email-smtp.ca-central-1.amazonaws.com") AWS_SES_ACCESS_KEY = os.getenv("AWS_SES_ACCESS_KEY") AWS_SES_SECRET_KEY = os.getenv("AWS_SES_SECRET_KEY") AWS_PINPOINT_REGION = os.getenv("AWS_PINPOINT_REGION", "us-west-2") diff --git a/app/models.py b/app/models.py index bd974aa359..37fff1fb19 100644 --- a/app/models.py +++ b/app/models.py @@ -613,6 +613,7 @@ class Service(BaseModel, Versioned): sms_annual_limit = db.Column(db.BigInteger, nullable=False, default=DEFAULT_SMS_ANNUAL_LIMIT) suspended_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey("users.id"), nullable=True) suspended_at = db.Column(db.DateTime, nullable=True) + smtp_user = db.Column(db.String(255), nullable=True, unique=False) email_branding = db.relationship( "EmailBranding", diff --git a/app/service/rest.py b/app/service/rest.py index 4a55b6d148..fddfe4db20 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -131,6 +131,9 @@ from app.user.users_schema import post_set_permissions_schema from app.utils import pagination_links +from app.smtp.aws import (smtp_add, smtp_get_user_key, smtp_remove) +from nanoid import generate + service_blueprint = Blueprint("service", __name__) register_errors(service_blueprint) @@ -1170,6 +1173,53 @@ def get_monthly_notification_data_by_service(): return jsonify(result) +@service_blueprint.route('//smtp', methods=['GET']) +def get_smtp_relay(service_id): + service = dao_fetch_service_by_id(service_id) + if service.smtp_user is not None: + credentials = { + "domain": service.smtp_user.split("-")[0], + "name": current_app.config["AWS_SES_SMTP"], + "port": "465", + "tls": "Yes", + "username": smtp_get_user_key(service.smtp_user), + } + return jsonify(credentials), 200 + else: + return jsonify({}), 200 + + +@service_blueprint.route('//smtp', methods=['POST']) +def create_smtp_relay(service_id): + service = dao_fetch_service_by_id(service_id) + + if service.smtp_user is None: + user_id = generate(size=6) + credentials = smtp_add(user_id) + service.smtp_user = credentials["iam"] + dao_update_service(service) + return jsonify(credentials), 201 + else: + raise InvalidRequest( + message="SMTP user already exists", + status_code=500) + + +@service_blueprint.route('//smtp', methods=['DELETE']) +def delete_smtp_relay(service_id): + service = dao_fetch_service_by_id(service_id) + + if service.smtp_user is not None: + smtp_remove(service.smtp_user) + service.smtp_user = None + dao_update_service(service) + return jsonify(True), 201 + else: + raise InvalidRequest( + message="SMTP user does not exist", + status_code=500) + + def check_unique_name_request_args(request): service_id = request.args.get("service_id") name = request.args.get("name", None) diff --git a/app/smtp/__init__.py b/app/smtp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/app/smtp/aws.py b/app/smtp/aws.py new file mode 100644 index 0000000000..5efd506d0c --- /dev/null +++ b/app/smtp/aws.py @@ -0,0 +1,236 @@ +import base64 # required to encode the computed key +import hashlib # required to create a SHA256 hash +import hmac # required to compute the HMAC key +import time + +import boto3 +from flask import current_app + + +def smtp_add(name): + ses_client = boto3.client( + 'ses', + aws_access_key_id=current_app.config["AWS_SES_ACCESS_KEY"], + aws_secret_access_key=current_app.config["AWS_SES_SECRET_KEY"], + region_name=current_app.config["AWS_SES_REGION"]) + r53_client = boto3.client( + 'route53', + aws_access_key_id=current_app.config["AWS_SES_ACCESS_KEY"], + aws_secret_access_key=current_app.config["AWS_SES_SECRET_KEY"], + region_name=current_app.config["AWS_SES_REGION"]) + iam_client = boto3.client( + 'iam', + aws_access_key_id=current_app.config["AWS_SES_ACCESS_KEY"], + aws_secret_access_key=current_app.config["AWS_SES_SECRET_KEY"], + region_name=current_app.config["AWS_SES_REGION"]) + + notify_email_domain = current_app.config["NOTIFY_EMAIL_DOMAIN"] + current_app.logger.info(f"The current notify email domain is {notify_email_domain}") + name = name + '.m.' + current_app.config["NOTIFY_EMAIL_DOMAIN"] + + token = create_domain_identity(ses_client, name) + add_record(r53_client, '_amazonses.' + name, "\"%s\"" % token, "TXT") + + tokens = get_dkim(ses_client, name) + for token in tokens: + add_record( + r53_client, + token + "._domainkey." + name, + token + ".dkim." + name, + ) + + add_record(r53_client, name, "\"v=spf1 include:amazonses.com ~all\"", "TXT") + add_record( + r53_client, + "_dmarc." + name, + "\"v=DMARC1; p=none; sp=none; rua=mailto:dmarc@cyber.gc.ca; ruf=mailto:dmarc@cyber.gc.ca\"", + "TXT") + + credentials = add_user(iam_client, name) + return credentials + + +def smtp_get_user_key(name): + try: + iam_client = boto3.client('iam', region_name=current_app.config["AWS_SES_REGION"]) + return iam_client.list_access_keys( + UserName=name, + )["AccessKeyMetadata"][0]["AccessKeyId"] + except Exception as e: + raise e + + +def smtp_remove(name): + try: + ses_client = boto3.client( + 'ses', + aws_access_key_id=current_app.config["AWS_SES_ACCESS_KEY"], + aws_secret_access_key=current_app.config["AWS_SES_SECRET_KEY"], + region_name=current_app.config["AWS_SES_REGION"]) + r53_client = boto3.client( + 'route53', + aws_access_key_id=current_app.config["AWS_SES_ACCESS_KEY"], + aws_secret_access_key=current_app.config["AWS_SES_SECRET_KEY"], + region_name=current_app.config["AWS_SES_REGION"]) + iam_client = boto3.client( + 'iam', + aws_access_key_id=current_app.config["AWS_SES_ACCESS_KEY"], + aws_secret_access_key=current_app.config["AWS_SES_SECRET_KEY"], + region_name=current_app.config["AWS_SES_REGION"]) + + [domain, _] = name.split("-") + + policies = iam_client.list_user_policies( + UserName=name, + )["PolicyNames"] + + for policy in policies: + iam_client.delete_user_policy( + UserName=name, + PolicyName=policy + ) + + keys = iam_client.list_access_keys( + UserName=name, + )["AccessKeyMetadata"] + + for key in keys: + iam_client.delete_access_key( + UserName=name, + AccessKeyId=key["AccessKeyId"] + ) + + iam_client.delete_user(UserName=name) + ses_client.delete_identity(Identity=domain) + records = r53_client.list_resource_record_sets( + HostedZoneId=current_app.config["AWS_ROUTE53_ZONE"], + StartRecordName=domain + )["ResourceRecordSets"] + + for record in records: + delete_record(r53_client, record) + + return True + + except Exception as e: + raise e + + +def create_domain_identity(client, name): + try: + return client.verify_domain_identity( + Domain=name + )['VerificationToken'] + except Exception as e: + raise e + + +def get_dkim(client, name): + try: + return client.verify_domain_dkim( + Domain=name + )['DkimTokens'] + except Exception as e: + raise e + + +def add_user(iam_client, name): + try: + user_name = name + "-" + str(int(time.time())) + iam_client.create_user( + Path='/notification-smtp/', + UserName=user_name, + Tags=[ + { + 'Key': 'SMTP-USER', + 'Value': name + }, + ] + ) + + iam_client.put_user_policy( + PolicyDocument=generate_user_policy(name), + PolicyName="SP-" + user_name, + UserName=user_name + ) + + response = iam_client.create_access_key( + UserName=user_name + ) + + credentials = { + "iam": user_name, + "domain": name, + "name": current_app.config["AWS_SES_SMTP"], + "port": "465", + "tls": "Yes", + "username": response["AccessKey"]["AccessKeyId"], + "password": munge(response["AccessKey"]["SecretAccessKey"]) + } + + return credentials + except Exception as e: + raise e + + +def add_record(client, name, value, record_type="CNAME"): + try: + client.change_resource_record_sets( + HostedZoneId=current_app.config["AWS_ROUTE53_ZONE"], + ChangeBatch={ + 'Comment': 'add %s -> %s' % (name, value), + 'Changes': [ + { + 'Action': 'UPSERT', + 'ResourceRecordSet': { + 'Name': name, + 'Type': record_type, + 'TTL': 300, + 'ResourceRecords': [{'Value': value}] + } + }] + }) + except Exception as e: + raise e + + +def delete_record(client, record): + try: + client.change_resource_record_sets( + HostedZoneId=current_app.config["AWS_ROUTE53_ZONE"], + ChangeBatch={ + 'Comment': 'Deleted', + 'Changes': [ + { + 'Action': 'DELETE', + 'ResourceRecordSet': record + } + ] + } + ) + except Exception as e: + raise e + + +def generate_user_policy(name): + policy = ( + '{"Version":"2012-10-17","Statement":' + '[{"Effect":"Allow","Action":["ses:SendRawEmail"],"Resource":"*",' + '"Condition":{"StringLike":{"ses:FromAddress":"*@%s"}}}]}' % name) + return policy + +# Taken from https://docs.aws.amazon.com/ses/latest/DeveloperGuide/example-create-smtp-credentials.html + + +def munge(secret): + message = 'SendRawEmail' + version = '\x02' + + # Compute an HMAC-SHA256 key from the AWS secret access key. + signatureInBytes = hmac.new(secret.encode('utf-8'), message.encode('utf-8'), hashlib.sha256).digest() + # Prepend the version number to the signature. + signatureAndVersion = version.encode('utf-8') + signatureInBytes + # Base64-encode the string that contains the version number and signature. + smtpPassword = base64.b64encode(signatureAndVersion) + # Decode the string and print it to the console. + return smtpPassword.decode('utf-8') diff --git a/migrations/versions/0493_smtp_columns.py b/migrations/versions/0493_smtp_columns.py new file mode 100644 index 0000000000..ef297d4ad6 --- /dev/null +++ b/migrations/versions/0493_smtp_columns.py @@ -0,0 +1,22 @@ +""" + +Revision ID: 0493_smtp_columns +Revises: 0492_add_service_del_template +Create Date: 2025-11-07 17:08:21.019759 + +""" +import sqlalchemy as sa +from alembic import op + +revision = "0493_smtp_columns" +down_revision = "0492_add_service_del_template" + + +def upgrade(): + op.add_column("services", sa.Column("smtp_user", sa.Text(), nullable=True)) + op.add_column("services_history", sa.Column("smtp_user", sa.Text(), nullable=True)) + + +def downgrade(): + op.drop_column("services", "smtp_user") + op.drop_column("services_history", "smtp_user") diff --git a/migrations/versions/0494_smtp_template.py b/migrations/versions/0494_smtp_template.py new file mode 100644 index 0000000000..1a78c5ec2f --- /dev/null +++ b/migrations/versions/0494_smtp_template.py @@ -0,0 +1,70 @@ +""" + +Revision ID: 0494_smtp_template +Revises: 0493_smtp_columns +Create Date: 2025-11-07 17:08:21.019759 + +""" +from alembic import op +from flask import current_app + +revision = "0494_smtp_template" +down_revision = "0493_smtp_columns" + + +templates = [ + { + "id": "3a4cab41-c47d-4d49-96ba-f4c4fa91d44b", + "name": "SMTP Message", + "type": "email", + "subject": "((subject))", + "content_lines": ["((message))"], + }, +] + + +def upgrade(): + insert = """ + INSERT INTO {} (id, name, template_type, created_at, content, archived, service_id, subject, + created_by_id, version, process_type, hidden) + VALUES ('{}', '{}', '{}', current_timestamp, '{}', False, '{}', '{}', '{}', 1, '{}', false) + """ + + for template in templates: + for table_name in "templates", "templates_history": + op.execute( + insert.format( + table_name, + template["id"], + template["name"], + template["type"], + "\n".join(template["content_lines"]), + current_app.config["NOTIFY_SERVICE_ID"], + template.get("subject"), + current_app.config["NOTIFY_USER_ID"], + "normal", + ) + ) + + op.execute( + """ + INSERT INTO template_redacted + ( + template_id, + redact_personalisation, + updated_at, + updated_by_id + ) VALUES ( '{}', false, current_timestamp, '{}' ) + """.format( + template["id"], current_app.config["NOTIFY_USER_ID"] + ) + ) + + +def downgrade(): + for template in templates: + op.execute("DELETE FROM notifications WHERE template_id = '{}'".format(template["id"])) + op.execute("DELETE FROM notification_history WHERE template_id = '{}'".format(template["id"])) + op.execute("DELETE FROM template_redacted WHERE template_id = '{}'".format(template["id"])) + op.execute("DELETE FROM templates WHERE id = '{}'".format(template["id"])) + op.execute("DELETE FROM templates_history WHERE id = '{}'".format(template["id"])) diff --git a/tests/app/clients/test_aws_smtp.py b/tests/app/clients/test_aws_smtp.py new file mode 100644 index 0000000000..a0f8858df6 --- /dev/null +++ b/tests/app/clients/test_aws_smtp.py @@ -0,0 +1,109 @@ +from app.smtp.aws import ( + add_record, + add_user, + create_domain_identity, + delete_record, + generate_user_policy, + get_dkim, + munge, + smtp_add, + smtp_get_user_key, + smtp_remove, +) + + +def test_smtp_add_adds_a_new_sender_domain(mocker, notify_api): + create_domain_identity_mock = mocker.patch("app.smtp.aws.create_domain_identity") + add_record_mock = mocker.patch("app.smtp.aws.add_record") + get_dkim_mock = mocker.patch("app.smtp.aws.get_dkim") + add_user_mock = mocker.patch("app.smtp.aws.add_user") + + with notify_api.app_context(): + smtp_add("foo") + + create_domain_identity_mock.assert_called_once() + add_record_mock.assert_called() + get_dkim_mock.assert_called_once() + add_user_mock.assert_called_once() + + +def test_smtp_get_user_key(mocker, notify_api): + boto_client = mocker.patch("app.smtp.aws.boto3") + boto_client.client.list_access_keys.return_value = mocker.Mock() + + with notify_api.app_context(): + smtp_get_user_key("foo") + + boto_client.client.assert_called() + + +def test_smtp_remove_deletes_a_sender_domain(mocker, notify_api): + boto_client = mocker.patch("app.smtp.aws.boto3") + + with notify_api.app_context(): + smtp_remove("foo-bbar") + + boto_client.client.assert_called() + + +def test_create_domain_identity_calls_verify_domain_identity(mocker, notify_api): + client = mocker.Mock() + client.verify_domain_identity.return_value = {"VerificationToken": ["FOO"]} + + with notify_api.app_context(): + create_domain_identity(client, "foo") + + client.verify_domain_identity.assert_called() + + +def test_get_dkim_calls_verify_domain_dkim(mocker, notify_api): + client = mocker.Mock() + client.verify_domain_dkim.return_value = {"DkimTokens": ["FOO"]} + + with notify_api.app_context(): + get_dkim(client, "foo") + + client.verify_domain_dkim.assert_called() + + +def test_add_user_creates_user_and_sets_policy(mocker, notify_api): + client = mocker.Mock() + client.create_access_key.return_value = {"AccessKey": {"AccessKeyId": "foo", "SecretAccessKey": "bar"}} + + with notify_api.app_context(): + add_user(client, "foo") + + client.create_user.assert_called() + client.put_user_policy.assert_called() + client.create_access_key.assert_called() + + +def test_add_record_calls_change_resource_record_sets(mocker, notify_api): + client = mocker.Mock() + + with notify_api.app_context(): + add_record(client, "foo", "bar") + + client.change_resource_record_sets.assert_called() + + +def test_delete_record_calls_change_resource_record_sets(mocker, notify_api): + client = mocker.Mock() + + with notify_api.app_context(): + delete_record(client, "foo") + + client.change_resource_record_sets.assert_called() + + +def test_generate_user_policy_restricts_policy_by_name(): + policy = ( + '{"Version":"2012-10-17","Statement":' + '[{"Effect":"Allow","Action":["ses:SendRawEmail"],"Resource":"*",' + '"Condition":{"StringLike":{"ses:FromAddress":"*@foo.bar"}}}]}') + + assert generate_user_policy("foo.bar") == policy + + +def test_munge_returns_an_smtp_secret(): + assert munge("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY") == "An60U4ZD3sd4fg+FvXUjayOipTt8LO4rUUmhpdX6ctDy" diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index b5a040eccf..2c70d5cf26 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -6,16 +6,6 @@ import pytest import pytest_mock -from flask import Flask, current_app, url_for -from freezegun import freeze_time -from notifications_utils.clients.redis import ( - daily_limit_cache_key, - near_daily_limit_cache_key, - near_email_daily_limit_cache_key, - over_daily_limit_cache_key, - over_email_daily_limit_cache_key, -) - from app.clients.salesforce.salesforce_engagement import ENGAGEMENT_STAGE_LIVE from app.dao.organisation_dao import dao_add_service_to_organisation from app.dao.service_sms_sender_dao import dao_get_sms_senders_by_service_id @@ -42,6 +32,16 @@ ServiceSmsSender, User, ) +from flask import Flask, current_app, url_for +from freezegun import freeze_time +from notifications_utils.clients.redis import ( + daily_limit_cache_key, + near_daily_limit_cache_key, + near_email_daily_limit_cache_key, + over_daily_limit_cache_key, + over_email_daily_limit_cache_key, +) + from tests import create_authorization_header from tests.app.conftest import ( create_sample_notification, @@ -3358,6 +3358,115 @@ def test_get_annual_limit_stats(self, admin_request, sample_service, mocker): mock_get_stats.assert_called_once_with((sample_service.id)) +class TestSmtpRelay: + + def test_create_smtp_relay_for_service_if_it_already_has_one(client, notify_db, notify_db_session): + service = create_service(service_name="ABCDEF", smtp_user="foo") + + resp = client.post( + '/service/{}/smtp'.format(service.id), + headers=[create_authorization_header()] + ) + + assert resp.status_code == 500 + + + def test_create_smtp_relay_for_service(mocker, client, notify_db, notify_db_session): + service = create_service(service_name="ABCDEF", smtp_user=None) + + credentials = { + "iam": "iam_username", + "domain": "domain", + "name": "smtp.relay", + "port": "465", + "tls": "Yes", + "username": "foo", + "password": "bar" + } + + add_mock = mocker.patch( + "app.service.rest.smtp_add", + return_value=credentials + ) + + resp = client.post( + '/service/{}/smtp'.format(service.id), + headers=[create_authorization_header()] + ) + + add_mock.assert_called_once() + assert resp.status_code == 201 + json_resp = json.loads(resp.get_data(as_text=True)) + assert json_resp == credentials + + + def test_get_smtp_relay_for_service(mocker, client, notify_db, notify_db_session): + service = create_service(service_name="ABCDEF", smtp_user="FOO-BAR") + + username_mock = mocker.patch( + "app.service.rest.smtp_get_user_key", + return_value="bar" + ) + + credentials = { + "domain": "FOO", + "name": "email-smtp.us-east-1.amazonaws.com", + "port": "465", + "tls": "Yes", + "username": "bar", + } + + resp = client.get( + '/service/{}/smtp'.format(service.id), + headers=[create_authorization_header()] + ) + + username_mock.assert_called_once() + assert resp.status_code == 200 + json_resp = json.loads(resp.get_data(as_text=True)) + assert json_resp == credentials + + + def test_get_smtp_relay_for_service_returns_empty_if_none(mocker, client, notify_db, notify_db_session): + service = create_service(service_name="ABCDEF", smtp_user=None) + + resp = client.get( + '/service/{}/smtp'.format(service.id), + headers=[create_authorization_header()] + ) + + assert resp.status_code == 200 + json_resp = json.loads(resp.get_data(as_text=True)) + assert json_resp == {} + + + def test_delete_smtp_relay_for_service_returns_500_if_none(mocker, client, notify_db, notify_db_session): + service = create_service(service_name="ABCDEF", smtp_user=None) + + resp = client.delete( + '/service/{}/smtp'.format(service.id), + headers=[create_authorization_header()] + ) + + assert resp.status_code == 500 + + + def test_delete_smtp_relay_for_service_returns_201_if_success(mocker, client, notify_db, notify_db_session): + service = create_service(service_name="ABCDEF", smtp_user="foo") + + delete_mock = mocker.patch( + "app.service.rest.smtp_remove" + ) + + resp = client.delete( + '/service/{}/smtp'.format(service.id), + headers=[create_authorization_header()] + ) + + delete_mock.assert_called_once() + assert resp.status_code == 201 + + class TestAddUserToService: def test_add_user_to_service_with_send_permissions_succeeds(self, notify_api, notify_db_session): """Test adding a user to a service with send permissions"""