Prod Deploy

README.rst

@@ -62,6 +62,7 @@ At the moment, boto supports:
 * Application Services
 
   * Amazon CloudSearch (Python 3)
+  * Amazon CloudSearch Domain (Python 3)
   * Amazon Elastic Transcoder (Python 3)
   * Amazon Simple Workflow Service (SWF) (Python 3)
   * Amazon Simple Queue Service (SQS) (Python 3)
@@ -179,7 +180,7 @@ boto config file.  See `this`_ for details.
 .. _github.com: http://github.com/boto/boto
 .. _Online documentation: http://docs.pythonboto.org
 .. _Python Cheese Shop: http://pypi.python.org/pypi/boto
-.. _this: http://code.google.com/p/boto/wiki/BotoConfig
+.. _this: http://docs.pythonboto.org/en/latest/boto_config_tut.html
 .. _gitflow: http://nvie.com/posts/a-successful-git-branching-model/
 .. _neo: https://github.com/boto/boto/tree/neo
 .. _boto-users Google Group: https://groups.google.com/forum/?fromgroups#!forum/boto-users

boto/__init__.py

@@ -664,6 +664,7 @@ def connect_cloudsearch(aws_access_key_id=None,
 
 def connect_cloudsearch2(aws_access_key_id=None,
                          aws_secret_access_key=None,
+                         sign_request=False,
                          **kwargs):
     """
     :type aws_access_key_id: string
@@ -672,14 +673,37 @@ def connect_cloudsearch2(aws_access_key_id=None,
     :type aws_secret_access_key: string
     :param aws_secret_access_key: Your AWS Secret Access Key
 
+    :type sign_request: bool
+    :param sign_request: whether or not to sign search and
+        upload requests
+
     :rtype: :class:`boto.cloudsearch2.layer2.Layer2`
     :return: A connection to Amazon's CloudSearch2 service
     """
     from boto.cloudsearch2.layer2 import Layer2
     return Layer2(aws_access_key_id, aws_secret_access_key,
+                  sign_request=sign_request,
                   **kwargs)
 
 
+def connect_cloudsearchdomain(aws_access_key_id=None,
+                              aws_secret_access_key=None,
+                              **kwargs):
+    """
+    :type aws_access_key_id: string
+    :param aws_access_key_id: Your AWS Access Key ID
+
+    :type aws_secret_access_key: string
+    :param aws_secret_access_key: Your AWS Secret Access Key
+
+    :rtype: :class:`boto.cloudsearchdomain.layer1.CloudSearchDomainConnection`
+    :return: A connection to Amazon's CloudSearch Domain service
+    """
+    from boto.cloudsearchdomain.layer1 import CloudSearchDomainConnection
+    return CloudSearchDomainConnection(aws_access_key_id,
+                                       aws_secret_access_key, **kwargs)
+
+
 def connect_beanstalk(aws_access_key_id=None,
                       aws_secret_access_key=None,
                       **kwargs):

boto/auth.py

@@ -51,6 +51,16 @@
     sha256 = None
 
 
+# Region detection strings to determine if SigV4 should be used
+# by default.
+SIGV4_DETECT = [
+    '.cn-',
+    # In eu-central we support both host styles for S3
+    '.eu-central',
+    '-eu-central',
+]
+
+
 class HmacKeys(object):
     """Key based Auth handler helper."""
 
@@ -359,7 +369,7 @@ def canonical_headers(self, headers_to_sign):
 
         for header in headers_to_sign:
             c_name = header.lower().strip()
-            raw_value = headers_to_sign[header]
+            raw_value = str(headers_to_sign[header])
             if '"' in raw_value:
                 c_value = raw_value.strip()
             else:
@@ -773,8 +783,11 @@ class QueryAuthHandler(AuthHandler):
     capability = ['pure-query']
 
     def _escape_value(self, value):
-        # Would normally be ``return urllib.parse.quote(value)``.
-        return value
+        # This is changed from a previous version because this string is
+        # being passed to the query string and query strings must
+        # be url encoded. In particular STS requires the saml_response to
+        # be urlencoded when calling assume_role_with_saml.
+        return urllib.parse.quote(value)
 
     def _build_query_string(self, params):
         keys = list(params.keys())
@@ -1000,9 +1013,9 @@ def _wrapper(self):
             # ``boto/iam/connection.py``, as several things there are also
             # endpoint-related.
             if getattr(self.region, 'endpoint', ''):
-                if '.cn-' in self.region.endpoint or \
-                        '.eu-central' in self.region.endpoint:
-                    return ['hmac-v4']
+                for test in SIGV4_DETECT:
+                    if test in self.region.endpoint:
+                        return ['hmac-v4']
 
         return func(self)
     return _wrapper
@@ -1020,8 +1033,9 @@ def _wrapper(self):
             # If you're making changes here, you should also check
             # ``boto/iam/connection.py``, as several things there are also
             # endpoint-related.
-            if '.cn-' in self.host or '.eu-central' in self.host:
-                return ['hmac-v4-s3']
+            for test in SIGV4_DETECT:
+                if test in self.host:
+                    return ['hmac-v4-s3']
 
         return func(self)
     return _wrapper

boto/cloudformation/connection.py

@@ -884,5 +884,4 @@ def set_stack_policy(self, stack_name_or_id, stack_policy_body=None,
             params['StackPolicyURL'] = stack_policy_url
 
         response = self._do_request('SetStackPolicy', params, '/', 'POST')
-        return response['SetStackPolicyResponse']\
-                       ['SetStackPolicyResult']
+        return response['SetStackPolicyResponse']

boto/cloudsearch2/document.py

@@ -25,6 +25,7 @@
 from boto.compat import json
 import requests
 import boto
+from boto.cloudsearchdomain.layer1 import CloudSearchDomainConnection
 
 
 class SearchServiceException(Exception):
@@ -93,11 +94,25 @@ def __init__(self, domain=None, endpoint=None):
         self.documents_batch = []
         self._sdf = None
 
-        # Copy proxy settings from connection
-        if self.domain and self.domain.layer1 and self.domain.layer1.use_proxy:
-            self.proxy = {'http': self.domain.layer1.get_proxy_url_with_auth()}
-        else:
-            self.proxy = {}
+        # Copy proxy settings from connection and check if request should be signed
+        self.proxy = {}
+        self.sign_request = False
+        if self.domain and self.domain.layer1:
+            if self.domain.layer1.use_proxy:
+                self.proxy = {'http': self.domain.layer1.get_proxy_url_with_auth()}
+
+            self.sign_request = getattr(self.domain.layer1, 'sign_request', False)
+
+            if self.sign_request:
+                # Create a domain connection to send signed requests
+                layer1 = self.domain.layer1
+                self.domain_connection = CloudSearchDomainConnection(
+                    host=self.endpoint,
+                    aws_access_key_id=layer1.aws_access_key_id,
+                    aws_secret_access_key=layer1.aws_secret_access_key,
+                    region=layer1.region,
+                    provider=layer1.provider
+                )
 
     def add(self, _id, fields):
         """
@@ -164,6 +179,26 @@ def add_sdf_from_s3(self, key_obj):
 
         self._sdf = key_obj.get_contents_as_string()
 
+    def _commit_with_auth(self, sdf, api_version):
+        return self.domain_connection.upload_documents(sdf, 'application/json')
+
+    def _commit_without_auth(self, sdf, api_version):
+        url = "http://%s/%s/documents/batch" % (self.endpoint, api_version)
+
+        # Keep-alive is automatic in a post-1.0 requests world.
+        session = requests.Session()
+        session.proxies = self.proxy
+        adapter = requests.adapters.HTTPAdapter(
+            pool_connections=20,
+            pool_maxsize=50,
+            max_retries=5
+        )
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
+
+        resp = session.post(url, data=sdf, headers={'Content-Type': 'application/json'})
+        return resp
+
     def commit(self):
         """
         Actually send an SDF to CloudSearch for processing
@@ -184,24 +219,15 @@ def commit(self):
             boto.log.error(sdf[index - 100:index + 100])
 
         api_version = '2013-01-01'
-        if self.domain:
+        if self.domain and self.domain.layer1:
             api_version = self.domain.layer1.APIVersion
-        url = "http://%s/%s/documents/batch" % (self.endpoint, api_version)
 
-        # Keep-alive is automatic in a post-1.0 requests world.
-        session = requests.Session()
-        session.proxies = self.proxy
-        adapter = requests.adapters.HTTPAdapter(
-            pool_connections=20,
-            pool_maxsize=50,
-            max_retries=5
-        )
-        session.mount('http://', adapter)
-        session.mount('https://', adapter)
-        r = session.post(url, data=sdf,
-                         headers={'Content-Type': 'application/json'})
+        if self.sign_request:
+            r = self._commit_with_auth(sdf, api_version)
+        else:
+            r = self._commit_without_auth(sdf, api_version)
 
-        return CommitResponse(r, self, sdf)
+        return CommitResponse(r, self, sdf, signed_request=self.sign_request)
 
 
 class CommitResponse(object):
@@ -219,20 +245,24 @@ class CommitResponse(object):
     :raises: :class:`boto.cloudsearch2.document.EncodingError`
     :raises: :class:`boto.cloudsearch2.document.ContentTooLongError`
     """
-    def __init__(self, response, doc_service, sdf):
+    def __init__(self, response, doc_service, sdf, signed_request=False):
         self.response = response
         self.doc_service = doc_service
         self.sdf = sdf
+        self.signed_request = signed_request
 
-        _body = response.content.decode('utf-8')
+        if self.signed_request:
+            self.content = response
+        else:
+            _body = response.content.decode('utf-8')
 
-        try:
-            self.content = json.loads(_body)
-        except:
-            boto.log.error('Error indexing documents.\nResponse Content:\n{0}'
-                           '\n\nSDF:\n{1}'.format(_body, self.sdf))
-            raise boto.exception.BotoServerError(self.response.status_code, '',
-                                                 body=_body)
+            try:
+                self.content = json.loads(_body)
+            except:
+                boto.log.error('Error indexing documents.\nResponse Content:\n{0}'
+                               '\n\nSDF:\n{1}'.format(_body, self.sdf))
+                raise boto.exception.BotoServerError(self.response.status_code, '',
+                                                     body=_body)
 
         self.status = self.content['status']
         if self.status == 'error':
@@ -266,7 +296,10 @@ def _check_num_ops(self, type_, response_num):
                           if d['type'] == type_])
 
         if response_num != commit_num:
-            boto.log.debug(self.response.content)
+            if self.signed_request:
+                boto.log.debug(self.response)
+            else:
+                boto.log.debug(self.response.content)
             # There will always be a commit mismatch error if there is any
             # errors on cloudsearch. self.errors gets lost when this
             # CommitMismatchError is raised. Whoever is using boto has no idea

boto/cloudsearch2/layer1.py

@@ -56,7 +56,6 @@ class CloudSearchConnection(AWSQueryConnection):
         "BaseException": exceptions.BaseException,
     }
 
-
     def __init__(self, **kwargs):
         region = kwargs.pop('region', None)
         if not region:
@@ -66,6 +65,9 @@ def __init__(self, **kwargs):
         if 'host' not in kwargs or kwargs['host'] is None:
             kwargs['host'] = region.endpoint
 
+        sign_request = kwargs.pop('sign_request', False)
+        self.sign_request = sign_request
+
         super(CloudSearchConnection, self).__init__(**kwargs)
         self.region = region
 

boto/cloudsearch2/layer2.py

@@ -32,7 +32,7 @@ class Layer2(object):
     def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
                  is_secure=True, port=None, proxy=None, proxy_port=None,
                  host=None, debug=0, session_token=None, region=None,
-                 validate_certs=True):
+                 validate_certs=True, sign_request=False):
 
         if isinstance(region, six.string_types):
             import boto.cloudsearch2
@@ -52,7 +52,8 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
             debug=debug,
             security_token=session_token,
             region=region,
-            validate_certs=validate_certs)
+            validate_certs=validate_certs,
+            sign_request=sign_request)
 
     def list_domains(self, domain_names=None):
         """