Merge pull request #222 from forrest-mao/master

support multi-zone

Author: longbai

CHANGELOG.md

@@ -1,5 +1,10 @@
 #Changelog
 
+## 7.0.9 (2016-10-09)
+### 增加
+* 多机房接口调用支持
+
+
 ## 7.0.8 (2016-07-05)
 ### 修正
 * 修复表单上传大于20M文件的400错误

qiniu/__init__.py

@@ -9,11 +9,12 @@
 
 # flake8: noqa
 
-__version__ = '7.0.8'
+__version__ = '7.0.9'
 
 from .auth import Auth
 
-from .config import set_default, Zone
+from .config import set_default
+from .zone import Zone
 
 from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, build_batch_delete
 from .services.storage.uploader import put_data, put_file, put_stream

qiniu/auth.py

@@ -58,6 +58,9 @@ def __init__(self, access_key, secret_key):
         self.__access_key = access_key
         self.__secret_key = b(secret_key)
 
+    def get_access_key(self):
+        return self.__access_key
+
     def __token(self, data):
         data = b(data)
         hashed = hmac.new(self.__secret_key, data, sha1)

qiniu/config.py

@@ -1,35 +1,17 @@
 # -*- coding: utf-8 -*-
 
-RS_HOST = 'rs.qbox.me'      # 管理操作Host
-IO_HOST = 'iovip.qbox.me'   # 七牛源站Host
-RSF_HOST = 'rsf.qbox.me'    # 列举操作Host
-API_HOST = 'api.qiniu.com'  # 数据处理操作Host
+from qiniu import zone
 
-_BLOCK_SIZE = 1024 * 1024 * 4  # 断点续上传分块大小，该参数为接口规格，暂不支持修改
-
-
-class Zone(object):
-    """七牛上传区域类
-
-    该类主要内容上传区域地址。
-
-    Attributes:
-        up_host: 首选上传地址
-        up_host_backup: 备用上传地址
-    """
-    def __init__(self, up_host, up_host_backup):
-        """初始化Zone类"""
-        self.up_host, self.up_host_backup = up_host, up_host_backup
+RS_HOST = 'http://rs.qbox.me'      # 管理操作Host
+RSF_HOST = 'http://rsf.qbox.me'    # 列举操作Host
+API_HOST = 'http://api.qiniu.com'  # 数据处理操作Host
 
+_BLOCK_SIZE = 1024 * 1024 * 4  # 断点续上传分块大小，该参数为接口规格，暂不支持修改
 
-zone0 = Zone('up.qiniu.com', 'upload.qiniu.com')
-zone1 = Zone('up-z1.qiniu.com', 'upload-z1.qiniu.com')
 
 _config = {
-    'default_up_host': zone0.up_host,  # 设置为默认上传Host
-    'default_up_host_backup': zone0.up_host_backup,
+    'default_zone': zone.Zone(),
     'default_rs_host': RS_HOST,
-    'default_io_host': IO_HOST,
     'default_rsf_host': RSF_HOST,
     'default_api_host': API_HOST,
     'connection_timeout': 30,        # 链接超时为时间为30s
@@ -44,15 +26,12 @@ def get_default(key):
 
 def set_default(
         default_zone=None, connection_retries=None, connection_pool=None,
-        connection_timeout=None, default_rs_host=None, default_io_host=None,
+        connection_timeout=None, default_rs_host=None,
         default_rsf_host=None, default_api_host=None):
     if default_zone:
-        _config['default_up_host'] = default_zone.up_host
-        _config['default_up_host_backup'] = default_zone.up_host_backup
+        _config['default_zone'] = default_zone
     if default_rs_host:
         _config['default_rs_host'] = default_rs_host
-    if default_io_host:
-        _config['default_io_host'] = default_io_host
     if default_rsf_host:
         _config['default_rsf_host'] = default_rsf_host
     if default_api_host:

qiniu/http.py

@@ -5,7 +5,7 @@
 from requests.auth import AuthBase
 
 from qiniu import config
-from .auth import RequestsAuth
+import qiniu.auth
 from . import __version__
 
 
@@ -49,7 +49,7 @@ def _post(url, data, files, auth):
 def _get(url, params, auth):
     try:
         r = requests.get(
-            url, params=params, auth=RequestsAuth(auth) if auth is not None else None,
+            url, params=params, auth=qiniu.auth.RequestsAuth(auth) if auth is not None else None,
             timeout=config.get_default('connection_timeout'), headers=_headers)
     except Exception as e:
         return None, ResponseInfo(None, e)
@@ -74,7 +74,7 @@ def _post_file(url, data, files):
 
 
 def _post_with_auth(url, data, auth):
-    return _post(url, data, None, RequestsAuth(auth))
+    return _post(url, data, None, qiniu.auth.RequestsAuth(auth))
 
 
 class ResponseInfo(object):

qiniu/services/processing/pfop.py

@@ -45,5 +45,5 @@ def execute(self, key, fops, force=None):
         if force == 1:
             data['force'] = 1
 
-        url = 'http://{0}/pfop'.format(config.get_default('default_api_host'))
+        url = '{0}/pfop'.format(config.get_default('default_api_host'))
         return http._post_with_auth(url, data, self.auth)

qiniu/services/storage/bucket.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 
 from qiniu import config
-from qiniu.utils import urlsafe_base64_encode, entry
 from qiniu import http
+from qiniu.utils import urlsafe_base64_encode, entry
 
 
 class BucketManager(object):
@@ -15,8 +15,12 @@ class BucketManager(object):
         auth: 账号管理密钥对，Auth对象
     """
 
-    def __init__(self, auth):
+    def __init__(self, auth, zone=None):
         self.auth = auth
+        if(zone is None):
+            self.zone = config.get_default('default_zone')
+        else:
+            self.zone = zone
 
     def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None):
         """前缀查询:
@@ -51,7 +55,7 @@ def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None):
         if delimiter is not None:
             options['delimiter'] = delimiter
 
-        url = 'http://{0}/list'.format(config.get_default('default_rsf_host'))
+        url = '{0}/list'.format(config.get_default('default_rsf_host'))
         ret, info = self.__get(url, options)
 
         eof = False
@@ -172,7 +176,7 @@ def fetch(self, url, bucket, key=None):
         """
         resource = urlsafe_base64_encode(url)
         to = entry(bucket, key)
-        return self.__io_do('fetch', resource, 'to/{0}'.format(to))
+        return self.__io_do(bucket, 'fetch', resource, 'to/{0}'.format(to))
 
     def prefetch(self, bucket, key):
         """镜像回源预取文件:
@@ -189,7 +193,7 @@ def prefetch(self, bucket, key):
             一个ResponseInfo对象
         """
         resource = entry(bucket, key)
-        return self.__io_do('prefetch', resource)
+        return self.__io_do(bucket, 'prefetch', resource)
 
     def change_mime(self, bucket, key, mime):
         """修改文件mimeType:
@@ -227,7 +231,7 @@ def batch(self, operations):
                 ]
             一个ResponseInfo对象
         """
-        url = 'http://{0}/batch'.format(config.get_default('default_rs_host'))
+        url = '{0}/batch'.format(config.get_default('default_rs_host'))
         return self.__post(url, dict(op=operations))
 
     def buckets(self):
@@ -245,12 +249,14 @@ def buckets(self):
     def __rs_do(self, operation, *args):
         return self.__server_do(config.get_default('default_rs_host'), operation, *args)
 
-    def __io_do(self, operation, *args):
-        return self.__server_do(config.get_default('default_io_host'), operation, *args)
+    def __io_do(self, bucket, operation, *args):
+        ak = self.auth.get_access_key()
+        io_host = self.zone.get_io_host(ak, bucket)
+        return self.__server_do(io_host, operation, *args)
 
     def __server_do(self, host, operation, *args):
         cmd = _build_op(operation, *args)
-        url = 'http://{0}/{1}'.format(host, cmd)
+        url = '{0}/{1}'.format(host, cmd)
         return self.__post(url)
 
     def __post(self, url, data=None):

qiniu/services/storage/uploader.py

@@ -77,7 +77,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None
         fields['key'] = key
 
     fields['token'] = up_token
-    url = 'http://' + config.get_default('default_up_host') + '/'
+    url = config.get_default('default_zone').get_up_host_by_token(up_token) + '/'
     # name = key if key else file_name
 
     fname = file_name
@@ -87,7 +87,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None
     r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)})
     if r is None and info.need_retry():
         if info.connect_failed:
-            url = 'http://' + config.get_default('default_up_host_backup') + '/'
+            url = config.get_default('default_zone').get_up_host_backup_by_token(up_token) + '/'
         if hasattr(data, 'read') is False:
             pass
         elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()):
@@ -170,7 +170,7 @@ def recovery_from_record(self):
     def upload(self):
         """上传操作"""
         self.blockStatus = []
-        host = config.get_default('default_up_host')
+        host = config.get_default('default_zone').get_up_host_by_token(self.up_token)
         offset = self.recovery_from_record()
         for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset):
             length = len(block)
@@ -179,7 +179,7 @@ def upload(self):
             if ret is None and not info.need_retry():
                 return ret, info
             if info.connect_failed():
-                host = config.get_default('default_up_host_backup')
+                host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token)
             if info.need_retry() or crc != ret['crc32']:
                 ret, info = self.make_block(block, length, host)
                 if ret is None or crc != ret['crc32']:
@@ -197,10 +197,10 @@ def make_block(self, block, block_size, host):
         return self.post(url, block)
 
     def block_url(self, host, size):
-        return 'http://{0}/mkblk/{1}'.format(host, size)
+        return '{0}/mkblk/{1}'.format(host, size)
 
     def file_url(self, host):
-        url = ['http://{0}/mkfile/{1}'.format(host, self.size)]
+        url = ['{0}/mkfile/{1}'.format(host, self.size)]
 
         if self.mime_type:
             url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type)))

qiniu/utils.py

@@ -3,8 +3,6 @@
 from hashlib import sha1
 from base64 import urlsafe_b64encode, urlsafe_b64decode
 
-from .config import _BLOCK_SIZE
-
 from .compat import b, s
 
 try:
@@ -14,6 +12,8 @@
     zlib = None
     import binascii
 
+_BLOCK_SIZE = 1024 * 1024 * 4
+
 
 def urlsafe_base64_encode(data):
     """urlsafe的base64编码: