From afd832263dbc61dc443202ddedb335fecb5b2bee Mon Sep 17 00:00:00 2001 From: ZhaoYu Date: Mon, 18 Nov 2013 13:45:24 +0800 Subject: [PATCH 001/478] 1. catch ret None issue in rsf.py 2. make param expires optional for GetPolicy in rs/rs_token.py --- qiniu/rs/rs_token.py | 4 ++-- qiniu/rsf.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/qiniu/rs/rs_token.py b/qiniu/rs/rs_token.py index fad90198..f0463c1b 100644 --- a/qiniu/rs/rs_token.py +++ b/qiniu/rs/rs_token.py @@ -77,8 +77,8 @@ def token(self, mac=None): class GetPolicy(object): expires = 3600 - def __init__(self): - pass + def __init__(self, expires=3600): + self.expires = expires def make_request(self, base_url, mac=None): ''' diff --git a/qiniu/rsf.py b/qiniu/rsf.py index 0bc51192..c4e5b679 100644 --- a/qiniu/rsf.py +++ b/qiniu/rsf.py @@ -36,6 +36,6 @@ def list_prefix(self, bucket, prefix=None, marker=None, limit=None): ops['prefix'] = prefix url = '%s?%s' % ('/list', urllib.urlencode(ops)) ret, err = self.conn.call_with(url, body=None, content_type='application/x-www-form-urlencoded') - if not ret.get('marker'): + if not ret or not ret.get('marker'): err = EOF return ret, err From 4c4a0201558a8b64192fcf3545cff58853da79b8 Mon Sep 17 00:00:00 2001 From: "Moody \"Kuuy\" Wizmann" Date: Sat, 14 Dec 2013 12:05:48 +0800 Subject: [PATCH 002/478] Update rsf.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``if not ret.get('marker'):`` 在ret为``None``时会报错。 --- qiniu/rsf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/rsf.py b/qiniu/rsf.py index 0bc51192..48d19f48 100644 --- a/qiniu/rsf.py +++ b/qiniu/rsf.py @@ -36,6 +36,6 @@ def list_prefix(self, bucket, prefix=None, marker=None, limit=None): ops['prefix'] = prefix url = '%s?%s' % ('/list', urllib.urlencode(ops)) ret, err = self.conn.call_with(url, body=None, content_type='application/x-www-form-urlencoded') - if not ret.get('marker'): + if ret and not ret.get('marker'): err = EOF return ret, err From c6b8609b7569ede4f18759c16a66095a4a01a1da Mon Sep 17 00:00:00 2001 From: dtynn Date: Wed, 26 Mar 2014 14:12:29 +0800 Subject: [PATCH 003/478] rio 4m bug fixed --- qiniu/resumable_io.py | 3 ++- qiniu/test/resumable_io_test.py | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index 85be9378..380043bd 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -14,6 +14,7 @@ _chunk_size = 256 * 1024 _try_times = 3 _block_size = 4 * 1024 * 1024 +_block_mask = _block_size - 1 class Error(Exception): value = None @@ -141,7 +142,7 @@ def resumable_block_put(block, index, extra, uptoken): def block_count(size): global _block_size - return size / _block_size + 1 + return (size + _block_mask) / _block_size def mkblock(client, block_size, first_chunk): url = "http://%s/mkblk/%s" % (conf.UP_HOST, block_size) diff --git a/qiniu/test/resumable_io_test.py b/qiniu/test/resumable_io_test.py index 19fbbf14..a08fb112 100644 --- a/qiniu/test/resumable_io_test.py +++ b/qiniu/test/resumable_io_test.py @@ -77,6 +77,32 @@ def test_put(self): self.assertEqual(ret["hash"], "FnyTMUqPNRTdk1Wou7oLqDHkBm_p", "hash not match") rs.Client().delete(bucket, key) + def test_put_4m(self): + src = urllib.urlopen("http://for-temp.qiniudn.com/FnIVmMd_oaUV3MLDM6F9in4RMz2U") + ostype = platform.system() + if ostype.lower().find("windows") != -1: + tmpf = "".join([os.getcwd(), os.tmpnam()]) + else: + tmpf = os.tmpnam() + dst = open(tmpf, 'wb') + shutil.copyfileobj(src, dst) + src.close() + + policy = rs.PutPolicy(bucket) + extra = resumable_io.PutExtra(bucket) + extra.bucket = bucket + extra.params = {"x:foo": "test"} + key = "sdk_py_resumable_block_4m_%s" % r(9) + localfile = dst.name + ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) + assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" + dst.close() + os.remove(tmpf) + + assert err is None, err + self.assertEqual(ret["hash"], "FnIVmMd_oaUV3MLDM6F9in4RMz2U", "hash not match") + rs.Client().delete(bucket, key) + if __name__ == "__main__": unittest.main() From f16ce3782c17fd11751e16aab1efb619f5783c27 Mon Sep 17 00:00:00 2001 From: dtynn Date: Wed, 26 Mar 2014 14:31:14 +0800 Subject: [PATCH 004/478] write 4m content --- qiniu/test/resumable_io_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/qiniu/test/resumable_io_test.py b/qiniu/test/resumable_io_test.py index a08fb112..f4effabd 100644 --- a/qiniu/test/resumable_io_test.py +++ b/qiniu/test/resumable_io_test.py @@ -78,21 +78,20 @@ def test_put(self): rs.Client().delete(bucket, key) def test_put_4m(self): - src = urllib.urlopen("http://for-temp.qiniudn.com/FnIVmMd_oaUV3MLDM6F9in4RMz2U") ostype = platform.system() if ostype.lower().find("windows") != -1: tmpf = "".join([os.getcwd(), os.tmpnam()]) else: tmpf = os.tmpnam() dst = open(tmpf, 'wb') - shutil.copyfileobj(src, dst) - src.close() + dst.write("abcd" * 1024 * 1024) + dst.flush() policy = rs.PutPolicy(bucket) extra = resumable_io.PutExtra(bucket) extra.bucket = bucket extra.params = {"x:foo": "test"} - key = "sdk_py_resumable_block_4m_%s" % r(9) + key = "sdk_py_resumable_block_6_%s" % r(9) localfile = dst.name ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" From dcc55f8d4bfefbb945562854843e709f952dc409 Mon Sep 17 00:00:00 2001 From: dtynn Date: Wed, 26 Mar 2014 17:23:26 +0800 Subject: [PATCH 005/478] rewrite --- qiniu/resumable_io_rewrite.py | 160 ++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 qiniu/resumable_io_rewrite.py diff --git a/qiniu/resumable_io_rewrite.py b/qiniu/resumable_io_rewrite.py new file mode 100644 index 00000000..0b3bb5bf --- /dev/null +++ b/qiniu/resumable_io_rewrite.py @@ -0,0 +1,160 @@ +#coding=utf-8 +import os +try: + import zlib as binascii +except ImportError: + import binascii +from base64 import urlsafe_b64encode + +from auth import up as auth_up +import conf + +_workers = 1 +_task_queue_size = _workers * 4 +_try_times = 3 +_block_bits = 22 +_block_size = 1 << _block_bits +_block_mask = _block_size - 1 +_chunk_size = _block_size # 简化模式，弃用 + + +class ResumableIoError(object): + value = None + + def __init__(self, value): + self.value = value + return + + def __str__(self): + return self.value + + +err_invalid_put_progress = ResumableIoError("invalid put progress") +err_put_failed = ResumableIoError("resumable put failed") +err_unmatched_checksum = ResumableIoError("unmatched checksum") +err_putExtra_type = ResumableIoError("extra must the instance of PutExtra") + + +def setup(chunk_size=0, try_times=0): + global _chunk_size, _try_times + _chunk_size = 1 << 22 if chunk_size <= 0 else chunk_size + _try_times = 3 if try_times == 0 else try_times + return + + +def gen_crc32(data): + return binascii.crc32(data) & 0xffffffff + + +class PutExtra(object): + params = None # 自定义用户变量, key需要x: 开头 + mimetype = None # 可选。在 uptoken 没有指定 DetectMime 时，用户客户端可自己指定 MimeType + chunk_size = None # 可选。每次上传的Chunk大小 简化模式，弃用 + try_times = None # 可选。尝试次数 + progresses = None # 可选。上传进度 + notify = lambda self, idx, size, ret: None # 可选。进度提示 + notify_err = lambda self, idx, size, err: None + + def __init__(self, bucket=None): + self.bucket = bucket + return + + +def put_file(uptoken, key, localfile, extra): + """ 上传文件 """ + f = open(localfile, "rb") + statinfo = os.stat(localfile) + ret = put(uptoken, key, f, statinfo.st_size, extra) + f.close() + return ret + + +def put(uptoken, key, f, fsize, extra): + """ 上传二进制流, 通过将data "切片" 分段上传 """ + if not isinstance(extra, PutExtra): + print("extra must the instance of PutExtra") + return + + block_cnt = block_count(fsize) + if extra.progresses is None: + extra.progresses = [None] * block_cnt + else: + if not len(extra.progresses) == block_cnt: + return None, err_invalid_put_progress + + if extra.try_times is None: + extra.try_times = _try_times + + if extra.chunk_size is None: + extra.chunk_size = _chunk_size + + for i in xrange(block_cnt): + try_time = extra.try_times + read_length = _block_size + if (i+1)*_block_size > fsize: + read_length = fsize - i*_block_size + data_slice = f.read(read_length) + while True: + err = resumable_block_put(data_slice, i, extra, uptoken) + if err is None: + break + + try_time -= 1 + if try_time <= 0: + return None, err_put_failed + print err, ".. retry" + + mkfile_client = auth_up.Client(uptoken, extra.progresses[-1]["host"]) + return mkfile(mkfile_client, key, fsize, extra) + + +def resumable_block_put(block, index, extra, uptoken): + block_size = len(block) + + mkblk_client = auth_up.Client(uptoken, conf.UP_HOST) + if extra.progresses[index] is None or "ctx" not in extra.progresses[index]: + crc32 = gen_crc32(block) + block = bytearray(block) + extra.progresses[index], err = mkblock(mkblk_client, block_size, block) + if err is not None: + extra.notify_err(index, block_size, err) + return err + if not extra.progresses[index]["crc32"] == crc32: + return err_unmatched_checksum + extra.notify(index, block_size, extra.progresses[index]) + return + + +def block_count(size): + global _block_size + return (size + _block_mask) / _block_size + + +def mkblock(client, block_size, first_chunk): + url = "http://%s/mkblk/%s" % (conf.UP_HOST, block_size) + content_type = "application/octet-stream" + return client.call_with(url, first_chunk, content_type, len(first_chunk)) + + +def putblock(client, block_ret, chunk): + url = "%s/bput/%s/%s" % (block_ret["host"], block_ret["ctx"], block_ret["offset"]) + content_type = "application/octet-stream" + return client.call_with(url, chunk, content_type, len(chunk)) + + +def mkfile(client, key, fsize, extra): + url = ["http://%s/mkfile/%s" % (conf.UP_HOST, fsize)] + + if extra.mimetype: + url.append("mimeType/%s" % urlsafe_b64encode(extra.mimetype)) + + if key is not None: + url.append("key/%s" % urlsafe_b64encode(key)) + + if extra.params: + for k, v in extra.params.iteritems(): + url.append("%s/%s" % (k, urlsafe_b64encode(v))) + + url = "/".join(url) + body = ",".join([i["ctx"] for i in extra.progresses]) + return client.call_with(url, body, "text/plain", len(body)) \ No newline at end of file From b30d2525c30e82adb3b6a2ac89d9544af971fef7 Mon Sep 17 00:00:00 2001 From: dtynn Date: Wed, 26 Mar 2014 17:53:13 +0800 Subject: [PATCH 006/478] =?UTF-8?q?=E6=9B=BF=E6=8D=A2=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/resumable_io.py | 252 ++++++++++++++++------------------ qiniu/resumable_io_rewrite.py | 160 --------------------- 2 files changed, 120 insertions(+), 292 deletions(-) delete mode 100644 qiniu/resumable_io_rewrite.py diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index 380043bd..0b3bb5bf 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -1,172 +1,160 @@ -# -*- coding: utf-8 -*- +#coding=utf-8 import os try: - import zlib as binascii + import zlib as binascii except ImportError: - import binascii + import binascii from base64 import urlsafe_b64encode -import auth.up +from auth import up as auth_up import conf _workers = 1 _task_queue_size = _workers * 4 -_chunk_size = 256 * 1024 _try_times = 3 -_block_size = 4 * 1024 * 1024 +_block_bits = 22 +_block_size = 1 << _block_bits _block_mask = _block_size - 1 +_chunk_size = _block_size # 简化模式，弃用 -class Error(Exception): - value = None - def __init__(self, value): - self.value = value - def __str__(self): - return self.value -err_invalid_put_progress = Error("invalid put progress") -err_put_failed = Error("resumable put failed") -err_unmatched_checksum = Error("unmatched checksum") +class ResumableIoError(object): + value = None -def setup(chunk_size=0, try_times=0): - """ - * chunk_size => 默认的Chunk大小，不设定则为256k - * try_times => 默认的尝试次数，不设定则为3 - """ - global _chunk_size, _try_times + def __init__(self, value): + self.value = value + return + + def __str__(self): + return self.value - if chunk_size == 0: - chunk_size = 1 << 18 - if try_times == 0: - try_times = 3 +err_invalid_put_progress = ResumableIoError("invalid put progress") +err_put_failed = ResumableIoError("resumable put failed") +err_unmatched_checksum = ResumableIoError("unmatched checksum") +err_putExtra_type = ResumableIoError("extra must the instance of PutExtra") + + +def setup(chunk_size=0, try_times=0): + global _chunk_size, _try_times + _chunk_size = 1 << 22 if chunk_size <= 0 else chunk_size + _try_times = 3 if try_times == 0 else try_times + return - _chunk_size, _try_times = chunk_size, try_times -# ---------------------------------------------------------- def gen_crc32(data): - return binascii.crc32(data) & 0xffffffff + return binascii.crc32(data) & 0xffffffff + class PutExtra(object): - params = None # 自定义用户变量, key需要x: 开头 - mimetype = None # 可选。在 uptoken 没有指定 DetectMime 时，用户客户端可自己指定 MimeType - chunk_size = None # 可选。每次上传的Chunk大小 - try_times = None # 可选。尝试次数 - progresses = None # 可选。上传进度 - notify = lambda self, idx, size, ret: None # 可选。进度提示 - notify_err = lambda self, idx, size, err: None + params = None # 自定义用户变量, key需要x: 开头 + mimetype = None # 可选。在 uptoken 没有指定 DetectMime 时，用户客户端可自己指定 MimeType + chunk_size = None # 可选。每次上传的Chunk大小 简化模式，弃用 + try_times = None # 可选。尝试次数 + progresses = None # 可选。上传进度 + notify = lambda self, idx, size, ret: None # 可选。进度提示 + notify_err = lambda self, idx, size, err: None + + def __init__(self, bucket=None): + self.bucket = bucket + return - def __init__(self, bucket): - self.bucket = bucket def put_file(uptoken, key, localfile, extra): - """ 上传文件 """ - f = open(localfile, "rb") - statinfo = os.stat(localfile) - ret = put(uptoken, key, f, statinfo.st_size, extra) - f.close() - return ret + """ 上传文件 """ + f = open(localfile, "rb") + statinfo = os.stat(localfile) + ret = put(uptoken, key, f, statinfo.st_size, extra) + f.close() + return ret + def put(uptoken, key, f, fsize, extra): - """ 上传二进制流, 通过将data "切片" 分段上传 """ - if not isinstance(extra, PutExtra): - print("extra must the instance of PutExtra") - return - - block_cnt = block_count(fsize) - if extra.progresses is None: - extra.progresses = [None for i in xrange(0, block_cnt)] - else: - if not len(extra.progresses) == block_cnt: - return None, err_invalid_put_progress - - if extra.try_times is None: - extra.try_times = _try_times - - if extra.chunk_size is None: - extra.chunk_size = _chunk_size - - for i in xrange(0, block_cnt): - try_time = extra.try_times - read_length = _block_size - if (i+1)*_block_size > fsize: - read_length = fsize - i*_block_size - data_slice = f.read(read_length) - while True: - err = resumable_block_put(data_slice, i, extra, uptoken) - if err is None: - break - - try_time -= 1 - if try_time <= 0: - return None, err_put_failed - print err, ".. retry" - - mkfile_client = auth.up.Client(uptoken, extra.progresses[-1]["host"]) - return mkfile(mkfile_client, key, fsize, extra) - -# ---------------------------------------------------------- + """ 上传二进制流, 通过将data "切片" 分段上传 """ + if not isinstance(extra, PutExtra): + print("extra must the instance of PutExtra") + return + + block_cnt = block_count(fsize) + if extra.progresses is None: + extra.progresses = [None] * block_cnt + else: + if not len(extra.progresses) == block_cnt: + return None, err_invalid_put_progress + + if extra.try_times is None: + extra.try_times = _try_times + + if extra.chunk_size is None: + extra.chunk_size = _chunk_size + + for i in xrange(block_cnt): + try_time = extra.try_times + read_length = _block_size + if (i+1)*_block_size > fsize: + read_length = fsize - i*_block_size + data_slice = f.read(read_length) + while True: + err = resumable_block_put(data_slice, i, extra, uptoken) + if err is None: + break + + try_time -= 1 + if try_time <= 0: + return None, err_put_failed + print err, ".. retry" + + mkfile_client = auth_up.Client(uptoken, extra.progresses[-1]["host"]) + return mkfile(mkfile_client, key, fsize, extra) + def resumable_block_put(block, index, extra, uptoken): - block_size = len(block) - - mkblk_client = auth.up.Client(uptoken, conf.UP_HOST) - if extra.progresses[index] is None or "ctx" not in extra.progresses[index]: - end_pos = extra.chunk_size-1 - if block_size < extra.chunk_size: - end_pos = block_size-1 - chunk = block[: end_pos] - crc32 = gen_crc32(chunk) - chunk = bytearray(chunk) - extra.progresses[index], err = mkblock(mkblk_client, block_size, chunk) - if not extra.progresses[index]["crc32"] == crc32: - return err_unmatched_checksum - if err is not None: - extra.notify_err(index, end_pos + 1, err) - return err - extra.notify(index, end_pos + 1, extra.progresses[index]) - - bput_client = auth.up.Client(uptoken, extra.progresses[index]["host"]) - while extra.progresses[index]["offset"] < block_size: - offset = extra.progresses[index]["offset"] - chunk = block[offset: offset+extra.chunk_size-1] - crc32 = gen_crc32(chunk) - chunk = bytearray(chunk) - - extra.progresses[index], err = putblock(bput_client, extra.progresses[index], chunk) - if not extra.progresses[index]["crc32"] == crc32: - return err_unmatched_checksum - if err is not None: - extra.notify_err(index, len(chunk), err) - return err - extra.notify(index, len(chunk), extra.progresses[index]) + block_size = len(block) + + mkblk_client = auth_up.Client(uptoken, conf.UP_HOST) + if extra.progresses[index] is None or "ctx" not in extra.progresses[index]: + crc32 = gen_crc32(block) + block = bytearray(block) + extra.progresses[index], err = mkblock(mkblk_client, block_size, block) + if err is not None: + extra.notify_err(index, block_size, err) + return err + if not extra.progresses[index]["crc32"] == crc32: + return err_unmatched_checksum + extra.notify(index, block_size, extra.progresses[index]) + return + def block_count(size): - global _block_size - return (size + _block_mask) / _block_size + global _block_size + return (size + _block_mask) / _block_size + def mkblock(client, block_size, first_chunk): - url = "http://%s/mkblk/%s" % (conf.UP_HOST, block_size) - content_type = "application/octet-stream" - return client.call_with(url, first_chunk, content_type, len(first_chunk)) + url = "http://%s/mkblk/%s" % (conf.UP_HOST, block_size) + content_type = "application/octet-stream" + return client.call_with(url, first_chunk, content_type, len(first_chunk)) + def putblock(client, block_ret, chunk): - url = "%s/bput/%s/%s" % (block_ret["host"], block_ret["ctx"], block_ret["offset"]) - content_type = "application/octet-stream" - return client.call_with(url, chunk, content_type, len(chunk)) + url = "%s/bput/%s/%s" % (block_ret["host"], block_ret["ctx"], block_ret["offset"]) + content_type = "application/octet-stream" + return client.call_with(url, chunk, content_type, len(chunk)) + def mkfile(client, key, fsize, extra): - url = ["http://%s/mkfile/%s" % (conf.UP_HOST, fsize)] + url = ["http://%s/mkfile/%s" % (conf.UP_HOST, fsize)] - if extra.mimetype: - url.append("mimeType/%s" % urlsafe_b64encode(extra.mimetype)) + if extra.mimetype: + url.append("mimeType/%s" % urlsafe_b64encode(extra.mimetype)) - if key is not None: - url.append("key/%s" % urlsafe_b64encode(key)) + if key is not None: + url.append("key/%s" % urlsafe_b64encode(key)) - if extra.params: - for k, v in extra.params.iteritems(): - url.append("%s/%s" % (k, urlsafe_b64encode(v))) + if extra.params: + for k, v in extra.params.iteritems(): + url.append("%s/%s" % (k, urlsafe_b64encode(v))) - url = "/".join(url) - body = ",".join([i["ctx"] for i in extra.progresses]) - return client.call_with(url, body, "text/plain", len(body)) + url = "/".join(url) + body = ",".join([i["ctx"] for i in extra.progresses]) + return client.call_with(url, body, "text/plain", len(body)) \ No newline at end of file diff --git a/qiniu/resumable_io_rewrite.py b/qiniu/resumable_io_rewrite.py deleted file mode 100644 index 0b3bb5bf..00000000 --- a/qiniu/resumable_io_rewrite.py +++ /dev/null @@ -1,160 +0,0 @@ -#coding=utf-8 -import os -try: - import zlib as binascii -except ImportError: - import binascii -from base64 import urlsafe_b64encode - -from auth import up as auth_up -import conf - -_workers = 1 -_task_queue_size = _workers * 4 -_try_times = 3 -_block_bits = 22 -_block_size = 1 << _block_bits -_block_mask = _block_size - 1 -_chunk_size = _block_size # 简化模式，弃用 - - -class ResumableIoError(object): - value = None - - def __init__(self, value): - self.value = value - return - - def __str__(self): - return self.value - - -err_invalid_put_progress = ResumableIoError("invalid put progress") -err_put_failed = ResumableIoError("resumable put failed") -err_unmatched_checksum = ResumableIoError("unmatched checksum") -err_putExtra_type = ResumableIoError("extra must the instance of PutExtra") - - -def setup(chunk_size=0, try_times=0): - global _chunk_size, _try_times - _chunk_size = 1 << 22 if chunk_size <= 0 else chunk_size - _try_times = 3 if try_times == 0 else try_times - return - - -def gen_crc32(data): - return binascii.crc32(data) & 0xffffffff - - -class PutExtra(object): - params = None # 自定义用户变量, key需要x: 开头 - mimetype = None # 可选。在 uptoken 没有指定 DetectMime 时，用户客户端可自己指定 MimeType - chunk_size = None # 可选。每次上传的Chunk大小 简化模式，弃用 - try_times = None # 可选。尝试次数 - progresses = None # 可选。上传进度 - notify = lambda self, idx, size, ret: None # 可选。进度提示 - notify_err = lambda self, idx, size, err: None - - def __init__(self, bucket=None): - self.bucket = bucket - return - - -def put_file(uptoken, key, localfile, extra): - """ 上传文件 """ - f = open(localfile, "rb") - statinfo = os.stat(localfile) - ret = put(uptoken, key, f, statinfo.st_size, extra) - f.close() - return ret - - -def put(uptoken, key, f, fsize, extra): - """ 上传二进制流, 通过将data "切片" 分段上传 """ - if not isinstance(extra, PutExtra): - print("extra must the instance of PutExtra") - return - - block_cnt = block_count(fsize) - if extra.progresses is None: - extra.progresses = [None] * block_cnt - else: - if not len(extra.progresses) == block_cnt: - return None, err_invalid_put_progress - - if extra.try_times is None: - extra.try_times = _try_times - - if extra.chunk_size is None: - extra.chunk_size = _chunk_size - - for i in xrange(block_cnt): - try_time = extra.try_times - read_length = _block_size - if (i+1)*_block_size > fsize: - read_length = fsize - i*_block_size - data_slice = f.read(read_length) - while True: - err = resumable_block_put(data_slice, i, extra, uptoken) - if err is None: - break - - try_time -= 1 - if try_time <= 0: - return None, err_put_failed - print err, ".. retry" - - mkfile_client = auth_up.Client(uptoken, extra.progresses[-1]["host"]) - return mkfile(mkfile_client, key, fsize, extra) - - -def resumable_block_put(block, index, extra, uptoken): - block_size = len(block) - - mkblk_client = auth_up.Client(uptoken, conf.UP_HOST) - if extra.progresses[index] is None or "ctx" not in extra.progresses[index]: - crc32 = gen_crc32(block) - block = bytearray(block) - extra.progresses[index], err = mkblock(mkblk_client, block_size, block) - if err is not None: - extra.notify_err(index, block_size, err) - return err - if not extra.progresses[index]["crc32"] == crc32: - return err_unmatched_checksum - extra.notify(index, block_size, extra.progresses[index]) - return - - -def block_count(size): - global _block_size - return (size + _block_mask) / _block_size - - -def mkblock(client, block_size, first_chunk): - url = "http://%s/mkblk/%s" % (conf.UP_HOST, block_size) - content_type = "application/octet-stream" - return client.call_with(url, first_chunk, content_type, len(first_chunk)) - - -def putblock(client, block_ret, chunk): - url = "%s/bput/%s/%s" % (block_ret["host"], block_ret["ctx"], block_ret["offset"]) - content_type = "application/octet-stream" - return client.call_with(url, chunk, content_type, len(chunk)) - - -def mkfile(client, key, fsize, extra): - url = ["http://%s/mkfile/%s" % (conf.UP_HOST, fsize)] - - if extra.mimetype: - url.append("mimeType/%s" % urlsafe_b64encode(extra.mimetype)) - - if key is not None: - url.append("key/%s" % urlsafe_b64encode(key)) - - if extra.params: - for k, v in extra.params.iteritems(): - url.append("%s/%s" % (k, urlsafe_b64encode(v))) - - url = "/".join(url) - body = ",".join([i["ctx"] for i in extra.progresses]) - return client.call_with(url, body, "text/plain", len(body)) \ No newline at end of file From 48a5f76fc2c72bf972881ddb290da9c7dd424f8a Mon Sep 17 00:00:00 2001 From: Sonic Xiao Date: Fri, 28 Mar 2014 01:00:01 +0800 Subject: [PATCH 007/478] fix mime_type typo --- qiniu/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/rpc.py b/qiniu/rpc.py index 4a224449..4ca701d2 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -112,7 +112,7 @@ def encode_multipart_formdata(self, fields, files): disposition = "Content-Disposition: form-data;" filename = _qiniu_escape(file_info.get('filename')) L.append('%s name="file"; filename="%s"' % (disposition, filename)) - L.append('Content-Type: %s' % file_info.get('content_type', 'application/octet-stream')) + L.append('Content-Type: %s' % file_info.get('mime_type', 'application/octet-stream')) L.append('') L.append('') b2 = CRLF.join(L) From b8969aa045545e6d5112ab5a00a36ceac32f337b Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 28 Mar 2014 14:16:33 +0800 Subject: [PATCH 008/478] update change log --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37273436..577ab80d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ ## CHANGE LOG +### v6.1.4 + +2013-10-24 issue [#95](https://github.com/qiniu/python-sdk/pull/95) + +- [#78] 增加 putpolicy 选项:saveKey,insertOnly,detectMime,fsizeLimit,persistentNotifyUrl,persistentOps +- [#80] 增加 gettoken 过期时间参数，增加 rsf 返回为空的EOF判断 +- [#86] 修正 断点续传的bug +- [#93] 修正 4M 分块计算bug +- [#96] 修正 mime_type typo + ### v6.1.3 2013-10-24 issue [#77](https://github.com/qiniu/python-sdk/pull/77) From a8525775c6d32f1e080a61e4ed24116c31adaa2a Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 28 Mar 2014 14:29:22 +0800 Subject: [PATCH 009/478] change log date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 577ab80d..a7782ec5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### v6.1.4 -2013-10-24 issue [#95](https://github.com/qiniu/python-sdk/pull/95) +2014-03-28 issue [#95](https://github.com/qiniu/python-sdk/pull/95) - [#78] 增加 putpolicy 选项:saveKey,insertOnly,detectMime,fsizeLimit,persistentNotifyUrl,persistentOps - [#80] 增加 gettoken 过期时间参数，增加 rsf 返回为空的EOF判断 From 580f755b884d4c2e2f37d19db2a3ef80bf0cf9e4 Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 28 Mar 2014 16:51:14 +0800 Subject: [PATCH 010/478] init version --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index f461c30e..8300d746 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -7,4 +7,4 @@ ''' # -*- coding: utf-8 -*- -__version__ = '6.1.3' +__version__ = '6.1.4' From f61f10de22e02f3d0ad60b04f112b99e2129ff63 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Thu, 3 Apr 2014 14:07:50 +0800 Subject: [PATCH 011/478] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7782ec5..4f290266 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ 2014-03-28 issue [#95](https://github.com/qiniu/python-sdk/pull/95) -- [#78] 增加 putpolicy 选项:saveKey,insertOnly,detectMime,fsizeLimit,persistentNotifyUrl,persistentOps +- #78 增加 putpolicy 选项:saveKey,insertOnly,detectMime,fsizeLimit,persistentNotifyUrl,persistentOps - [#80] 增加 gettoken 过期时间参数，增加 rsf 返回为空的EOF判断 - [#86] 修正 断点续传的bug - [#93] 修正 4M 分块计算bug From 57eadd0ffe2488f588dd7f1dbfa89709a01f5f3a Mon Sep 17 00:00:00 2001 From: Bai Long Date: Thu, 3 Apr 2014 14:08:16 +0800 Subject: [PATCH 012/478] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f290266..a7782ec5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ 2014-03-28 issue [#95](https://github.com/qiniu/python-sdk/pull/95) -- #78 增加 putpolicy 选项:saveKey,insertOnly,detectMime,fsizeLimit,persistentNotifyUrl,persistentOps +- [#78] 增加 putpolicy 选项:saveKey,insertOnly,detectMime,fsizeLimit,persistentNotifyUrl,persistentOps - [#80] 增加 gettoken 过期时间参数，增加 rsf 返回为空的EOF判断 - [#86] 修正 断点续传的bug - [#93] 修正 4M 分块计算bug From d27b6768140a7b627dcb6892ee5426dfbe0197ce Mon Sep 17 00:00:00 2001 From: dtynn Date: Fri, 4 Apr 2014 16:52:04 +0800 Subject: [PATCH 013/478] gist:new api --- docs/gist/fetch.py | 31 +++++++++++++++++++++++++++++++ docs/gist/pfop.py | 37 +++++++++++++++++++++++++++++++++++++ docs/gist/prefetch.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 docs/gist/fetch.py create mode 100644 docs/gist/pfop.py create mode 100644 docs/gist/prefetch.py diff --git a/docs/gist/fetch.py b/docs/gist/fetch.py new file mode 100644 index 00000000..ba125ed0 --- /dev/null +++ b/docs/gist/fetch.py @@ -0,0 +1,31 @@ +#coding=utf-8 +import sys +sys.path.insert(0, "../../") + +from base64 import urlsafe_b64encode as b64e +from qiniu.auth import digest + +access_key = "" +secret_key = "" + +target_url = "" +dest_bucket = "" +dest_key = "" + +encoded_url = b64e(target_url) +dest_entry = "%s:%s" % (dest_bucket, dest_key) +encoded_entry = b64e(dest_entry) + + +api_host = "iovip.qbox.me" +api_path = "/fetch/%s/to/%s" % (encoded_url, encoded_entry) + +mac = digest.Mac(access=access_key, secret=secret_key) +client = digest.Client(host=api_host, mac=mac) + +ret, err = client.call(path=api_path) +if err is not None: + print "failed" + print err +else: + print "success" \ No newline at end of file diff --git a/docs/gist/pfop.py b/docs/gist/pfop.py new file mode 100644 index 00000000..e993d370 --- /dev/null +++ b/docs/gist/pfop.py @@ -0,0 +1,37 @@ +#coding=utf-8 +import sys +sys.path.insert(0, "../../") + +from urllib import quote +from qiniu.auth import digest + +access_key = "" +secret_key = "" + +bucket = "" +key = "" +fops = "" +notify_url = "" +force = False + +api_host = "api.qiniu.com" +api_path = "/pfop/" +body = "bucket=%s&key=%s&fops=%s¬ifyURL=%s" % \ + (quote(bucket), quote(key), quote(fops), quote(notify_url)) + +body = "%s&force=1" % (body,) if force is not False else body + +content_type = "application/x-www-form-urlencoded" +content_length = len(body) + +mac = digest.Mac(access=access_key, secret=secret_key) +client = digest.Client(host=api_host, mac=mac) + +ret, err = client.call_with(path=api_path, body=body, + content_type=content_type, content_length=content_length) +if err is not None: + print "failed" + print err +else: + print "success" + print ret \ No newline at end of file diff --git a/docs/gist/prefetch.py b/docs/gist/prefetch.py new file mode 100644 index 00000000..ec184965 --- /dev/null +++ b/docs/gist/prefetch.py @@ -0,0 +1,29 @@ +#coding=utf-8 +import sys +sys.path.insert(0, "../../") + +from base64 import urlsafe_b64encode as b64e +from qiniu.auth import digest + +access_key = "" +secret_key = "" + +bucket = "" +key = "" + +entry = "%s:%s" % (bucket, key) +encoded_entry = b64e(entry) + + +api_host = "iovip.qbox.me" +api_path = "/prefetch/%s" % (encoded_entry,) + +mac = digest.Mac(access=access_key, secret=secret_key) +client = digest.Client(host=api_host, mac=mac) + +ret, err = client.call(path=api_path) +if err is not None: + print "failed" + print err +else: + print "success" \ No newline at end of file From 1673ec75adde514546e726068468856652125afc Mon Sep 17 00:00:00 2001 From: dtynn Date: Tue, 8 Apr 2014 09:29:22 +0800 Subject: [PATCH 014/478] changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7782ec5..13abd233 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## CHANGE LOG +### v6.1.5 + +2014-04-08 issue [#98](https://github.com/qiniu/python-sdk/pull/98) + +- [#98] 增加fetch、prefetch、pfop三个接口的范例代码 + ### v6.1.4 2014-03-28 issue [#95](https://github.com/qiniu/python-sdk/pull/95) From b087a3e16f8d1c86ae2c0a5e80738117c053c4ee Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 9 Apr 2014 00:42:38 +0800 Subject: [PATCH 015/478] update readme --- README.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6c95d9e1..31e3a64a 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,35 @@ Qiniu Resource Storage SDK for Python === -[![Build Status](https://api.travis-ci.org/qiniu/python-sdk.png?branch=develop)](https://travis-ci.org/qiniu/python-sdk) +[![Build Status](https://api.travis-ci.org/qiniu/python-sdk.png?branch=master)](https://travis-ci.org/qiniu/python-sdk) -[![Qiniu Logo](http://qiniutek.com/images/logo-2.png)](http://qiniu.com/) +[![Qiniu Logo](http://qiniu-brand.qiniudn.com/5/logo-white-195x105.png)](http://www.qiniu.com/) ## 使用 参考文档：[七牛云存储 Python SDK 使用指南](https://github.com/qiniu/python-sdk/blob/develop/docs/README.md) +## 准备开发环境 + +### 安装 + +* 直接安装: + + pip install qiniu + #或 + easy_install qiniu + +Python-SDK可以使用`pip`或`easy_install`从PyPI服务器上安装，但不包括文档和样例。如果需要，请下载源码并安装。 + +* 源码安装： + +从[release](https://github.com/qiniu/python-sdk/releases)下载源码： + + tar xvzf python-sdk-$VERSION.tar.gz + cd python-sdk-$VERSION + python setup.py install + + ## 单元测试 1. 测试环境 @@ -36,7 +57,7 @@ Qiniu Resource Storage SDK for Python ## 许可证 -Copyright (c) 2013 qiniu.com +Copyright (c) 2012-2014 qiniu.com 基于 MIT 协议发布: From 218719cde6ce293c684188ed51459790a09db6f0 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 9 Apr 2014 00:42:55 +0800 Subject: [PATCH 016/478] rename sample var --- docs/gist/fetch.py | 7 +++---- docs/gist/prefetch.py | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/gist/fetch.py b/docs/gist/fetch.py index ba125ed0..cbc74ff4 100644 --- a/docs/gist/fetch.py +++ b/docs/gist/fetch.py @@ -8,15 +8,14 @@ access_key = "" secret_key = "" -target_url = "" +src_url = "" dest_bucket = "" dest_key = "" -encoded_url = b64e(target_url) +encoded_url = b64e(src_url) dest_entry = "%s:%s" % (dest_bucket, dest_key) encoded_entry = b64e(dest_entry) - api_host = "iovip.qbox.me" api_path = "/fetch/%s/to/%s" % (encoded_url, encoded_entry) @@ -28,4 +27,4 @@ print "failed" print err else: - print "success" \ No newline at end of file + print "success" diff --git a/docs/gist/prefetch.py b/docs/gist/prefetch.py index ec184965..d384bf2d 100644 --- a/docs/gist/prefetch.py +++ b/docs/gist/prefetch.py @@ -16,7 +16,7 @@ api_host = "iovip.qbox.me" -api_path = "/prefetch/%s" % (encoded_entry,) +api_path = "/prefetch/%s" % (encoded_entry) mac = digest.Mac(access=access_key, secret=secret_key) client = digest.Client(host=api_host, mac=mac) @@ -26,4 +26,4 @@ print "failed" print err else: - print "success" \ No newline at end of file + print "success" From 1d41b577d2e2585fc5dbcb61459236393b288434 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Wed, 9 Apr 2014 00:46:38 +0800 Subject: [PATCH 017/478] Update README.md update format --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 31e3a64a..9dbec30a 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,9 @@ Qiniu Resource Storage SDK for Python * 直接安装: - pip install qiniu - #或 - easy_install qiniu + pip install qiniu + 或 + easy_install qiniu Python-SDK可以使用`pip`或`easy_install`从PyPI服务器上安装，但不包括文档和样例。如果需要，请下载源码并安装。 From 61a39e135116fb6d4ba1b0689f7d3547536a6ecf Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 10 Apr 2014 20:02:34 +0800 Subject: [PATCH 018/478] use autopep8 format --- docs/gist/demo.py | 575 +++++++++++++++++--------------- docs/gist/fetch.py | 2 +- docs/gist/pfop.py | 4 +- docs/gist/prefetch.py | 2 +- qiniu/auth/digest.py | 94 +++--- qiniu/auth/up.py | 22 +- qiniu/fop.py | 47 +-- qiniu/httplib_chunk.py | 211 ++++++------ qiniu/io.py | 95 +++--- qiniu/resumable_io.py | 11 +- qiniu/rpc.py | 374 ++++++++++----------- qiniu/rs/__init__.py | 4 +- qiniu/rs/rs.py | 138 ++++---- qiniu/rs/rs_token.py | 154 ++++----- qiniu/rs/test/__init__.py | 23 +- qiniu/rs/test/rs_test.py | 147 ++++---- qiniu/rs/test/rs_token_test.py | 110 +++--- qiniu/rsf.py | 64 ++-- qiniu/test/conf_test.py | 10 +- qiniu/test/fop_test.py | 44 +-- qiniu/test/io_test.py | 305 ++++++++--------- qiniu/test/resumable_io_test.py | 164 ++++----- qiniu/test/rpc_test.py | 283 ++++++++-------- qiniu/test/rsf_test.py | 14 +- setup.py | 53 +-- 25 files changed, 1523 insertions(+), 1427 deletions(-) diff --git a/docs/gist/demo.py b/docs/gist/demo.py index 78afaf05..0fe26a9a 100644 --- a/docs/gist/demo.py +++ b/docs/gist/demo.py @@ -30,331 +30,354 @@ # ---------------------------------------------------------- + def setup(access_key, secret_key, bucketname, bucket_domain, pickey): - global bucket_name, uptoken, key, key2, domain, key3, pic_key - qiniu.conf.ACCESS_KEY = access_key - qiniu.conf.SECRET_KEY = secret_key - bucket_name = bucketname - domain = bucket_domain - pic_key = pickey - # @gist uptoken - policy = qiniu.rs.PutPolicy(bucket_name) - uptoken = policy.token() - # @endgist - key = "python-demo-put-file" - key2 = "python-demo-put-file-2" - key3 = "python-demo-put-file-3" + global bucket_name, uptoken, key, key2, domain, key3, pic_key + qiniu.conf.ACCESS_KEY = access_key + qiniu.conf.SECRET_KEY = secret_key + bucket_name = bucketname + domain = bucket_domain + pic_key = pickey + # @gist uptoken + policy = qiniu.rs.PutPolicy(bucket_name) + uptoken = policy.token() + # @endgist + key = "python-demo-put-file" + key2 = "python-demo-put-file-2" + key3 = "python-demo-put-file-3" + def _setup(): - ''' 根据环境变量配置信息 ''' - access_key = getenv("QINIU_ACCESS_KEY") - if access_key is None: - exit("请配置环境变量 QINIU_ACCESS_KEY") - secret_key = getenv("QINIU_SECRET_KEY") - bucket_name = getenv("QINIU_TEST_BUCKET") - domain = getenv("QINIU_TEST_DOMAIN") - pickey = 'QINIU_UNIT_TEST_PIC' - setup(access_key, secret_key, bucket_name, domain, pickey) + ''' 根据环境变量配置信息 ''' + access_key = getenv("QINIU_ACCESS_KEY") + if access_key is None: + exit("请配置环境变量 QINIU_ACCESS_KEY") + secret_key = getenv("QINIU_SECRET_KEY") + bucket_name = getenv("QINIU_TEST_BUCKET") + domain = getenv("QINIU_TEST_DOMAIN") + pickey = 'QINIU_UNIT_TEST_PIC' + setup(access_key, secret_key, bucket_name, domain, pickey) + def getenv(name): - env = os.getenv(name) - if env is None: - sys.stderr.write("请配置环境变量 %s\n" % name) - exit(1) - return env + env = os.getenv(name) + if env is None: + sys.stderr.write("请配置环境变量 %s\n" % name) + exit(1) + return env + def get_demo_list(): - return [put_file, put_binary, - resumable_put, resumable_put_file, - stat, copy, move, delete, batch, - image_info, image_exif, image_view, - list_prefix, list_prefix_all, - ] + return [put_file, put_binary, + resumable_put, resumable_put_file, + stat, copy, move, delete, batch, + image_info, image_exif, image_view, + list_prefix, list_prefix_all, + ] + def run_demos(demos): - for i, demo in enumerate(demos): - print '%s.%s ' % (i+1, demo.__doc__), - demo() - print + for i, demo in enumerate(demos): + print '%s.%s ' % (i + 1, demo.__doc__), + demo() + print # ---------------------------------------------------------- + + def make_private_url(domain, key): - ''' 生成私有下载链接 ''' - # @gist dntoken - base_url = qiniu.rs.make_base_url(domain, key) - policy = qiniu.rs.GetPolicy() - private_url = policy.make_request(base_url) - # @endgist - return private_url + ''' 生成私有下载链接 ''' + # @gist dntoken + base_url = qiniu.rs.make_base_url(domain, key) + policy = qiniu.rs.GetPolicy() + private_url = policy.make_request(base_url) + # @endgist + return private_url + def put_file(): - ''' 演示上传文件的过程 ''' - # 尝试删除 - qiniu.rs.Client().delete(bucket_name, key) - - # @gist put_file - localfile = "%s" % __file__ - - ret, err = qiniu.io.put_file(uptoken, key, localfile) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - # @endgist - + ''' 演示上传文件的过程 ''' + # 尝试删除 + qiniu.rs.Client().delete(bucket_name, key) + + # @gist put_file + localfile = "%s" % __file__ + + ret, err = qiniu.io.put_file(uptoken, key, localfile) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + # @endgist + def put_binary(): - ''' 上传二进制数据 ''' - # 尝试删除 - qiniu.rs.Client().delete(bucket_name, key) - - # @gist put - extra = qiniu.io.PutExtra() - extra.mime_type = "text/plain" - - # data 可以是str或read()able对象 - data = StringIO.StringIO("hello!") - ret, err = qiniu.io.put(uptoken, key, data, extra) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - # @endgist + ''' 上传二进制数据 ''' + # 尝试删除 + qiniu.rs.Client().delete(bucket_name, key) + + # @gist put + extra = qiniu.io.PutExtra() + extra.mime_type = "text/plain" + + # data 可以是str或read()able对象 + data = StringIO.StringIO("hello!") + ret, err = qiniu.io.put(uptoken, key, data, extra) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + # @endgist + def resumable_put(): - ''' 断点续上传 ''' - # 尝试删除 - qiniu.rs.Client().delete(bucket_name, key) - - # @gist resumable_put - a = "resumable upload string" - extra = rio.PutExtra(bucket_name) - extra.mime_type = "text/plain" - ret, err = rio.put(uptoken, key, StringIO.StringIO(a), len(a), extra) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - print ret, - # @endgist - + ''' 断点续上传 ''' + # 尝试删除 + qiniu.rs.Client().delete(bucket_name, key) + + # @gist resumable_put + a = "resumable upload string" + extra = rio.PutExtra(bucket_name) + extra.mime_type = "text/plain" + ret, err = rio.put(uptoken, key, StringIO.StringIO(a), len(a), extra) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + print ret, + # @endgist + def resumable_put_file(): - ''' 断点续上传文件 ''' - # 尝试删除 - qiniu.rs.Client().delete(bucket_name, key) - - # @gist resumable_put_file - localfile = "%s" % __file__ - extra = rio.PutExtra(bucket_name) - - ret, err = rio.put_file(uptoken, key, localfile, extra) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - print ret, - # @endgist - + ''' 断点续上传文件 ''' + # 尝试删除 + qiniu.rs.Client().delete(bucket_name, key) + + # @gist resumable_put_file + localfile = "%s" % __file__ + extra = rio.PutExtra(bucket_name) + + ret, err = rio.put_file(uptoken, key, localfile, extra) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + print ret, + # @endgist + def stat(): - ''' 查看上传文件的内容 ''' - # @gist stat - ret, err = qiniu.rs.Client().stat(bucket_name, key) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - print ret, - # @endgist + ''' 查看上传文件的内容 ''' + # @gist stat + ret, err = qiniu.rs.Client().stat(bucket_name, key) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + print ret, + # @endgist + def copy(): - ''' 复制文件 ''' - # 初始化 - qiniu.rs.Client().delete(bucket_name, key2) - - # @gist copy - ret, err = qiniu.rs.Client().copy(bucket_name, key, bucket_name, key2) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - # @endgist - - stat, err = qiniu.rs.Client().stat(bucket_name, key2) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - print 'new file:', stat, + ''' 复制文件 ''' + # 初始化 + qiniu.rs.Client().delete(bucket_name, key2) + + # @gist copy + ret, err = qiniu.rs.Client().copy(bucket_name, key, bucket_name, key2) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + # @endgist + + stat, err = qiniu.rs.Client().stat(bucket_name, key2) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + print 'new file:', stat, + def move(): - ''' 移动文件 ''' - # 初始化 - qiniu.rs.Client().delete(bucket_name, key3) - - # @gist move - ret, err = qiniu.rs.Client().move(bucket_name, key2, bucket_name, key3) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - # @endgist - - # 查看文件是否移动成功 - ret, err = qiniu.rs.Client().stat(bucket_name, key3) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - - # 查看文件是否被删除 - ret, err = qiniu.rs.Client().stat(bucket_name, key2) - if err is None: - sys.stderr.write('error: %s ' % "删除失败") - return + ''' 移动文件 ''' + # 初始化 + qiniu.rs.Client().delete(bucket_name, key3) + + # @gist move + ret, err = qiniu.rs.Client().move(bucket_name, key2, bucket_name, key3) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + # @endgist + + # 查看文件是否移动成功 + ret, err = qiniu.rs.Client().stat(bucket_name, key3) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + + # 查看文件是否被删除 + ret, err = qiniu.rs.Client().stat(bucket_name, key2) + if err is None: + sys.stderr.write('error: %s ' % "删除失败") + return + def delete(): - ''' 删除文件 ''' - # @gist delete - ret, err = qiniu.rs.Client().delete(bucket_name, key3) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - # @endgist - - ret, err = qiniu.rs.Client().stat(bucket_name, key3) - if err is None: - sys.stderr.write('error: %s ' % "删除失败") - return + ''' 删除文件 ''' + # @gist delete + ret, err = qiniu.rs.Client().delete(bucket_name, key3) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + # @endgist + + ret, err = qiniu.rs.Client().stat(bucket_name, key3) + if err is None: + sys.stderr.write('error: %s ' % "删除失败") + return + def image_info(): - ''' 查看图片的信息 ''' - - # @gist image_info - # 生成base_url - url = qiniu.rs.make_base_url(domain, pic_key) + ''' 查看图片的信息 ''' + + # @gist image_info + # 生成base_url + url = qiniu.rs.make_base_url(domain, pic_key) - # 生成fop_url - image_info = qiniu.fop.ImageInfo() - url = image_info.make_request(url) + # 生成fop_url + image_info = qiniu.fop.ImageInfo() + url = image_info.make_request(url) - # 对其签名，生成private_url。如果是公有bucket此步可以省略 - policy = qiniu.rs.GetPolicy() - url = policy.make_request(url) + # 对其签名，生成private_url。如果是公有bucket此步可以省略 + policy = qiniu.rs.GetPolicy() + url = policy.make_request(url) + + print '可以在浏览器浏览: %s' % url + # @endgist - print '可以在浏览器浏览: %s' % url - # @endgist def image_exif(): - ''' 查看图片的exif信息 ''' - # @gist exif - # 生成base_url - url = qiniu.rs.make_base_url(domain, pic_key) + ''' 查看图片的exif信息 ''' + # @gist exif + # 生成base_url + url = qiniu.rs.make_base_url(domain, pic_key) + + # 生成fop_url + image_exif = qiniu.fop.Exif() + url = image_exif.make_request(url) - # 生成fop_url - image_exif = qiniu.fop.Exif() - url = image_exif.make_request(url) + # 对其签名，生成private_url。如果是公有bucket此步可以省略 + policy = qiniu.rs.GetPolicy() + url = policy.make_request(url) - # 对其签名，生成private_url。如果是公有bucket此步可以省略 - policy = qiniu.rs.GetPolicy() - url = policy.make_request(url) + print '可以在浏览器浏览: %s' % url + # @endgist - print '可以在浏览器浏览: %s' % url - # @endgist def image_view(): - ''' 对图片进行预览处理 ''' - # @gist image_view - iv = qiniu.fop.ImageView() - iv.width = 100 - - # 生成base_url - url = qiniu.rs.make_base_url(domain, pic_key) - # 生成fop_url - url = iv.make_request(url) - # 对其签名，生成private_url。如果是公有bucket此步可以省略 - policy = qiniu.rs.GetPolicy() - url = policy.make_request(url) - print '可以在浏览器浏览: %s' % url - # @endgist + ''' 对图片进行预览处理 ''' + # @gist image_view + iv = qiniu.fop.ImageView() + iv.width = 100 + + # 生成base_url + url = qiniu.rs.make_base_url(domain, pic_key) + # 生成fop_url + url = iv.make_request(url) + # 对其签名，生成private_url。如果是公有bucket此步可以省略 + policy = qiniu.rs.GetPolicy() + url = policy.make_request(url) + print '可以在浏览器浏览: %s' % url + # @endgist + def batch(): - ''' 文件处理的批量操作 ''' - # @gist batch_path - path_1 = qiniu.rs.EntryPath(bucket_name, key) - path_2 = qiniu.rs.EntryPath(bucket_name, key2) - path_3 = qiniu.rs.EntryPath(bucket_name, key3) - # @endgist - - # 查看状态 - # @gist batch_stat - rets, err = qiniu.rs.Client().batch_stat([path_1, path_2, path_3]) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - # @endgist - if not [ret['code'] for ret in rets] == [200, 612, 612]: - sys.stderr.write('error: %s ' % "批量获取状态与预期不同") - return - - # 复制 - # @gist batch_copy - pair_1 = qiniu.rs.EntryPathPair(path_1, path_3) - rets, err = qiniu.rs.Client().batch_copy([pair_1]) - if not rets[0]['code'] == 200: - sys.stderr.write('error: %s ' % "复制失败") - return - # @endgist - - qiniu.rs.Client().batch_delete([path_2]) - # @gist batch_move - pair_2 = qiniu.rs.EntryPathPair(path_3, path_2) - rets, err = qiniu.rs.Client().batch_move([pair_2]) - if not rets[0]['code'] == 200: - sys.stderr.write('error: %s ' % "移动失败") - return - # @endgist - - # 删除残留文件 - # @gist batch_delete - rets, err = qiniu.rs.Client().batch_delete([path_1, path_2]) - if not [ret['code'] for ret in rets] == [200, 200]: - sys.stderr.write('error: %s ' % "删除失败") - return - # @endgist + ''' 文件处理的批量操作 ''' + # @gist batch_path + path_1 = qiniu.rs.EntryPath(bucket_name, key) + path_2 = qiniu.rs.EntryPath(bucket_name, key2) + path_3 = qiniu.rs.EntryPath(bucket_name, key3) + # @endgist + + # 查看状态 + # @gist batch_stat + rets, err = qiniu.rs.Client().batch_stat([path_1, path_2, path_3]) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + # @endgist + if not [ret['code'] for ret in rets] == [200, 612, 612]: + sys.stderr.write('error: %s ' % "批量获取状态与预期不同") + return + + # 复制 + # @gist batch_copy + pair_1 = qiniu.rs.EntryPathPair(path_1, path_3) + rets, err = qiniu.rs.Client().batch_copy([pair_1]) + if not rets[0]['code'] == 200: + sys.stderr.write('error: %s ' % "复制失败") + return + # @endgist + + qiniu.rs.Client().batch_delete([path_2]) + # @gist batch_move + pair_2 = qiniu.rs.EntryPathPair(path_3, path_2) + rets, err = qiniu.rs.Client().batch_move([pair_2]) + if not rets[0]['code'] == 200: + sys.stderr.write('error: %s ' % "移动失败") + return + # @endgist + + # 删除残留文件 + # @gist batch_delete + rets, err = qiniu.rs.Client().batch_delete([path_1, path_2]) + if not [ret['code'] for ret in rets] == [200, 200]: + sys.stderr.write('error: %s ' % "删除失败") + return + # @endgist + def list_prefix(): - ''' 列出文件操作 ''' - # @gist list_prefix - rets, err = qiniu.rsf.Client().list_prefix(bucket_name, prefix="test", limit=2) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - print rets - - # 从上一次list_prefix的位置继续列出文件 - rets2, err = qiniu.rsf.Client().list_prefix(bucket_name, prefix="test", limit=1, marker=rets['marker']) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - print rets2 - # @endgist + ''' 列出文件操作 ''' + # @gist list_prefix + rets, err = qiniu.rsf.Client().list_prefix( + bucket_name, prefix="test", limit=2) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + print rets + + # 从上一次list_prefix的位置继续列出文件 + rets2, err = qiniu.rsf.Client().list_prefix( + bucket_name, prefix="test", limit=1, marker=rets['marker']) + if err is not None: + sys.stderr.write('error: %s ' % err) + return + print rets2 + # @endgist + def list_prefix_all(): - ''' 列出所有 ''' - list_all(bucket_name, prefix='test_Z', limit=10) + ''' 列出所有 ''' + list_all(bucket_name, prefix='test_Z', limit=10) # @gist list_all + + def list_all(bucket, rs=None, prefix=None, limit=None): - if rs is None: - rs = qiniu.rsf.Client() - marker = None - err = None - while err is None: - ret, err = rs.list_prefix(bucket_name, prefix=prefix, limit=limit, marker=marker) - marker = ret.get('marker', None) - for item in ret['items']: - #do something - pass - if err is not qiniu.rsf.EOF: - # 错误处理 - pass + if rs is None: + rs = qiniu.rsf.Client() + marker = None + err = None + while err is None: + ret, err = rs.list_prefix( + bucket_name, prefix=prefix, limit=limit, marker=marker) + marker = ret.get('marker', None) + for item in ret['items']: + # do something + pass + if err is not qiniu.rsf.EOF: + # 错误处理 + pass # @endgist if __name__ == "__main__": - _setup() - - demos = get_demo_list() - run_demos(demos) + _setup() + + demos = get_demo_list() + run_demos(demos) diff --git a/docs/gist/fetch.py b/docs/gist/fetch.py index cbc74ff4..caeb73c4 100644 --- a/docs/gist/fetch.py +++ b/docs/gist/fetch.py @@ -1,4 +1,4 @@ -#coding=utf-8 +# coding=utf-8 import sys sys.path.insert(0, "../../") diff --git a/docs/gist/pfop.py b/docs/gist/pfop.py index e993d370..bb09463e 100644 --- a/docs/gist/pfop.py +++ b/docs/gist/pfop.py @@ -1,4 +1,4 @@ -#coding=utf-8 +# coding=utf-8 import sys sys.path.insert(0, "../../") @@ -34,4 +34,4 @@ print err else: print "success" - print ret \ No newline at end of file + print ret diff --git a/docs/gist/prefetch.py b/docs/gist/prefetch.py index d384bf2d..968004ec 100644 --- a/docs/gist/prefetch.py +++ b/docs/gist/prefetch.py @@ -1,4 +1,4 @@ -#coding=utf-8 +# coding=utf-8 import sys sys.path.insert(0, "../../") diff --git a/qiniu/auth/digest.py b/qiniu/auth/digest.py index 7ef3a542..5d2788f3 100644 --- a/qiniu/auth/digest.py +++ b/qiniu/auth/digest.py @@ -7,52 +7,56 @@ from .. import rpc from .. import conf + class Mac(object): - access = None - secret = None - def __init__(self, access=None, secret=None): - if access is None and secret is None: - access, secret = conf.ACCESS_KEY, conf.SECRET_KEY - self.access, self.secret = access, secret - - def __sign(self, data): - hashed = hmac.new(self.secret, data, sha1) - return urlsafe_b64encode(hashed.digest()) - - def sign(self, data): - return '%s:%s' % (self.access, self.__sign(data)) - - def sign_with_data(self, b): - data = urlsafe_b64encode(b) - return '%s:%s:%s' % (self.access, self.__sign(data), data) - - def sign_request(self, path, body, content_type): - parsedurl = urlparse(path) - p_query = parsedurl.query - p_path = parsedurl.path - data = p_path - if p_query != "": - data = ''.join([data, '?', p_query]) - data = ''.join([data, "\n"]) - - if body: - incBody = [ - "application/x-www-form-urlencoded", - ] - if content_type in incBody: - data += body - - return '%s:%s' % (self.access, self.__sign(data)) + access = None + secret = None + + def __init__(self, access=None, secret=None): + if access is None and secret is None: + access, secret = conf.ACCESS_KEY, conf.SECRET_KEY + self.access, self.secret = access, secret + + def __sign(self, data): + hashed = hmac.new(self.secret, data, sha1) + return urlsafe_b64encode(hashed.digest()) + + def sign(self, data): + return '%s:%s' % (self.access, self.__sign(data)) + + def sign_with_data(self, b): + data = urlsafe_b64encode(b) + return '%s:%s:%s' % (self.access, self.__sign(data), data) + + def sign_request(self, path, body, content_type): + parsedurl = urlparse(path) + p_query = parsedurl.query + p_path = parsedurl.path + data = p_path + if p_query != "": + data = ''.join([data, '?', p_query]) + data = ''.join([data, "\n"]) + + if body: + incBody = [ + "application/x-www-form-urlencoded", + ] + if content_type in incBody: + data += body + + return '%s:%s' % (self.access, self.__sign(data)) class Client(rpc.Client): - def __init__(self, host, mac=None): - if mac is None: - mac = Mac() - super(Client, self).__init__(host) - self.mac = mac - - def round_tripper(self, method, path, body): - token = self.mac.sign_request(path, body, self._header.get("Content-Type")) - self.set_header("Authorization", "QBox %s" % token) - return super(Client, self).round_tripper(method, path, body) + + def __init__(self, host, mac=None): + if mac is None: + mac = Mac() + super(Client, self).__init__(host) + self.mac = mac + + def round_tripper(self, method, path, body): + token = self.mac.sign_request( + path, body, self._header.get("Content-Type")) + self.set_header("Authorization", "QBox %s" % token) + return super(Client, self).round_tripper(method, path, body) diff --git a/qiniu/auth/up.py b/qiniu/auth/up.py index b13aa629..de43b223 100644 --- a/qiniu/auth/up.py +++ b/qiniu/auth/up.py @@ -4,16 +4,16 @@ class Client(rpc.Client): - up_token = None + up_token = None - def __init__(self, up_token, host=None): - if host is None: - host = conf.UP_HOST - if host.startswith("http://"): - host = host[7:] - self.up_token = up_token - super(Client, self).__init__(host) + def __init__(self, up_token, host=None): + if host is None: + host = conf.UP_HOST + if host.startswith("http://"): + host = host[7:] + self.up_token = up_token + super(Client, self).__init__(host) - def round_tripper(self, method, path, body): - self.set_header("Authorization", "UpToken %s" % self.up_token) - return super(Client, self).round_tripper(method, path, body) + def round_tripper(self, method, path, body): + self.set_header("Authorization", "UpToken %s" % self.up_token) + return super(Client, self).round_tripper(method, path, body) diff --git a/qiniu/fop.py b/qiniu/fop.py index be9a7c19..e04b084a 100644 --- a/qiniu/fop.py +++ b/qiniu/fop.py @@ -1,37 +1,40 @@ # -*- coding:utf-8 -*- import json + class Exif(object): - def make_request(self, url): - return '%s?exif' % url + + def make_request(self, url): + return '%s?exif' % url class ImageView(object): - mode = 1 # 1或2 - width = None # width 默认为0，表示不限定宽度 - height = None - quality = None # 图片质量, 1-100 - format = None # 输出格式, jpg, gif, png, tif 等图片格式 + mode = 1 # 1或2 + width = None # width 默认为0，表示不限定宽度 + height = None + quality = None # 图片质量, 1-100 + format = None # 输出格式, jpg, gif, png, tif 等图片格式 - def make_request(self, url): - target = [] - target.append('%s' % self.mode) - - if self.width is not None: - target.append("w/%s" % self.width) + def make_request(self, url): + target = [] + target.append('%s' % self.mode) - if self.height is not None: - target.append("h/%s" % self.height) + if self.width is not None: + target.append("w/%s" % self.width) - if self.quality is not None: - target.append("q/%s" % self.quality) + if self.height is not None: + target.append("h/%s" % self.height) - if self.format is not None: - target.append("format/%s" % self.format) + if self.quality is not None: + target.append("q/%s" % self.quality) - return "%s?imageView/%s" % (url, '/'.join(target)) + if self.format is not None: + target.append("format/%s" % self.format) + + return "%s?imageView/%s" % (url, '/'.join(target)) class ImageInfo(object): - def make_request(self, url): - return '%s?imageInfo' % url + + def make_request(self, url): + return '%s?imageInfo' % url diff --git a/qiniu/httplib_chunk.py b/qiniu/httplib_chunk.py index d08b0ebb..3aef95f4 100644 --- a/qiniu/httplib_chunk.py +++ b/qiniu/httplib_chunk.py @@ -13,111 +13,110 @@ import os from array import array -class HTTPConnection(httplib.HTTPConnection): - def send(self, data, is_chunked=False): - """Send `data' to the server.""" - if self.sock is None: - if self.auto_open: - self.connect() - else: - raise NotConnected() - - if self.debuglevel > 0: - print "send:", repr(data) - blocksize = 8192 - if hasattr(data,'read') and not isinstance(data, array): - if self.debuglevel > 0: print "sendIng a read()able" - datablock = data.read(blocksize) - while datablock: - if self.debuglevel > 0: - print 'chunked:', is_chunked - if is_chunked: - if self.debuglevel > 0: print 'send: with trunked data' - lenstr = string.upper(hex(len(datablock))[2:]) - self.sock.sendall('%s\r\n%s\r\n' % (lenstr, datablock)) - else: - self.sock.sendall(datablock) - datablock = data.read(blocksize) - if is_chunked: - self.sock.sendall('0\r\n\r\n') - else: - self.sock.sendall(data) - - - def _set_content_length(self, body): - # Set the content-length based on the body. - thelen = None - try: - thelen = str(len(body)) - except (TypeError, AttributeError), te: - # Don't send a length if this failed - if self.debuglevel > 0: print "Cannot stat!!" - - if thelen is not None: - self.putheader('Content-Length', thelen) - return True - return False - - - def _send_request(self, method, url, body, headers): - # Honor explicitly requested Host: and Accept-Encoding: headers. - header_names = dict.fromkeys([k.lower() for k in headers]) - skips = {} - if 'host' in header_names: - skips['skip_host'] = 1 - if 'accept-encoding' in header_names: - skips['skip_accept_encoding'] = 1 - - self.putrequest(method, url, **skips) - - is_chunked = False - if body and header_names.get('Transfer-Encoding') == 'chunked': - is_chunked = True - elif body and ('content-length' not in header_names): - is_chunked = not self._set_content_length(body) - if is_chunked: - self.putheader('Transfer-Encoding', 'chunked') - for hdr, value in headers.iteritems(): - self.putheader(hdr, value) - - self.endheaders(body, is_chunked=is_chunked) - - - def endheaders(self, message_body=None, is_chunked=False): - """Indicate that the last header line has been sent to the server. - - This method sends the request to the server. The optional - message_body argument can be used to pass a message body - associated with the request. The message body will be sent in - the same packet as the message headers if it is string, otherwise it is - sent as a separate packet. - """ - if self.__state == _CS_REQ_STARTED: - self.__state = _CS_REQ_SENT - else: - raise CannotSendHeader() - self._send_output(message_body, is_chunked=is_chunked) - - - def _send_output(self, message_body=None, is_chunked=False): - """Send the currently buffered request and clear the buffer. - - Appends an extra \\r\\n to the buffer. - A message_body may be specified, to be appended to the request. - """ - self._buffer.extend(("", "")) - msg = "\r\n".join(self._buffer) - del self._buffer[:] - # If msg and message_body are sent in a single send() call, - # it will avoid performance problems caused by the interaction - # between delayed ack and the Nagle algorithm. - if isinstance(message_body, str): - msg += message_body - message_body = None - self.send(msg) - if message_body is not None: - #message_body was not a string (i.e. it is a file) and - #we must run the risk of Nagle - self.send(message_body, is_chunked=is_chunked) +class HTTPConnection(httplib.HTTPConnection): + def send(self, data, is_chunked=False): + """Send `data' to the server.""" + if self.sock is None: + if self.auto_open: + self.connect() + else: + raise NotConnected() + + if self.debuglevel > 0: + print "send:", repr(data) + blocksize = 8192 + if hasattr(data, 'read') and not isinstance(data, array): + if self.debuglevel > 0: + print "sendIng a read()able" + datablock = data.read(blocksize) + while datablock: + if self.debuglevel > 0: + print 'chunked:', is_chunked + if is_chunked: + if self.debuglevel > 0: + print 'send: with trunked data' + lenstr = string.upper(hex(len(datablock))[2:]) + self.sock.sendall('%s\r\n%s\r\n' % (lenstr, datablock)) + else: + self.sock.sendall(datablock) + datablock = data.read(blocksize) + if is_chunked: + self.sock.sendall('0\r\n\r\n') + else: + self.sock.sendall(data) + + def _set_content_length(self, body): + # Set the content-length based on the body. + thelen = None + try: + thelen = str(len(body)) + except (TypeError, AttributeError), te: + # Don't send a length if this failed + if self.debuglevel > 0: + print "Cannot stat!!" + + if thelen is not None: + self.putheader('Content-Length', thelen) + return True + return False + + def _send_request(self, method, url, body, headers): + # Honor explicitly requested Host: and Accept-Encoding: headers. + header_names = dict.fromkeys([k.lower() for k in headers]) + skips = {} + if 'host' in header_names: + skips['skip_host'] = 1 + if 'accept-encoding' in header_names: + skips['skip_accept_encoding'] = 1 + + self.putrequest(method, url, **skips) + + is_chunked = False + if body and header_names.get('Transfer-Encoding') == 'chunked': + is_chunked = True + elif body and ('content-length' not in header_names): + is_chunked = not self._set_content_length(body) + if is_chunked: + self.putheader('Transfer-Encoding', 'chunked') + for hdr, value in headers.iteritems(): + self.putheader(hdr, value) + + self.endheaders(body, is_chunked=is_chunked) + + def endheaders(self, message_body=None, is_chunked=False): + """Indicate that the last header line has been sent to the server. + + This method sends the request to the server. The optional + message_body argument can be used to pass a message body + associated with the request. The message body will be sent in + the same packet as the message headers if it is string, otherwise it is + sent as a separate packet. + """ + if self.__state == _CS_REQ_STARTED: + self.__state = _CS_REQ_SENT + else: + raise CannotSendHeader() + self._send_output(message_body, is_chunked=is_chunked) + + def _send_output(self, message_body=None, is_chunked=False): + """Send the currently buffered request and clear the buffer. + + Appends an extra \\r\\n to the buffer. + A message_body may be specified, to be appended to the request. + """ + self._buffer.extend(("", "")) + msg = "\r\n".join(self._buffer) + del self._buffer[:] + # If msg and message_body are sent in a single send() call, + # it will avoid performance problems caused by the interaction + # between delayed ack and the Nagle algorithm. + if isinstance(message_body, str): + msg += message_body + message_body = None + self.send(msg) + if message_body is not None: + # message_body was not a string (i.e. it is a file) and + # we must run the risk of Nagle + self.send(message_body, is_chunked=is_chunked) diff --git a/qiniu/io.py b/qiniu/io.py index 2576fc0b..afaacc9d 100644 --- a/qiniu/io.py +++ b/qiniu/io.py @@ -5,78 +5,79 @@ import random import string try: - import zlib as binascii + import zlib as binascii except ImportError: - import binascii + import binascii # @gist PutExtra class PutExtra(object): - params = {} - mime_type = 'application/octet-stream' - crc32 = "" - check_crc = 0 + params = {} + mime_type = 'application/octet-stream' + crc32 = "" + check_crc = 0 # @endgist def put(uptoken, key, data, extra=None): - """ put your data to Qiniu + """ put your data to Qiniu - If key is None, the server will generate one. - data may be str or read()able object. - """ - fields = { - } + If key is None, the server will generate one. + data may be str or read()able object. + """ + fields = { + } - if not extra: - extra = PutExtra() + if not extra: + extra = PutExtra() - if extra.params: - for k in extra.params: - fields[k] = str(extra.params[k]) + if extra.params: + for k in extra.params: + fields[k] = str(extra.params[k]) - if extra.check_crc: - fields["crc32"] = str(extra.crc32) + if extra.check_crc: + fields["crc32"] = str(extra.crc32) - if key is not None: - fields['key'] = key + if key is not None: + fields['key'] = key - fields["token"] = uptoken + fields["token"] = uptoken - fname = key - if fname is None: - fname = _random_str(9) - elif fname is '': - fname = 'index.html' - files = [ - {'filename': fname, 'data': data, 'mime_type': extra.mime_type}, - ] - return rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files) + fname = key + if fname is None: + fname = _random_str(9) + elif fname is '': + fname = 'index.html' + files = [ + {'filename': fname, 'data': data, 'mime_type': extra.mime_type}, + ] + return rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files) def put_file(uptoken, key, localfile, extra=None): - """ put a file to Qiniu + """ put a file to Qiniu - If key is None, the server will generate one. - """ - if extra is not None and extra.check_crc == 1: - extra.crc32 = _get_file_crc32(localfile) - with open(localfile, 'rb') as f: - return put(uptoken, key, f, extra) + If key is None, the server will generate one. + """ + if extra is not None and extra.check_crc == 1: + extra.crc32 = _get_file_crc32(localfile) + with open(localfile, 'rb') as f: + return put(uptoken, key, f, extra) _BLOCK_SIZE = 1024 * 1024 * 4 + def _get_file_crc32(filepath): - with open(filepath, 'rb') as f: - block = f.read(_BLOCK_SIZE) - crc = 0 - while len(block) != 0: - crc = binascii.crc32(block, crc) & 0xFFFFFFFF - block = f.read(_BLOCK_SIZE) - return crc + with open(filepath, 'rb') as f: + block = f.read(_BLOCK_SIZE) + crc = 0 + while len(block) != 0: + crc = binascii.crc32(block, crc) & 0xFFFFFFFF + block = f.read(_BLOCK_SIZE) + return crc def _random_str(length): - lib = string.ascii_lowercase - return ''.join([random.choice(lib) for i in range(0, length)]) + lib = string.ascii_lowercase + return ''.join([random.choice(lib) for i in range(0, length)]) diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index 0b3bb5bf..e8feee18 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -1,4 +1,4 @@ -#coding=utf-8 +# coding=utf-8 import os try: import zlib as binascii @@ -91,8 +91,8 @@ def put(uptoken, key, f, fsize, extra): for i in xrange(block_cnt): try_time = extra.try_times read_length = _block_size - if (i+1)*_block_size > fsize: - read_length = fsize - i*_block_size + if (i + 1) * _block_size > fsize: + read_length = fsize - i * _block_size data_slice = f.read(read_length) while True: err = resumable_block_put(data_slice, i, extra, uptoken) @@ -137,7 +137,8 @@ def mkblock(client, block_size, first_chunk): def putblock(client, block_ret, chunk): - url = "%s/bput/%s/%s" % (block_ret["host"], block_ret["ctx"], block_ret["offset"]) + url = "%s/bput/%s/%s" % (block_ret["host"], + block_ret["ctx"], block_ret["offset"]) content_type = "application/octet-stream" return client.call_with(url, chunk, content_type, len(chunk)) @@ -157,4 +158,4 @@ def mkfile(client, key, fsize, extra): url = "/".join(url) body = ",".join([i["ctx"] for i in extra.progresses]) - return client.call_with(url, body, "text/plain", len(body)) \ No newline at end of file + return client.call_with(url, body, "text/plain", len(body)) diff --git a/qiniu/rpc.py b/qiniu/rpc.py index 4ca701d2..2e78c806 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -6,197 +6,199 @@ class Client(object): - _conn = None - _header = None - - def __init__(self, host): - self._conn = httplib.HTTPConnection(host) - self._header = {} - - def round_tripper(self, method, path, body): - self._conn.request(method, path, body, self._header) - resp = self._conn.getresponse() - return resp - - def call(self, path): - return self.call_with(path, None) - - def call_with(self, path, body, content_type=None, content_length=None): - ret = None - - self.set_header("User-Agent", conf.USER_AGENT) - if content_type is not None: - self.set_header("Content-Type", content_type) - - if content_length is not None: - self.set_header("Content-Length", content_length) - - resp = self.round_tripper("POST", path, body) - try: - ret = resp.read() - ret = json.loads(ret) - except IOError, e: - return None, e - except ValueError: - pass - - if resp.status / 100 != 2: - err_msg = ret if "error" not in ret else ret["error"] - detail = resp.getheader("x-log", None) - if detail is not None: - err_msg += ", detail:%s" % detail - - return None, err_msg - - return ret, None - - def call_with_multipart(self, path, fields=None, files=None): - """ - * fields => {key} - * files => [{filename, data, content_type}] - """ - content_type, mr = self.encode_multipart_formdata(fields, files) - return self.call_with(path, mr, content_type, mr.length()) - - def call_with_form(self, path, ops): - """ - * ops => {"key": value/list()} - """ - - body = [] - for i in ops: - if isinstance(ops[i], (list, tuple)): - data = ('&%s=' % i).join(ops[i]) - else: - data = ops[i] - - body.append('%s=%s' % (i, data)) - body = '&'.join(body) - - content_type = "application/x-www-form-urlencoded" - return self.call_with(path, body, content_type, len(body)) - - def set_header(self, field, value): - self._header[field] = value - - def set_headers(self, headers): - self._header.update(headers) - - def encode_multipart_formdata(self, fields, files): - """ - * fields => {key} - * files => [{filename, data, content_type}] - * return content_type, content_length, body - """ - if files is None: - files = [] - if fields is None: - fields = {} - - readers = [] - BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$' - CRLF = '\r\n' - L1 = [] - for key in fields: - L1.append('--' + BOUNDARY) - L1.append('Content-Disposition: form-data; name="%s"' % key) - L1.append('') - L1.append(fields[key]) - b1 = CRLF.join(L1) - readers.append(b1) - - for file_info in files: - L = [] - L.append('') - L.append('--' + BOUNDARY) - disposition = "Content-Disposition: form-data;" - filename = _qiniu_escape(file_info.get('filename')) - L.append('%s name="file"; filename="%s"' % (disposition, filename)) - L.append('Content-Type: %s' % file_info.get('mime_type', 'application/octet-stream')) - L.append('') - L.append('') - b2 = CRLF.join(L) - readers.append(b2) - - data = file_info.get('data') - readers.append(data) - - L3 = ['', '--' + BOUNDARY + '--', ''] - b3 = CRLF.join(L3) - readers.append(b3) - - content_type = 'multipart/form-data; boundary=%s' % BOUNDARY - return content_type, MultiReader(readers) + _conn = None + _header = None + + def __init__(self, host): + self._conn = httplib.HTTPConnection(host) + self._header = {} + + def round_tripper(self, method, path, body): + self._conn.request(method, path, body, self._header) + resp = self._conn.getresponse() + return resp + + def call(self, path): + return self.call_with(path, None) + + def call_with(self, path, body, content_type=None, content_length=None): + ret = None + + self.set_header("User-Agent", conf.USER_AGENT) + if content_type is not None: + self.set_header("Content-Type", content_type) + + if content_length is not None: + self.set_header("Content-Length", content_length) + + resp = self.round_tripper("POST", path, body) + try: + ret = resp.read() + ret = json.loads(ret) + except IOError, e: + return None, e + except ValueError: + pass + + if resp.status / 100 != 2: + err_msg = ret if "error" not in ret else ret["error"] + detail = resp.getheader("x-log", None) + if detail is not None: + err_msg += ", detail:%s" % detail + + return None, err_msg + + return ret, None + + def call_with_multipart(self, path, fields=None, files=None): + """ + * fields => {key} + * files => [{filename, data, content_type}] + """ + content_type, mr = self.encode_multipart_formdata(fields, files) + return self.call_with(path, mr, content_type, mr.length()) + + def call_with_form(self, path, ops): + """ + * ops => {"key": value/list()} + """ + + body = [] + for i in ops: + if isinstance(ops[i], (list, tuple)): + data = ('&%s=' % i).join(ops[i]) + else: + data = ops[i] + + body.append('%s=%s' % (i, data)) + body = '&'.join(body) + + content_type = "application/x-www-form-urlencoded" + return self.call_with(path, body, content_type, len(body)) + + def set_header(self, field, value): + self._header[field] = value + + def set_headers(self, headers): + self._header.update(headers) + + def encode_multipart_formdata(self, fields, files): + """ + * fields => {key} + * files => [{filename, data, content_type}] + * return content_type, content_length, body + """ + if files is None: + files = [] + if fields is None: + fields = {} + + readers = [] + BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$' + CRLF = '\r\n' + L1 = [] + for key in fields: + L1.append('--' + BOUNDARY) + L1.append('Content-Disposition: form-data; name="%s"' % key) + L1.append('') + L1.append(fields[key]) + b1 = CRLF.join(L1) + readers.append(b1) + + for file_info in files: + L = [] + L.append('') + L.append('--' + BOUNDARY) + disposition = "Content-Disposition: form-data;" + filename = _qiniu_escape(file_info.get('filename')) + L.append('%s name="file"; filename="%s"' % (disposition, filename)) + L.append('Content-Type: %s' % + file_info.get('mime_type', 'application/octet-stream')) + L.append('') + L.append('') + b2 = CRLF.join(L) + readers.append(b2) + + data = file_info.get('data') + readers.append(data) + + L3 = ['', '--' + BOUNDARY + '--', ''] + b3 = CRLF.join(L3) + readers.append(b3) + + content_type = 'multipart/form-data; boundary=%s' % BOUNDARY + return content_type, MultiReader(readers) + def _qiniu_escape(s): - edits = [('\\', '\\\\'), ('\"', '\\\"')] - for (search, replace) in edits: - s = s.replace(search, replace) - return s + edits = [('\\', '\\\\'), ('\"', '\\\"')] + for (search, replace) in edits: + s = s.replace(search, replace) + return s class MultiReader(object): - """ class MultiReader([readers...]) - - MultiReader returns a read()able object that's the logical concatenation of - the provided input readers. They're read sequentially. - """ - - def __init__(self, readers): - self.readers = [] - self.content_length = 0 - self.valid_content_length = True - for r in readers: - if hasattr(r, 'read'): - if self.valid_content_length: - length = self._get_content_length(r) - if length is not None: - self.content_length += length - else: - self.valid_content_length = False - else: - buf = r - if not isinstance(buf, basestring): - buf = str(buf) - buf = encode_unicode(buf) - r = cStringIO.StringIO(buf) - self.content_length += len(buf) - self.readers.append(r) - - - # don't name it __len__, because the length of MultiReader is not alway valid. - def length(self): - return self.content_length if self.valid_content_length else None - - - def _get_content_length(self, reader): - data_len = None - if hasattr(reader, 'seek') and hasattr(reader, 'tell'): - try: - reader.seek(0, 2) - data_len= reader.tell() - reader.seek(0, 0) - except OSError: - # Don't send a length if this failed - data_len = None - return data_len - - def read(self, n=-1): - if n is None or n == -1: - return ''.join([encode_unicode(r.read()) for r in self.readers]) - else: - L = [] - while len(self.readers) > 0 and n > 0: - b = self.readers[0].read(n) - if len(b) == 0: - self.readers = self.readers[1:] - else: - L.append(encode_unicode(b)) - n -= len(b) - return ''.join(L) + + """ class MultiReader([readers...]) + + MultiReader returns a read()able object that's the logical concatenation of + the provided input readers. They're read sequentially. + """ + + def __init__(self, readers): + self.readers = [] + self.content_length = 0 + self.valid_content_length = True + for r in readers: + if hasattr(r, 'read'): + if self.valid_content_length: + length = self._get_content_length(r) + if length is not None: + self.content_length += length + else: + self.valid_content_length = False + else: + buf = r + if not isinstance(buf, basestring): + buf = str(buf) + buf = encode_unicode(buf) + r = cStringIO.StringIO(buf) + self.content_length += len(buf) + self.readers.append(r) + + # don't name it __len__, because the length of MultiReader is not alway + # valid. + def length(self): + return self.content_length if self.valid_content_length else None + + def _get_content_length(self, reader): + data_len = None + if hasattr(reader, 'seek') and hasattr(reader, 'tell'): + try: + reader.seek(0, 2) + data_len = reader.tell() + reader.seek(0, 0) + except OSError: + # Don't send a length if this failed + data_len = None + return data_len + + def read(self, n=-1): + if n is None or n == -1: + return ''.join([encode_unicode(r.read()) for r in self.readers]) + else: + L = [] + while len(self.readers) > 0 and n > 0: + b = self.readers[0].read(n) + if len(b) == 0: + self.readers = self.readers[1:] + else: + L.append(encode_unicode(b)) + n -= len(b) + return ''.join(L) def encode_unicode(u): - if isinstance(u, unicode): - u = u.encode('utf8') - return u + if isinstance(u, unicode): + u = u.encode('utf8') + return u diff --git a/qiniu/rs/__init__.py b/qiniu/rs/__init__.py index 5eed5702..861d4264 100644 --- a/qiniu/rs/__init__.py +++ b/qiniu/rs/__init__.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- __all__ = [ - "Client", "EntryPath", "EntryPathPair", "uri_stat", "uri_delete", "uri_move", "uri_copy", - "PutPolicy", "GetPolicy", "make_base_url", + "Client", "EntryPath", "EntryPathPair", "uri_stat", "uri_delete", "uri_move", "uri_copy", + "PutPolicy", "GetPolicy", "make_base_url", ] from .rs import * diff --git a/qiniu/rs/rs.py b/qiniu/rs/rs.py index 38a86bd5..b84d538a 100644 --- a/qiniu/rs/rs.py +++ b/qiniu/rs/rs.py @@ -4,80 +4,90 @@ from ..auth import digest from .. import conf + class Client(object): - conn = None - def __init__(self, mac=None): - if mac is None: - mac = digest.Mac() - self.conn = digest.Client(host=conf.RS_HOST, mac=mac) - - def stat(self, bucket, key): - return self.conn.call(uri_stat(bucket, key)) - - def delete(self, bucket, key): - return self.conn.call(uri_delete(bucket, key)) - - def move(self, bucket_src, key_src, bucket_dest, key_dest): - return self.conn.call(uri_move(bucket_src, key_src, bucket_dest, key_dest)) - - def copy(self, bucket_src, key_src, bucket_dest, key_dest): - return self.conn.call(uri_copy(bucket_src, key_src, bucket_dest, key_dest)) - - def batch(self, ops): - return self.conn.call_with_form("/batch", dict(op=ops)) - - def batch_stat(self, entries): - ops = [] - for entry in entries: - ops.append(uri_stat(entry.bucket, entry.key)) - return self.batch(ops) - - def batch_delete(self, entries): - ops = [] - for entry in entries: - ops.append(uri_delete(entry.bucket, entry.key)) - return self.batch(ops) - - def batch_move(self, entries): - ops = [] - for entry in entries: - ops.append(uri_move(entry.src.bucket, entry.src.key, - entry.dest.bucket, entry.dest.key)) - return self.batch(ops) - - def batch_copy(self, entries): - ops = [] - for entry in entries: - ops.append(uri_copy(entry.src.bucket, entry.src.key, - entry.dest.bucket, entry.dest.key)) - return self.batch(ops) + conn = None + + def __init__(self, mac=None): + if mac is None: + mac = digest.Mac() + self.conn = digest.Client(host=conf.RS_HOST, mac=mac) + + def stat(self, bucket, key): + return self.conn.call(uri_stat(bucket, key)) + + def delete(self, bucket, key): + return self.conn.call(uri_delete(bucket, key)) + + def move(self, bucket_src, key_src, bucket_dest, key_dest): + return self.conn.call(uri_move(bucket_src, key_src, bucket_dest, key_dest)) + + def copy(self, bucket_src, key_src, bucket_dest, key_dest): + return self.conn.call(uri_copy(bucket_src, key_src, bucket_dest, key_dest)) + + def batch(self, ops): + return self.conn.call_with_form("/batch", dict(op=ops)) + + def batch_stat(self, entries): + ops = [] + for entry in entries: + ops.append(uri_stat(entry.bucket, entry.key)) + return self.batch(ops) + + def batch_delete(self, entries): + ops = [] + for entry in entries: + ops.append(uri_delete(entry.bucket, entry.key)) + return self.batch(ops) + + def batch_move(self, entries): + ops = [] + for entry in entries: + ops.append(uri_move(entry.src.bucket, entry.src.key, + entry.dest.bucket, entry.dest.key)) + return self.batch(ops) + + def batch_copy(self, entries): + ops = [] + for entry in entries: + ops.append(uri_copy(entry.src.bucket, entry.src.key, + entry.dest.bucket, entry.dest.key)) + return self.batch(ops) + class EntryPath(object): - bucket = None - key = None - def __init__(self, bucket, key): - self.bucket = bucket - self.key = key + bucket = None + key = None + + def __init__(self, bucket, key): + self.bucket = bucket + self.key = key + class EntryPathPair: - src = None - dest = None - def __init__(self, src, dest): - self.src = src - self.dest = dest + src = None + dest = None + + def __init__(self, src, dest): + self.src = src + self.dest = dest + def uri_stat(bucket, key): - return "/stat/%s" % urlsafe_b64encode("%s:%s" % (bucket, key)) + return "/stat/%s" % urlsafe_b64encode("%s:%s" % (bucket, key)) + def uri_delete(bucket, key): - return "/delete/%s" % urlsafe_b64encode("%s:%s" % (bucket, key)) + return "/delete/%s" % urlsafe_b64encode("%s:%s" % (bucket, key)) + def uri_move(bucket_src, key_src, bucket_dest, key_dest): - src = urlsafe_b64encode("%s:%s" % (bucket_src, key_src)) - dest = urlsafe_b64encode("%s:%s" % (bucket_dest, key_dest)) - return "/move/%s/%s" % (src, dest) + src = urlsafe_b64encode("%s:%s" % (bucket_src, key_src)) + dest = urlsafe_b64encode("%s:%s" % (bucket_dest, key_dest)) + return "/move/%s/%s" % (src, dest) + def uri_copy(bucket_src, key_src, bucket_dest, key_dest): - src = urlsafe_b64encode("%s:%s" % (bucket_src, key_src)) - dest = urlsafe_b64encode("%s:%s" % (bucket_dest, key_dest)) - return "/copy/%s/%s" % (src, dest) + src = urlsafe_b64encode("%s:%s" % (bucket_src, key_src)) + dest = urlsafe_b64encode("%s:%s" % (bucket_dest, key_dest)) + return "/copy/%s/%s" % (src, dest) diff --git a/qiniu/rs/rs_token.py b/qiniu/rs/rs_token.py index f0463c1b..2250e484 100644 --- a/qiniu/rs/rs_token.py +++ b/qiniu/rs/rs_token.py @@ -7,102 +7,106 @@ from ..import rpc # @gist PutPolicy + + class PutPolicy(object): - scope = None # 可以是 bucketName 或者 bucketName:key - expires = 3600 # 默认是 3600 秒 - callbackUrl = None - callbackBody = None - returnUrl = None - returnBody = None - endUser = None - asyncOps = None - - saveKey = None - insertOnly = None - detectMime = None - fsizeLimit = None - persistentNotifyUrl = None - persistentOps = None - - def __init__(self, scope): - self.scope = scope + scope = None # 可以是 bucketName 或者 bucketName:key + expires = 3600 # 默认是 3600 秒 + callbackUrl = None + callbackBody = None + returnUrl = None + returnBody = None + endUser = None + asyncOps = None + + saveKey = None + insertOnly = None + detectMime = None + fsizeLimit = None + persistentNotifyUrl = None + persistentOps = None + + def __init__(self, scope): + self.scope = scope # @endgist - def token(self, mac=None): - if mac is None: - mac = digest.Mac() - token = dict( - scope = self.scope, - deadline = int(time.time()) + self.expires, - ) + def token(self, mac=None): + if mac is None: + mac = digest.Mac() + token = dict( + scope=self.scope, + deadline=int(time.time()) + self.expires, + ) - if self.callbackUrl is not None: - token["callbackUrl"] = self.callbackUrl + if self.callbackUrl is not None: + token["callbackUrl"] = self.callbackUrl - if self.callbackBody is not None: - token["callbackBody"] = self.callbackBody + if self.callbackBody is not None: + token["callbackBody"] = self.callbackBody - if self.returnUrl is not None: - token["returnUrl"] = self.returnUrl + if self.returnUrl is not None: + token["returnUrl"] = self.returnUrl - if self.returnBody is not None: - token["returnBody"] = self.returnBody + if self.returnBody is not None: + token["returnBody"] = self.returnBody - if self.endUser is not None: - token["endUser"] = self.endUser + if self.endUser is not None: + token["endUser"] = self.endUser - if self.asyncOps is not None: - token["asyncOps"] = self.asyncOps + if self.asyncOps is not None: + token["asyncOps"] = self.asyncOps - if self.saveKey is not None: - token["saveKey"] = self.saveKey + if self.saveKey is not None: + token["saveKey"] = self.saveKey - if self.insertOnly is not None: - token["exclusive"] = self.insertOnly + if self.insertOnly is not None: + token["exclusive"] = self.insertOnly - if self.detectMime is not None: - token["detectMime"] = self.detectMime + if self.detectMime is not None: + token["detectMime"] = self.detectMime - if self.fsizeLimit is not None: - token["fsizeLimit"] = self.fsizeLimit + if self.fsizeLimit is not None: + token["fsizeLimit"] = self.fsizeLimit - if self.persistentOps is not None: - token["persistentOps"] = self.persistentOps + if self.persistentOps is not None: + token["persistentOps"] = self.persistentOps - if self.persistentNotifyUrl is not None: - token["persistentNotifyUrl"] = self.persistentNotifyUrl + if self.persistentNotifyUrl is not None: + token["persistentNotifyUrl"] = self.persistentNotifyUrl + + b = json.dumps(token, separators=(',', ':')) + return mac.sign_with_data(b) - b = json.dumps(token, separators=(',',':')) - return mac.sign_with_data(b) class GetPolicy(object): - expires = 3600 - def __init__(self, expires=3600): - self.expires = expires + expires = 3600 + + def __init__(self, expires=3600): + self.expires = expires - def make_request(self, base_url, mac=None): - ''' - * return private_url - ''' - if mac is None: - mac = digest.Mac() + def make_request(self, base_url, mac=None): + ''' + * return private_url + ''' + if mac is None: + mac = digest.Mac() - deadline = int(time.time()) + self.expires - if '?' in base_url: - base_url += '&' - else: - base_url += '?' - base_url = '%se=%s' % (base_url, str(deadline)) + deadline = int(time.time()) + self.expires + if '?' in base_url: + base_url += '&' + else: + base_url += '?' + base_url = '%se=%s' % (base_url, str(deadline)) - token = mac.sign(base_url) - return '%s&token=%s' % (base_url, token) + token = mac.sign(base_url) + return '%s&token=%s' % (base_url, token) def make_base_url(domain, key): - ''' - * domain => str - * key => str - * return base_url - ''' - key = rpc.encode_unicode(key) - return 'http://%s/%s' % (domain, urllib.quote(key)) + ''' + * domain => str + * key => str + * return base_url + ''' + key = rpc.encode_unicode(key) + return 'http://%s/%s' % (domain, urllib.quote(key)) diff --git a/qiniu/rs/test/__init__.py b/qiniu/rs/test/__init__.py index f290dd77..5704743b 100644 --- a/qiniu/rs/test/__init__.py +++ b/qiniu/rs/test/__init__.py @@ -9,17 +9,18 @@ pic = "http://cheneya.qiniudn.com/hello_jpg" key = 'QINIU_UNIT_TEST_PIC' + def setUp(): - qiniu.conf.ACCESS_KEY = os.getenv("QINIU_ACCESS_KEY") - qiniu.conf.SECRET_KEY = os.getenv("QINIU_SECRET_KEY") - bucket_name = os.getenv("QINIU_TEST_BUCKET") + qiniu.conf.ACCESS_KEY = os.getenv("QINIU_ACCESS_KEY") + qiniu.conf.SECRET_KEY = os.getenv("QINIU_SECRET_KEY") + bucket_name = os.getenv("QINIU_TEST_BUCKET") - policy = qiniu.rs.PutPolicy(bucket_name) - uptoken = policy.token() + policy = qiniu.rs.PutPolicy(bucket_name) + uptoken = policy.token() - f = urllib.urlopen(pic) - _, err = qiniu.io.put(uptoken, key, f) - f.close() - if err is None or err.startswith('file exists'): - print err - assert err is None or err.startswith('file exists') + f = urllib.urlopen(pic) + _, err = qiniu.io.put(uptoken, key, f) + f.close() + if err is None or err.startswith('file exists'): + print err + assert err is None or err.startswith('file exists') diff --git a/qiniu/rs/test/rs_test.py b/qiniu/rs/test/rs_test.py index a18cec66..32714ff5 100644 --- a/qiniu/rs/test/rs_test.py +++ b/qiniu/rs/test/rs_test.py @@ -7,9 +7,10 @@ from qiniu import rs from qiniu import conf + def r(length): - lib = string.ascii_uppercase - return ''.join([random.choice(lib) for i in range(0, length)]) + lib = string.ascii_uppercase + return ''.join([random.choice(lib) for i in range(0, length)]) conf.ACCESS_KEY = os.getenv("QINIU_ACCESS_KEY") conf.SECRET_KEY = os.getenv("QINIU_SECRET_KEY") @@ -20,76 +21,78 @@ def r(length): key3 = "rs_demo_test_key_2_" + r(5) key4 = "rs_demo_test_key_3_" + r(5) + class TestRs(unittest.TestCase): - def test_stat(self): - r = rs.Client() - ret, err = r.stat(bucket_name, key) - assert err is None - assert ret is not None - - # error - _, err = r.stat(bucket_name, noexist_key) - assert err is not None - - def test_delete_move_copy(self): - r = rs.Client() - r.delete(bucket_name, key2) - r.delete(bucket_name, key3) - - ret, err = r.copy(bucket_name, key, bucket_name, key2) - assert err is None, err - - ret, err = r.move(bucket_name, key2, bucket_name, key3) - assert err is None, err - - ret, err = r.delete(bucket_name, key3) - assert err is None, err - - # error - _, err = r.delete(bucket_name, key2) - assert err is not None - - _, err = r.delete(bucket_name, key3) - assert err is not None - - def test_batch_stat(self): - r = rs.Client() - entries = [ - rs.EntryPath(bucket_name, key), - rs.EntryPath(bucket_name, key2), - ] - ret, err = r.batch_stat(entries) - assert err is None - self.assertEqual(ret[0]["code"], 200) - self.assertEqual(ret[1]["code"], 612) - - def test_batch_delete_move_copy(self): - r = rs.Client() - e1 = rs.EntryPath(bucket_name, key) - e2 = rs.EntryPath(bucket_name, key2) - e3 = rs.EntryPath(bucket_name, key3) - e4 = rs.EntryPath(bucket_name, key4) - r.batch_delete([e2, e3, e4]) - - # copy - entries = [ - rs.EntryPathPair(e1, e2), - rs.EntryPathPair(e1, e3), - ] - ret, err = r.batch_copy(entries) - assert err is None - self.assertEqual(ret[0]["code"], 200) - self.assertEqual(ret[1]["code"], 200) - - ret, err = r.batch_move([rs.EntryPathPair(e2, e4)]) - assert err is None - self.assertEqual(ret[0]["code"], 200) - - ret, err = r.batch_delete([e3, e4]) - assert err is None - self.assertEqual(ret[0]["code"], 200) - - r.batch_delete([e2, e3, e4]) + + def test_stat(self): + r = rs.Client() + ret, err = r.stat(bucket_name, key) + assert err is None + assert ret is not None + + # error + _, err = r.stat(bucket_name, noexist_key) + assert err is not None + + def test_delete_move_copy(self): + r = rs.Client() + r.delete(bucket_name, key2) + r.delete(bucket_name, key3) + + ret, err = r.copy(bucket_name, key, bucket_name, key2) + assert err is None, err + + ret, err = r.move(bucket_name, key2, bucket_name, key3) + assert err is None, err + + ret, err = r.delete(bucket_name, key3) + assert err is None, err + + # error + _, err = r.delete(bucket_name, key2) + assert err is not None + + _, err = r.delete(bucket_name, key3) + assert err is not None + + def test_batch_stat(self): + r = rs.Client() + entries = [ + rs.EntryPath(bucket_name, key), + rs.EntryPath(bucket_name, key2), + ] + ret, err = r.batch_stat(entries) + assert err is None + self.assertEqual(ret[0]["code"], 200) + self.assertEqual(ret[1]["code"], 612) + + def test_batch_delete_move_copy(self): + r = rs.Client() + e1 = rs.EntryPath(bucket_name, key) + e2 = rs.EntryPath(bucket_name, key2) + e3 = rs.EntryPath(bucket_name, key3) + e4 = rs.EntryPath(bucket_name, key4) + r.batch_delete([e2, e3, e4]) + + # copy + entries = [ + rs.EntryPathPair(e1, e2), + rs.EntryPathPair(e1, e3), + ] + ret, err = r.batch_copy(entries) + assert err is None + self.assertEqual(ret[0]["code"], 200) + self.assertEqual(ret[1]["code"], 200) + + ret, err = r.batch_move([rs.EntryPathPair(e2, e4)]) + assert err is None + self.assertEqual(ret[0]["code"], 200) + + ret, err = r.batch_delete([e3, e4]) + assert err is None + self.assertEqual(ret[0]["code"], 200) + + r.batch_delete([e2, e3, e4]) if __name__ == "__main__": - unittest.main() + unittest.main() diff --git a/qiniu/rs/test/rs_token_test.py b/qiniu/rs/test/rs_token_test.py index 66af6dfc..9e1a95a5 100644 --- a/qiniu/rs/test/rs_token_test.py +++ b/qiniu/rs/test/rs_token_test.py @@ -18,62 +18,66 @@ domain = os.getenv("QINIU_TEST_DOMAIN") key = 'QINIU_UNIT_TEST_PIC' + class TestToken(unittest.TestCase): - def test_put_policy(self): - policy = rs.PutPolicy(bucket_name) - policy.endUser = "hello!" - policy.returnUrl = "http://localhost:1234/path?query=hello" - policy.returnBody = "$(sha1)" - # Do not specify the returnUrl and callbackUrl at the same time - policy.callbackUrl = "http://1.2.3.4/callback" - policy.callbackBody = "$(bucket)" - - policy.saveKey = "$(sha1)" - policy.insertOnly = 1 - policy.detectMime = 1 - policy.fsizeLimit = 1024 - policy.persistentNotifyUrl = "http://4.3.2.1/persistentNotifyUrl" - policy.persistentOps = "avthumb/flash" - - tokens = policy.token().split(':') - - # chcek first part of token - self.assertEqual(conf.ACCESS_KEY, tokens[0]) - data = json.loads(decode(tokens[2])) - - # check if same - self.assertEqual(data["scope"], bucket_name) - self.assertEqual(data["endUser"], policy.endUser) - self.assertEqual(data["returnUrl"], policy.returnUrl) - self.assertEqual(data["returnBody"], policy.returnBody) - self.assertEqual(data["callbackUrl"], policy.callbackUrl) - self.assertEqual(data["callbackBody"], policy.callbackBody) - self.assertEqual(data["saveKey"], policy.saveKey) - self.assertEqual(data["exclusive"], policy.insertOnly) - self.assertEqual(data["detectMime"], policy.detectMime) - self.assertEqual(data["fsizeLimit"], policy.fsizeLimit) - self.assertEqual(data["persistentNotifyUrl"], policy.persistentNotifyUrl) - self.assertEqual(data["persistentOps"], policy.persistentOps) - - new_hmac = encode(hmac.new(conf.SECRET_KEY, tokens[2], sha1).digest()) - self.assertEqual(new_hmac, tokens[1]) - - def test_get_policy(self): - base_url = rs.make_base_url(domain, key) - policy = rs.GetPolicy() - private_url = policy.make_request(base_url) - - f = urllib.urlopen(private_url) - body = f.read() - f.close() - self.assertEqual(len(body)>100, True) + + def test_put_policy(self): + policy = rs.PutPolicy(bucket_name) + policy.endUser = "hello!" + policy.returnUrl = "http://localhost:1234/path?query=hello" + policy.returnBody = "$(sha1)" + # Do not specify the returnUrl and callbackUrl at the same time + policy.callbackUrl = "http://1.2.3.4/callback" + policy.callbackBody = "$(bucket)" + + policy.saveKey = "$(sha1)" + policy.insertOnly = 1 + policy.detectMime = 1 + policy.fsizeLimit = 1024 + policy.persistentNotifyUrl = "http://4.3.2.1/persistentNotifyUrl" + policy.persistentOps = "avthumb/flash" + + tokens = policy.token().split(':') + + # chcek first part of token + self.assertEqual(conf.ACCESS_KEY, tokens[0]) + data = json.loads(decode(tokens[2])) + + # check if same + self.assertEqual(data["scope"], bucket_name) + self.assertEqual(data["endUser"], policy.endUser) + self.assertEqual(data["returnUrl"], policy.returnUrl) + self.assertEqual(data["returnBody"], policy.returnBody) + self.assertEqual(data["callbackUrl"], policy.callbackUrl) + self.assertEqual(data["callbackBody"], policy.callbackBody) + self.assertEqual(data["saveKey"], policy.saveKey) + self.assertEqual(data["exclusive"], policy.insertOnly) + self.assertEqual(data["detectMime"], policy.detectMime) + self.assertEqual(data["fsizeLimit"], policy.fsizeLimit) + self.assertEqual( + data["persistentNotifyUrl"], policy.persistentNotifyUrl) + self.assertEqual(data["persistentOps"], policy.persistentOps) + + new_hmac = encode(hmac.new(conf.SECRET_KEY, tokens[2], sha1).digest()) + self.assertEqual(new_hmac, tokens[1]) + + def test_get_policy(self): + base_url = rs.make_base_url(domain, key) + policy = rs.GetPolicy() + private_url = policy.make_request(base_url) + + f = urllib.urlopen(private_url) + body = f.read() + f.close() + self.assertEqual(len(body) > 100, True) class Test_make_base_url(unittest.TestCase): - def test_unicode(self): - url1 = rs.make_base_url('1.com', '你好') - url2 = rs.make_base_url('1.com', u'你好') - assert url1 == url2 + + def test_unicode(self): + url1 = rs.make_base_url('1.com', '你好') + url2 = rs.make_base_url('1.com', u'你好') + assert url1 == url2 if __name__ == "__main__": - unittest.main() + unittest.main() diff --git a/qiniu/rsf.py b/qiniu/rsf.py index 48d19f48..eb0aeac2 100644 --- a/qiniu/rsf.py +++ b/qiniu/rsf.py @@ -7,35 +7,37 @@ class Client(object): - conn = None - def __init__(self, mac=None): - if mac is None: - mac = auth.digest.Mac() - self.conn = auth.digest.Client(host=conf.RSF_HOST, mac=mac) - - def list_prefix(self, bucket, prefix=None, marker=None, limit=None): - '''前缀查询: - * bucket => str - * prefix => str - * marker => str - * limit => int - * return ret => {'items': items, 'marker': markerOut}, err => str + conn = None - 1. 首次请求 marker = None - 2. 无论 err 值如何，均应该先看 ret.get('items') 是否有内容 - 3. 如果后续没有更多数据，err 返回 EOF，markerOut 返回 None（但不通过该特征来判断是否结束） - ''' - ops = { - 'bucket': bucket, - } - if marker is not None: - ops['marker'] = marker - if limit is not None: - ops['limit'] = limit - if prefix is not None: - ops['prefix'] = prefix - url = '%s?%s' % ('/list', urllib.urlencode(ops)) - ret, err = self.conn.call_with(url, body=None, content_type='application/x-www-form-urlencoded') - if ret and not ret.get('marker'): - err = EOF - return ret, err + def __init__(self, mac=None): + if mac is None: + mac = auth.digest.Mac() + self.conn = auth.digest.Client(host=conf.RSF_HOST, mac=mac) + + def list_prefix(self, bucket, prefix=None, marker=None, limit=None): + '''前缀查询: + * bucket => str + * prefix => str + * marker => str + * limit => int + * return ret => {'items': items, 'marker': markerOut}, err => str + + 1. 首次请求 marker = None + 2. 无论 err 值如何，均应该先看 ret.get('items') 是否有内容 + 3. 如果后续没有更多数据，err 返回 EOF，markerOut 返回 None（但不通过该特征来判断是否结束） + ''' + ops = { + 'bucket': bucket, + } + if marker is not None: + ops['marker'] = marker + if limit is not None: + ops['limit'] = limit + if prefix is not None: + ops['prefix'] = prefix + url = '%s?%s' % ('/list', urllib.urlencode(ops)) + ret, err = self.conn.call_with( + url, body=None, content_type='application/x-www-form-urlencoded') + if ret and not ret.get('marker'): + err = EOF + return ret, err diff --git a/qiniu/test/conf_test.py b/qiniu/test/conf_test.py index daf0ed3b..fd200c6d 100644 --- a/qiniu/test/conf_test.py +++ b/qiniu/test/conf_test.py @@ -2,9 +2,11 @@ import unittest from qiniu import conf + class TestConfig(unittest.TestCase): - def test_USER_AGENT(self): - assert len(conf.USER_AGENT) >= len('qiniu python-sdk') - + + def test_USER_AGENT(self): + assert len(conf.USER_AGENT) >= len('qiniu python-sdk') + if __name__ == '__main__': - unittest.main() + unittest.main() diff --git a/qiniu/test/fop_test.py b/qiniu/test/fop_test.py index 43741fd8..b25d5a1a 100644 --- a/qiniu/test/fop_test.py +++ b/qiniu/test/fop_test.py @@ -5,28 +5,30 @@ pic = "http://cheneya.qiniudn.com/hello_jpg" + class TestFop(unittest.TestCase): - def test_exif(self): - ie = fop.Exif() - ret = ie.make_request(pic) - self.assertEqual(ret, "%s?exif" % pic) - - def test_imageView(self): - iv = fop.ImageView() - iv.height = 100 - ret = iv.make_request(pic) - self.assertEqual(ret, "%s?imageView/1/h/100" % pic) - - iv.quality = 20 - iv.format = "png" - ret = iv.make_request(pic) - self.assertEqual(ret, "%s?imageView/1/h/100/q/20/format/png" % pic) - - def test_imageInfo(self): - ii = fop.ImageInfo() - ret = ii.make_request(pic) - self.assertEqual(ret, "%s?imageInfo" % pic) + + def test_exif(self): + ie = fop.Exif() + ret = ie.make_request(pic) + self.assertEqual(ret, "%s?exif" % pic) + + def test_imageView(self): + iv = fop.ImageView() + iv.height = 100 + ret = iv.make_request(pic) + self.assertEqual(ret, "%s?imageView/1/h/100" % pic) + + iv.quality = 20 + iv.format = "png" + ret = iv.make_request(pic) + self.assertEqual(ret, "%s?imageView/1/h/100/q/20/format/png" % pic) + + def test_imageInfo(self): + ii = fop.ImageInfo() + ret = ii.make_request(pic) + self.assertEqual(ret, "%s?imageInfo" % pic) if __name__ == '__main__': - unittest.main() + unittest.main() diff --git a/qiniu/test/io_test.py b/qiniu/test/io_test.py index 503dce5e..18c9bc9f 100644 --- a/qiniu/test/io_test.py +++ b/qiniu/test/io_test.py @@ -5,9 +5,9 @@ import random import urllib try: - import zlib as binascii + import zlib as binascii except ImportError: - import binascii + import binascii import cStringIO from qiniu import conf @@ -21,161 +21,168 @@ policy = rs.PutPolicy(bucket_name) extra = io.PutExtra() extra.mime_type = "text/plain" -extra.params = {'x:a':'a'} +extra.params = {'x:a': 'a'} + def r(length): - lib = string.ascii_uppercase - return ''.join([random.choice(lib) for i in range(0, length)]) + lib = string.ascii_uppercase + return ''.join([random.choice(lib) for i in range(0, length)]) + class TestUp(unittest.TestCase): - def test(self): - def test_put(): - key = "test_%s" % r(9) - params = "op=3" - data = "hello bubby!" - extra.check_crc = 2 - extra.crc32 = binascii.crc32(data) & 0xFFFFFFFF - ret, err = io.put(policy.token(), key, data, extra) - assert err is None - assert ret['key'] == key - - def test_put_same_crc(): - key = "test_%s" % r(9) - data = "hello bubby!" - extra.check_crc = 2 - ret, err = io.put(policy.token(), key, data, extra) - assert err is None - assert ret['key'] == key - - def test_put_no_key(): - data = r(100) - extra.check_crc = 0 - ret, err = io.put(policy.token(), key=None, data=data, extra=extra) - assert err is None - assert ret['hash'] == ret['key'] - - def test_put_quote_key(): - data = r(100) - key = 'a\\b\\c"你好' + r(9) - ret, err = io.put(policy.token(), key, data) - print err - assert err is None - assert ret['key'].encode('utf8') == key - - data = r(100) - key = u'a\\b\\c"你好' + r(9) - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret['key'] == key - - def test_put_unicode1(): - key = "test_%s" % r(9) + '你好' - data = key - ret, err = io.put(policy.token(), key, data, extra) - assert err is None - assert ret[u'key'].endswith(u'你好') - - def test_put_unicode2(): - key = "test_%s" % r(9) + '你好' - data = key - data = data.decode('utf8') - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret[u'key'].endswith(u'你好') - - def test_put_unicode3(): - key = "test_%s" % r(9) + '你好' - data = key - key = key.decode('utf8') - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret[u'key'].endswith(u'你好') - - def test_put_unicode4(): - key = "test_%s" % r(9) + '你好' - data = key - key = key.decode('utf8') - data = data.decode('utf8') - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret[u'key'].endswith(u'你好') - - def test_put_StringIO(): - key = "test_%s" % r(9) - data = cStringIO.StringIO('hello buddy!') - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret['key'] == key - - def test_put_urlopen(): - key = "test_%s" % r(9) - data = urllib.urlopen('http://cheneya.qiniudn.com/hello_jpg') - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret['key'] == key - - def test_put_no_length(): - class test_reader(object): - def __init__(self): - self.data = 'abc' - self.pos = 0 - def read(self, n=None): - if n is None or n < 0: - newpos = len(self.data) - else: - newpos = min(self.pos+n, len(self.data)) - r = self.data[self.pos: newpos] - self.pos = newpos - return r - key = "test_%s" % r(9) - data = test_reader() - - extra.check_crc = 2 - extra.crc32 = binascii.crc32('abc') & 0xFFFFFFFF - ret, err = io.put(policy.token(), key, data, extra) - assert err is None - assert ret['key'] == key - - test_put() - test_put_same_crc() - test_put_no_key() - test_put_quote_key() - test_put_unicode1() - test_put_unicode2() - test_put_unicode3() - test_put_unicode4() - test_put_StringIO() - test_put_urlopen() - test_put_no_length() - - def test_put_file(self): - localfile = "%s" % __file__ - key = "test_%s" % r(9) - - extra.check_crc = 1 - ret, err = io.put_file(policy.token(), key, localfile, extra) - assert err is None - assert ret['key'] == key - - def test_put_crc_fail(self): - key = "test_%s" % r(9) - data = "hello bubby!" - extra.check_crc = 2 - extra.crc32 = "wrong crc32" - ret, err = io.put(policy.token(), key, data, extra) - assert err is not None + + def test(self): + def test_put(): + key = "test_%s" % r(9) + params = "op=3" + data = "hello bubby!" + extra.check_crc = 2 + extra.crc32 = binascii.crc32(data) & 0xFFFFFFFF + ret, err = io.put(policy.token(), key, data, extra) + assert err is None + assert ret['key'] == key + + def test_put_same_crc(): + key = "test_%s" % r(9) + data = "hello bubby!" + extra.check_crc = 2 + ret, err = io.put(policy.token(), key, data, extra) + assert err is None + assert ret['key'] == key + + def test_put_no_key(): + data = r(100) + extra.check_crc = 0 + ret, err = io.put(policy.token(), key=None, data=data, extra=extra) + assert err is None + assert ret['hash'] == ret['key'] + + def test_put_quote_key(): + data = r(100) + key = 'a\\b\\c"你好' + r(9) + ret, err = io.put(policy.token(), key, data) + print err + assert err is None + assert ret['key'].encode('utf8') == key + + data = r(100) + key = u'a\\b\\c"你好' + r(9) + ret, err = io.put(policy.token(), key, data) + assert err is None + assert ret['key'] == key + + def test_put_unicode1(): + key = "test_%s" % r(9) + '你好' + data = key + ret, err = io.put(policy.token(), key, data, extra) + assert err is None + assert ret[u'key'].endswith(u'你好') + + def test_put_unicode2(): + key = "test_%s" % r(9) + '你好' + data = key + data = data.decode('utf8') + ret, err = io.put(policy.token(), key, data) + assert err is None + assert ret[u'key'].endswith(u'你好') + + def test_put_unicode3(): + key = "test_%s" % r(9) + '你好' + data = key + key = key.decode('utf8') + ret, err = io.put(policy.token(), key, data) + assert err is None + assert ret[u'key'].endswith(u'你好') + + def test_put_unicode4(): + key = "test_%s" % r(9) + '你好' + data = key + key = key.decode('utf8') + data = data.decode('utf8') + ret, err = io.put(policy.token(), key, data) + assert err is None + assert ret[u'key'].endswith(u'你好') + + def test_put_StringIO(): + key = "test_%s" % r(9) + data = cStringIO.StringIO('hello buddy!') + ret, err = io.put(policy.token(), key, data) + assert err is None + assert ret['key'] == key + + def test_put_urlopen(): + key = "test_%s" % r(9) + data = urllib.urlopen('http://cheneya.qiniudn.com/hello_jpg') + ret, err = io.put(policy.token(), key, data) + assert err is None + assert ret['key'] == key + + def test_put_no_length(): + class test_reader(object): + + def __init__(self): + self.data = 'abc' + self.pos = 0 + + def read(self, n=None): + if n is None or n < 0: + newpos = len(self.data) + else: + newpos = min(self.pos + n, len(self.data)) + r = self.data[self.pos: newpos] + self.pos = newpos + return r + key = "test_%s" % r(9) + data = test_reader() + + extra.check_crc = 2 + extra.crc32 = binascii.crc32('abc') & 0xFFFFFFFF + ret, err = io.put(policy.token(), key, data, extra) + assert err is None + assert ret['key'] == key + + test_put() + test_put_same_crc() + test_put_no_key() + test_put_quote_key() + test_put_unicode1() + test_put_unicode2() + test_put_unicode3() + test_put_unicode4() + test_put_StringIO() + test_put_urlopen() + test_put_no_length() + + def test_put_file(self): + localfile = "%s" % __file__ + key = "test_%s" % r(9) + + extra.check_crc = 1 + ret, err = io.put_file(policy.token(), key, localfile, extra) + assert err is None + assert ret['key'] == key + + def test_put_crc_fail(self): + key = "test_%s" % r(9) + data = "hello bubby!" + extra.check_crc = 2 + extra.crc32 = "wrong crc32" + ret, err = io.put(policy.token(), key, data, extra) + assert err is not None class Test_get_file_crc32(unittest.TestCase): - def test_get_file_crc32(self): - file_path = '%s' % __file__ - data = None - with open(file_path, 'rb') as f: - data = f.read() - io._BLOCK_SIZE = 4 - assert binascii.crc32(data) & 0xFFFFFFFF == io._get_file_crc32(file_path) + def test_get_file_crc32(self): + file_path = '%s' % __file__ + + data = None + with open(file_path, 'rb') as f: + data = f.read() + io._BLOCK_SIZE = 4 + assert binascii.crc32( + data) & 0xFFFFFFFF == io._get_file_crc32(file_path) if __name__ == "__main__": - unittest.main() + unittest.main() diff --git a/qiniu/test/resumable_io_test.py b/qiniu/test/resumable_io_test.py index f4effabd..d4e97487 100644 --- a/qiniu/test/resumable_io_test.py +++ b/qiniu/test/resumable_io_test.py @@ -5,9 +5,9 @@ import random import platform try: - import zlib as binascii + import zlib as binascii except ImportError: - import binascii + import binascii import urllib import tempfile import shutil @@ -23,85 +23,91 @@ def r(length): - lib = string.ascii_uppercase - return ''.join([random.choice(lib) for i in range(0, length)]) + lib = string.ascii_uppercase + return ''.join([random.choice(lib) for i in range(0, length)]) + class TestBlock(unittest.TestCase): - def test_block(self): - policy = rs.PutPolicy(bucket) - uptoken = policy.token() - client = up.Client(uptoken) - - rets = [0, 0] - data_slice_2 = "\nbye!" - ret, err = resumable_io.mkblock(client, len(data_slice_2), data_slice_2) - assert err is None, err - self.assertEqual(ret["crc32"], binascii.crc32(data_slice_2)) - - extra = resumable_io.PutExtra(bucket) - extra.mimetype = "text/plain" - extra.progresses = [ret] - lens = 0 - for i in xrange(0, len(extra.progresses)): - lens += extra.progresses[i]["offset"] - - key = u"sdk_py_resumable_block_4_%s" % r(9) - ret, err = resumable_io.mkfile(client, key, lens, extra) - assert err is None, err - self.assertEqual(ret["hash"], "FtCFo0mQugW98uaPYgr54Vb1QsO0", "hash not match") - rs.Client().delete(bucket, key) - - def test_put(self): - src = urllib.urlopen("http://cheneya.qiniudn.com/hello_jpg") - ostype = platform.system() - if ostype.lower().find("windows") != -1: - tmpf = "".join([os.getcwd(), os.tmpnam()]) - else: - tmpf = os.tmpnam() - dst = open(tmpf, 'wb') - shutil.copyfileobj(src, dst) - src.close() - - policy = rs.PutPolicy(bucket) - extra = resumable_io.PutExtra(bucket) - extra.bucket = bucket - extra.params = {"x:foo": "test"} - key = "sdk_py_resumable_block_5_%s" % r(9) - localfile = dst.name - ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) - assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" - dst.close() - os.remove(tmpf) - - assert err is None, err - self.assertEqual(ret["hash"], "FnyTMUqPNRTdk1Wou7oLqDHkBm_p", "hash not match") - rs.Client().delete(bucket, key) - - def test_put_4m(self): - ostype = platform.system() - if ostype.lower().find("windows") != -1: - tmpf = "".join([os.getcwd(), os.tmpnam()]) - else: - tmpf = os.tmpnam() - dst = open(tmpf, 'wb') - dst.write("abcd" * 1024 * 1024) - dst.flush() - - policy = rs.PutPolicy(bucket) - extra = resumable_io.PutExtra(bucket) - extra.bucket = bucket - extra.params = {"x:foo": "test"} - key = "sdk_py_resumable_block_6_%s" % r(9) - localfile = dst.name - ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) - assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" - dst.close() - os.remove(tmpf) - - assert err is None, err - self.assertEqual(ret["hash"], "FnIVmMd_oaUV3MLDM6F9in4RMz2U", "hash not match") - rs.Client().delete(bucket, key) + + def test_block(self): + policy = rs.PutPolicy(bucket) + uptoken = policy.token() + client = up.Client(uptoken) + + rets = [0, 0] + data_slice_2 = "\nbye!" + ret, err = resumable_io.mkblock( + client, len(data_slice_2), data_slice_2) + assert err is None, err + self.assertEqual(ret["crc32"], binascii.crc32(data_slice_2)) + + extra = resumable_io.PutExtra(bucket) + extra.mimetype = "text/plain" + extra.progresses = [ret] + lens = 0 + for i in xrange(0, len(extra.progresses)): + lens += extra.progresses[i]["offset"] + + key = u"sdk_py_resumable_block_4_%s" % r(9) + ret, err = resumable_io.mkfile(client, key, lens, extra) + assert err is None, err + self.assertEqual( + ret["hash"], "FtCFo0mQugW98uaPYgr54Vb1QsO0", "hash not match") + rs.Client().delete(bucket, key) + + def test_put(self): + src = urllib.urlopen("http://cheneya.qiniudn.com/hello_jpg") + ostype = platform.system() + if ostype.lower().find("windows") != -1: + tmpf = "".join([os.getcwd(), os.tmpnam()]) + else: + tmpf = os.tmpnam() + dst = open(tmpf, 'wb') + shutil.copyfileobj(src, dst) + src.close() + + policy = rs.PutPolicy(bucket) + extra = resumable_io.PutExtra(bucket) + extra.bucket = bucket + extra.params = {"x:foo": "test"} + key = "sdk_py_resumable_block_5_%s" % r(9) + localfile = dst.name + ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) + assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" + dst.close() + os.remove(tmpf) + + assert err is None, err + self.assertEqual( + ret["hash"], "FnyTMUqPNRTdk1Wou7oLqDHkBm_p", "hash not match") + rs.Client().delete(bucket, key) + + def test_put_4m(self): + ostype = platform.system() + if ostype.lower().find("windows") != -1: + tmpf = "".join([os.getcwd(), os.tmpnam()]) + else: + tmpf = os.tmpnam() + dst = open(tmpf, 'wb') + dst.write("abcd" * 1024 * 1024) + dst.flush() + + policy = rs.PutPolicy(bucket) + extra = resumable_io.PutExtra(bucket) + extra.bucket = bucket + extra.params = {"x:foo": "test"} + key = "sdk_py_resumable_block_6_%s" % r(9) + localfile = dst.name + ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) + assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" + dst.close() + os.remove(tmpf) + + assert err is None, err + self.assertEqual( + ret["hash"], "FnIVmMd_oaUV3MLDM6F9in4RMz2U", "hash not match") + rs.Client().delete(bucket, key) if __name__ == "__main__": - unittest.main() + unittest.main() diff --git a/qiniu/test/rpc_test.py b/qiniu/test/rpc_test.py index db35b11a..08acfa04 100644 --- a/qiniu/test/rpc_test.py +++ b/qiniu/test/rpc_test.py @@ -5,154 +5,173 @@ from qiniu import rpc from qiniu import conf + def round_tripper(client, method, path, body): - pass + pass + class ClsTestClient(rpc.Client): - def round_tripper(self, method, path, body): - round_tripper(self, method, path, body) - return super(ClsTestClient, self).round_tripper(method, path, body) + + def round_tripper(self, method, path, body): + round_tripper(self, method, path, body) + return super(ClsTestClient, self).round_tripper(method, path, body) client = ClsTestClient(conf.RS_HOST) + class TestClient(unittest.TestCase): - def test_call(self): - global round_tripper - - def tripper(client, method, path, body): - self.assertEqual(path, "/hello") - assert body is None - - round_tripper = tripper - client.call("/hello") - - def test_call_with(self): - global round_tripper - def tripper(client, method, path, body): - self.assertEqual(body, "body") - - round_tripper = tripper - client.call_with("/hello", "body") - - def test_call_with_multipart(self): - global round_tripper - def tripper(client, method, path, body): - target_type = "multipart/form-data" - self.assertTrue(client._header["Content-Type"].startswith(target_type)) - start_index = client._header["Content-Type"].find("boundary") - boundary = client._header["Content-Type"][start_index + 9: ] - dispostion = 'Content-Disposition: form-data; name="auth"' - tpl = "--%s\r\n%s\r\n\r\n%s\r\n--%s--\r\n" % (boundary, dispostion, - "auth_string", boundary) - self.assertEqual(len(tpl), client._header["Content-Length"]) - self.assertEqual(len(tpl), body.length()) - - round_tripper = tripper - client.call_with_multipart("/hello", fields={"auth": "auth_string"}) - - def test_call_with_form(self): - global round_tripper - def tripper(client, method, path, body): - self.assertEqual(body, "action=a&op=a&op=b") - target_type = "application/x-www-form-urlencoded" - self.assertEqual(client._header["Content-Type"], target_type) - self.assertEqual(client._header["Content-Length"], len(body)) - - round_tripper = tripper - client.call_with_form("/hello", dict(op=["a", "b"], action="a")) + + def test_call(self): + global round_tripper + + def tripper(client, method, path, body): + self.assertEqual(path, "/hello") + assert body is None + + round_tripper = tripper + client.call("/hello") + + def test_call_with(self): + global round_tripper + + def tripper(client, method, path, body): + self.assertEqual(body, "body") + + round_tripper = tripper + client.call_with("/hello", "body") + + def test_call_with_multipart(self): + global round_tripper + + def tripper(client, method, path, body): + target_type = "multipart/form-data" + self.assertTrue( + client._header["Content-Type"].startswith(target_type)) + start_index = client._header["Content-Type"].find("boundary") + boundary = client._header["Content-Type"][start_index + 9:] + dispostion = 'Content-Disposition: form-data; name="auth"' + tpl = "--%s\r\n%s\r\n\r\n%s\r\n--%s--\r\n" % (boundary, dispostion, + "auth_string", boundary) + self.assertEqual(len(tpl), client._header["Content-Length"]) + self.assertEqual(len(tpl), body.length()) + + round_tripper = tripper + client.call_with_multipart("/hello", fields={"auth": "auth_string"}) + + def test_call_with_form(self): + global round_tripper + + def tripper(client, method, path, body): + self.assertEqual(body, "action=a&op=a&op=b") + target_type = "application/x-www-form-urlencoded" + self.assertEqual(client._header["Content-Type"], target_type) + self.assertEqual(client._header["Content-Length"], len(body)) + + round_tripper = tripper + client.call_with_form("/hello", dict(op=["a", "b"], action="a")) class TestMultiReader(unittest.TestCase): - def test_multi_reader1(self): - a = StringIO.StringIO('你好') - b = StringIO.StringIO('abcdefg') - c = StringIO.StringIO(u'悲剧') - mr = rpc.MultiReader([a, b, c]) - data = mr.read() - assert data.index('悲剧') > data.index('abcdefg') - - def test_multi_reader2(self): - a = StringIO.StringIO('你好') - b = StringIO.StringIO('abcdefg') - c = StringIO.StringIO(u'悲剧') - mr = rpc.MultiReader([a, b, c]) - data = mr.read(8) - assert len(data) is 8 + + def test_multi_reader1(self): + a = StringIO.StringIO('你好') + b = StringIO.StringIO('abcdefg') + c = StringIO.StringIO(u'悲剧') + mr = rpc.MultiReader([a, b, c]) + data = mr.read() + assert data.index('悲剧') > data.index('abcdefg') + + def test_multi_reader2(self): + a = StringIO.StringIO('你好') + b = StringIO.StringIO('abcdefg') + c = StringIO.StringIO(u'悲剧') + mr = rpc.MultiReader([a, b, c]) + data = mr.read(8) + assert len(data) is 8 def encode_multipart_formdata2(fields, files): - if files is None: - files = [] - if fields is None: - fields = [] - - BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$' - CRLF = '\r\n' - L = [] - for (key, value) in fields: - L.append('--' + BOUNDARY) - L.append('Content-Disposition: form-data; name="%s"' % key) - L.append('') - L.append(value) - for (key, filename, value) in files: - L.append('--' + BOUNDARY) - disposition = "Content-Disposition: form-data;" - L.append('%s name="%s"; filename="%s"' % (disposition, key, filename)) - L.append('Content-Type: application/octet-stream') - L.append('') - L.append(value) - L.append('--' + BOUNDARY + '--') - L.append('') - body = CRLF.join(L) - content_type = 'multipart/form-data; boundary=%s' % BOUNDARY - return content_type, body + if files is None: + files = [] + if fields is None: + fields = [] + + BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$' + CRLF = '\r\n' + L = [] + for (key, value) in fields: + L.append('--' + BOUNDARY) + L.append('Content-Disposition: form-data; name="%s"' % key) + L.append('') + L.append(value) + for (key, filename, value) in files: + L.append('--' + BOUNDARY) + disposition = "Content-Disposition: form-data;" + L.append('%s name="%s"; filename="%s"' % (disposition, key, filename)) + L.append('Content-Type: application/octet-stream') + L.append('') + L.append(value) + L.append('--' + BOUNDARY + '--') + L.append('') + body = CRLF.join(L) + content_type = 'multipart/form-data; boundary=%s' % BOUNDARY + return content_type, body class TestEncodeMultipartFormdata(unittest.TestCase): - def test_encode(self): - fields = {'a': '1', 'b': '2'} - files = [ - { - 'filename': 'key1', - 'data': 'data1', - 'mime_type': 'application/octet-stream', - }, - { - 'filename': 'key2', - 'data': 'data2', - 'mime_type': 'application/octet-stream', - } - ] - content_type, mr = rpc.Client('localhost').encode_multipart_formdata(fields, files) - t, b = encode_multipart_formdata2( - [('a', '1'), ('b', '2')], - [('file', 'key1', 'data1'), ('file', 'key2', 'data2')] - ) - assert t == content_type - assert len(b) == mr.length() - - def test_unicode(self): - def test1(): - files = [{'filename': '你好', 'data': '你好', 'mime_type': ''}] - _, body = rpc.Client('localhost').encode_multipart_formdata(None, files) - return len(body.read()) - def test2(): - files = [{'filename': u'你好', 'data': '你好', 'mime_type': ''}] - _, body = rpc.Client('localhost').encode_multipart_formdata(None, files) - return len(body.read()) - def test3(): - files = [{'filename': '你好', 'data': u'你好', 'mime_type': ''}] - _, body = rpc.Client('localhost').encode_multipart_formdata(None, files) - return len(body.read()) - def test4(): - files = [{'filename': u'你好', 'data': u'你好', 'mime_type': ''}] - _, body = rpc.Client('localhost').encode_multipart_formdata(None, files) - return len(body.read()) - - assert test1() == test2() - assert test2() == test3() - assert test3() == test4() + + def test_encode(self): + fields = {'a': '1', 'b': '2'} + files = [ + { + 'filename': 'key1', + 'data': 'data1', + 'mime_type': 'application/octet-stream', + }, + { + 'filename': 'key2', + 'data': 'data2', + 'mime_type': 'application/octet-stream', + } + ] + content_type, mr = rpc.Client( + 'localhost').encode_multipart_formdata(fields, files) + t, b = encode_multipart_formdata2( + [('a', '1'), ('b', '2')], + [('file', 'key1', 'data1'), ('file', 'key2', 'data2')] + ) + assert t == content_type + assert len(b) == mr.length() + + def test_unicode(self): + def test1(): + files = [{'filename': '你好', 'data': '你好', 'mime_type': ''}] + _, body = rpc.Client( + 'localhost').encode_multipart_formdata(None, files) + return len(body.read()) + + def test2(): + files = [{'filename': u'你好', 'data': '你好', 'mime_type': ''}] + _, body = rpc.Client( + 'localhost').encode_multipart_formdata(None, files) + return len(body.read()) + + def test3(): + files = [{'filename': '你好', 'data': u'你好', 'mime_type': ''}] + _, body = rpc.Client( + 'localhost').encode_multipart_formdata(None, files) + return len(body.read()) + + def test4(): + files = [{'filename': u'你好', 'data': u'你好', 'mime_type': ''}] + _, body = rpc.Client( + 'localhost').encode_multipart_formdata(None, files) + return len(body.read()) + + assert test1() == test2() + assert test2() == test3() + assert test3() == test4() if __name__ == "__main__": - unittest.main() + unittest.main() diff --git a/qiniu/test/rsf_test.py b/qiniu/test/rsf_test.py index 3f0c5fd7..7b7573d0 100644 --- a/qiniu/test/rsf_test.py +++ b/qiniu/test/rsf_test.py @@ -8,13 +8,15 @@ conf.SECRET_KEY = os.getenv("QINIU_SECRET_KEY") bucket_name = os.getenv("QINIU_TEST_BUCKET") + class TestRsf(unittest.TestCase): - def test_list_prefix(self): - c = rsf.Client() - ret, err = c.list_prefix(bucket_name, limit = 4) - self.assertEqual(err is rsf.EOF or err is None, True) - assert len(ret.get('items')) == 4 + + def test_list_prefix(self): + c = rsf.Client() + ret, err = c.list_prefix(bucket_name, limit=4) + self.assertEqual(err is rsf.EOF or err is None, True) + assert len(ret.get('items')) == 4 if __name__ == "__main__": - unittest.main() + unittest.main() diff --git a/setup.py b/setup.py index 734d7d6e..3554f54c 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,9 @@ # -*- coding: utf-8 -*- try: - from setuptools import setup -except ImportError: - from distutils.core import setup + from setuptools import setup +except ImportError: + from distutils.core import setup PACKAGE = 'qiniu' NAME = 'qiniu' @@ -18,27 +18,28 @@ setup( - name=NAME, - version=VERSION, - description=DESCRIPTION, - long_description=LONG_DESCRIPTION, - author=AUTHOR, - author_email=AUTHOR_EMAIL, - maintainer_email=MAINTAINER_EMAIL, - license='MIT', - url=URL, - packages=['qiniu', 'qiniu.test', 'qiniu.auth', 'qiniu.rs', 'qiniu.rs.test'], - platforms='any', - classifiers=[ - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', - 'Topic :: Software Development :: Libraries :: Python Modules' - ], - test_suite = 'nose.collector' + name=NAME, + version=VERSION, + description=DESCRIPTION, + long_description=LONG_DESCRIPTION, + author=AUTHOR, + author_email=AUTHOR_EMAIL, + maintainer_email=MAINTAINER_EMAIL, + license='MIT', + url=URL, + packages=['qiniu', 'qiniu.test', 'qiniu.auth', + 'qiniu.rs', 'qiniu.rs.test'], + platforms='any', + classifiers=[ + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', + 'Topic :: Software Development :: Libraries :: Python Modules' + ], + test_suite='nose.collector' ) From 6c8d67a964992d7b75aa413676bd207c1df6c4ca Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 10 Apr 2014 20:06:21 +0800 Subject: [PATCH 019/478] rm useless space --- qiniu/rsf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/rsf.py b/qiniu/rsf.py index eb0aeac2..ea9be409 100644 --- a/qiniu/rsf.py +++ b/qiniu/rsf.py @@ -24,7 +24,7 @@ def list_prefix(self, bucket, prefix=None, marker=None, limit=None): 1. 首次请求 marker = None 2. 无论 err 值如何，均应该先看 ret.get('items') 是否有内容 - 3. 如果后续没有更多数据，err 返回 EOF，markerOut 返回 None（但不通过该特征来判断是否结束） + 3. 如果后续没有更多数据，err 返回 EOF，markerOut 返回 None（但不通过该特征来判断是否结束） ''' ops = { 'bucket': bucket, From fc7f70a10e9c0a073df2cda4e0e0ab27aaa7039d Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 10 Apr 2014 20:14:38 +0800 Subject: [PATCH 020/478] pep8 check in travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index f8e1d6fd..1f2f8329 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,9 @@ python: - "2.7" install: - "pip install coverage --use-mirrors" + - "pip install pep8 --use-mirrors" before_script: + - "pep8 --max-line-length=160 ." - export QINIU_ACCESS_KEY="X0XpjFmLMTJpHB_ESHjeolCtipk-1U3Ok7LVTdoN" - export QINIU_SECRET_KEY="wenlwkU1AYwNBf7Q9cCoG4VT_GYyrHE9AS_R2u81" - export QINIU_TEST_BUCKET="pysdk" From e0e14a3975ddf303a1631aca81d41756eff197cf Mon Sep 17 00:00:00 2001 From: dtynn Date: Thu, 10 Apr 2014 21:17:55 +0800 Subject: [PATCH 021/478] pyflakes --- qiniu/fop.py | 1 - qiniu/httplib_chunk.py | 2 +- qiniu/io.py | 5 +++-- qiniu/resumable_io.py | 4 +++- qiniu/rs/__init__.py | 4 ++-- qiniu/rs/test/rs_token_test.py | 1 - qiniu/test/fop_test.py | 1 - qiniu/test/io_test.py | 6 ++++-- qiniu/test/resumable_io_test.py | 9 +++++---- setup.py | 4 +++- 10 files changed, 21 insertions(+), 16 deletions(-) diff --git a/qiniu/fop.py b/qiniu/fop.py index e04b084a..350b2857 100644 --- a/qiniu/fop.py +++ b/qiniu/fop.py @@ -1,5 +1,4 @@ # -*- coding:utf-8 -*- -import json class Exif(object): diff --git a/qiniu/httplib_chunk.py b/qiniu/httplib_chunk.py index 3aef95f4..8fb43134 100644 --- a/qiniu/httplib_chunk.py +++ b/qiniu/httplib_chunk.py @@ -10,7 +10,6 @@ import httplib from httplib import _CS_REQ_STARTED, _CS_REQ_SENT, CannotSendHeader, NotConnected import string -import os from array import array @@ -56,6 +55,7 @@ def _set_content_length(self, body): # Don't send a length if this failed if self.debuglevel > 0: print "Cannot stat!!" + print te if thelen is not None: self.putheader('Content-Length', thelen) diff --git a/qiniu/io.py b/qiniu/io.py index afaacc9d..efe042ad 100644 --- a/qiniu/io.py +++ b/qiniu/io.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- -from base64 import urlsafe_b64encode import rpc import conf import random import string try: - import zlib as binascii + import zlib + binascii = zlib except ImportError: + zlib = None import binascii diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index e8feee18..974a5819 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -1,8 +1,10 @@ # coding=utf-8 import os try: - import zlib as binascii + import zlib + binascii = zlib except ImportError: + zlib = None import binascii from base64 import urlsafe_b64encode diff --git a/qiniu/rs/__init__.py b/qiniu/rs/__init__.py index 861d4264..54f748aa 100644 --- a/qiniu/rs/__init__.py +++ b/qiniu/rs/__init__.py @@ -5,5 +5,5 @@ "PutPolicy", "GetPolicy", "make_base_url", ] -from .rs import * -from .rs_token import * +from .rs import Client, EntryPath, EntryPathPair, uri_stat, uri_delete, uri_move, uri_copy +from .rs_token import PutPolicy, GetPolicy, make_base_url diff --git a/qiniu/rs/test/rs_token_test.py b/qiniu/rs/test/rs_token_test.py index 9e1a95a5..9aef1c00 100644 --- a/qiniu/rs/test/rs_token_test.py +++ b/qiniu/rs/test/rs_token_test.py @@ -9,7 +9,6 @@ import urllib from qiniu import conf -from qiniu import rpc from qiniu import rs conf.ACCESS_KEY = os.getenv("QINIU_ACCESS_KEY") diff --git a/qiniu/test/fop_test.py b/qiniu/test/fop_test.py index b25d5a1a..c2664882 100644 --- a/qiniu/test/fop_test.py +++ b/qiniu/test/fop_test.py @@ -1,6 +1,5 @@ # -*- coding:utf-8 -*- import unittest -import os from qiniu import fop pic = "http://cheneya.qiniudn.com/hello_jpg" diff --git a/qiniu/test/io_test.py b/qiniu/test/io_test.py index 18c9bc9f..892f10ad 100644 --- a/qiniu/test/io_test.py +++ b/qiniu/test/io_test.py @@ -5,8 +5,10 @@ import random import urllib try: - import zlib as binascii + import zlib + binascii = zlib except ImportError: + zlib = None import binascii import cStringIO @@ -34,7 +36,7 @@ class TestUp(unittest.TestCase): def test(self): def test_put(): key = "test_%s" % r(9) - params = "op=3" + #params = "op=3" data = "hello bubby!" extra.check_crc = 2 extra.crc32 = binascii.crc32(data) & 0xFFFFFFFF diff --git a/qiniu/test/resumable_io_test.py b/qiniu/test/resumable_io_test.py index d4e97487..04961906 100644 --- a/qiniu/test/resumable_io_test.py +++ b/qiniu/test/resumable_io_test.py @@ -5,11 +5,12 @@ import random import platform try: - import zlib as binascii + import zlib + binascii = zlib except ImportError: + zlib = None import binascii import urllib -import tempfile import shutil from qiniu import conf @@ -24,7 +25,7 @@ def r(length): lib = string.ascii_uppercase - return ''.join([random.choice(lib) for i in range(0, length)]) + return ''.join([random.choice(lib) for _ in range(0, length)]) class TestBlock(unittest.TestCase): @@ -34,7 +35,7 @@ def test_block(self): uptoken = policy.token() client = up.Client(uptoken) - rets = [0, 0] + #rets = [0, 0] data_slice_2 = "\nbye!" ret, err = resumable_io.mkblock( client, len(data_slice_2), data_slice_2) diff --git a/setup.py b/setup.py index 3554f54c..5ce3f277 100644 --- a/setup.py +++ b/setup.py @@ -2,8 +2,10 @@ # -*- coding: utf-8 -*- try: - from setuptools import setup + import setuptools + setup = setuptools.setup except ImportError: + setuptools = None from distutils.core import setup PACKAGE = 'qiniu' From 351775359718172d1bae5ec74e5aea3d6f6481a7 Mon Sep 17 00:00:00 2001 From: dtynn Date: Thu, 10 Apr 2014 21:19:46 +0800 Subject: [PATCH 022/478] pyflakes --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1f2f8329..2db91e85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,10 @@ python: install: - "pip install coverage --use-mirrors" - "pip install pep8 --use-mirrors" + - "pip install pyflakes" before_script: - "pep8 --max-line-length=160 ." + - "pyflakes ." - export QINIU_ACCESS_KEY="X0XpjFmLMTJpHB_ESHjeolCtipk-1U3Ok7LVTdoN" - export QINIU_SECRET_KEY="wenlwkU1AYwNBf7Q9cCoG4VT_GYyrHE9AS_R2u81" - export QINIU_TEST_BUCKET="pysdk" From 33d7e757971a20a4bfc417cd13fda8e598778007 Mon Sep 17 00:00:00 2001 From: dtynn Date: Thu, 10 Apr 2014 21:39:28 +0800 Subject: [PATCH 023/478] pep8 E265 --- qiniu/test/io_test.py | 2 +- qiniu/test/resumable_io_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/test/io_test.py b/qiniu/test/io_test.py index 892f10ad..31fa5a1e 100644 --- a/qiniu/test/io_test.py +++ b/qiniu/test/io_test.py @@ -36,7 +36,7 @@ class TestUp(unittest.TestCase): def test(self): def test_put(): key = "test_%s" % r(9) - #params = "op=3" + # params = "op=3" data = "hello bubby!" extra.check_crc = 2 extra.crc32 = binascii.crc32(data) & 0xFFFFFFFF diff --git a/qiniu/test/resumable_io_test.py b/qiniu/test/resumable_io_test.py index 04961906..5e718dfe 100644 --- a/qiniu/test/resumable_io_test.py +++ b/qiniu/test/resumable_io_test.py @@ -35,7 +35,7 @@ def test_block(self): uptoken = policy.token() client = up.Client(uptoken) - #rets = [0, 0] + # rets = [0, 0] data_slice_2 = "\nbye!" ret, err = resumable_io.mkblock( client, len(data_slice_2), data_slice_2) From 24b8ddc5fd386454d9d2850af34e5ad76bf43dea Mon Sep 17 00:00:00 2001 From: dtynn Date: Thu, 10 Apr 2014 22:27:12 +0800 Subject: [PATCH 024/478] pip install use mirrors --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2db91e85..7c147aed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: install: - "pip install coverage --use-mirrors" - "pip install pep8 --use-mirrors" - - "pip install pyflakes" + - "pip install pyflakes --use-mirrors" before_script: - "pep8 --max-line-length=160 ." - "pyflakes ." From d4732b8d7adb469c6171514a5de393e4318ebdbc Mon Sep 17 00:00:00 2001 From: dtynn Date: Thu, 10 Apr 2014 22:31:42 +0800 Subject: [PATCH 025/478] author info --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5ce3f277..179ef86c 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,8 @@ DESCRIPTION = 'Qiniu Resource Storage SDK for Python 2.X.' LONG_DESCRIPTION = 'see:\nhttps://github.com/qiniu/python-sdk\n' AUTHOR = 'Shanghai Qiniu Information Technologies Co., Ltd.' -AUTHOR_EMAIL = 'support@qiniu.com' -MAINTAINER_EMAIL = 'fengliyuan@qiniu.com' +AUTHOR_EMAIL = 'sdk@qiniu.com' +MAINTAINER_EMAIL = 'support@qiniu.com' URL = 'https://github.com/qiniu/python-sdk' VERSION = __import__(PACKAGE).__version__ From 69a338fc77ba7d3486f460a0185c900c2d2d743a Mon Sep 17 00:00:00 2001 From: dtynn Date: Fri, 11 Apr 2014 14:05:27 +0800 Subject: [PATCH 026/478] =?UTF-8?q?travis=E7=8E=AF=E5=A2=83=E4=B8=8B?= =?UTF-8?q?=E5=8F=96=E6=B6=88=E5=88=87=E7=89=87=E4=B8=8A=E4=BC=A0=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/test/resumable_io_test.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/qiniu/test/resumable_io_test.py b/qiniu/test/resumable_io_test.py index 5e718dfe..a6a82976 100644 --- a/qiniu/test/resumable_io_test.py +++ b/qiniu/test/resumable_io_test.py @@ -21,6 +21,8 @@ bucket = os.getenv("QINIU_TEST_BUCKET") conf.ACCESS_KEY = os.getenv("QINIU_ACCESS_KEY") conf.SECRET_KEY = os.getenv("QINIU_SECRET_KEY") +test_env = os.getenv("QINIU_TEST_ENV") +is_travis = test_env == "travis" def r(length): @@ -31,6 +33,8 @@ def r(length): class TestBlock(unittest.TestCase): def test_block(self): + if is_travis: + return policy = rs.PutPolicy(bucket) uptoken = policy.token() client = up.Client(uptoken) @@ -57,6 +61,8 @@ def test_block(self): rs.Client().delete(bucket, key) def test_put(self): + if is_travis: + return src = urllib.urlopen("http://cheneya.qiniudn.com/hello_jpg") ostype = platform.system() if ostype.lower().find("windows") != -1: @@ -84,6 +90,8 @@ def test_put(self): rs.Client().delete(bucket, key) def test_put_4m(self): + if is_travis: + return ostype = platform.system() if ostype.lower().find("windows") != -1: tmpf = "".join([os.getcwd(), os.tmpnam()]) @@ -111,4 +119,5 @@ def test_put_4m(self): if __name__ == "__main__": - unittest.main() + if not is_travis: + unittest.main() From fa385520f8243e804e331212b64480a97652d7c0 Mon Sep 17 00:00:00 2001 From: dtynn Date: Fri, 11 Apr 2014 14:08:19 +0800 Subject: [PATCH 027/478] =?UTF-8?q?travis=E7=8E=AF=E5=A2=83=E4=B8=8B?= =?UTF-8?q?=E5=8F=96=E6=B6=88=E5=88=87=E7=89=87=E4=B8=8A=E4=BC=A0=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 7c147aed..3e8d4cee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ before_script: - export QINIU_SECRET_KEY="wenlwkU1AYwNBf7Q9cCoG4VT_GYyrHE9AS_R2u81" - export QINIU_TEST_BUCKET="pysdk" - export QINIU_TEST_DOMAIN="pysdk.qiniudn.com" + - export QINIU_TEST_ENV="travis" - export PYTHONPATH="$PYTHONPATH:." script: - python setup.py nosetests From 15d4ae0889bde702b9352aeec2f5724960fc572e Mon Sep 17 00:00:00 2001 From: dtynn Date: Mon, 14 Apr 2014 17:27:50 +0800 Subject: [PATCH 028/478] =?UTF-8?q?=E8=AF=B7=E6=B1=82=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E4=B8=AD=E5=8A=A0=E5=85=A5reqid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/rpc.py | 3 +++ qiniu/test/io_test.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/qiniu/rpc.py b/qiniu/rpc.py index 2e78c806..8bc62295 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -42,7 +42,10 @@ def call_with(self, path, body, content_type=None, content_length=None): if resp.status / 100 != 2: err_msg = ret if "error" not in ret else ret["error"] + reqid = resp.getheader("X-Reqid", None) detail = resp.getheader("x-log", None) + if reqid is not None: + err_msg += ", reqid:%s" % reqid if detail is not None: err_msg += ", detail:%s" % detail diff --git a/qiniu/test/io_test.py b/qiniu/test/io_test.py index 31fa5a1e..28fa3e31 100644 --- a/qiniu/test/io_test.py +++ b/qiniu/test/io_test.py @@ -172,6 +172,12 @@ def test_put_crc_fail(self): ret, err = io.put(policy.token(), key, data, extra) assert err is not None + def test_put_fail_reqid(self): + key = "test_%s" % r(9) + data = "hello bubby!" + ret, err = io.put("", key, data, extra) + assert "reqid" in err + class Test_get_file_crc32(unittest.TestCase): From 2750fa07fe412846728948b16dcaf3d0cdc2407d Mon Sep 17 00:00:00 2001 From: dtynn Date: Tue, 15 Apr 2014 15:09:41 +0800 Subject: [PATCH 029/478] useragent --- qiniu/__init__.py | 2 +- qiniu/conf.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 8300d746..31496ad4 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -7,4 +7,4 @@ ''' # -*- coding: utf-8 -*- -__version__ = '6.1.4' +__version__ = '6.1.6' diff --git a/qiniu/conf.py b/qiniu/conf.py index d43b3144..a603a736 100644 --- a/qiniu/conf.py +++ b/qiniu/conf.py @@ -8,4 +8,9 @@ UP_HOST = "up.qiniu.com" from . import __version__ -USER_AGENT = "qiniu python-sdk v%s" % __version__ +import platform + +sys_info = "%s/%s" % (platform.system(), platform.machine()) +py_ver = platform.python_version() + +USER_AGENT = "QiniuPython/%s (%s) Python/%s" % (__version__, sys_info, py_ver) From dee5087523a1533ee39c11480ff960d6464064f5 Mon Sep 17 00:00:00 2001 From: dtynn Date: Tue, 15 Apr 2014 15:13:31 +0800 Subject: [PATCH 030/478] =?UTF-8?q?=E6=9A=82=E6=97=B6=E5=8E=BB=E6=8E=89det?= =?UTF-8?q?ail=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/rpc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qiniu/rpc.py b/qiniu/rpc.py index 8bc62295..2d35e3ff 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -46,8 +46,6 @@ def call_with(self, path, body, content_type=None, content_length=None): detail = resp.getheader("x-log", None) if reqid is not None: err_msg += ", reqid:%s" % reqid - if detail is not None: - err_msg += ", detail:%s" % detail return None, err_msg From bd0d4a5dc7e5107e01ebc35f4380f583298353ad Mon Sep 17 00:00:00 2001 From: dtynn Date: Tue, 15 Apr 2014 15:13:49 +0800 Subject: [PATCH 031/478] =?UTF-8?q?=E6=9A=82=E6=97=B6=E5=8E=BB=E6=8E=89det?= =?UTF-8?q?ail=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/rpc.py b/qiniu/rpc.py index 2d35e3ff..8c092376 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -43,7 +43,7 @@ def call_with(self, path, body, content_type=None, content_length=None): if resp.status / 100 != 2: err_msg = ret if "error" not in ret else ret["error"] reqid = resp.getheader("X-Reqid", None) - detail = resp.getheader("x-log", None) + # detail = resp.getheader("x-log", None) if reqid is not None: err_msg += ", reqid:%s" % reqid From 1df7c338ea7917bbe197dea957262cc8bffc7529 Mon Sep 17 00:00:00 2001 From: dtynn Date: Tue, 15 Apr 2014 15:38:26 +0800 Subject: [PATCH 032/478] =?UTF-8?q?=E8=A1=A5=E5=85=85changelog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13abd233..f5e368e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ## CHANGE LOG +### v6.1.6 + +2014-04-15 issue [#104](https://github.com/qiniu/python-sdk/pull/104) +- [#100] 遵循PEP8语法规范 +- [#101] 增加pyflakes语法检测 +- [#103] 错误信息中加入Reqid信息，以便追溯 +- [#104] 完善User-Agent头信息，以便追溯 + ### v6.1.5 2014-04-08 issue [#98](https://github.com/qiniu/python-sdk/pull/98) From 5be1e258fccd8ec41f5e76b2e8f31d2d28929b11 Mon Sep 17 00:00:00 2001 From: longbai Date: Mon, 28 Apr 2014 11:08:59 +0800 Subject: [PATCH 033/478] update changelog --- CHANGELOG.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5e368e8..6b36350f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,12 @@ ## CHANGE LOG -### v6.1.6 +### v6.1.5 -2014-04-15 issue [#104](https://github.com/qiniu/python-sdk/pull/104) +2014-04-28 issue [#105](https://github.com/qiniu/python-sdk/pull/105) - [#100] 遵循PEP8语法规范 - [#101] 增加pyflakes语法检测 - [#103] 错误信息中加入Reqid信息，以便追溯 - [#104] 完善User-Agent头信息，以便追溯 - -### v6.1.5 - -2014-04-08 issue [#98](https://github.com/qiniu/python-sdk/pull/98) - - [#98] 增加fetch、prefetch、pfop三个接口的范例代码 ### v6.1.4 From 0770a86eec5db53bce622019fab102bdbc043074 Mon Sep 17 00:00:00 2001 From: Cloverstd Date: Mon, 12 May 2014 13:42:10 +0800 Subject: [PATCH 034/478] Update rs_token.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 增加“限定用户上传的文件类型”字段 --- qiniu/rs/rs_token.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qiniu/rs/rs_token.py b/qiniu/rs/rs_token.py index 2250e484..62e824b1 100644 --- a/qiniu/rs/rs_token.py +++ b/qiniu/rs/rs_token.py @@ -22,6 +22,7 @@ class PutPolicy(object): saveKey = None insertOnly = None detectMime = None + mimeLimit = None fsizeLimit = None persistentNotifyUrl = None persistentOps = None @@ -64,6 +65,9 @@ def token(self, mac=None): if self.detectMime is not None: token["detectMime"] = self.detectMime + + if self.mimeLimit is not None: + token["mimeLimit"] = self.mimeLimit if self.fsizeLimit is not None: token["fsizeLimit"] = self.fsizeLimit From 9f01a7ae12cc7489158750dea6fbf56a6ec58ab7 Mon Sep 17 00:00:00 2001 From: Cloverstd Date: Tue, 13 May 2014 22:26:41 +0800 Subject: [PATCH 035/478] Update rs_token.py pep8 --- qiniu/rs/rs_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/rs/rs_token.py b/qiniu/rs/rs_token.py index 62e824b1..478744e3 100644 --- a/qiniu/rs/rs_token.py +++ b/qiniu/rs/rs_token.py @@ -65,7 +65,7 @@ def token(self, mac=None): if self.detectMime is not None: token["detectMime"] = self.detectMime - + if self.mimeLimit is not None: token["mimeLimit"] = self.mimeLimit From 1774db1a7d016630501a198763b352a2d4dee4a3 Mon Sep 17 00:00:00 2001 From: SunRunAway Date: Tue, 3 Jun 2014 12:08:39 +0800 Subject: [PATCH 036/478] resumblePut can put 0-size file --- qiniu/resumable_io.py | 3 ++- qiniu/test/resumable_io_test.py | 24 ++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index 974a5819..c0622e1f 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -106,7 +106,8 @@ def put(uptoken, key, f, fsize, extra): return None, err_put_failed print err, ".. retry" - mkfile_client = auth_up.Client(uptoken, extra.progresses[-1]["host"]) + mkfile_host = extra.progresses[-1]["host"] if block_cnt else conf.UP_HOST + mkfile_client = auth_up.Client(uptoken, mkfile_host) return mkfile(mkfile_client, key, fsize, extra) diff --git a/qiniu/test/resumable_io_test.py b/qiniu/test/resumable_io_test.py index a6a82976..bacb0a61 100644 --- a/qiniu/test/resumable_io_test.py +++ b/qiniu/test/resumable_io_test.py @@ -12,6 +12,7 @@ import binascii import urllib import shutil +import StringIO from qiniu import conf from qiniu.auth import up @@ -80,11 +81,11 @@ def test_put(self): key = "sdk_py_resumable_block_5_%s" % r(9) localfile = dst.name ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) - assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" dst.close() os.remove(tmpf) assert err is None, err + assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" self.assertEqual( ret["hash"], "FnyTMUqPNRTdk1Wou7oLqDHkBm_p", "hash not match") rs.Client().delete(bucket, key) @@ -108,15 +109,34 @@ def test_put_4m(self): key = "sdk_py_resumable_block_6_%s" % r(9) localfile = dst.name ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) - assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" dst.close() os.remove(tmpf) assert err is None, err + assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" self.assertEqual( ret["hash"], "FnIVmMd_oaUV3MLDM6F9in4RMz2U", "hash not match") rs.Client().delete(bucket, key) + def test_put_0(self): + if is_travis: + return + + f = StringIO.StringIO('') + + policy = rs.PutPolicy(bucket) + extra = resumable_io.PutExtra(bucket) + extra.bucket = bucket + extra.params = {"x:foo": "test"} + key = "sdk_py_resumable_block_7_%s" % r(9) + ret, err = resumable_io.put(policy.token(), key, f, 0, extra) + + assert err is None, err + assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" + self.assertEqual( + ret["hash"], "Fg==", "hash not match") + rs.Client().delete(bucket, key) + if __name__ == "__main__": if not is_travis: From ad8abfa444b1da6521135b1f1de6fa3d3c5d7b89 Mon Sep 17 00:00:00 2001 From: Gully Chen Date: Fri, 20 Jun 2014 22:43:52 +0800 Subject: [PATCH 037/478] Support Google AppEngine, fix import error GAE httplib implementation uses google.appengine.api.urlfetch --- qiniu/rpc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/qiniu/rpc.py b/qiniu/rpc.py index 8c092376..5ec9a921 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -1,5 +1,10 @@ # -*- coding: utf-8 -*- -import httplib_chunk as httplib + +import httplib + +if not getattr(httplib, "_IMPLEMENTATION", False): # httplib._IMPLEMENTATION is "gae" on GAE + import httplib_chunk as httplib + import json import cStringIO import conf From 2d87e8bc1d4e144a6855ab392a79b9aee75ceac5 Mon Sep 17 00:00:00 2001 From: Gully Chen Date: Fri, 20 Jun 2014 23:02:18 +0800 Subject: [PATCH 038/478] fix for GAE import --- qiniu/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/rpc.py b/qiniu/rpc.py index 5ec9a921..45b6bff4 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -2,7 +2,7 @@ import httplib -if not getattr(httplib, "_IMPLEMENTATION", False): # httplib._IMPLEMENTATION is "gae" on GAE +if getattr(httplib, "_IMPLEMENTATION") != "gae": # httplib._IMPLEMENTATION is "gae" on GAE import httplib_chunk as httplib import json From adcae2d3761a93446403d724ed36f9711dbc5660 Mon Sep 17 00:00:00 2001 From: Gully Chen Date: Fri, 20 Jun 2014 23:38:38 +0800 Subject: [PATCH 039/478] fix for GAE import --- qiniu/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/rpc.py b/qiniu/rpc.py index 45b6bff4..c2d8afff 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -2,7 +2,7 @@ import httplib -if getattr(httplib, "_IMPLEMENTATION") != "gae": # httplib._IMPLEMENTATION is "gae" on GAE +if getattr(httplib, "_IMPLEMENTATION", None) != "gae": # httplib._IMPLEMENTATION is "gae" on GAE import httplib_chunk as httplib import json From f07756182eb561746e2f7c954cc6c1d65bd8f1a3 Mon Sep 17 00:00:00 2001 From: Philip Tzou Date: Fri, 27 Jun 2014 17:56:26 +0800 Subject: [PATCH 040/478] blank out headers after sending request, or the next request will be polluted. --- qiniu/auth/digest.py | 8 +++---- qiniu/auth/up.py | 6 ++--- qiniu/rpc.py | 21 ++++++++++++------ qiniu/test/resumable_io_test.py | 9 ++++---- qiniu/test/rpc_test.py | 39 +++++++++++++++++++++------------ 5 files changed, 51 insertions(+), 32 deletions(-) diff --git a/qiniu/auth/digest.py b/qiniu/auth/digest.py index 5d2788f3..dcee6fbc 100644 --- a/qiniu/auth/digest.py +++ b/qiniu/auth/digest.py @@ -55,8 +55,8 @@ def __init__(self, host, mac=None): super(Client, self).__init__(host) self.mac = mac - def round_tripper(self, method, path, body): + def round_tripper(self, method, path, body, header={}): token = self.mac.sign_request( - path, body, self._header.get("Content-Type")) - self.set_header("Authorization", "QBox %s" % token) - return super(Client, self).round_tripper(method, path, body) + path, body, header.get("Content-Type")) + header["Authorization"] = "QBox %s" % token + return super(Client, self).round_tripper(method, path, body, header) diff --git a/qiniu/auth/up.py b/qiniu/auth/up.py index de43b223..a2a979f3 100644 --- a/qiniu/auth/up.py +++ b/qiniu/auth/up.py @@ -14,6 +14,6 @@ def __init__(self, up_token, host=None): self.up_token = up_token super(Client, self).__init__(host) - def round_tripper(self, method, path, body): - self.set_header("Authorization", "UpToken %s" % self.up_token) - return super(Client, self).round_tripper(method, path, body) + def round_tripper(self, method, path, body, header={}): + header["Authorization"] = "UpToken %s" % self.up_token + return super(Client, self).round_tripper(method, path, body, header) diff --git a/qiniu/rpc.py b/qiniu/rpc.py index c2d8afff..cf2cf9c0 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -2,7 +2,8 @@ import httplib -if getattr(httplib, "_IMPLEMENTATION", None) != "gae": # httplib._IMPLEMENTATION is "gae" on GAE +if getattr(httplib, "_IMPLEMENTATION", None) != "gae": + # httplib._IMPLEMENTATION is "gae" on GAE import httplib_chunk as httplib import json @@ -18,25 +19,31 @@ def __init__(self, host): self._conn = httplib.HTTPConnection(host) self._header = {} - def round_tripper(self, method, path, body): - self._conn.request(method, path, body, self._header) + def round_tripper(self, method, path, body, header={}): + header = self.merged_headers(header) + self._conn.request(method, path, body, header) resp = self._conn.getresponse() return resp + def merged_headers(self, header): + _header = self._header.copy() + _header.update(header) + return _header + def call(self, path): return self.call_with(path, None) def call_with(self, path, body, content_type=None, content_length=None): ret = None - self.set_header("User-Agent", conf.USER_AGENT) + header = {"User-Agent": conf.USER_AGENT} if content_type is not None: - self.set_header("Content-Type", content_type) + header["Content-Type"] = content_type if content_length is not None: - self.set_header("Content-Length", content_length) + header["Content-Length"] = content_length - resp = self.round_tripper("POST", path, body) + resp = self.round_tripper("POST", path, body, header) try: ret = resp.read() ret = json.loads(ret) diff --git a/qiniu/test/resumable_io_test.py b/qiniu/test/resumable_io_test.py index bacb0a61..addb8f3a 100644 --- a/qiniu/test/resumable_io_test.py +++ b/qiniu/test/resumable_io_test.py @@ -13,6 +13,7 @@ import urllib import shutil import StringIO +from tempfile import mktemp from qiniu import conf from qiniu.auth import up @@ -67,9 +68,9 @@ def test_put(self): src = urllib.urlopen("http://cheneya.qiniudn.com/hello_jpg") ostype = platform.system() if ostype.lower().find("windows") != -1: - tmpf = "".join([os.getcwd(), os.tmpnam()]) + tmpf = "".join([os.getcwd(), mktemp()]) else: - tmpf = os.tmpnam() + tmpf = mktemp() dst = open(tmpf, 'wb') shutil.copyfileobj(src, dst) src.close() @@ -95,9 +96,9 @@ def test_put_4m(self): return ostype = platform.system() if ostype.lower().find("windows") != -1: - tmpf = "".join([os.getcwd(), os.tmpnam()]) + tmpf = "".join([os.getcwd(), mktemp()]) else: - tmpf = os.tmpnam() + tmpf = mktemp() dst = open(tmpf, 'wb') dst.write("abcd" * 1024 * 1024) dst.flush() diff --git a/qiniu/test/rpc_test.py b/qiniu/test/rpc_test.py index 08acfa04..ae326484 100644 --- a/qiniu/test/rpc_test.py +++ b/qiniu/test/rpc_test.py @@ -6,15 +6,15 @@ from qiniu import conf -def round_tripper(client, method, path, body): +def round_tripper(client, method, path, body, header={}): pass class ClsTestClient(rpc.Client): - def round_tripper(self, method, path, body): - round_tripper(self, method, path, body) - return super(ClsTestClient, self).round_tripper(method, path, body) + def round_tripper(self, method, path, body, header={}): + round_tripper(self, method, path, body, header) + return super(ClsTestClient, self).round_tripper(method, path, body, header) client = ClsTestClient(conf.RS_HOST) @@ -24,7 +24,7 @@ class TestClient(unittest.TestCase): def test_call(self): global round_tripper - def tripper(client, method, path, body): + def tripper(client, method, path, body, header={}): self.assertEqual(path, "/hello") assert body is None @@ -34,7 +34,7 @@ def tripper(client, method, path, body): def test_call_with(self): global round_tripper - def tripper(client, method, path, body): + def tripper(client, method, path, body, header={}): self.assertEqual(body, "body") round_tripper = tripper @@ -43,16 +43,16 @@ def tripper(client, method, path, body): def test_call_with_multipart(self): global round_tripper - def tripper(client, method, path, body): + def tripper(client, method, path, body, header={}): target_type = "multipart/form-data" self.assertTrue( - client._header["Content-Type"].startswith(target_type)) - start_index = client._header["Content-Type"].find("boundary") - boundary = client._header["Content-Type"][start_index + 9:] + header["Content-Type"].startswith(target_type)) + start_index = header["Content-Type"].find("boundary") + boundary = header["Content-Type"][start_index + 9:] dispostion = 'Content-Disposition: form-data; name="auth"' tpl = "--%s\r\n%s\r\n\r\n%s\r\n--%s--\r\n" % (boundary, dispostion, "auth_string", boundary) - self.assertEqual(len(tpl), client._header["Content-Length"]) + self.assertEqual(len(tpl), header["Content-Length"]) self.assertEqual(len(tpl), body.length()) round_tripper = tripper @@ -61,15 +61,26 @@ def tripper(client, method, path, body): def test_call_with_form(self): global round_tripper - def tripper(client, method, path, body): + def tripper(client, method, path, body, header={}): self.assertEqual(body, "action=a&op=a&op=b") target_type = "application/x-www-form-urlencoded" - self.assertEqual(client._header["Content-Type"], target_type) - self.assertEqual(client._header["Content-Length"], len(body)) + self.assertEqual(header["Content-Type"], target_type) + self.assertEqual(header["Content-Length"], len(body)) round_tripper = tripper client.call_with_form("/hello", dict(op=["a", "b"], action="a")) + def test_call_after_call_with_form(self): + # test case for https://github.com/qiniu/python-sdk/issues/112 + global round_tripper + + def tripper(client, method, path, body, header={}): + pass + + round_tripper = tripper + client.call_with_form("/hello", dict(op=["a", "b"], action="a")) + client.call("/hello") + class TestMultiReader(unittest.TestCase): From 8200fdf91448433846f9997efe7948591bf03b49 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Mon, 30 Jun 2014 11:17:20 +0800 Subject: [PATCH 041/478] Update CHANGELOG.md --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b36350f..0f6a338f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ## CHANGE LOG +### v6.1.6 + +2014-06-30 issue [#118](https://github.com/qiniu/python-sdk/pull/118) +- [#108] 增加限定用户上传类型字段 +- [#110] 0字节文件上传支持 +- [#114] 支持GAE +- [#115] 请求完成后清除http header + ### v6.1.5 2014-04-28 issue [#105](https://github.com/qiniu/python-sdk/pull/105) From 76d5bb5d432231114af856d07f8f2987908b1826 Mon Sep 17 00:00:00 2001 From: longbai Date: Mon, 30 Jun 2014 12:11:43 +0800 Subject: [PATCH 042/478] add uphost2 --- qiniu/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/conf.py b/qiniu/conf.py index a603a736..ff84fafa 100644 --- a/qiniu/conf.py +++ b/qiniu/conf.py @@ -6,6 +6,7 @@ RS_HOST = "rs.qbox.me" RSF_HOST = "rsf.qbox.me" UP_HOST = "up.qiniu.com" +UP_HOST2 = "up.qbox.me" from . import __version__ import platform From e1b16038b7492150f569862db481a8dcf988c217 Mon Sep 17 00:00:00 2001 From: longbai Date: Mon, 30 Jun 2014 12:12:26 +0800 Subject: [PATCH 043/478] callwith add code return --- qiniu/rpc.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/qiniu/rpc.py b/qiniu/rpc.py index cf2cf9c0..7be6446b 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -31,7 +31,8 @@ def merged_headers(self, header): return _header def call(self, path): - return self.call_with(path, None) + ret, err, code = self.call_with(path, None) + return ret, err def call_with(self, path, body, content_type=None, content_length=None): ret = None @@ -59,9 +60,9 @@ def call_with(self, path, body, content_type=None, content_length=None): if reqid is not None: err_msg += ", reqid:%s" % reqid - return None, err_msg + return None, err_msg, resp.status - return ret, None + return ret, None, resp.status def call_with_multipart(self, path, fields=None, files=None): """ @@ -87,7 +88,8 @@ def call_with_form(self, path, ops): body = '&'.join(body) content_type = "application/x-www-form-urlencoded" - return self.call_with(path, body, content_type, len(body)) + ret, err, code = self.call_with(path, body, content_type, len(body)) + return ret, err def set_header(self, field, value): self._header[field] = value From 1b0871e2a7b439583bbabf7ee4db70ad0381961e Mon Sep 17 00:00:00 2001 From: longbai Date: Mon, 30 Jun 2014 12:12:56 +0800 Subject: [PATCH 044/478] change by rpc --- qiniu/rsf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/rsf.py b/qiniu/rsf.py index ea9be409..cb183e95 100644 --- a/qiniu/rsf.py +++ b/qiniu/rsf.py @@ -36,7 +36,7 @@ def list_prefix(self, bucket, prefix=None, marker=None, limit=None): if prefix is not None: ops['prefix'] = prefix url = '%s?%s' % ('/list', urllib.urlencode(ops)) - ret, err = self.conn.call_with( + ret, err, code = self.conn.call_with( url, body=None, content_type='application/x-www-form-urlencoded') if ret and not ret.get('marker'): err = EOF From bbf9756765c59738f59f35b6d143d3f069de046b Mon Sep 17 00:00:00 2001 From: longbai Date: Mon, 30 Jun 2014 12:13:23 +0800 Subject: [PATCH 045/478] add up host2 --- qiniu/io.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qiniu/io.py b/qiniu/io.py index efe042ad..8f4649f9 100644 --- a/qiniu/io.py +++ b/qiniu/io.py @@ -52,8 +52,12 @@ def put(uptoken, key, data, extra=None): files = [ {'filename': fname, 'data': data, 'mime_type': extra.mime_type}, ] - return rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files) + ret, err, code = rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files) + if err is None or code == 571 or code == 614: + return ret, err + ret, err, code = rpc.Client(conf.UP_HOST2).call_with_multipart("/", fields, files) + return ret, err def put_file(uptoken, key, localfile, extra=None): """ put a file to Qiniu From 7f49b76ac87422f6b3738729dcf3a2241f63e689 Mon Sep 17 00:00:00 2001 From: longbai Date: Mon, 30 Jun 2014 12:13:44 +0800 Subject: [PATCH 046/478] change new rpc --- qiniu/resumable_io.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index c0622e1f..cf3346fd 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -71,12 +71,15 @@ def put_file(uptoken, key, localfile, extra): return ret -def put(uptoken, key, f, fsize, extra): +def put(uptoken, key, f, fsize, extra, host=None): """ 上传二进制流, 通过将data "切片" 分段上传 """ if not isinstance(extra, PutExtra): print("extra must the instance of PutExtra") return + if host is None: + host = conf.UP_HOST + block_cnt = block_count(fsize) if extra.progresses is None: extra.progresses = [None] * block_cnt @@ -97,7 +100,7 @@ def put(uptoken, key, f, fsize, extra): read_length = fsize - i * _block_size data_slice = f.read(read_length) while True: - err = resumable_block_put(data_slice, i, extra, uptoken) + err = resumable_block_put(data_slice, i, extra, uptoken, host) if err is None: break @@ -106,19 +109,19 @@ def put(uptoken, key, f, fsize, extra): return None, err_put_failed print err, ".. retry" - mkfile_host = extra.progresses[-1]["host"] if block_cnt else conf.UP_HOST + mkfile_host = extra.progresses[-1]["host"] if block_cnt else host mkfile_client = auth_up.Client(uptoken, mkfile_host) - return mkfile(mkfile_client, key, fsize, extra) + return mkfile(mkfile_client, key, fsize, extra, mkfile_host) -def resumable_block_put(block, index, extra, uptoken): +def resumable_block_put(block, index, extra, uptoken, host): block_size = len(block) - mkblk_client = auth_up.Client(uptoken, conf.UP_HOST) + mkblk_client = auth_up.Client(uptoken, host) if extra.progresses[index] is None or "ctx" not in extra.progresses[index]: crc32 = gen_crc32(block) block = bytearray(block) - extra.progresses[index], err = mkblock(mkblk_client, block_size, block) + extra.progresses[index], err = mkblock(mkblk_client, block_size, block, host) if err is not None: extra.notify_err(index, block_size, err) return err @@ -133,8 +136,8 @@ def block_count(size): return (size + _block_mask) / _block_size -def mkblock(client, block_size, first_chunk): - url = "http://%s/mkblk/%s" % (conf.UP_HOST, block_size) +def mkblock(client, block_size, first_chunk, host): + url = "http://%s/mkblk/%s" % (host, block_size) content_type = "application/octet-stream" return client.call_with(url, first_chunk, content_type, len(first_chunk)) @@ -146,8 +149,8 @@ def putblock(client, block_ret, chunk): return client.call_with(url, chunk, content_type, len(chunk)) -def mkfile(client, key, fsize, extra): - url = ["http://%s/mkfile/%s" % (conf.UP_HOST, fsize)] +def mkfile(client, key, fsize, extra, host): + url = ["http://%s/mkfile/%s" % (host, fsize)] if extra.mimetype: url.append("mimeType/%s" % urlsafe_b64encode(extra.mimetype)) From edf1714bf2dcbaa79b6c5388c943af09ccd7809e Mon Sep 17 00:00:00 2001 From: longbai Date: Tue, 1 Jul 2014 23:12:26 +0800 Subject: [PATCH 047/478] add backup host --- qiniu/io.py | 1 + qiniu/resumable_io.py | 20 ++++++++++++-------- qiniu/test/io_test.py | 15 ++++++++++++++- qiniu/test/resumable_io_test.py | 16 ++++++++-------- 4 files changed, 35 insertions(+), 17 deletions(-) diff --git a/qiniu/io.py b/qiniu/io.py index 8f4649f9..4669b8d3 100644 --- a/qiniu/io.py +++ b/qiniu/io.py @@ -53,6 +53,7 @@ def put(uptoken, key, data, extra=None): {'filename': fname, 'data': data, 'mime_type': extra.mime_type}, ] ret, err, code = rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files) + print err if err is None or code == 571 or code == 614: return ret, err diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index cf3346fd..bba371fd 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -66,20 +66,25 @@ def put_file(uptoken, key, localfile, extra): """ 上传文件 """ f = open(localfile, "rb") statinfo = os.stat(localfile) - ret = put(uptoken, key, f, statinfo.st_size, extra) + ret, err = put(uptoken, key, f, statinfo.st_size, extra) f.close() - return ret + return ret, err - -def put(uptoken, key, f, fsize, extra, host=None): +def put(uptoken, key, f, fsize, extra): """ 上传二进制流, 通过将data "切片" 分段上传 """ if not isinstance(extra, PutExtra): print("extra must the instance of PutExtra") return + host = conf.UP_HOST + ret, err, code = put_with_host(uptoken, key, f, fsize, extra, host) + if err is None or code == 571 or code == 614: + return ret, err + + ret, err, code = put_with_host(uptoken, key, f, fsize, extra, conf.UP_HOST2) + return ret, err - if host is None: - host = conf.UP_HOST +def put_with_host(uptoken, key, f, fsize, extra, host): block_cnt = block_count(fsize) if extra.progresses is None: extra.progresses = [None] * block_cnt @@ -113,7 +118,6 @@ def put(uptoken, key, f, fsize, extra, host=None): mkfile_client = auth_up.Client(uptoken, mkfile_host) return mkfile(mkfile_client, key, fsize, extra, mkfile_host) - def resumable_block_put(block, index, extra, uptoken, host): block_size = len(block) @@ -121,7 +125,7 @@ def resumable_block_put(block, index, extra, uptoken, host): if extra.progresses[index] is None or "ctx" not in extra.progresses[index]: crc32 = gen_crc32(block) block = bytearray(block) - extra.progresses[index], err = mkblock(mkblk_client, block_size, block, host) + extra.progresses[index], err, code = mkblock(mkblk_client, block_size, block, host) if err is not None: extra.notify_err(index, block_size, err) return err diff --git a/qiniu/test/io_test.py b/qiniu/test/io_test.py index 28fa3e31..d3d34378 100644 --- a/qiniu/test/io_test.py +++ b/qiniu/test/io_test.py @@ -114,7 +114,7 @@ def test_put_StringIO(): def test_put_urlopen(): key = "test_%s" % r(9) - data = urllib.urlopen('http://cheneya.qiniudn.com/hello_jpg') + data = urllib.urlopen('http://pythonsdk.qiniudn.com/hello.jpg') ret, err = io.put(policy.token(), key, data) assert err is None assert ret['key'] == key @@ -155,6 +155,7 @@ def read(self, n=None): test_put_urlopen() test_put_no_length() + def test_put_file(self): localfile = "%s" % __file__ key = "test_%s" % r(9) @@ -178,6 +179,18 @@ def test_put_fail_reqid(self): ret, err = io.put("", key, data, extra) assert "reqid" in err + def test_put_with_uphost2(self): + #mistake up host + conf.UP_HOST = "api.qiniu.com" + localfile = "%s" % __file__ + key = "test_up2_%s" % r(9) + + extra.check_crc = 1 + ret, err = io.put_file(policy.token(), key, localfile, extra) + assert err is None + assert ret['key'] == key + conf.UP_HOST = "up.qiniu.com" + class Test_get_file_crc32(unittest.TestCase): diff --git a/qiniu/test/resumable_io_test.py b/qiniu/test/resumable_io_test.py index addb8f3a..f387e66d 100644 --- a/qiniu/test/resumable_io_test.py +++ b/qiniu/test/resumable_io_test.py @@ -37,14 +37,15 @@ class TestBlock(unittest.TestCase): def test_block(self): if is_travis: return + host = conf.UP_HOST policy = rs.PutPolicy(bucket) uptoken = policy.token() client = up.Client(uptoken) # rets = [0, 0] data_slice_2 = "\nbye!" - ret, err = resumable_io.mkblock( - client, len(data_slice_2), data_slice_2) + ret, err, code = resumable_io.mkblock( + client, len(data_slice_2), data_slice_2, host) assert err is None, err self.assertEqual(ret["crc32"], binascii.crc32(data_slice_2)) @@ -56,8 +57,8 @@ def test_block(self): lens += extra.progresses[i]["offset"] key = u"sdk_py_resumable_block_4_%s" % r(9) - ret, err = resumable_io.mkfile(client, key, lens, extra) - assert err is None, err + ret, err, code = resumable_io.mkfile(client, key, lens, extra, host) + assert err is None self.assertEqual( ret["hash"], "FtCFo0mQugW98uaPYgr54Vb1QsO0", "hash not match") rs.Client().delete(bucket, key) @@ -65,7 +66,7 @@ def test_block(self): def test_put(self): if is_travis: return - src = urllib.urlopen("http://cheneya.qiniudn.com/hello_jpg") + src = urllib.urlopen("http://pythonsdk.qiniudn.com/hello.jpg") ostype = platform.system() if ostype.lower().find("windows") != -1: tmpf = "".join([os.getcwd(), mktemp()]) @@ -84,8 +85,7 @@ def test_put(self): ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) dst.close() os.remove(tmpf) - - assert err is None, err + assert err is None assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" self.assertEqual( ret["hash"], "FnyTMUqPNRTdk1Wou7oLqDHkBm_p", "hash not match") @@ -112,7 +112,7 @@ def test_put_4m(self): ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) dst.close() os.remove(tmpf) - + print err assert err is None, err assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" self.assertEqual( From 8eaaaa034e4d57185e49bd573b478965b7178a95 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 2 Jul 2014 00:45:07 +0800 Subject: [PATCH 048/478] fixed mkfile host, add exept catch --- qiniu/io.py | 3 +-- qiniu/resumable_io.py | 16 ++++++++++------ qiniu/rpc.py | 2 +- qiniu/test/resumable_io_test.py | 5 ++--- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/qiniu/io.py b/qiniu/io.py index 4669b8d3..ccbf6eec 100644 --- a/qiniu/io.py +++ b/qiniu/io.py @@ -53,8 +53,7 @@ def put(uptoken, key, data, extra=None): {'filename': fname, 'data': data, 'mime_type': extra.mime_type}, ] ret, err, code = rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files) - print err - if err is None or code == 571 or code == 614: + if err is None or code == 571 or code == 614 or code == 301: return ret, err ret, err, code = rpc.Client(conf.UP_HOST2).call_with_multipart("/", fields, files) diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index bba371fd..d8c41f05 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -76,9 +76,12 @@ def put(uptoken, key, f, fsize, extra): print("extra must the instance of PutExtra") return host = conf.UP_HOST - ret, err, code = put_with_host(uptoken, key, f, fsize, extra, host) - if err is None or code == 571 or code == 614: - return ret, err + try: + ret, err, code = put_with_host(uptoken, key, f, fsize, extra, host) + if err is None or code == 571 or code == 614 or code == 301: + return ret, err + except: + pass ret, err, code = put_with_host(uptoken, key, f, fsize, extra, conf.UP_HOST2) return ret, err @@ -90,7 +93,7 @@ def put_with_host(uptoken, key, f, fsize, extra, host): extra.progresses = [None] * block_cnt else: if not len(extra.progresses) == block_cnt: - return None, err_invalid_put_progress + return None, err_invalid_put_progress, 0 if extra.try_times is None: extra.try_times = _try_times @@ -111,12 +114,13 @@ def put_with_host(uptoken, key, f, fsize, extra, host): try_time -= 1 if try_time <= 0: - return None, err_put_failed + return None, err_put_failed, 0 print err, ".. retry" mkfile_host = extra.progresses[-1]["host"] if block_cnt else host mkfile_client = auth_up.Client(uptoken, mkfile_host) - return mkfile(mkfile_client, key, fsize, extra, mkfile_host) + + return mkfile(mkfile_client, key, fsize, extra, host) def resumable_block_put(block, index, extra, uptoken, host): block_size = len(block) diff --git a/qiniu/rpc.py b/qiniu/rpc.py index 7be6446b..c9f9b17e 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -53,7 +53,7 @@ def call_with(self, path, body, content_type=None, content_length=None): except ValueError: pass - if resp.status / 100 != 2: + if resp.status >= 400: err_msg = ret if "error" not in ret else ret["error"] reqid = resp.getheader("X-Reqid", None) # detail = resp.getheader("x-log", None) diff --git a/qiniu/test/resumable_io_test.py b/qiniu/test/resumable_io_test.py index f387e66d..dad56ee0 100644 --- a/qiniu/test/resumable_io_test.py +++ b/qiniu/test/resumable_io_test.py @@ -58,7 +58,7 @@ def test_block(self): key = u"sdk_py_resumable_block_4_%s" % r(9) ret, err, code = resumable_io.mkfile(client, key, lens, extra, host) - assert err is None + assert err is None, err self.assertEqual( ret["hash"], "FtCFo0mQugW98uaPYgr54Vb1QsO0", "hash not match") rs.Client().delete(bucket, key) @@ -85,7 +85,7 @@ def test_put(self): ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) dst.close() os.remove(tmpf) - assert err is None + assert err is None, err assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" self.assertEqual( ret["hash"], "FnyTMUqPNRTdk1Wou7oLqDHkBm_p", "hash not match") @@ -112,7 +112,6 @@ def test_put_4m(self): ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) dst.close() os.remove(tmpf) - print err assert err is None, err assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" self.assertEqual( From cb1bcd7a932a7c1d0c95df2ec80f14c93a71fb9e Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 2 Jul 2014 01:05:03 +0800 Subject: [PATCH 049/478] pep8 --- qiniu/io.py | 1 + qiniu/resumable_io.py | 2 ++ qiniu/test/io_test.py | 4 +--- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/qiniu/io.py b/qiniu/io.py index ccbf6eec..f8d1504a 100644 --- a/qiniu/io.py +++ b/qiniu/io.py @@ -59,6 +59,7 @@ def put(uptoken, key, data, extra=None): ret, err, code = rpc.Client(conf.UP_HOST2).call_with_multipart("/", fields, files) return ret, err + def put_file(uptoken, key, localfile, extra=None): """ put a file to Qiniu diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index d8c41f05..f4fa2fcc 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -70,6 +70,7 @@ def put_file(uptoken, key, localfile, extra): f.close() return ret, err + def put(uptoken, key, f, fsize, extra): """ 上传二进制流, 通过将data "切片" 分段上传 """ if not isinstance(extra, PutExtra): @@ -122,6 +123,7 @@ def put_with_host(uptoken, key, f, fsize, extra, host): return mkfile(mkfile_client, key, fsize, extra, host) + def resumable_block_put(block, index, extra, uptoken, host): block_size = len(block) diff --git a/qiniu/test/io_test.py b/qiniu/test/io_test.py index d3d34378..2f16be29 100644 --- a/qiniu/test/io_test.py +++ b/qiniu/test/io_test.py @@ -155,7 +155,6 @@ def read(self, n=None): test_put_urlopen() test_put_no_length() - def test_put_file(self): localfile = "%s" % __file__ key = "test_%s" % r(9) @@ -180,8 +179,7 @@ def test_put_fail_reqid(self): assert "reqid" in err def test_put_with_uphost2(self): - #mistake up host - conf.UP_HOST = "api.qiniu.com" + conf.UP_HOST = "api.qiniu.com" # mistake up host localfile = "%s" % __file__ key = "test_up2_%s" % r(9) From 778268f9ea13976b2cb3c96b41d8deff727ffa0b Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 2 Jul 2014 01:07:03 +0800 Subject: [PATCH 050/478] remove useless err --- qiniu/test/io_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qiniu/test/io_test.py b/qiniu/test/io_test.py index 2f16be29..438b785b 100644 --- a/qiniu/test/io_test.py +++ b/qiniu/test/io_test.py @@ -63,7 +63,6 @@ def test_put_quote_key(): data = r(100) key = 'a\\b\\c"你好' + r(9) ret, err = io.put(policy.token(), key, data) - print err assert err is None assert ret['key'].encode('utf8') == key From b812e99a0d870ee621db1a42d3128e99187cffe3 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Wed, 2 Jul 2014 01:17:00 +0800 Subject: [PATCH 051/478] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f6a338f..dbdc1ba9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - [#110] 0字节文件上传支持 - [#114] 支持GAE - [#115] 请求完成后清除http header +- [#120] 增加第二个up host重试 ### v6.1.5 From 77636b1abcd28ad9358759c8608098aac60ee1ee Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 2 Jul 2014 11:33:27 +0800 Subject: [PATCH 052/478] ignore 4xx retry --- qiniu/io.py | 2 +- qiniu/resumable_io.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/io.py b/qiniu/io.py index f8d1504a..e74e3208 100644 --- a/qiniu/io.py +++ b/qiniu/io.py @@ -53,7 +53,7 @@ def put(uptoken, key, data, extra=None): {'filename': fname, 'data': data, 'mime_type': extra.mime_type}, ] ret, err, code = rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files) - if err is None or code == 571 or code == 614 or code == 301: + if err is None or code < 500 or code == 571 or code == 614: return ret, err ret, err, code = rpc.Client(conf.UP_HOST2).call_with_multipart("/", fields, files) diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index f4fa2fcc..45ab33e9 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -79,7 +79,7 @@ def put(uptoken, key, f, fsize, extra): host = conf.UP_HOST try: ret, err, code = put_with_host(uptoken, key, f, fsize, extra, host) - if err is None or code == 571 or code == 614 or code == 301: + if err is None or code < 500 or code == 571 or code == 614: return ret, err except: pass From e82ca8dbe561618f639c0659716f2e3d87e4e458 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 2 Jul 2014 11:38:36 +0800 Subject: [PATCH 053/478] optimize code --- qiniu/io.py | 2 +- qiniu/resumable_io.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/io.py b/qiniu/io.py index e74e3208..573ef20f 100644 --- a/qiniu/io.py +++ b/qiniu/io.py @@ -53,7 +53,7 @@ def put(uptoken, key, data, extra=None): {'filename': fname, 'data': data, 'mime_type': extra.mime_type}, ] ret, err, code = rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files) - if err is None or code < 500 or code == 571 or code == 614: + if err is None or code / 100 == 4 or code == 571 or code == 614: return ret, err ret, err, code = rpc.Client(conf.UP_HOST2).call_with_multipart("/", fields, files) diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index 45ab33e9..145d4c44 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -79,7 +79,7 @@ def put(uptoken, key, f, fsize, extra): host = conf.UP_HOST try: ret, err, code = put_with_host(uptoken, key, f, fsize, extra, host) - if err is None or code < 500 or code == 571 or code == 614: + if err is None or code / 100 == 4 or code == 571 or code == 614: return ret, err except: pass From 96ee043115910682f5a5e7aab7b086f4f272779a Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 2 Jul 2014 11:45:24 +0800 Subject: [PATCH 054/478] fixed callback failed code --- qiniu/io.py | 2 +- qiniu/resumable_io.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/io.py b/qiniu/io.py index 573ef20f..6d07621f 100644 --- a/qiniu/io.py +++ b/qiniu/io.py @@ -53,7 +53,7 @@ def put(uptoken, key, data, extra=None): {'filename': fname, 'data': data, 'mime_type': extra.mime_type}, ] ret, err, code = rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files) - if err is None or code / 100 == 4 or code == 571 or code == 614: + if err is None or code / 100 == 4 or code == 579 or code == 614: return ret, err ret, err, code = rpc.Client(conf.UP_HOST2).call_with_multipart("/", fields, files) diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index 145d4c44..53ef55e2 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -79,7 +79,7 @@ def put(uptoken, key, f, fsize, extra): host = conf.UP_HOST try: ret, err, code = put_with_host(uptoken, key, f, fsize, extra, host) - if err is None or code / 100 == 4 or code == 571 or code == 614: + if err is None or code / 100 == 4 or code == 579 or code == 614: return ret, err except: pass From 554d13ea682ad2f78c533e2fb9689ae03cea2a20 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 2 Jul 2014 11:52:54 +0800 Subject: [PATCH 055/478] add 6xx, 7xx code --- qiniu/io.py | 2 +- qiniu/resumable_io.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/io.py b/qiniu/io.py index 6d07621f..afe6dcda 100644 --- a/qiniu/io.py +++ b/qiniu/io.py @@ -53,7 +53,7 @@ def put(uptoken, key, data, extra=None): {'filename': fname, 'data': data, 'mime_type': extra.mime_type}, ] ret, err, code = rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files) - if err is None or code / 100 == 4 or code == 579 or code == 614: + if err is None or code / 100 == 4 or code == 579 or code / 100 == 6 or code / 100 == 7: return ret, err ret, err, code = rpc.Client(conf.UP_HOST2).call_with_multipart("/", fields, files) diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py index 53ef55e2..cc544b86 100644 --- a/qiniu/resumable_io.py +++ b/qiniu/resumable_io.py @@ -79,7 +79,7 @@ def put(uptoken, key, f, fsize, extra): host = conf.UP_HOST try: ret, err, code = put_with_host(uptoken, key, f, fsize, extra, host) - if err is None or code / 100 == 4 or code == 579 or code == 614: + if err is None or code / 100 == 4 or code == 579 or code / 100 == 6 or code / 100 == 7: return ret, err except: pass From 9cc383bb2f999ca1f25a56fb8cfa777ca0bd3b45 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 2 Jul 2014 14:11:26 +0800 Subject: [PATCH 056/478] remove a mistake test --- qiniu/test/io_test.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/qiniu/test/io_test.py b/qiniu/test/io_test.py index 438b785b..60538dc9 100644 --- a/qiniu/test/io_test.py +++ b/qiniu/test/io_test.py @@ -177,17 +177,6 @@ def test_put_fail_reqid(self): ret, err = io.put("", key, data, extra) assert "reqid" in err - def test_put_with_uphost2(self): - conf.UP_HOST = "api.qiniu.com" # mistake up host - localfile = "%s" % __file__ - key = "test_up2_%s" % r(9) - - extra.check_crc = 1 - ret, err = io.put_file(policy.token(), key, localfile, extra) - assert err is None - assert ret['key'] == key - conf.UP_HOST = "up.qiniu.com" - class Test_get_file_crc32(unittest.TestCase): From eab6a37b9adacd7df34511716375d1c2c89c0969 Mon Sep 17 00:00:00 2001 From: xuzhaokui Date: Wed, 9 Jul 2014 11:15:40 +0800 Subject: [PATCH 057/478] #9891 --- qiniu/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/conf.py b/qiniu/conf.py index ff84fafa..005a4bce 100644 --- a/qiniu/conf.py +++ b/qiniu/conf.py @@ -5,7 +5,7 @@ RS_HOST = "rs.qbox.me" RSF_HOST = "rsf.qbox.me" -UP_HOST = "up.qiniu.com" +UP_HOST = "upload.qiniu.com" UP_HOST2 = "up.qbox.me" from . import __version__ From 995e3cbd1eb98dc54860c7b2e4618d8a2c9cfa42 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Fri, 25 Jul 2014 10:13:27 +0800 Subject: [PATCH 058/478] update version [ci skip] --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 31496ad4..0a65e7cd 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -7,4 +7,4 @@ ''' # -*- coding: utf-8 -*- -__version__ = '6.1.6' +__version__ = '6.1.7' From 8ef5f1c1764bfb3eebd689fa549bfbf6d970ae0a Mon Sep 17 00:00:00 2001 From: Bai Long Date: Fri, 25 Jul 2014 10:14:43 +0800 Subject: [PATCH 059/478] update version [ci skip] --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 31496ad4..0a65e7cd 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -7,4 +7,4 @@ ''' # -*- coding: utf-8 -*- -__version__ = '6.1.6' +__version__ = '6.1.7' From 7cecd5f83f86a93c7d8443149d9136b207605d82 Mon Sep 17 00:00:00 2001 From: longbai Date: Tue, 5 Aug 2014 22:01:50 +0800 Subject: [PATCH 060/478] catch io exception --- qiniu/rpc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/qiniu/rpc.py b/qiniu/rpc.py index c9f9b17e..81421598 100644 --- a/qiniu/rpc.py +++ b/qiniu/rpc.py @@ -44,14 +44,15 @@ def call_with(self, path, body, content_type=None, content_length=None): if content_length is not None: header["Content-Length"] = content_length - resp = self.round_tripper("POST", path, body, header) try: + resp = self.round_tripper("POST", path, body, header) ret = resp.read() ret = json.loads(ret) - except IOError, e: - return None, e except ValueError: + # ignore empty body when success pass + except Exception, e: + return None, str(e)+path, 0 if resp.status >= 400: err_msg = ret if "error" not in ret else ret["error"] From b63a77983a18677dd25b0487f951a5c6c3fcc155 Mon Sep 17 00:00:00 2001 From: longbai Date: Tue, 5 Aug 2014 22:18:13 +0800 Subject: [PATCH 061/478] version up [ci skip] --- CHANGELOG.md | 11 +++++++++++ qiniu/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbdc1ba9..18943e62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ ## CHANGE LOG +### v6.1.8 + +2014-08-05 issue [#126](https://github.com/qiniu/python-sdk/pull/126) +- [#125] 网络出错不抛异常 + +### v6.1.7 + +2014-07-10 issue [#123](https://github.com/qiniu/python-sdk/pull/123) +- [#121] 上传多host重试 +- [#122] 修改上传域名 + ### v6.1.6 2014-06-30 issue [#118](https://github.com/qiniu/python-sdk/pull/118) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 0a65e7cd..2db8aa64 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -7,4 +7,4 @@ ''' # -*- coding: utf-8 -*- -__version__ = '6.1.7' +__version__ = '6.1.8' From 7530e6922d3b81689074ba332098e2754e808076 Mon Sep 17 00:00:00 2001 From: greenmoon55 Date: Mon, 25 Aug 2014 21:57:38 +0800 Subject: [PATCH 062/478] Fix variable name in doc --- docs/gist/demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gist/demo.py b/docs/gist/demo.py index 0fe26a9a..3ae0d9b1 100644 --- a/docs/gist/demo.py +++ b/docs/gist/demo.py @@ -359,7 +359,7 @@ def list_prefix_all(): # @gist list_all -def list_all(bucket, rs=None, prefix=None, limit=None): +def list_all(bucket_name, rs=None, prefix=None, limit=None): if rs is None: rs = qiniu.rsf.Client() marker = None From a15bc91c3465b97353aa08b0d2a77a18f352725a Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 15 Oct 2014 00:45:13 +0800 Subject: [PATCH 063/478] add attname for download private file --- qiniu/rs/rs_token.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/qiniu/rs/rs_token.py b/qiniu/rs/rs_token.py index 478744e3..965313b4 100644 --- a/qiniu/rs/rs_token.py +++ b/qiniu/rs/rs_token.py @@ -88,7 +88,7 @@ class GetPolicy(object): def __init__(self, expires=3600): self.expires = expires - def make_request(self, base_url, mac=None): + def make_request(self, base_url, mac=None, attname=None): ''' * return private_url ''' @@ -100,7 +100,10 @@ def make_request(self, base_url, mac=None): base_url += '&' else: base_url += '?' - base_url = '%se=%s' % (base_url, str(deadline)) + if attname is None: + base_url = '%se=%s' % (base_url, str(deadline)) + else: + base_url = '%se=%s&attname=%s' % (base_url, str(deadline), attname) token = mac.sign(base_url) return '%s&token=%s' % (base_url, token) From 46f00cb312d1614cb698c75dd76f0d80247c609b Mon Sep 17 00:00:00 2001 From: Yufei Li Date: Mon, 27 Oct 2014 20:15:20 +0800 Subject: [PATCH 064/478] Add delimiter to rsf.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RESTFul API有Delimiter参数，加入到SDK中。 RESTFul API include the argument named delimiter, add it to SDK. --- qiniu/rsf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qiniu/rsf.py b/qiniu/rsf.py index cb183e95..87428aff 100644 --- a/qiniu/rsf.py +++ b/qiniu/rsf.py @@ -35,6 +35,8 @@ def list_prefix(self, bucket, prefix=None, marker=None, limit=None): ops['limit'] = limit if prefix is not None: ops['prefix'] = prefix + if delimiter is not None: + ops['delimiter'] = delimiter url = '%s?%s' % ('/list', urllib.urlencode(ops)) ret, err, code = self.conn.call_with( url, body=None, content_type='application/x-www-form-urlencoded') From 9c9c4a4dd836743f8bbc91d1b8aa9540c4712f41 Mon Sep 17 00:00:00 2001 From: Yufei Li Date: Mon, 27 Oct 2014 20:22:29 +0800 Subject: [PATCH 065/478] Merge previous commit. Description is ame to previous commit; --- qiniu/rsf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qiniu/rsf.py b/qiniu/rsf.py index 87428aff..9b5c5b8b 100644 --- a/qiniu/rsf.py +++ b/qiniu/rsf.py @@ -14,12 +14,13 @@ def __init__(self, mac=None): mac = auth.digest.Mac() self.conn = auth.digest.Client(host=conf.RSF_HOST, mac=mac) - def list_prefix(self, bucket, prefix=None, marker=None, limit=None): + def list_prefix(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): '''前缀查询: * bucket => str * prefix => str * marker => str * limit => int + * delimiter => str * return ret => {'items': items, 'marker': markerOut}, err => str 1. 首次请求 marker = None @@ -37,6 +38,7 @@ def list_prefix(self, bucket, prefix=None, marker=None, limit=None): ops['prefix'] = prefix if delimiter is not None: ops['delimiter'] = delimiter + url = '%s?%s' % ('/list', urllib.urlencode(ops)) ret, err, code = self.conn.call_with( url, body=None, content_type='application/x-www-form-urlencoded') From 5f26708ca2babe05eebdb2c22d48705d7c0aa80a Mon Sep 17 00:00:00 2001 From: Bai Long Date: Thu, 13 Nov 2014 17:36:08 +0800 Subject: [PATCH 066/478] [ci skip] --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18943e62..9f3076cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ## CHANGE LOG +### v6.1.9 + +2014-11-13 +- 增加文件列表的delimiter，模拟目录方式 + ### v6.1.8 2014-08-05 issue [#126](https://github.com/qiniu/python-sdk/pull/126) From 362e80478ea534dde9bf97b14d3d4bc7c3d3fa69 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Thu, 13 Nov 2014 17:36:50 +0800 Subject: [PATCH 067/478] update version [ci skip] --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 2db8aa64..cb83be37 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -7,4 +7,4 @@ ''' # -*- coding: utf-8 -*- -__version__ = '6.1.8' +__version__ = '6.1.9' From 11ca2c8398bb1019a12a4eee47e45e4ad4228e24 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Thu, 13 Nov 2014 17:37:50 +0800 Subject: [PATCH 068/478] [ci skip] --- qiniu/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/conf.py b/qiniu/conf.py index 005a4bce..03092b84 100644 --- a/qiniu/conf.py +++ b/qiniu/conf.py @@ -5,8 +5,8 @@ RS_HOST = "rs.qbox.me" RSF_HOST = "rsf.qbox.me" -UP_HOST = "upload.qiniu.com" -UP_HOST2 = "up.qbox.me" +UP_HOST = "up.qiniu.com" +UP_HOST2 = "upload.qbox.me" from . import __version__ import platform From 2f418dca9bea9b4211758d092f847891c058a601 Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 13 Nov 2014 18:00:21 +0800 Subject: [PATCH 069/478] remove 6.x version --- CHANGELOG.md | 95 ----- README.md | 64 --- docs/Makefile | 3 - docs/README.gist.md | 499 ----------------------- docs/README.md | 695 -------------------------------- docs/gist/conf.py | 6 - docs/gist/demo.py | 383 ------------------ docs/gist/fetch.py | 30 -- docs/gist/pfop.py | 37 -- docs/gist/prefetch.py | 29 -- qiniu/__init__.py | 10 - qiniu/auth/__init__.py | 0 qiniu/auth/digest.py | 62 --- qiniu/auth/up.py | 19 - qiniu/conf.py | 17 - qiniu/fop.py | 39 -- qiniu/httplib_chunk.py | 122 ------ qiniu/io.py | 89 ---- qiniu/resumable_io.py | 177 -------- qiniu/rpc.py | 220 ---------- qiniu/rs/__init__.py | 9 - qiniu/rs/rs.py | 93 ----- qiniu/rs/rs_token.py | 119 ------ qiniu/rs/test/__init__.py | 26 -- qiniu/rs/test/rs_test.py | 98 ----- qiniu/rs/test/rs_token_test.py | 82 ---- qiniu/rsf.py | 47 --- qiniu/test/__init__.py | 0 qiniu/test/conf_test.py | 12 - qiniu/test/fop_test.py | 33 -- qiniu/test/io_test.py | 195 --------- qiniu/test/resumable_io_test.py | 143 ------- qiniu/test/rpc_test.py | 188 --------- qiniu/test/rsf_test.py | 22 - setup.py | 47 --- test-env.sh | 4 - 36 files changed, 3714 deletions(-) delete mode 100644 CHANGELOG.md delete mode 100644 README.md delete mode 100644 docs/Makefile delete mode 100644 docs/README.gist.md delete mode 100644 docs/README.md delete mode 100644 docs/gist/conf.py delete mode 100644 docs/gist/demo.py delete mode 100644 docs/gist/fetch.py delete mode 100644 docs/gist/pfop.py delete mode 100644 docs/gist/prefetch.py delete mode 100644 qiniu/__init__.py delete mode 100644 qiniu/auth/__init__.py delete mode 100644 qiniu/auth/digest.py delete mode 100644 qiniu/auth/up.py delete mode 100644 qiniu/conf.py delete mode 100644 qiniu/fop.py delete mode 100644 qiniu/httplib_chunk.py delete mode 100644 qiniu/io.py delete mode 100644 qiniu/resumable_io.py delete mode 100644 qiniu/rpc.py delete mode 100644 qiniu/rs/__init__.py delete mode 100644 qiniu/rs/rs.py delete mode 100644 qiniu/rs/rs_token.py delete mode 100644 qiniu/rs/test/__init__.py delete mode 100644 qiniu/rs/test/rs_test.py delete mode 100644 qiniu/rs/test/rs_token_test.py delete mode 100644 qiniu/rsf.py delete mode 100644 qiniu/test/__init__.py delete mode 100644 qiniu/test/conf_test.py delete mode 100644 qiniu/test/fop_test.py delete mode 100644 qiniu/test/io_test.py delete mode 100644 qiniu/test/resumable_io_test.py delete mode 100644 qiniu/test/rpc_test.py delete mode 100644 qiniu/test/rsf_test.py delete mode 100644 setup.py delete mode 100644 test-env.sh diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 9f3076cd..00000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,95 +0,0 @@ -## CHANGE LOG - -### v6.1.9 - -2014-11-13 -- 增加文件列表的delimiter，模拟目录方式 - -### v6.1.8 - -2014-08-05 issue [#126](https://github.com/qiniu/python-sdk/pull/126) -- [#125] 网络出错不抛异常 - -### v6.1.7 - -2014-07-10 issue [#123](https://github.com/qiniu/python-sdk/pull/123) -- [#121] 上传多host重试 -- [#122] 修改上传域名 - -### v6.1.6 - -2014-06-30 issue [#118](https://github.com/qiniu/python-sdk/pull/118) -- [#108] 增加限定用户上传类型字段 -- [#110] 0字节文件上传支持 -- [#114] 支持GAE -- [#115] 请求完成后清除http header -- [#120] 增加第二个up host重试 - -### v6.1.5 - -2014-04-28 issue [#105](https://github.com/qiniu/python-sdk/pull/105) -- [#100] 遵循PEP8语法规范 -- [#101] 增加pyflakes语法检测 -- [#103] 错误信息中加入Reqid信息，以便追溯 -- [#104] 完善User-Agent头信息，以便追溯 -- [#98] 增加fetch、prefetch、pfop三个接口的范例代码 - -### v6.1.4 - -2014-03-28 issue [#95](https://github.com/qiniu/python-sdk/pull/95) - -- [#78] 增加 putpolicy 选项:saveKey,insertOnly,detectMime,fsizeLimit,persistentNotifyUrl,persistentOps -- [#80] 增加 gettoken 过期时间参数，增加 rsf 返回为空的EOF判断 -- [#86] 修正 断点续传的bug -- [#93] 修正 4M 分块计算bug -- [#96] 修正 mime_type typo - -### v6.1.3 - -2013-10-24 issue [#77](https://github.com/qiniu/python-sdk/pull/77) - -- bug fix, httplib_thunk.py 中的无效符号引用 -- PutPolicy：增加 saveKey、persistentOps/persistentNotifyUrl、fsizeLimit（文件大小限制）等支持 -- 断点续传：使用新的 mkfile 协议 - - -### v6.1.2 - -2013-08-01 issue [#66](https://github.com/qiniu/python-sdk/pull/66) - -- 修复在Windows环境下put_file无法读取文件的bug -- 修复在Windows环境下创建临时文件的权限问题 -- 修复在Windows环境下对二进制文件计算crc32的bug - - -### v6.1.1 - -2013-07-05 issue [#60](https://github.com/qiniu/python-sdk/pull/60) - -- 整理文档 - - -### v6.1.0 - -2013-07-03 issue [#58](https://github.com/qiniu/python-sdk/pull/58) - -- 实现最新版的上传API， - - io.PutExtra更新，废弃callback_params，bucket，和custom_meta，新增params -- 修复[#16](https://github.com/qiniu/python-sdk/issues/16) - - put接口可以传入类文件对象（file-like object） -- 修复[#52](https://github.com/qiniu/python-sdk/issues/52) - - -### v6.0.1 - -2013-06-27 issue [#43](https://github.com/qiniu/python-sdk/pull/43) - -- 遵循 [sdkspec v6.0.2](https://github.com/qiniu/sdkspec/tree/v6.0.2) - - 现在，rsf.list_prefix在没有更多数据时，err 会返回 rsf.EOF - - -### v6.0.0 - -2013-06-26 issue [#42](https://github.com/qiniu/python-sdk/pull/42) - -- 遵循 [sdkspec v6.0.1](https://github.com/qiniu/sdkspec/tree/v6.0.1) diff --git a/README.md b/README.md deleted file mode 100644 index 9dbec30a..00000000 --- a/README.md +++ /dev/null @@ -1,64 +0,0 @@ -Qiniu Resource Storage SDK for Python -=== - -[![Build Status](https://api.travis-ci.org/qiniu/python-sdk.png?branch=master)](https://travis-ci.org/qiniu/python-sdk) - -[![Qiniu Logo](http://qiniu-brand.qiniudn.com/5/logo-white-195x105.png)](http://www.qiniu.com/) - -## 使用 - -参考文档：[七牛云存储 Python SDK 使用指南](https://github.com/qiniu/python-sdk/blob/develop/docs/README.md) - -## 准备开发环境 - -### 安装 - -* 直接安装: - - pip install qiniu - 或 - easy_install qiniu - -Python-SDK可以使用`pip`或`easy_install`从PyPI服务器上安装，但不包括文档和样例。如果需要，请下载源码并安装。 - -* 源码安装： - -从[release](https://github.com/qiniu/python-sdk/releases)下载源码： - - tar xvzf python-sdk-$VERSION.tar.gz - cd python-sdk-$VERSION - python setup.py install - - -## 单元测试 - -1. 测试环境 - - 1. [开通七牛开发者帐号](https://portal.qiniu.com/signup) - 2. [登录七牛开发者自助平台，查看 Access Key 和 Secret Key](https://portal.qiniu.com/setting/key) 。 - 3. 在开发者后台新建一个空间 - - 然后将在`test-env.sh`中填入相关信息。 - -2. 需安装[nosetests](https://nose.readthedocs.org/en/latest/)测试工具。 - -运行测试： - - source test-env.sh - nosetests - -## 贡献代码 - -1. Fork -2. 创建您的特性分支 (`git checkout -b my-new-feature`) -3. 提交您的改动 (`git commit -am 'Added some feature'`) -4. 将您的修改记录提交到远程 `git` 仓库 (`git push origin my-new-feature`) -5. 然后到 github 网站的该 `git` 远程仓库的 `my-new-feature` 分支下发起 Pull Request - -## 许可证 - -Copyright (c) 2012-2014 qiniu.com - -基于 MIT 协议发布: - -* [www.opensource.org/licenses/MIT](http://www.opensource.org/licenses/MIT) diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 1ef535fd..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -all: - gist README.gist.md > README.md - diff --git a/docs/README.gist.md b/docs/README.gist.md deleted file mode 100644 index 0e148bea..00000000 --- a/docs/README.gist.md +++ /dev/null @@ -1,499 +0,0 @@ ---- -title: Python SDK 使用指南 ---- - -此 Python SDK 适用于2.x版本，基于 [七牛云存储官方API](http://docs.qiniu.com/) 构建。使用此 SDK 构建您的网络应用程序，能让您以非常便捷地方式将数据安全地存储到七牛云存储上。无论您的网络应用是一个网站程序，还是包括从云端（服务端程序）到终端（手持设备应用）的架构的服务或应用，通过七牛云存储及其 SDK，都能让您应用程序的终端用户高速上传和下载，同时也让您的服务端更加轻盈。 - -SDK 下载地址： - -**文档大纲** - -- [概述](#overview) -- [准备开发环境](#prepare) - - [安装](#install) - - [ACCESS_KEY 和 SECRET_KEY](#appkey) -- [使用SDK](#sdk-usage) - - [初始化环境](#init) - - [上传文件](#io-put) - - [上传流程](#io-put-flow) - - [上传策略](#io-put-policy) - - [上传凭证](#upload-token) - - [PutExtra](#put-extra) - - [上传文件](#upload-do) - - [断点续上传、分块并行上传](#resumable-io-put) - - [下载文件](#io-get) - - [下载公有文件](#io-get-public) - - [下载私有文件](#io-get-private) - - [断点续下载](#resumable-io-get) - - [资源操作](#rs) - - [获取文件信息](#rs-stat) - - [复制文件](#rs-copy) - - [移动文件](#rs-move) - - [删除文件](#rs-delete) - - [批量操作](#rs-batch) - - [批量获取文件信息](#batch-stat) - - [批量复制文件](#batch-copy) - - [批量移动文件](#batch-move) - - [批量删除文件](#batch-delete) - - [高级管理操作](#rsf) - - [列出文件](#list-prefix) - - [云处理](#fop) - - [图像](#fop-image) - - [查看图像属性](#fop-image-info) - - [查看图片EXIF信息](#fop-exif) - - [生成图片预览](#fop-image-view) -- [贡献代码](#contribution) -- [许可证](#license) - - - -## 概述 - -七牛云存储的 Python 语言版本 SDK（本文以下称 Python-SDK）是对七牛云存储API协议的一层封装，以提供一套对于 Python 开发者而言简单易用的开发工具。Python 开发者在对接 Python-SDK 时无需理解七牛云存储 API 协议的细节，原则上也不需要对 HTTP 协议和原理做非常深入的了解，但如果拥有基础的 HTTP 知识，对于出错场景的处理可以更加高效。 - -Python-SDK 被设计为同时适合服务器端和客户端使用。服务端是指开发者自己的业务服务器，客户端是指开发者提供给终端用户的软件，通常运行在 Windows/Mac/Linux 这样的桌面平台上。服务端因为有七牛颁发的 AccessKey/SecretKey，可以做很多客户端做不了的事情，比如删除文件、移动/复制文件等操作。一般而言，客服端操作文件需要获得服务端的授权。客户端上传文件需要获得服务端颁发的 [uptoken（上传授权凭证）](http://docs.qiniu.com/api/put.html#uploadToken)，客户端下载文件（包括下载处理过的文件，比如下载图片的缩略图）需要获得服务端颁发的 [dntoken（下载授权凭证）](http://docs.qiniu.com/api/get.html#download-token)。但开发者也可以将 bucket 设置为公开，此时文件有永久有效的访问地址，不需要业务服务器的授权，这对网站的静态文件（如图片、js、css、html）托管非常方便。 - -从 v5.0.0 版本开始，我们对 SDK 的内容进行了精简。所有管理操作，比如：创建/删除 bucket、为 bucket 绑定域名（publish）、设置数据处理的样式分隔符（fop seperator）、新增数据处理样式（fop style）等都去除了，统一建议到[开发者后台](https://portal.qiniu.com/)来完成。另外，此前服务端还有自己独有的上传 API，现在也推荐统一成基于客户端上传的工作方式。 - -从内容上来说，Python-SDK 主要包含如下几方面的内容： - -* 公共部分，所有用况下都用到：qiniu/rpc.py, qiniu/httplib_chunk.py -* 客户端上传文件：qiniu/io.py -* 客户端断点续上传：qiniu/resumable_io.py -* 数据处理：qiniu/fop.py -* 服务端操作：qiniu/auth/digest.py, qiniu/auth/up.py (授权), qiniu/rs/rs.py, qiniu/rs/rs_token.py (资源操作, uptoken/dntoken颁发) - - - - - -## 准备开发环境 - - - - -### 安装 - -直接安装: - - pip install qiniu - #或 - easy_install qiniu - -Python-SDK可以使用`pip`或`easy_install`从PyPI服务器上安装，但不包括文档和样例。如果需要，请下载源码并安装。 - -源码安装： - -从[Python-SDK下载地址](https://github.com/qiniu/python-sdk/releases)下载源码： - - tar xvzf python-sdk-$VERSION.tar.gz - cd python-sdk-$VERSION - python setup.py install - - - - -### ACCESS_KEY 和 SECRET_KEY - -在使用SDK 前，您需要拥有一对有效的 AccessKey 和 SecretKey 用来进行签名授权。 - -可以通过如下步骤获得： - -1. [开通七牛开发者帐号](https://portal.qiniu.com/signup) -2. [登录七牛开发者自助平台，查看 Access Key 和 Secret Key](https://portal.qiniu.com/setting/key) 。 - - - -## 使用SDK - - - -### 初始化环境 - -在获取到 Access Key 和 Secret Key 之后，您可以在您的程序中调用如下两行代码进行初始化对接, 要确保`ACCESS_KEY` 和 `SECRET_KEY` 在调用所有七牛API服务之前均已赋值： - -```{python} -@gist(gist/conf.py#config) -``` - - - -### 上传文件 - -为了尽可能地改善终端用户的上传体验，七牛云存储首创了客户端直传功能。一般云存储的上传流程是： - - 客户端（终端用户） => 业务服务器 => 云存储服务 - -这样多了一次上传的流程，和本地存储相比，会相对慢一些。但七牛引入了客户端直传，将整个上传过程调整为： - - 客户端（终端用户） => 七牛 => 业务服务器 - -客户端（终端用户）直接上传到七牛的服务器，通过DNS智能解析，七牛会选择到离终端用户最近的ISP服务商节点，速度会比本地存储快很多。文件上传成功以后，七牛的服务器使用回调功能，只需要将非常少的数据（比如Key）传给应用服务器，应用服务器进行保存即可。 - - - -#### 上传流程 - -在七牛云存储中，整个上传流程大体分为这样几步： - -1. 业务服务器颁发 [uptoken（上传授权凭证）](http://docs.qiniu.com/api/put.html#uploadToken)给客户端（终端用户） -2. 客户端凭借 [uptoken](http://docs.qiniu.com/api/put.html#uploadToken) 上传文件到七牛 -3. 在七牛获得完整数据后，发起一个 HTTP 请求回调到业务服务器 -4. 业务服务器保存相关信息，并返回一些信息给七牛 -5. 七牛原封不动地将这些信息转发给客户端（终端用户） - -需要注意的是，回调到业务服务器的过程是可选的，它取决于业务服务器颁发的 [uptoken](http://docs.qiniu.com/api/put.html#uploadToken)。如果没有回调，七牛会返回一些标准的信息（比如文件的 hash）给客户端。如果上传发生在业务服务器，以上流程可以自然简化为： - -1. 业务服务器生成 uptoken（不设置回调，自己回调到自己这里没有意义） -2. 凭借 [uptoken](http://docs.qiniu.com/api/put.html#uploadToken) 上传文件到七牛 -3. 善后工作，比如保存相关的一些信息 - - - -##### 上传策略 - -[uptoken](http://docs.qiniu.com/api/put.html#uploadToken) 实际上是用 AccessKey/SecretKey 进行数字签名的上传策略(`qiniu.rs.PutPolicy`)，它控制则整个上传流程的行为。让我们快速过一遍你都能够决策啥： - -```{python} -@gist(../qiniu/rs/rs_token.py#PutPolicy) -``` - -* `scope` 限定客户端的权限。如果 `scope` 是 bucket，则客户端只能新增文件到指定的 bucket，不能修改文件。如果 `scope` 为 bucket:key，则客户端可以修改指定的文件。**注意： key必须采用utf8编码，如使用非utf8编码访问七牛云存储将反馈错误** -* `callbackUrl` 设定业务服务器的回调地址，这样业务服务器才能感知到上传行为的发生。 -* `callbackBody` 设定业务服务器的回调信息。文件上传成功后，七牛向业务服务器的callbackUrl发送的POST请求携带的数据。支持 [魔法变量](http://docs.qiniu.com/api/put.html#MagicVariables) 和 [自定义变量](http://docs.qiniu.com/api/put.html#xVariables)。 -* `returnUrl` 设置用于浏览器端文件上传成功后，浏览器执行301跳转的URL，一般为 HTML Form 上传时使用。文件上传成功后浏览器会自动跳转到 `returnUrl?upload_ret=returnBody`。 -* `returnBody` 可调整返回给客户端的数据包，支持 [魔法变量](http://docs.qiniu.com/api/put.html#MagicVariables) 和 [自定义变量](http://docs.qiniu.com/api/put.html#xVariables)。`returnBody` 只在没有 `callbackUrl` 时有效（否则直接返回 `callbackUrl` 返回的结果）。不同情形下默认返回的 `returnBody` 并不相同。在一般情况下返回的是文件内容的 `hash`，也就是下载该文件时的 `etag`；但指定 `returnUrl` 时默认的 `returnBody` 会带上更多的信息。 -* `asyncOps` 可指定上传完成后，需要自动执行哪些数据处理。这是因为有些数据处理操作（比如音视频转码）比较慢，如果不进行预转可能第一次访问的时候效果不理想，预转可以很大程度改善这一点。 - -关于上传策略更完整的说明，请参考 [uptoken](http://docs.qiniu.com/api/put.html#uploadToken)。 - - - -##### 上传凭证 - -服务端生成 [uptoken](http://docs.qiniu.com/api/put.html#uploadToken) 代码如下： - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#uptoken) -``` - - - -##### PutExtra - -PutExtra是上传时的可选信息，默认为None - -```{python} -@gist(../qiniu/io.py#PutExtra) -``` - -* `params` 是一个字典。[自定义变量](http://docs.qiniu.com/api/put.html#xVariables)，key必须以 x: 开头命名，不限个数。可以在 uploadToken 的 callbackBody 选项中求值。 -* `mime_type` 表示数据的MimeType，当不指定时七牛服务器会自动检测。 -* `crc32` 待检查的crc32值 -* `check_crc` 可选值为0, 1, 2。 - `check_crc == 0`: 表示不进行 crc32 校验。 - `check_crc == 1`: 上传二进制数据时等同于 `check_crc=2`；上传本地文件时会自动计算 crc32 值。 - `check_crc == 2`: 表示进行 crc32 校验，且 crc32 值就是上面的 `crc32` 变量 - - - -##### 上传文件 - -上传文件到七牛（通常是客户端完成，但也可以发生在服务端）： - -直接上传二进制流 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_io) - -@gist(gist/demo.py#put) -``` - -上传本地文件 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_io) - -@gist(gist/demo.py#put_file) -``` - -ret是一个字典，含有`hash`，`key`等信息。 - - - -##### 断点续上传、分块并行上传 - -除了基本的上传外，七牛还支持你将文件切成若干块（除最后一块外，每个块固定为4M大小），每个块可独立上传，互不干扰；每个分块块内则能够做到断点上续传。 - -我们来看支持了断点上续传、分块并行上传的基本样例： - -上传二进制流 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_resumable_io) - -@gist(gist/demo.py#resumable_put) -``` - -上传本地文件 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_resumable_io) - -@gist(gist/demo.py#resumable_put_file) -``` - - - -### 下载文件 - - - -#### 下载公有文件 - -每个 bucket 都会绑定一个或多个域名（domain）。如果这个 bucket 是公开的，那么该 bucket 中的所有文件可以通过一个公开的下载 url 可以访问到： - - http:/// - -其中\是bucket所对应的域名。七牛云存储为每一个bucket提供一个默认域名。默认域名可以到[七牛云存储开发者平台](https://portal.qiniu.com/)中，空间设置的域名设置一节查询。 - -假设某个 bucket 既绑定了七牛的二级域名，如 hello.qiniudn.com，也绑定了自定义域名（需要备案），如 hello.com。那么该 bucket 中 key 为 a/b/c.htm 的文件可以通过 http://hello.qiniudn.com/a/b/c.htm 或 http://hello.com/a/b/c.htm 中任意一个 url 进行访问。 - -**注意： key必须采用utf8编码，如使用非utf8编码访问七牛云存储将反馈错误** - - - -#### 下载私有文件 - -如果某个 bucket 是私有的，那么这个 bucket 中的所有文件只能通过一个的临时有效的 downloadUrl 访问： - - http:///?e=&token= - -其中 dntoken 是由业务服务器签发的一个[临时下载授权凭证](http://docs.qiniu.com/api/get.html#download-token)，deadline 是 dntoken 的有效期。dntoken不需要单独生成，SDK 提供了生成完整 downloadUrl 的方法（包含了 dntoken），示例代码如下： - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#dntoken) -``` - -生成 downloadUrl 后，服务端下发 downloadUrl 给客户端。客户端收到 downloadUrl 后，和公有资源类似，直接用任意的 HTTP 客户端就可以下载该资源了。唯一需要注意的是，在 downloadUrl 失效却还没有完成下载时，需要重新向服务器申请授权。 - -无论公有资源还是私有资源，下载过程中客户端并不需要七牛 SDK 参与其中。 - - - -#### 断点续下载 - -无论是公有资源还是私有资源，获得的下载 url 支持标准的 HTTP 断点续传协议。考虑到多数语言都有相应的断点续下载支持的成熟方法，七牛 Python-SDK 并不提供断点续下载相关代码。 - - - -### 资源操作 - - - - -#### 获取文件信息 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#stat) -``` - - - -#### 复制文件 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#copy) -``` - - - -#### 移动文件 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#move) -``` - - - -#### 删除文件 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#delete) -``` - - - -#### 批量操作 - -当您需要一次性进行多个操作时, 可以使用批量操作。 - - - -##### 批量获取文件信息 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#batch_path) - -@gist(gist/demo.py#batch_stat) -``` - - -##### 批量复制文件 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#batch_path) - -@gist(gist/demo.py#batch_copy) -``` - - -##### 批量移动文件 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#batch_path) - -@gist(gist/demo.py#batch_move) -``` - - -##### 批量删除文件 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#batch_path) - -@gist(gist/demo.py#batch_delete) -``` - - - -### 高级管理操作 - - -#### 列出文件 - -请求某个存储空间（bucket）下的文件列表，如果有前缀，可以按前缀（prefix）进行过滤；如果前一次返回marker就表示还有资源，下一步请求需要将marker参数填上。 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_rsf) - -@gist(gist/demo.py#list_prefix) -``` - -一个典型的对整个bucket遍历的操作为： - -```{python} -@gist(gist/demo.py#list_all) -``` - - -### 云处理 - - -#### 图像 - - -##### 查看图像属性 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_fop) -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#image_info) -``` - - -##### 查看图片EXIF信息 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_fop) -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#exif) -``` - - - -##### 生成图片预览 - -```{python} -@gist(gist/conf.py#config) - -@gist(gist/demo.py#import_fop) -@gist(gist/demo.py#import_rs) - -@gist(gist/demo.py#image_view) -``` - - -## 贡献代码 - -+ Fork -+ 创建您的特性分支 (git checkout -b my-new-feature) -+ 提交您的改动 (git commit -am 'Added some feature') -+ 将您的修改记录提交到远程 git 仓库 (git push origin my-new-feature) -+ 然后到 github 网站的该 git 远程仓库的 my-new-feature 分支下发起 Pull Request - - -## 许可证 - -> Copyright (c) 2013 qiniu.com - -基于 MIT 协议发布: - -> [www.opensource.org/licenses/MIT](http://www.opensource.org/licenses/MIT) - diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index b27d0d9a..00000000 --- a/docs/README.md +++ /dev/null @@ -1,695 +0,0 @@ ---- -title: Python SDK 使用指南 ---- - -# Python SDK 使用指南 - -此 Python SDK 适用于2.x版本，基于 [七牛云存储官方API](http://docs.qiniu.com/) 构建。使用此 SDK 构建您的网络应用程序，能让您以非常便捷地方式将数据安全地存储到七牛云存储上。无论您的网络应用是一个网站程序，还是包括从云端（服务端程序）到终端（手持设备应用）的架构的服务或应用，通过七牛云存储及其 SDK，都能让您应用程序的终端用户高速上传和下载，同时也让您的服务端更加轻盈。 - -SDK 下载地址： - -**文档大纲** - -- [概述](#overview) -- [准备开发环境](#prepare) - - [安装](#install) - - [ACCESS_KEY 和 SECRET_KEY](#appkey) -- [使用SDK](#sdk-usage) - - [初始化环境](#init) - - [上传文件](#io-put) - - [上传流程](#io-put-flow) - - [上传策略](#io-put-policy) - - [上传凭证](#upload-token) - - [PutExtra](#put-extra) - - [上传文件](#upload-do) - - [断点续上传、分块并行上传](#resumable-io-put) - - [下载文件](#io-get) - - [下载公有文件](#io-get-public) - - [下载私有文件](#io-get-private) - - [断点续下载](#resumable-io-get) - - [资源操作](#rs) - - [获取文件信息](#rs-stat) - - [复制文件](#rs-copy) - - [移动文件](#rs-move) - - [删除文件](#rs-delete) - - [批量操作](#rs-batch) - - [批量获取文件信息](#batch-stat) - - [批量复制文件](#batch-copy) - - [批量移动文件](#batch-move) - - [批量删除文件](#batch-delete) - - [高级管理操作](#rsf) - - [列出文件](#list-prefix) - - [云处理](#fop) - - [图像](#fop-image) - - [查看图像属性](#fop-image-info) - - [查看图片EXIF信息](#fop-exif) - - [生成图片预览](#fop-image-view) -- [贡献代码](#contribution) -- [许可证](#license) - - - -## 概述 - -七牛云存储的 Python 语言版本 SDK（本文以下称 Python-SDK）是对七牛云存储API协议的一层封装，以提供一套对于 Python 开发者而言简单易用的开发工具。Python 开发者在对接 Python-SDK 时无需理解七牛云存储 API 协议的细节，原则上也不需要对 HTTP 协议和原理做非常深入的了解，但如果拥有基础的 HTTP 知识，对于出错场景的处理可以更加高效。 - -Python-SDK 被设计为同时适合服务器端和客户端使用。服务端是指开发者自己的业务服务器，客户端是指开发者提供给终端用户的软件，通常运行在 Windows/Mac/Linux 这样的桌面平台上。服务端因为有七牛颁发的 AccessKey/SecretKey，可以做很多客户端做不了的事情，比如删除文件、移动/复制文件等操作。一般而言，客服端操作文件需要获得服务端的授权。客户端上传文件需要获得服务端颁发的 [uptoken（上传授权凭证）](http://docs.qiniu.com/api/put.html#uploadToken)，客户端下载文件（包括下载处理过的文件，比如下载图片的缩略图）需要获得服务端颁发的 [dntoken（下载授权凭证）](http://docs.qiniu.com/api/get.html#download-token)。但开发者也可以将 bucket 设置为公开，此时文件有永久有效的访问地址，不需要业务服务器的授权，这对网站的静态文件（如图片、js、css、html）托管非常方便。 - -从 v5.0.0 版本开始，我们对 SDK 的内容进行了精简。所有管理操作，比如：创建/删除 bucket、为 bucket 绑定域名（publish）、设置数据处理的样式分隔符（fop seperator）、新增数据处理样式（fop style）等都去除了，统一建议到[开发者后台](https://portal.qiniu.com/)来完成。另外，此前服务端还有自己独有的上传 API，现在也推荐统一成基于客户端上传的工作方式。 - -从内容上来说，Python-SDK 主要包含如下几方面的内容： - -* 公共部分，所有用况下都用到：qiniu/rpc.py, qiniu/httplib_chunk.py -* 客户端上传文件：qiniu/io.py -* 客户端断点续上传：qiniu/resumable_io.py -* 数据处理：qiniu/fop.py -* 服务端操作：qiniu/auth/digest.py, qiniu/auth/up.py (授权), qiniu/rs/rs.py, qiniu/rs/rs_token.py (资源操作, uptoken/dntoken颁发) - - - - - -## 准备开发环境 - - - - -### 安装 - -直接安装: - - pip install qiniu - #或 - easy_install qiniu - -Python-SDK可以使用`pip`或`easy_install`从PyPI服务器上安装，但不包括文档和样例。如果需要，请下载源码并安装。 - -源码安装： - -从[Python-SDK下载地址](https://github.com/qiniu/python-sdk/releases)下载源码： - - tar xvzf python-sdk-$VERSION.tar.gz - cd python-sdk-$VERSION - python setup.py install - - - - -### ACCESS_KEY 和 SECRET_KEY - -在使用SDK 前，您需要拥有一对有效的 AccessKey 和 SecretKey 用来进行签名授权。 - -可以通过如下步骤获得： - -1. [开通七牛开发者帐号](https://portal.qiniu.com/signup) -2. [登录七牛开发者自助平台，查看 Access Key 和 Secret Key](https://portal.qiniu.com/setting/key) 。 - - - -## 使用SDK - - - -### 初始化环境 - -在获取到 Access Key 和 Secret Key 之后，您可以在您的程序中调用如下两行代码进行初始化对接, 要确保`ACCESS_KEY` 和 `SECRET_KEY` 在调用所有七牛API服务之前均已赋值： - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" -``` - - - -### 上传文件 - -为了尽可能地改善终端用户的上传体验，七牛云存储首创了客户端直传功能。一般云存储的上传流程是： - - 客户端（终端用户） => 业务服务器 => 云存储服务 - -这样多了一次上传的流程，和本地存储相比，会相对慢一些。但七牛引入了客户端直传，将整个上传过程调整为： - - 客户端（终端用户） => 七牛 => 业务服务器 - -客户端（终端用户）直接上传到七牛的服务器，通过DNS智能解析，七牛会选择到离终端用户最近的ISP服务商节点，速度会比本地存储快很多。文件上传成功以后，七牛的服务器使用回调功能，只需要将非常少的数据（比如Key）传给应用服务器，应用服务器进行保存即可。 - - - -#### 上传流程 - -在七牛云存储中，整个上传流程大体分为这样几步： - -1. 业务服务器颁发 [uptoken（上传授权凭证）](http://docs.qiniu.com/api/put.html#uploadToken)给客户端（终端用户） -2. 客户端凭借 [uptoken](http://docs.qiniu.com/api/put.html#uploadToken) 上传文件到七牛 -3. 在七牛获得完整数据后，发起一个 HTTP 请求回调到业务服务器 -4. 业务服务器保存相关信息，并返回一些信息给七牛 -5. 七牛原封不动地将这些信息转发给客户端（终端用户） - -需要注意的是，回调到业务服务器的过程是可选的，它取决于业务服务器颁发的 [uptoken](http://docs.qiniu.com/api/put.html#uploadToken)。如果没有回调，七牛会返回一些标准的信息（比如文件的 hash）给客户端。如果上传发生在业务服务器，以上流程可以自然简化为： - -1. 业务服务器生成 uptoken（不设置回调，自己回调到自己这里没有意义） -2. 凭借 [uptoken](http://docs.qiniu.com/api/put.html#uploadToken) 上传文件到七牛 -3. 善后工作，比如保存相关的一些信息 - - - -##### 上传策略 - -[uptoken](http://docs.qiniu.com/api/put.html#uploadToken) 实际上是用 AccessKey/SecretKey 进行数字签名的上传策略(`qiniu.rs.PutPolicy`)，它控制则整个上传流程的行为。让我们快速过一遍你都能够决策啥： - -```{python} -class PutPolicy(object): - scope = None # 可以是 bucketName 或者 bucketName:key - expires = 3600 # 默认是 3600 秒 - callbackUrl = None - callbackBody = None - returnUrl = None - returnBody = None - endUser = None - asyncOps = None - - def __init__(self, scope): - self.scope = scope -``` - -* `scope` 限定客户端的权限。如果 `scope` 是 bucket，则客户端只能新增文件到指定的 bucket，不能修改文件。如果 `scope` 为 bucket:key，则客户端可以修改指定的文件。**注意： key必须采用utf8编码，如使用非utf8编码访问七牛云存储将反馈错误** -* `callbackUrl` 设定业务服务器的回调地址，这样业务服务器才能感知到上传行为的发生。 -* `callbackBody` 设定业务服务器的回调信息。文件上传成功后，七牛向业务服务器的callbackUrl发送的POST请求携带的数据。支持 [魔法变量](http://docs.qiniu.com/api/put.html#MagicVariables) 和 [自定义变量](http://docs.qiniu.com/api/put.html#xVariables)。 -* `returnUrl` 设置用于浏览器端文件上传成功后，浏览器执行301跳转的URL，一般为 HTML Form 上传时使用。文件上传成功后浏览器会自动跳转到 `returnUrl?upload_ret=returnBody`。 -* `returnBody` 可调整返回给客户端的数据包，支持 [魔法变量](http://docs.qiniu.com/api/put.html#MagicVariables) 和 [自定义变量](http://docs.qiniu.com/api/put.html#xVariables)。`returnBody` 只在没有 `callbackUrl` 时有效（否则直接返回 `callbackUrl` 返回的结果）。不同情形下默认返回的 `returnBody` 并不相同。在一般情况下返回的是文件内容的 `hash`，也就是下载该文件时的 `etag`；但指定 `returnUrl` 时默认的 `returnBody` 会带上更多的信息。 -* `asyncOps` 可指定上传完成后，需要自动执行哪些数据处理。这是因为有些数据处理操作（比如音视频转码）比较慢，如果不进行预转可能第一次访问的时候效果不理想，预转可以很大程度改善这一点。 - -关于上传策略更完整的说明，请参考 [uptoken](http://docs.qiniu.com/api/put.html#uploadToken)。 - - - -##### 上传凭证 - -服务端生成 [uptoken](http://docs.qiniu.com/api/put.html#uploadToken) 代码如下： - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.rs - -policy = qiniu.rs.PutPolicy(bucket_name) -uptoken = policy.token() -``` - - - -##### PutExtra - -PutExtra是上传时的可选信息，默认为None - -```{python} -class PutExtra(object): - params = {} - mime_type = 'application/octet-stream' - crc32 = "" - check_crc = 0 -``` - -* `params` 是一个字典。[自定义变量](http://docs.qiniu.com/api/put.html#xVariables)，key必须以 x: 开头命名，不限个数。可以在 uploadToken 的 callbackBody 选项中求值。 -* `mime_type` 表示数据的MimeType，当不指定时七牛服务器会自动检测。 -* `crc32` 待检查的crc32值 -* `check_crc` 可选值为0, 1, 2。 - `check_crc == 0`: 表示不进行 crc32 校验。 - `check_crc == 1`: 上传二进制数据时等同于 `check_crc=2`；上传本地文件时会自动计算 crc32 值。 - `check_crc == 2`: 表示进行 crc32 校验，且 crc32 值就是上面的 `crc32` 变量 - - - -##### 上传文件 - -上传文件到七牛（通常是客户端完成，但也可以发生在服务端）： - -直接上传二进制流 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.io - -extra = qiniu.io.PutExtra() -extra.mime_type = "text/plain" - -# data 可以是str或read()able对象 -data = StringIO.StringIO("hello!") -ret, err = qiniu.io.put(uptoken, key, data, extra) -if err is not None: - sys.stderr.write('error: %s ' % err) - return -``` - -上传本地文件 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.io - -localfile = "%s" % __file__ - -ret, err = qiniu.io.put_file(uptoken, key, localfile) -if err is not None: - sys.stderr.write('error: %s ' % err) - return -``` - -ret是一个字典，含有`hash`，`key`等信息。 - - - -##### 断点续上传、分块并行上传 - -除了基本的上传外，七牛还支持你将文件切成若干块（除最后一块外，每个块固定为4M大小），每个块可独立上传，互不干扰；每个分块块内则能够做到断点上续传。 - -我们来看支持了断点上续传、分块并行上传的基本样例： - -上传二进制流 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.resumable_io as rio - -a = "resumable upload string" -extra = rio.PutExtra(bucket_name) -extra.mime_type = "text/plain" -ret, err = rio.put(uptoken, key, StringIO.StringIO(a), len(a), extra) -if err is not None: - sys.stderr.write('error: %s ' % err) - return -print ret, -``` - -上传本地文件 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.resumable_io as rio - -localfile = "%s" % __file__ -extra = rio.PutExtra(bucket_name) - -ret, err = rio.put_file(uptoken, key, localfile, extra) -if err is not None: - sys.stderr.write('error: %s ' % err) - return -print ret, -``` - - - -### 下载文件 - - - -#### 下载公有文件 - -每个 bucket 都会绑定一个或多个域名（domain）。如果这个 bucket 是公开的，那么该 bucket 中的所有文件可以通过一个公开的下载 url 可以访问到： - - http:/// - -其中\是bucket所对应的域名。七牛云存储为每一个bucket提供一个默认域名。默认域名可以到[七牛云存储开发者平台](https://portal.qiniu.com/)中，空间设置的域名设置一节查询。 - -假设某个 bucket 既绑定了七牛的二级域名，如 hello.qiniudn.com，也绑定了自定义域名（需要备案），如 hello.com。那么该 bucket 中 key 为 a/b/c.htm 的文件可以通过 http://hello.qiniudn.com/a/b/c.htm 或 http://hello.com/a/b/c.htm 中任意一个 url 进行访问。 - -**注意： key必须采用utf8编码，如使用非utf8编码访问七牛云存储将反馈错误** - - - -#### 下载私有文件 - -如果某个 bucket 是私有的，那么这个 bucket 中的所有文件只能通过一个的临时有效的 downloadUrl 访问： - - http:///?e=&token= - -其中 dntoken 是由业务服务器签发的一个[临时下载授权凭证](http://docs.qiniu.com/api/get.html#download-token)，deadline 是 dntoken 的有效期。dntoken不需要单独生成，SDK 提供了生成完整 downloadUrl 的方法（包含了 dntoken），示例代码如下： - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.rs - -base_url = qiniu.rs.make_base_url(domain, key) -policy = qiniu.rs.GetPolicy() -private_url = policy.make_request(base_url) -``` - -生成 downloadUrl 后，服务端下发 downloadUrl 给客户端。客户端收到 downloadUrl 后，和公有资源类似，直接用任意的 HTTP 客户端就可以下载该资源了。唯一需要注意的是，在 downloadUrl 失效却还没有完成下载时，需要重新向服务器申请授权。 - -无论公有资源还是私有资源，下载过程中客户端并不需要七牛 SDK 参与其中。 - - - -#### 断点续下载 - -无论是公有资源还是私有资源，获得的下载 url 支持标准的 HTTP 断点续传协议。考虑到多数语言都有相应的断点续下载支持的成熟方法，七牛 Python-SDK 并不提供断点续下载相关代码。 - - - -### 资源操作 - - - - -#### 获取文件信息 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.rs - -ret, err = qiniu.rs.Client().stat(bucket_name, key) -if err is not None: - sys.stderr.write('error: %s ' % err) - return -print ret, -``` - - - -#### 复制文件 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.rs - -ret, err = qiniu.rs.Client().copy(bucket_name, key, bucket_name, key2) -if err is not None: - sys.stderr.write('error: %s ' % err) - return -``` - - - -#### 移动文件 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.rs - -ret, err = qiniu.rs.Client().move(bucket_name, key2, bucket_name, key3) -if err is not None: - sys.stderr.write('error: %s ' % err) - return -``` - - - -#### 删除文件 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.rs - -ret, err = qiniu.rs.Client().delete(bucket_name, key3) -if err is not None: - sys.stderr.write('error: %s ' % err) - return -``` - - - -#### 批量操作 - -当您需要一次性进行多个操作时, 可以使用批量操作。 - - - -##### 批量获取文件信息 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.rs - -path_1 = qiniu.rs.EntryPath(bucket_name, key) -path_2 = qiniu.rs.EntryPath(bucket_name, key2) -path_3 = qiniu.rs.EntryPath(bucket_name, key3) - -rets, err = qiniu.rs.Client().batch_stat([path_1, path_2, path_3]) -if err is not None: - sys.stderr.write('error: %s ' % err) - return -``` - - -##### 批量复制文件 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.rs - -path_1 = qiniu.rs.EntryPath(bucket_name, key) -path_2 = qiniu.rs.EntryPath(bucket_name, key2) -path_3 = qiniu.rs.EntryPath(bucket_name, key3) - -pair_1 = qiniu.rs.EntryPathPair(path_1, path_3) -rets, err = qiniu.rs.Client().batch_copy([pair_1]) -if not rets[0]['code'] == 200: - sys.stderr.write('error: %s ' % "复制失败") - return -``` - - -##### 批量移动文件 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.rs - -path_1 = qiniu.rs.EntryPath(bucket_name, key) -path_2 = qiniu.rs.EntryPath(bucket_name, key2) -path_3 = qiniu.rs.EntryPath(bucket_name, key3) - -pair_2 = qiniu.rs.EntryPathPair(path_3, path_2) -rets, err = qiniu.rs.Client().batch_move([pair_2]) -if not rets[0]['code'] == 200: - sys.stderr.write('error: %s ' % "移动失败") - return -``` - - -##### 批量删除文件 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.rs - -path_1 = qiniu.rs.EntryPath(bucket_name, key) -path_2 = qiniu.rs.EntryPath(bucket_name, key2) -path_3 = qiniu.rs.EntryPath(bucket_name, key3) - -rets, err = qiniu.rs.Client().batch_delete([path_1, path_2]) -if not [ret['code'] for ret in rets] == [200, 200]: - sys.stderr.write('error: %s ' % "删除失败") - return -``` - - - -### 高级管理操作 - - -#### 列出文件 - -请求某个存储空间（bucket）下的文件列表，如果有前缀，可以按前缀（prefix）进行过滤；如果前一次返回marker就表示还有资源，下一步请求需要将marker参数填上。 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.rsf - -rets, err = qiniu.rsf.Client().list_prefix(bucket_name, prefix="test", limit=2) -if err is not None: - sys.stderr.write('error: %s ' % err) - return -print rets - -# 从上一次list_prefix的位置继续列出文件 -rets2, err = qiniu.rsf.Client().list_prefix(bucket_name, prefix="test", limit=1, marker=rets['marker']) -if err is not None: - sys.stderr.write('error: %s ' % err) - return -print rets2 -``` - -一个典型的对整个bucket遍历的操作为： - -```{python} -def list_all(bucket, rs=None, prefix=None, limit=None): - if rs is None: - rs = qiniu.rsf.Client() - marker = None - err = None - while err is None: - ret, err = rs.list_prefix(bucket_name, prefix=prefix, limit=limit, marker=marker) - marker = ret.get('marker', None) - for item in ret['items']: - #do something - pass - if err is not qiniu.rsf.EOF: - # 错误处理 - pass -``` - - -### 云处理 - - -#### 图像 - - -##### 查看图像属性 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.fop -import qiniu.rs - -# 生成base_url -url = qiniu.rs.make_base_url(domain, pic_key) - -# 生成fop_url -image_info = qiniu.fop.ImageInfo() -url = image_info.make_request(url) - -# 对其签名，生成private_url。如果是公有bucket此步可以省略 -policy = qiniu.rs.GetPolicy() -url = policy.make_request(url) - -print '可以在浏览器浏览: %s' % url -``` - - -##### 查看图片EXIF信息 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.fop -import qiniu.rs - -# 生成base_url -url = qiniu.rs.make_base_url(domain, pic_key) - -# 生成fop_url -image_exif = qiniu.fop.Exif() -url = image_exif.make_request(url) - -# 对其签名，生成private_url。如果是公有bucket此步可以省略 -policy = qiniu.rs.GetPolicy() -url = policy.make_request(url) - -print '可以在浏览器浏览: %s' % url -``` - - - -##### 生成图片预览 - -```{python} -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" - -import qiniu.fop -import qiniu.rs - -iv = qiniu.fop.ImageView() -iv.width = 100 - -# 生成base_url -url = qiniu.rs.make_base_url(domain, pic_key) -# 生成fop_url -url = iv.make_request(url) -# 对其签名，生成private_url。如果是公有bucket此步可以省略 -policy = qiniu.rs.GetPolicy() -url = policy.make_request(url) -print '可以在浏览器浏览: %s' % url -``` - - -## 贡献代码 - -+ Fork -+ 创建您的特性分支 (git checkout -b my-new-feature) -+ 提交您的改动 (git commit -am 'Added some feature') -+ 将您的修改记录提交到远程 git 仓库 (git push origin my-new-feature) -+ 然后到 github 网站的该 git 远程仓库的 my-new-feature 分支下发起 Pull Request - - -## 许可证 - -> Copyright (c) 2013 qiniu.com - -基于 MIT 协议发布: - -> [www.opensource.org/licenses/MIT](http://www.opensource.org/licenses/MIT) - - diff --git a/docs/gist/conf.py b/docs/gist/conf.py deleted file mode 100644 index c9190dd7..00000000 --- a/docs/gist/conf.py +++ /dev/null @@ -1,6 +0,0 @@ -# @gist config -import qiniu.conf - -qiniu.conf.ACCESS_KEY = "" -qiniu.conf.SECRET_KEY = "" -# @endgist diff --git a/docs/gist/demo.py b/docs/gist/demo.py deleted file mode 100644 index 3ae0d9b1..00000000 --- a/docs/gist/demo.py +++ /dev/null @@ -1,383 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import sys -import StringIO - -# @gist import_io -import qiniu.io -# @endgist -import qiniu.conf -# @gist import_rs -import qiniu.rs -# @endgist -# @gist import_fop -import qiniu.fop -# @endgist -# @gist import_resumable_io -import qiniu.resumable_io as rio -# @endgist -# @gist import_rsf -import qiniu.rsf -# @endgist - -bucket_name = None -uptoken = None -key = None -key2 = None -key3 = None -domain = None -pic_key = None - -# ---------------------------------------------------------- - - -def setup(access_key, secret_key, bucketname, bucket_domain, pickey): - global bucket_name, uptoken, key, key2, domain, key3, pic_key - qiniu.conf.ACCESS_KEY = access_key - qiniu.conf.SECRET_KEY = secret_key - bucket_name = bucketname - domain = bucket_domain - pic_key = pickey - # @gist uptoken - policy = qiniu.rs.PutPolicy(bucket_name) - uptoken = policy.token() - # @endgist - key = "python-demo-put-file" - key2 = "python-demo-put-file-2" - key3 = "python-demo-put-file-3" - - -def _setup(): - ''' 根据环境变量配置信息 ''' - access_key = getenv("QINIU_ACCESS_KEY") - if access_key is None: - exit("请配置环境变量 QINIU_ACCESS_KEY") - secret_key = getenv("QINIU_SECRET_KEY") - bucket_name = getenv("QINIU_TEST_BUCKET") - domain = getenv("QINIU_TEST_DOMAIN") - pickey = 'QINIU_UNIT_TEST_PIC' - setup(access_key, secret_key, bucket_name, domain, pickey) - - -def getenv(name): - env = os.getenv(name) - if env is None: - sys.stderr.write("请配置环境变量 %s\n" % name) - exit(1) - return env - - -def get_demo_list(): - return [put_file, put_binary, - resumable_put, resumable_put_file, - stat, copy, move, delete, batch, - image_info, image_exif, image_view, - list_prefix, list_prefix_all, - ] - - -def run_demos(demos): - for i, demo in enumerate(demos): - print '%s.%s ' % (i + 1, demo.__doc__), - demo() - print - -# ---------------------------------------------------------- - - -def make_private_url(domain, key): - ''' 生成私有下载链接 ''' - # @gist dntoken - base_url = qiniu.rs.make_base_url(domain, key) - policy = qiniu.rs.GetPolicy() - private_url = policy.make_request(base_url) - # @endgist - return private_url - - -def put_file(): - ''' 演示上传文件的过程 ''' - # 尝试删除 - qiniu.rs.Client().delete(bucket_name, key) - - # @gist put_file - localfile = "%s" % __file__ - - ret, err = qiniu.io.put_file(uptoken, key, localfile) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - # @endgist - - -def put_binary(): - ''' 上传二进制数据 ''' - # 尝试删除 - qiniu.rs.Client().delete(bucket_name, key) - - # @gist put - extra = qiniu.io.PutExtra() - extra.mime_type = "text/plain" - - # data 可以是str或read()able对象 - data = StringIO.StringIO("hello!") - ret, err = qiniu.io.put(uptoken, key, data, extra) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - # @endgist - - -def resumable_put(): - ''' 断点续上传 ''' - # 尝试删除 - qiniu.rs.Client().delete(bucket_name, key) - - # @gist resumable_put - a = "resumable upload string" - extra = rio.PutExtra(bucket_name) - extra.mime_type = "text/plain" - ret, err = rio.put(uptoken, key, StringIO.StringIO(a), len(a), extra) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - print ret, - # @endgist - - -def resumable_put_file(): - ''' 断点续上传文件 ''' - # 尝试删除 - qiniu.rs.Client().delete(bucket_name, key) - - # @gist resumable_put_file - localfile = "%s" % __file__ - extra = rio.PutExtra(bucket_name) - - ret, err = rio.put_file(uptoken, key, localfile, extra) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - print ret, - # @endgist - - -def stat(): - ''' 查看上传文件的内容 ''' - # @gist stat - ret, err = qiniu.rs.Client().stat(bucket_name, key) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - print ret, - # @endgist - - -def copy(): - ''' 复制文件 ''' - # 初始化 - qiniu.rs.Client().delete(bucket_name, key2) - - # @gist copy - ret, err = qiniu.rs.Client().copy(bucket_name, key, bucket_name, key2) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - # @endgist - - stat, err = qiniu.rs.Client().stat(bucket_name, key2) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - print 'new file:', stat, - - -def move(): - ''' 移动文件 ''' - # 初始化 - qiniu.rs.Client().delete(bucket_name, key3) - - # @gist move - ret, err = qiniu.rs.Client().move(bucket_name, key2, bucket_name, key3) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - # @endgist - - # 查看文件是否移动成功 - ret, err = qiniu.rs.Client().stat(bucket_name, key3) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - - # 查看文件是否被删除 - ret, err = qiniu.rs.Client().stat(bucket_name, key2) - if err is None: - sys.stderr.write('error: %s ' % "删除失败") - return - - -def delete(): - ''' 删除文件 ''' - # @gist delete - ret, err = qiniu.rs.Client().delete(bucket_name, key3) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - # @endgist - - ret, err = qiniu.rs.Client().stat(bucket_name, key3) - if err is None: - sys.stderr.write('error: %s ' % "删除失败") - return - - -def image_info(): - ''' 查看图片的信息 ''' - - # @gist image_info - # 生成base_url - url = qiniu.rs.make_base_url(domain, pic_key) - - # 生成fop_url - image_info = qiniu.fop.ImageInfo() - url = image_info.make_request(url) - - # 对其签名，生成private_url。如果是公有bucket此步可以省略 - policy = qiniu.rs.GetPolicy() - url = policy.make_request(url) - - print '可以在浏览器浏览: %s' % url - # @endgist - - -def image_exif(): - ''' 查看图片的exif信息 ''' - # @gist exif - # 生成base_url - url = qiniu.rs.make_base_url(domain, pic_key) - - # 生成fop_url - image_exif = qiniu.fop.Exif() - url = image_exif.make_request(url) - - # 对其签名，生成private_url。如果是公有bucket此步可以省略 - policy = qiniu.rs.GetPolicy() - url = policy.make_request(url) - - print '可以在浏览器浏览: %s' % url - # @endgist - - -def image_view(): - ''' 对图片进行预览处理 ''' - # @gist image_view - iv = qiniu.fop.ImageView() - iv.width = 100 - - # 生成base_url - url = qiniu.rs.make_base_url(domain, pic_key) - # 生成fop_url - url = iv.make_request(url) - # 对其签名，生成private_url。如果是公有bucket此步可以省略 - policy = qiniu.rs.GetPolicy() - url = policy.make_request(url) - print '可以在浏览器浏览: %s' % url - # @endgist - - -def batch(): - ''' 文件处理的批量操作 ''' - # @gist batch_path - path_1 = qiniu.rs.EntryPath(bucket_name, key) - path_2 = qiniu.rs.EntryPath(bucket_name, key2) - path_3 = qiniu.rs.EntryPath(bucket_name, key3) - # @endgist - - # 查看状态 - # @gist batch_stat - rets, err = qiniu.rs.Client().batch_stat([path_1, path_2, path_3]) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - # @endgist - if not [ret['code'] for ret in rets] == [200, 612, 612]: - sys.stderr.write('error: %s ' % "批量获取状态与预期不同") - return - - # 复制 - # @gist batch_copy - pair_1 = qiniu.rs.EntryPathPair(path_1, path_3) - rets, err = qiniu.rs.Client().batch_copy([pair_1]) - if not rets[0]['code'] == 200: - sys.stderr.write('error: %s ' % "复制失败") - return - # @endgist - - qiniu.rs.Client().batch_delete([path_2]) - # @gist batch_move - pair_2 = qiniu.rs.EntryPathPair(path_3, path_2) - rets, err = qiniu.rs.Client().batch_move([pair_2]) - if not rets[0]['code'] == 200: - sys.stderr.write('error: %s ' % "移动失败") - return - # @endgist - - # 删除残留文件 - # @gist batch_delete - rets, err = qiniu.rs.Client().batch_delete([path_1, path_2]) - if not [ret['code'] for ret in rets] == [200, 200]: - sys.stderr.write('error: %s ' % "删除失败") - return - # @endgist - - -def list_prefix(): - ''' 列出文件操作 ''' - # @gist list_prefix - rets, err = qiniu.rsf.Client().list_prefix( - bucket_name, prefix="test", limit=2) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - print rets - - # 从上一次list_prefix的位置继续列出文件 - rets2, err = qiniu.rsf.Client().list_prefix( - bucket_name, prefix="test", limit=1, marker=rets['marker']) - if err is not None: - sys.stderr.write('error: %s ' % err) - return - print rets2 - # @endgist - - -def list_prefix_all(): - ''' 列出所有 ''' - list_all(bucket_name, prefix='test_Z', limit=10) - -# @gist list_all - - -def list_all(bucket_name, rs=None, prefix=None, limit=None): - if rs is None: - rs = qiniu.rsf.Client() - marker = None - err = None - while err is None: - ret, err = rs.list_prefix( - bucket_name, prefix=prefix, limit=limit, marker=marker) - marker = ret.get('marker', None) - for item in ret['items']: - # do something - pass - if err is not qiniu.rsf.EOF: - # 错误处理 - pass -# @endgist - -if __name__ == "__main__": - _setup() - - demos = get_demo_list() - run_demos(demos) diff --git a/docs/gist/fetch.py b/docs/gist/fetch.py deleted file mode 100644 index caeb73c4..00000000 --- a/docs/gist/fetch.py +++ /dev/null @@ -1,30 +0,0 @@ -# coding=utf-8 -import sys -sys.path.insert(0, "../../") - -from base64 import urlsafe_b64encode as b64e -from qiniu.auth import digest - -access_key = "" -secret_key = "" - -src_url = "" -dest_bucket = "" -dest_key = "" - -encoded_url = b64e(src_url) -dest_entry = "%s:%s" % (dest_bucket, dest_key) -encoded_entry = b64e(dest_entry) - -api_host = "iovip.qbox.me" -api_path = "/fetch/%s/to/%s" % (encoded_url, encoded_entry) - -mac = digest.Mac(access=access_key, secret=secret_key) -client = digest.Client(host=api_host, mac=mac) - -ret, err = client.call(path=api_path) -if err is not None: - print "failed" - print err -else: - print "success" diff --git a/docs/gist/pfop.py b/docs/gist/pfop.py deleted file mode 100644 index bb09463e..00000000 --- a/docs/gist/pfop.py +++ /dev/null @@ -1,37 +0,0 @@ -# coding=utf-8 -import sys -sys.path.insert(0, "../../") - -from urllib import quote -from qiniu.auth import digest - -access_key = "" -secret_key = "" - -bucket = "" -key = "" -fops = "" -notify_url = "" -force = False - -api_host = "api.qiniu.com" -api_path = "/pfop/" -body = "bucket=%s&key=%s&fops=%s¬ifyURL=%s" % \ - (quote(bucket), quote(key), quote(fops), quote(notify_url)) - -body = "%s&force=1" % (body,) if force is not False else body - -content_type = "application/x-www-form-urlencoded" -content_length = len(body) - -mac = digest.Mac(access=access_key, secret=secret_key) -client = digest.Client(host=api_host, mac=mac) - -ret, err = client.call_with(path=api_path, body=body, - content_type=content_type, content_length=content_length) -if err is not None: - print "failed" - print err -else: - print "success" - print ret diff --git a/docs/gist/prefetch.py b/docs/gist/prefetch.py deleted file mode 100644 index 968004ec..00000000 --- a/docs/gist/prefetch.py +++ /dev/null @@ -1,29 +0,0 @@ -# coding=utf-8 -import sys -sys.path.insert(0, "../../") - -from base64 import urlsafe_b64encode as b64e -from qiniu.auth import digest - -access_key = "" -secret_key = "" - -bucket = "" -key = "" - -entry = "%s:%s" % (bucket, key) -encoded_entry = b64e(entry) - - -api_host = "iovip.qbox.me" -api_path = "/prefetch/%s" % (encoded_entry) - -mac = digest.Mac(access=access_key, secret=secret_key) -client = digest.Client(host=api_host, mac=mac) - -ret, err = client.call(path=api_path) -if err is not None: - print "failed" - print err -else: - print "success" diff --git a/qiniu/__init__.py b/qiniu/__init__.py deleted file mode 100644 index cb83be37..00000000 --- a/qiniu/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -''' -Qiniu Resource Storage SDK for Python -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -For detailed document, please see: - -''' - -# -*- coding: utf-8 -*- -__version__ = '6.1.9' diff --git a/qiniu/auth/__init__.py b/qiniu/auth/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/qiniu/auth/digest.py b/qiniu/auth/digest.py deleted file mode 100644 index dcee6fbc..00000000 --- a/qiniu/auth/digest.py +++ /dev/null @@ -1,62 +0,0 @@ -# -*- coding: utf-8 -*- -from urlparse import urlparse -import hmac -from hashlib import sha1 -from base64 import urlsafe_b64encode - -from .. import rpc -from .. import conf - - -class Mac(object): - access = None - secret = None - - def __init__(self, access=None, secret=None): - if access is None and secret is None: - access, secret = conf.ACCESS_KEY, conf.SECRET_KEY - self.access, self.secret = access, secret - - def __sign(self, data): - hashed = hmac.new(self.secret, data, sha1) - return urlsafe_b64encode(hashed.digest()) - - def sign(self, data): - return '%s:%s' % (self.access, self.__sign(data)) - - def sign_with_data(self, b): - data = urlsafe_b64encode(b) - return '%s:%s:%s' % (self.access, self.__sign(data), data) - - def sign_request(self, path, body, content_type): - parsedurl = urlparse(path) - p_query = parsedurl.query - p_path = parsedurl.path - data = p_path - if p_query != "": - data = ''.join([data, '?', p_query]) - data = ''.join([data, "\n"]) - - if body: - incBody = [ - "application/x-www-form-urlencoded", - ] - if content_type in incBody: - data += body - - return '%s:%s' % (self.access, self.__sign(data)) - - -class Client(rpc.Client): - - def __init__(self, host, mac=None): - if mac is None: - mac = Mac() - super(Client, self).__init__(host) - self.mac = mac - - def round_tripper(self, method, path, body, header={}): - token = self.mac.sign_request( - path, body, header.get("Content-Type")) - header["Authorization"] = "QBox %s" % token - return super(Client, self).round_tripper(method, path, body, header) diff --git a/qiniu/auth/up.py b/qiniu/auth/up.py deleted file mode 100644 index a2a979f3..00000000 --- a/qiniu/auth/up.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from .. import conf -from .. import rpc - - -class Client(rpc.Client): - up_token = None - - def __init__(self, up_token, host=None): - if host is None: - host = conf.UP_HOST - if host.startswith("http://"): - host = host[7:] - self.up_token = up_token - super(Client, self).__init__(host) - - def round_tripper(self, method, path, body, header={}): - header["Authorization"] = "UpToken %s" % self.up_token - return super(Client, self).round_tripper(method, path, body, header) diff --git a/qiniu/conf.py b/qiniu/conf.py deleted file mode 100644 index 03092b84..00000000 --- a/qiniu/conf.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- - -ACCESS_KEY = "" -SECRET_KEY = "" - -RS_HOST = "rs.qbox.me" -RSF_HOST = "rsf.qbox.me" -UP_HOST = "up.qiniu.com" -UP_HOST2 = "upload.qbox.me" - -from . import __version__ -import platform - -sys_info = "%s/%s" % (platform.system(), platform.machine()) -py_ver = platform.python_version() - -USER_AGENT = "QiniuPython/%s (%s) Python/%s" % (__version__, sys_info, py_ver) diff --git a/qiniu/fop.py b/qiniu/fop.py deleted file mode 100644 index 350b2857..00000000 --- a/qiniu/fop.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding:utf-8 -*- - - -class Exif(object): - - def make_request(self, url): - return '%s?exif' % url - - -class ImageView(object): - mode = 1 # 1或2 - width = None # width 默认为0，表示不限定宽度 - height = None - quality = None # 图片质量, 1-100 - format = None # 输出格式, jpg, gif, png, tif 等图片格式 - - def make_request(self, url): - target = [] - target.append('%s' % self.mode) - - if self.width is not None: - target.append("w/%s" % self.width) - - if self.height is not None: - target.append("h/%s" % self.height) - - if self.quality is not None: - target.append("q/%s" % self.quality) - - if self.format is not None: - target.append("format/%s" % self.format) - - return "%s?imageView/%s" % (url, '/'.join(target)) - - -class ImageInfo(object): - - def make_request(self, url): - return '%s?imageInfo' % url diff --git a/qiniu/httplib_chunk.py b/qiniu/httplib_chunk.py deleted file mode 100644 index 8fb43134..00000000 --- a/qiniu/httplib_chunk.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -Modified from standard httplib - -1. HTTPConnection can send trunked data. -2. Remove httplib's automatic Content-Length insertion when data is a file-like object. -""" - -# -*- coding: utf-8 -*- - -import httplib -from httplib import _CS_REQ_STARTED, _CS_REQ_SENT, CannotSendHeader, NotConnected -import string -from array import array - - -class HTTPConnection(httplib.HTTPConnection): - - def send(self, data, is_chunked=False): - """Send `data' to the server.""" - if self.sock is None: - if self.auto_open: - self.connect() - else: - raise NotConnected() - - if self.debuglevel > 0: - print "send:", repr(data) - blocksize = 8192 - if hasattr(data, 'read') and not isinstance(data, array): - if self.debuglevel > 0: - print "sendIng a read()able" - datablock = data.read(blocksize) - while datablock: - if self.debuglevel > 0: - print 'chunked:', is_chunked - if is_chunked: - if self.debuglevel > 0: - print 'send: with trunked data' - lenstr = string.upper(hex(len(datablock))[2:]) - self.sock.sendall('%s\r\n%s\r\n' % (lenstr, datablock)) - else: - self.sock.sendall(datablock) - datablock = data.read(blocksize) - if is_chunked: - self.sock.sendall('0\r\n\r\n') - else: - self.sock.sendall(data) - - def _set_content_length(self, body): - # Set the content-length based on the body. - thelen = None - try: - thelen = str(len(body)) - except (TypeError, AttributeError), te: - # Don't send a length if this failed - if self.debuglevel > 0: - print "Cannot stat!!" - print te - - if thelen is not None: - self.putheader('Content-Length', thelen) - return True - return False - - def _send_request(self, method, url, body, headers): - # Honor explicitly requested Host: and Accept-Encoding: headers. - header_names = dict.fromkeys([k.lower() for k in headers]) - skips = {} - if 'host' in header_names: - skips['skip_host'] = 1 - if 'accept-encoding' in header_names: - skips['skip_accept_encoding'] = 1 - - self.putrequest(method, url, **skips) - - is_chunked = False - if body and header_names.get('Transfer-Encoding') == 'chunked': - is_chunked = True - elif body and ('content-length' not in header_names): - is_chunked = not self._set_content_length(body) - if is_chunked: - self.putheader('Transfer-Encoding', 'chunked') - for hdr, value in headers.iteritems(): - self.putheader(hdr, value) - - self.endheaders(body, is_chunked=is_chunked) - - def endheaders(self, message_body=None, is_chunked=False): - """Indicate that the last header line has been sent to the server. - - This method sends the request to the server. The optional - message_body argument can be used to pass a message body - associated with the request. The message body will be sent in - the same packet as the message headers if it is string, otherwise it is - sent as a separate packet. - """ - if self.__state == _CS_REQ_STARTED: - self.__state = _CS_REQ_SENT - else: - raise CannotSendHeader() - self._send_output(message_body, is_chunked=is_chunked) - - def _send_output(self, message_body=None, is_chunked=False): - """Send the currently buffered request and clear the buffer. - - Appends an extra \\r\\n to the buffer. - A message_body may be specified, to be appended to the request. - """ - self._buffer.extend(("", "")) - msg = "\r\n".join(self._buffer) - del self._buffer[:] - # If msg and message_body are sent in a single send() call, - # it will avoid performance problems caused by the interaction - # between delayed ack and the Nagle algorithm. - if isinstance(message_body, str): - msg += message_body - message_body = None - self.send(msg) - if message_body is not None: - # message_body was not a string (i.e. it is a file) and - # we must run the risk of Nagle - self.send(message_body, is_chunked=is_chunked) diff --git a/qiniu/io.py b/qiniu/io.py deleted file mode 100644 index afe6dcda..00000000 --- a/qiniu/io.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- -import rpc -import conf -import random -import string -try: - import zlib - binascii = zlib -except ImportError: - zlib = None - import binascii - - -# @gist PutExtra -class PutExtra(object): - params = {} - mime_type = 'application/octet-stream' - crc32 = "" - check_crc = 0 -# @endgist - - -def put(uptoken, key, data, extra=None): - """ put your data to Qiniu - - If key is None, the server will generate one. - data may be str or read()able object. - """ - fields = { - } - - if not extra: - extra = PutExtra() - - if extra.params: - for k in extra.params: - fields[k] = str(extra.params[k]) - - if extra.check_crc: - fields["crc32"] = str(extra.crc32) - - if key is not None: - fields['key'] = key - - fields["token"] = uptoken - - fname = key - if fname is None: - fname = _random_str(9) - elif fname is '': - fname = 'index.html' - files = [ - {'filename': fname, 'data': data, 'mime_type': extra.mime_type}, - ] - ret, err, code = rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files) - if err is None or code / 100 == 4 or code == 579 or code / 100 == 6 or code / 100 == 7: - return ret, err - - ret, err, code = rpc.Client(conf.UP_HOST2).call_with_multipart("/", fields, files) - return ret, err - - -def put_file(uptoken, key, localfile, extra=None): - """ put a file to Qiniu - - If key is None, the server will generate one. - """ - if extra is not None and extra.check_crc == 1: - extra.crc32 = _get_file_crc32(localfile) - with open(localfile, 'rb') as f: - return put(uptoken, key, f, extra) - - -_BLOCK_SIZE = 1024 * 1024 * 4 - - -def _get_file_crc32(filepath): - with open(filepath, 'rb') as f: - block = f.read(_BLOCK_SIZE) - crc = 0 - while len(block) != 0: - crc = binascii.crc32(block, crc) & 0xFFFFFFFF - block = f.read(_BLOCK_SIZE) - return crc - - -def _random_str(length): - lib = string.ascii_lowercase - return ''.join([random.choice(lib) for i in range(0, length)]) diff --git a/qiniu/resumable_io.py b/qiniu/resumable_io.py deleted file mode 100644 index cc544b86..00000000 --- a/qiniu/resumable_io.py +++ /dev/null @@ -1,177 +0,0 @@ -# coding=utf-8 -import os -try: - import zlib - binascii = zlib -except ImportError: - zlib = None - import binascii -from base64 import urlsafe_b64encode - -from auth import up as auth_up -import conf - -_workers = 1 -_task_queue_size = _workers * 4 -_try_times = 3 -_block_bits = 22 -_block_size = 1 << _block_bits -_block_mask = _block_size - 1 -_chunk_size = _block_size # 简化模式，弃用 - - -class ResumableIoError(object): - value = None - - def __init__(self, value): - self.value = value - return - - def __str__(self): - return self.value - - -err_invalid_put_progress = ResumableIoError("invalid put progress") -err_put_failed = ResumableIoError("resumable put failed") -err_unmatched_checksum = ResumableIoError("unmatched checksum") -err_putExtra_type = ResumableIoError("extra must the instance of PutExtra") - - -def setup(chunk_size=0, try_times=0): - global _chunk_size, _try_times - _chunk_size = 1 << 22 if chunk_size <= 0 else chunk_size - _try_times = 3 if try_times == 0 else try_times - return - - -def gen_crc32(data): - return binascii.crc32(data) & 0xffffffff - - -class PutExtra(object): - params = None # 自定义用户变量, key需要x: 开头 - mimetype = None # 可选。在 uptoken 没有指定 DetectMime 时，用户客户端可自己指定 MimeType - chunk_size = None # 可选。每次上传的Chunk大小 简化模式，弃用 - try_times = None # 可选。尝试次数 - progresses = None # 可选。上传进度 - notify = lambda self, idx, size, ret: None # 可选。进度提示 - notify_err = lambda self, idx, size, err: None - - def __init__(self, bucket=None): - self.bucket = bucket - return - - -def put_file(uptoken, key, localfile, extra): - """ 上传文件 """ - f = open(localfile, "rb") - statinfo = os.stat(localfile) - ret, err = put(uptoken, key, f, statinfo.st_size, extra) - f.close() - return ret, err - - -def put(uptoken, key, f, fsize, extra): - """ 上传二进制流, 通过将data "切片" 分段上传 """ - if not isinstance(extra, PutExtra): - print("extra must the instance of PutExtra") - return - host = conf.UP_HOST - try: - ret, err, code = put_with_host(uptoken, key, f, fsize, extra, host) - if err is None or code / 100 == 4 or code == 579 or code / 100 == 6 or code / 100 == 7: - return ret, err - except: - pass - - ret, err, code = put_with_host(uptoken, key, f, fsize, extra, conf.UP_HOST2) - return ret, err - - -def put_with_host(uptoken, key, f, fsize, extra, host): - block_cnt = block_count(fsize) - if extra.progresses is None: - extra.progresses = [None] * block_cnt - else: - if not len(extra.progresses) == block_cnt: - return None, err_invalid_put_progress, 0 - - if extra.try_times is None: - extra.try_times = _try_times - - if extra.chunk_size is None: - extra.chunk_size = _chunk_size - - for i in xrange(block_cnt): - try_time = extra.try_times - read_length = _block_size - if (i + 1) * _block_size > fsize: - read_length = fsize - i * _block_size - data_slice = f.read(read_length) - while True: - err = resumable_block_put(data_slice, i, extra, uptoken, host) - if err is None: - break - - try_time -= 1 - if try_time <= 0: - return None, err_put_failed, 0 - print err, ".. retry" - - mkfile_host = extra.progresses[-1]["host"] if block_cnt else host - mkfile_client = auth_up.Client(uptoken, mkfile_host) - - return mkfile(mkfile_client, key, fsize, extra, host) - - -def resumable_block_put(block, index, extra, uptoken, host): - block_size = len(block) - - mkblk_client = auth_up.Client(uptoken, host) - if extra.progresses[index] is None or "ctx" not in extra.progresses[index]: - crc32 = gen_crc32(block) - block = bytearray(block) - extra.progresses[index], err, code = mkblock(mkblk_client, block_size, block, host) - if err is not None: - extra.notify_err(index, block_size, err) - return err - if not extra.progresses[index]["crc32"] == crc32: - return err_unmatched_checksum - extra.notify(index, block_size, extra.progresses[index]) - return - - -def block_count(size): - global _block_size - return (size + _block_mask) / _block_size - - -def mkblock(client, block_size, first_chunk, host): - url = "http://%s/mkblk/%s" % (host, block_size) - content_type = "application/octet-stream" - return client.call_with(url, first_chunk, content_type, len(first_chunk)) - - -def putblock(client, block_ret, chunk): - url = "%s/bput/%s/%s" % (block_ret["host"], - block_ret["ctx"], block_ret["offset"]) - content_type = "application/octet-stream" - return client.call_with(url, chunk, content_type, len(chunk)) - - -def mkfile(client, key, fsize, extra, host): - url = ["http://%s/mkfile/%s" % (host, fsize)] - - if extra.mimetype: - url.append("mimeType/%s" % urlsafe_b64encode(extra.mimetype)) - - if key is not None: - url.append("key/%s" % urlsafe_b64encode(key)) - - if extra.params: - for k, v in extra.params.iteritems(): - url.append("%s/%s" % (k, urlsafe_b64encode(v))) - - url = "/".join(url) - body = ",".join([i["ctx"] for i in extra.progresses]) - return client.call_with(url, body, "text/plain", len(body)) diff --git a/qiniu/rpc.py b/qiniu/rpc.py deleted file mode 100644 index 81421598..00000000 --- a/qiniu/rpc.py +++ /dev/null @@ -1,220 +0,0 @@ -# -*- coding: utf-8 -*- - -import httplib - -if getattr(httplib, "_IMPLEMENTATION", None) != "gae": - # httplib._IMPLEMENTATION is "gae" on GAE - import httplib_chunk as httplib - -import json -import cStringIO -import conf - - -class Client(object): - _conn = None - _header = None - - def __init__(self, host): - self._conn = httplib.HTTPConnection(host) - self._header = {} - - def round_tripper(self, method, path, body, header={}): - header = self.merged_headers(header) - self._conn.request(method, path, body, header) - resp = self._conn.getresponse() - return resp - - def merged_headers(self, header): - _header = self._header.copy() - _header.update(header) - return _header - - def call(self, path): - ret, err, code = self.call_with(path, None) - return ret, err - - def call_with(self, path, body, content_type=None, content_length=None): - ret = None - - header = {"User-Agent": conf.USER_AGENT} - if content_type is not None: - header["Content-Type"] = content_type - - if content_length is not None: - header["Content-Length"] = content_length - - try: - resp = self.round_tripper("POST", path, body, header) - ret = resp.read() - ret = json.loads(ret) - except ValueError: - # ignore empty body when success - pass - except Exception, e: - return None, str(e)+path, 0 - - if resp.status >= 400: - err_msg = ret if "error" not in ret else ret["error"] - reqid = resp.getheader("X-Reqid", None) - # detail = resp.getheader("x-log", None) - if reqid is not None: - err_msg += ", reqid:%s" % reqid - - return None, err_msg, resp.status - - return ret, None, resp.status - - def call_with_multipart(self, path, fields=None, files=None): - """ - * fields => {key} - * files => [{filename, data, content_type}] - """ - content_type, mr = self.encode_multipart_formdata(fields, files) - return self.call_with(path, mr, content_type, mr.length()) - - def call_with_form(self, path, ops): - """ - * ops => {"key": value/list()} - """ - - body = [] - for i in ops: - if isinstance(ops[i], (list, tuple)): - data = ('&%s=' % i).join(ops[i]) - else: - data = ops[i] - - body.append('%s=%s' % (i, data)) - body = '&'.join(body) - - content_type = "application/x-www-form-urlencoded" - ret, err, code = self.call_with(path, body, content_type, len(body)) - return ret, err - - def set_header(self, field, value): - self._header[field] = value - - def set_headers(self, headers): - self._header.update(headers) - - def encode_multipart_formdata(self, fields, files): - """ - * fields => {key} - * files => [{filename, data, content_type}] - * return content_type, content_length, body - """ - if files is None: - files = [] - if fields is None: - fields = {} - - readers = [] - BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$' - CRLF = '\r\n' - L1 = [] - for key in fields: - L1.append('--' + BOUNDARY) - L1.append('Content-Disposition: form-data; name="%s"' % key) - L1.append('') - L1.append(fields[key]) - b1 = CRLF.join(L1) - readers.append(b1) - - for file_info in files: - L = [] - L.append('') - L.append('--' + BOUNDARY) - disposition = "Content-Disposition: form-data;" - filename = _qiniu_escape(file_info.get('filename')) - L.append('%s name="file"; filename="%s"' % (disposition, filename)) - L.append('Content-Type: %s' % - file_info.get('mime_type', 'application/octet-stream')) - L.append('') - L.append('') - b2 = CRLF.join(L) - readers.append(b2) - - data = file_info.get('data') - readers.append(data) - - L3 = ['', '--' + BOUNDARY + '--', ''] - b3 = CRLF.join(L3) - readers.append(b3) - - content_type = 'multipart/form-data; boundary=%s' % BOUNDARY - return content_type, MultiReader(readers) - - -def _qiniu_escape(s): - edits = [('\\', '\\\\'), ('\"', '\\\"')] - for (search, replace) in edits: - s = s.replace(search, replace) - return s - - -class MultiReader(object): - - """ class MultiReader([readers...]) - - MultiReader returns a read()able object that's the logical concatenation of - the provided input readers. They're read sequentially. - """ - - def __init__(self, readers): - self.readers = [] - self.content_length = 0 - self.valid_content_length = True - for r in readers: - if hasattr(r, 'read'): - if self.valid_content_length: - length = self._get_content_length(r) - if length is not None: - self.content_length += length - else: - self.valid_content_length = False - else: - buf = r - if not isinstance(buf, basestring): - buf = str(buf) - buf = encode_unicode(buf) - r = cStringIO.StringIO(buf) - self.content_length += len(buf) - self.readers.append(r) - - # don't name it __len__, because the length of MultiReader is not alway - # valid. - def length(self): - return self.content_length if self.valid_content_length else None - - def _get_content_length(self, reader): - data_len = None - if hasattr(reader, 'seek') and hasattr(reader, 'tell'): - try: - reader.seek(0, 2) - data_len = reader.tell() - reader.seek(0, 0) - except OSError: - # Don't send a length if this failed - data_len = None - return data_len - - def read(self, n=-1): - if n is None or n == -1: - return ''.join([encode_unicode(r.read()) for r in self.readers]) - else: - L = [] - while len(self.readers) > 0 and n > 0: - b = self.readers[0].read(n) - if len(b) == 0: - self.readers = self.readers[1:] - else: - L.append(encode_unicode(b)) - n -= len(b) - return ''.join(L) - - -def encode_unicode(u): - if isinstance(u, unicode): - u = u.encode('utf8') - return u diff --git a/qiniu/rs/__init__.py b/qiniu/rs/__init__.py deleted file mode 100644 index 54f748aa..00000000 --- a/qiniu/rs/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- - -__all__ = [ - "Client", "EntryPath", "EntryPathPair", "uri_stat", "uri_delete", "uri_move", "uri_copy", - "PutPolicy", "GetPolicy", "make_base_url", -] - -from .rs import Client, EntryPath, EntryPathPair, uri_stat, uri_delete, uri_move, uri_copy -from .rs_token import PutPolicy, GetPolicy, make_base_url diff --git a/qiniu/rs/rs.py b/qiniu/rs/rs.py deleted file mode 100644 index b84d538a..00000000 --- a/qiniu/rs/rs.py +++ /dev/null @@ -1,93 +0,0 @@ -# -*- coding: utf-8 -*- -from base64 import urlsafe_b64encode - -from ..auth import digest -from .. import conf - - -class Client(object): - conn = None - - def __init__(self, mac=None): - if mac is None: - mac = digest.Mac() - self.conn = digest.Client(host=conf.RS_HOST, mac=mac) - - def stat(self, bucket, key): - return self.conn.call(uri_stat(bucket, key)) - - def delete(self, bucket, key): - return self.conn.call(uri_delete(bucket, key)) - - def move(self, bucket_src, key_src, bucket_dest, key_dest): - return self.conn.call(uri_move(bucket_src, key_src, bucket_dest, key_dest)) - - def copy(self, bucket_src, key_src, bucket_dest, key_dest): - return self.conn.call(uri_copy(bucket_src, key_src, bucket_dest, key_dest)) - - def batch(self, ops): - return self.conn.call_with_form("/batch", dict(op=ops)) - - def batch_stat(self, entries): - ops = [] - for entry in entries: - ops.append(uri_stat(entry.bucket, entry.key)) - return self.batch(ops) - - def batch_delete(self, entries): - ops = [] - for entry in entries: - ops.append(uri_delete(entry.bucket, entry.key)) - return self.batch(ops) - - def batch_move(self, entries): - ops = [] - for entry in entries: - ops.append(uri_move(entry.src.bucket, entry.src.key, - entry.dest.bucket, entry.dest.key)) - return self.batch(ops) - - def batch_copy(self, entries): - ops = [] - for entry in entries: - ops.append(uri_copy(entry.src.bucket, entry.src.key, - entry.dest.bucket, entry.dest.key)) - return self.batch(ops) - - -class EntryPath(object): - bucket = None - key = None - - def __init__(self, bucket, key): - self.bucket = bucket - self.key = key - - -class EntryPathPair: - src = None - dest = None - - def __init__(self, src, dest): - self.src = src - self.dest = dest - - -def uri_stat(bucket, key): - return "/stat/%s" % urlsafe_b64encode("%s:%s" % (bucket, key)) - - -def uri_delete(bucket, key): - return "/delete/%s" % urlsafe_b64encode("%s:%s" % (bucket, key)) - - -def uri_move(bucket_src, key_src, bucket_dest, key_dest): - src = urlsafe_b64encode("%s:%s" % (bucket_src, key_src)) - dest = urlsafe_b64encode("%s:%s" % (bucket_dest, key_dest)) - return "/move/%s/%s" % (src, dest) - - -def uri_copy(bucket_src, key_src, bucket_dest, key_dest): - src = urlsafe_b64encode("%s:%s" % (bucket_src, key_src)) - dest = urlsafe_b64encode("%s:%s" % (bucket_dest, key_dest)) - return "/copy/%s/%s" % (src, dest) diff --git a/qiniu/rs/rs_token.py b/qiniu/rs/rs_token.py deleted file mode 100644 index 965313b4..00000000 --- a/qiniu/rs/rs_token.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- coding: utf-8 -*- -import json -import time -import urllib - -from ..auth import digest -from ..import rpc - -# @gist PutPolicy - - -class PutPolicy(object): - scope = None # 可以是 bucketName 或者 bucketName:key - expires = 3600 # 默认是 3600 秒 - callbackUrl = None - callbackBody = None - returnUrl = None - returnBody = None - endUser = None - asyncOps = None - - saveKey = None - insertOnly = None - detectMime = None - mimeLimit = None - fsizeLimit = None - persistentNotifyUrl = None - persistentOps = None - - def __init__(self, scope): - self.scope = scope -# @endgist - - def token(self, mac=None): - if mac is None: - mac = digest.Mac() - token = dict( - scope=self.scope, - deadline=int(time.time()) + self.expires, - ) - - if self.callbackUrl is not None: - token["callbackUrl"] = self.callbackUrl - - if self.callbackBody is not None: - token["callbackBody"] = self.callbackBody - - if self.returnUrl is not None: - token["returnUrl"] = self.returnUrl - - if self.returnBody is not None: - token["returnBody"] = self.returnBody - - if self.endUser is not None: - token["endUser"] = self.endUser - - if self.asyncOps is not None: - token["asyncOps"] = self.asyncOps - - if self.saveKey is not None: - token["saveKey"] = self.saveKey - - if self.insertOnly is not None: - token["exclusive"] = self.insertOnly - - if self.detectMime is not None: - token["detectMime"] = self.detectMime - - if self.mimeLimit is not None: - token["mimeLimit"] = self.mimeLimit - - if self.fsizeLimit is not None: - token["fsizeLimit"] = self.fsizeLimit - - if self.persistentOps is not None: - token["persistentOps"] = self.persistentOps - - if self.persistentNotifyUrl is not None: - token["persistentNotifyUrl"] = self.persistentNotifyUrl - - b = json.dumps(token, separators=(',', ':')) - return mac.sign_with_data(b) - - -class GetPolicy(object): - expires = 3600 - - def __init__(self, expires=3600): - self.expires = expires - - def make_request(self, base_url, mac=None, attname=None): - ''' - * return private_url - ''' - if mac is None: - mac = digest.Mac() - - deadline = int(time.time()) + self.expires - if '?' in base_url: - base_url += '&' - else: - base_url += '?' - if attname is None: - base_url = '%se=%s' % (base_url, str(deadline)) - else: - base_url = '%se=%s&attname=%s' % (base_url, str(deadline), attname) - - token = mac.sign(base_url) - return '%s&token=%s' % (base_url, token) - - -def make_base_url(domain, key): - ''' - * domain => str - * key => str - * return base_url - ''' - key = rpc.encode_unicode(key) - return 'http://%s/%s' % (domain, urllib.quote(key)) diff --git a/qiniu/rs/test/__init__.py b/qiniu/rs/test/__init__.py deleted file mode 100644 index 5704743b..00000000 --- a/qiniu/rs/test/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import urllib - -import qiniu.io -import qiniu.rs -import qiniu.conf - -pic = "http://cheneya.qiniudn.com/hello_jpg" -key = 'QINIU_UNIT_TEST_PIC' - - -def setUp(): - qiniu.conf.ACCESS_KEY = os.getenv("QINIU_ACCESS_KEY") - qiniu.conf.SECRET_KEY = os.getenv("QINIU_SECRET_KEY") - bucket_name = os.getenv("QINIU_TEST_BUCKET") - - policy = qiniu.rs.PutPolicy(bucket_name) - uptoken = policy.token() - - f = urllib.urlopen(pic) - _, err = qiniu.io.put(uptoken, key, f) - f.close() - if err is None or err.startswith('file exists'): - print err - assert err is None or err.startswith('file exists') diff --git a/qiniu/rs/test/rs_test.py b/qiniu/rs/test/rs_test.py deleted file mode 100644 index 32714ff5..00000000 --- a/qiniu/rs/test/rs_test.py +++ /dev/null @@ -1,98 +0,0 @@ -# -*- coding: utf-8 -*- -import unittest -import os -import random -import string - -from qiniu import rs -from qiniu import conf - - -def r(length): - lib = string.ascii_uppercase - return ''.join([random.choice(lib) for i in range(0, length)]) - -conf.ACCESS_KEY = os.getenv("QINIU_ACCESS_KEY") -conf.SECRET_KEY = os.getenv("QINIU_SECRET_KEY") -key = 'QINIU_UNIT_TEST_PIC' -bucket_name = os.getenv("QINIU_TEST_BUCKET") -noexist_key = 'QINIU_UNIT_TEST_NOEXIST' + r(30) -key2 = "rs_demo_test_key_1_" + r(5) -key3 = "rs_demo_test_key_2_" + r(5) -key4 = "rs_demo_test_key_3_" + r(5) - - -class TestRs(unittest.TestCase): - - def test_stat(self): - r = rs.Client() - ret, err = r.stat(bucket_name, key) - assert err is None - assert ret is not None - - # error - _, err = r.stat(bucket_name, noexist_key) - assert err is not None - - def test_delete_move_copy(self): - r = rs.Client() - r.delete(bucket_name, key2) - r.delete(bucket_name, key3) - - ret, err = r.copy(bucket_name, key, bucket_name, key2) - assert err is None, err - - ret, err = r.move(bucket_name, key2, bucket_name, key3) - assert err is None, err - - ret, err = r.delete(bucket_name, key3) - assert err is None, err - - # error - _, err = r.delete(bucket_name, key2) - assert err is not None - - _, err = r.delete(bucket_name, key3) - assert err is not None - - def test_batch_stat(self): - r = rs.Client() - entries = [ - rs.EntryPath(bucket_name, key), - rs.EntryPath(bucket_name, key2), - ] - ret, err = r.batch_stat(entries) - assert err is None - self.assertEqual(ret[0]["code"], 200) - self.assertEqual(ret[1]["code"], 612) - - def test_batch_delete_move_copy(self): - r = rs.Client() - e1 = rs.EntryPath(bucket_name, key) - e2 = rs.EntryPath(bucket_name, key2) - e3 = rs.EntryPath(bucket_name, key3) - e4 = rs.EntryPath(bucket_name, key4) - r.batch_delete([e2, e3, e4]) - - # copy - entries = [ - rs.EntryPathPair(e1, e2), - rs.EntryPathPair(e1, e3), - ] - ret, err = r.batch_copy(entries) - assert err is None - self.assertEqual(ret[0]["code"], 200) - self.assertEqual(ret[1]["code"], 200) - - ret, err = r.batch_move([rs.EntryPathPair(e2, e4)]) - assert err is None - self.assertEqual(ret[0]["code"], 200) - - ret, err = r.batch_delete([e3, e4]) - assert err is None - self.assertEqual(ret[0]["code"], 200) - - r.batch_delete([e2, e3, e4]) - -if __name__ == "__main__": - unittest.main() diff --git a/qiniu/rs/test/rs_token_test.py b/qiniu/rs/test/rs_token_test.py deleted file mode 100644 index 9aef1c00..00000000 --- a/qiniu/rs/test/rs_token_test.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- -import unittest -import os -import json -from base64 import urlsafe_b64decode as decode -from base64 import urlsafe_b64encode as encode -from hashlib import sha1 -import hmac -import urllib - -from qiniu import conf -from qiniu import rs - -conf.ACCESS_KEY = os.getenv("QINIU_ACCESS_KEY") -conf.SECRET_KEY = os.getenv("QINIU_SECRET_KEY") -bucket_name = os.getenv("QINIU_TEST_BUCKET") -domain = os.getenv("QINIU_TEST_DOMAIN") -key = 'QINIU_UNIT_TEST_PIC' - - -class TestToken(unittest.TestCase): - - def test_put_policy(self): - policy = rs.PutPolicy(bucket_name) - policy.endUser = "hello!" - policy.returnUrl = "http://localhost:1234/path?query=hello" - policy.returnBody = "$(sha1)" - # Do not specify the returnUrl and callbackUrl at the same time - policy.callbackUrl = "http://1.2.3.4/callback" - policy.callbackBody = "$(bucket)" - - policy.saveKey = "$(sha1)" - policy.insertOnly = 1 - policy.detectMime = 1 - policy.fsizeLimit = 1024 - policy.persistentNotifyUrl = "http://4.3.2.1/persistentNotifyUrl" - policy.persistentOps = "avthumb/flash" - - tokens = policy.token().split(':') - - # chcek first part of token - self.assertEqual(conf.ACCESS_KEY, tokens[0]) - data = json.loads(decode(tokens[2])) - - # check if same - self.assertEqual(data["scope"], bucket_name) - self.assertEqual(data["endUser"], policy.endUser) - self.assertEqual(data["returnUrl"], policy.returnUrl) - self.assertEqual(data["returnBody"], policy.returnBody) - self.assertEqual(data["callbackUrl"], policy.callbackUrl) - self.assertEqual(data["callbackBody"], policy.callbackBody) - self.assertEqual(data["saveKey"], policy.saveKey) - self.assertEqual(data["exclusive"], policy.insertOnly) - self.assertEqual(data["detectMime"], policy.detectMime) - self.assertEqual(data["fsizeLimit"], policy.fsizeLimit) - self.assertEqual( - data["persistentNotifyUrl"], policy.persistentNotifyUrl) - self.assertEqual(data["persistentOps"], policy.persistentOps) - - new_hmac = encode(hmac.new(conf.SECRET_KEY, tokens[2], sha1).digest()) - self.assertEqual(new_hmac, tokens[1]) - - def test_get_policy(self): - base_url = rs.make_base_url(domain, key) - policy = rs.GetPolicy() - private_url = policy.make_request(base_url) - - f = urllib.urlopen(private_url) - body = f.read() - f.close() - self.assertEqual(len(body) > 100, True) - - -class Test_make_base_url(unittest.TestCase): - - def test_unicode(self): - url1 = rs.make_base_url('1.com', '你好') - url2 = rs.make_base_url('1.com', u'你好') - assert url1 == url2 - -if __name__ == "__main__": - unittest.main() diff --git a/qiniu/rsf.py b/qiniu/rsf.py deleted file mode 100644 index 9b5c5b8b..00000000 --- a/qiniu/rsf.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -import auth.digest -import conf -import urllib - -EOF = 'EOF' - - -class Client(object): - conn = None - - def __init__(self, mac=None): - if mac is None: - mac = auth.digest.Mac() - self.conn = auth.digest.Client(host=conf.RSF_HOST, mac=mac) - - def list_prefix(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): - '''前缀查询: - * bucket => str - * prefix => str - * marker => str - * limit => int - * delimiter => str - * return ret => {'items': items, 'marker': markerOut}, err => str - - 1. 首次请求 marker = None - 2. 无论 err 值如何，均应该先看 ret.get('items') 是否有内容 - 3. 如果后续没有更多数据，err 返回 EOF，markerOut 返回 None（但不通过该特征来判断是否结束） - ''' - ops = { - 'bucket': bucket, - } - if marker is not None: - ops['marker'] = marker - if limit is not None: - ops['limit'] = limit - if prefix is not None: - ops['prefix'] = prefix - if delimiter is not None: - ops['delimiter'] = delimiter - - url = '%s?%s' % ('/list', urllib.urlencode(ops)) - ret, err, code = self.conn.call_with( - url, body=None, content_type='application/x-www-form-urlencoded') - if ret and not ret.get('marker'): - err = EOF - return ret, err diff --git a/qiniu/test/__init__.py b/qiniu/test/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/qiniu/test/conf_test.py b/qiniu/test/conf_test.py deleted file mode 100644 index fd200c6d..00000000 --- a/qiniu/test/conf_test.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -import unittest -from qiniu import conf - - -class TestConfig(unittest.TestCase): - - def test_USER_AGENT(self): - assert len(conf.USER_AGENT) >= len('qiniu python-sdk') - -if __name__ == '__main__': - unittest.main() diff --git a/qiniu/test/fop_test.py b/qiniu/test/fop_test.py deleted file mode 100644 index c2664882..00000000 --- a/qiniu/test/fop_test.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding:utf-8 -*- -import unittest -from qiniu import fop - -pic = "http://cheneya.qiniudn.com/hello_jpg" - - -class TestFop(unittest.TestCase): - - def test_exif(self): - ie = fop.Exif() - ret = ie.make_request(pic) - self.assertEqual(ret, "%s?exif" % pic) - - def test_imageView(self): - iv = fop.ImageView() - iv.height = 100 - ret = iv.make_request(pic) - self.assertEqual(ret, "%s?imageView/1/h/100" % pic) - - iv.quality = 20 - iv.format = "png" - ret = iv.make_request(pic) - self.assertEqual(ret, "%s?imageView/1/h/100/q/20/format/png" % pic) - - def test_imageInfo(self): - ii = fop.ImageInfo() - ret = ii.make_request(pic) - self.assertEqual(ret, "%s?imageInfo" % pic) - - -if __name__ == '__main__': - unittest.main() diff --git a/qiniu/test/io_test.py b/qiniu/test/io_test.py deleted file mode 100644 index 60538dc9..00000000 --- a/qiniu/test/io_test.py +++ /dev/null @@ -1,195 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import unittest -import string -import random -import urllib -try: - import zlib - binascii = zlib -except ImportError: - zlib = None - import binascii -import cStringIO - -from qiniu import conf -from qiniu import rs -from qiniu import io - -conf.ACCESS_KEY = os.getenv("QINIU_ACCESS_KEY") -conf.SECRET_KEY = os.getenv("QINIU_SECRET_KEY") -bucket_name = os.getenv("QINIU_TEST_BUCKET") - -policy = rs.PutPolicy(bucket_name) -extra = io.PutExtra() -extra.mime_type = "text/plain" -extra.params = {'x:a': 'a'} - - -def r(length): - lib = string.ascii_uppercase - return ''.join([random.choice(lib) for i in range(0, length)]) - - -class TestUp(unittest.TestCase): - - def test(self): - def test_put(): - key = "test_%s" % r(9) - # params = "op=3" - data = "hello bubby!" - extra.check_crc = 2 - extra.crc32 = binascii.crc32(data) & 0xFFFFFFFF - ret, err = io.put(policy.token(), key, data, extra) - assert err is None - assert ret['key'] == key - - def test_put_same_crc(): - key = "test_%s" % r(9) - data = "hello bubby!" - extra.check_crc = 2 - ret, err = io.put(policy.token(), key, data, extra) - assert err is None - assert ret['key'] == key - - def test_put_no_key(): - data = r(100) - extra.check_crc = 0 - ret, err = io.put(policy.token(), key=None, data=data, extra=extra) - assert err is None - assert ret['hash'] == ret['key'] - - def test_put_quote_key(): - data = r(100) - key = 'a\\b\\c"你好' + r(9) - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret['key'].encode('utf8') == key - - data = r(100) - key = u'a\\b\\c"你好' + r(9) - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret['key'] == key - - def test_put_unicode1(): - key = "test_%s" % r(9) + '你好' - data = key - ret, err = io.put(policy.token(), key, data, extra) - assert err is None - assert ret[u'key'].endswith(u'你好') - - def test_put_unicode2(): - key = "test_%s" % r(9) + '你好' - data = key - data = data.decode('utf8') - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret[u'key'].endswith(u'你好') - - def test_put_unicode3(): - key = "test_%s" % r(9) + '你好' - data = key - key = key.decode('utf8') - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret[u'key'].endswith(u'你好') - - def test_put_unicode4(): - key = "test_%s" % r(9) + '你好' - data = key - key = key.decode('utf8') - data = data.decode('utf8') - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret[u'key'].endswith(u'你好') - - def test_put_StringIO(): - key = "test_%s" % r(9) - data = cStringIO.StringIO('hello buddy!') - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret['key'] == key - - def test_put_urlopen(): - key = "test_%s" % r(9) - data = urllib.urlopen('http://pythonsdk.qiniudn.com/hello.jpg') - ret, err = io.put(policy.token(), key, data) - assert err is None - assert ret['key'] == key - - def test_put_no_length(): - class test_reader(object): - - def __init__(self): - self.data = 'abc' - self.pos = 0 - - def read(self, n=None): - if n is None or n < 0: - newpos = len(self.data) - else: - newpos = min(self.pos + n, len(self.data)) - r = self.data[self.pos: newpos] - self.pos = newpos - return r - key = "test_%s" % r(9) - data = test_reader() - - extra.check_crc = 2 - extra.crc32 = binascii.crc32('abc') & 0xFFFFFFFF - ret, err = io.put(policy.token(), key, data, extra) - assert err is None - assert ret['key'] == key - - test_put() - test_put_same_crc() - test_put_no_key() - test_put_quote_key() - test_put_unicode1() - test_put_unicode2() - test_put_unicode3() - test_put_unicode4() - test_put_StringIO() - test_put_urlopen() - test_put_no_length() - - def test_put_file(self): - localfile = "%s" % __file__ - key = "test_%s" % r(9) - - extra.check_crc = 1 - ret, err = io.put_file(policy.token(), key, localfile, extra) - assert err is None - assert ret['key'] == key - - def test_put_crc_fail(self): - key = "test_%s" % r(9) - data = "hello bubby!" - extra.check_crc = 2 - extra.crc32 = "wrong crc32" - ret, err = io.put(policy.token(), key, data, extra) - assert err is not None - - def test_put_fail_reqid(self): - key = "test_%s" % r(9) - data = "hello bubby!" - ret, err = io.put("", key, data, extra) - assert "reqid" in err - - -class Test_get_file_crc32(unittest.TestCase): - - def test_get_file_crc32(self): - file_path = '%s' % __file__ - - data = None - with open(file_path, 'rb') as f: - data = f.read() - io._BLOCK_SIZE = 4 - assert binascii.crc32( - data) & 0xFFFFFFFF == io._get_file_crc32(file_path) - - -if __name__ == "__main__": - unittest.main() diff --git a/qiniu/test/resumable_io_test.py b/qiniu/test/resumable_io_test.py deleted file mode 100644 index dad56ee0..00000000 --- a/qiniu/test/resumable_io_test.py +++ /dev/null @@ -1,143 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import unittest -import string -import random -import platform -try: - import zlib - binascii = zlib -except ImportError: - zlib = None - import binascii -import urllib -import shutil -import StringIO -from tempfile import mktemp - -from qiniu import conf -from qiniu.auth import up -from qiniu import resumable_io -from qiniu import rs - -bucket = os.getenv("QINIU_TEST_BUCKET") -conf.ACCESS_KEY = os.getenv("QINIU_ACCESS_KEY") -conf.SECRET_KEY = os.getenv("QINIU_SECRET_KEY") -test_env = os.getenv("QINIU_TEST_ENV") -is_travis = test_env == "travis" - - -def r(length): - lib = string.ascii_uppercase - return ''.join([random.choice(lib) for _ in range(0, length)]) - - -class TestBlock(unittest.TestCase): - - def test_block(self): - if is_travis: - return - host = conf.UP_HOST - policy = rs.PutPolicy(bucket) - uptoken = policy.token() - client = up.Client(uptoken) - - # rets = [0, 0] - data_slice_2 = "\nbye!" - ret, err, code = resumable_io.mkblock( - client, len(data_slice_2), data_slice_2, host) - assert err is None, err - self.assertEqual(ret["crc32"], binascii.crc32(data_slice_2)) - - extra = resumable_io.PutExtra(bucket) - extra.mimetype = "text/plain" - extra.progresses = [ret] - lens = 0 - for i in xrange(0, len(extra.progresses)): - lens += extra.progresses[i]["offset"] - - key = u"sdk_py_resumable_block_4_%s" % r(9) - ret, err, code = resumable_io.mkfile(client, key, lens, extra, host) - assert err is None, err - self.assertEqual( - ret["hash"], "FtCFo0mQugW98uaPYgr54Vb1QsO0", "hash not match") - rs.Client().delete(bucket, key) - - def test_put(self): - if is_travis: - return - src = urllib.urlopen("http://pythonsdk.qiniudn.com/hello.jpg") - ostype = platform.system() - if ostype.lower().find("windows") != -1: - tmpf = "".join([os.getcwd(), mktemp()]) - else: - tmpf = mktemp() - dst = open(tmpf, 'wb') - shutil.copyfileobj(src, dst) - src.close() - - policy = rs.PutPolicy(bucket) - extra = resumable_io.PutExtra(bucket) - extra.bucket = bucket - extra.params = {"x:foo": "test"} - key = "sdk_py_resumable_block_5_%s" % r(9) - localfile = dst.name - ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) - dst.close() - os.remove(tmpf) - assert err is None, err - assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" - self.assertEqual( - ret["hash"], "FnyTMUqPNRTdk1Wou7oLqDHkBm_p", "hash not match") - rs.Client().delete(bucket, key) - - def test_put_4m(self): - if is_travis: - return - ostype = platform.system() - if ostype.lower().find("windows") != -1: - tmpf = "".join([os.getcwd(), mktemp()]) - else: - tmpf = mktemp() - dst = open(tmpf, 'wb') - dst.write("abcd" * 1024 * 1024) - dst.flush() - - policy = rs.PutPolicy(bucket) - extra = resumable_io.PutExtra(bucket) - extra.bucket = bucket - extra.params = {"x:foo": "test"} - key = "sdk_py_resumable_block_6_%s" % r(9) - localfile = dst.name - ret, err = resumable_io.put_file(policy.token(), key, localfile, extra) - dst.close() - os.remove(tmpf) - assert err is None, err - assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" - self.assertEqual( - ret["hash"], "FnIVmMd_oaUV3MLDM6F9in4RMz2U", "hash not match") - rs.Client().delete(bucket, key) - - def test_put_0(self): - if is_travis: - return - - f = StringIO.StringIO('') - - policy = rs.PutPolicy(bucket) - extra = resumable_io.PutExtra(bucket) - extra.bucket = bucket - extra.params = {"x:foo": "test"} - key = "sdk_py_resumable_block_7_%s" % r(9) - ret, err = resumable_io.put(policy.token(), key, f, 0, extra) - - assert err is None, err - assert ret.get("x:foo") == "test", "return data not contains 'x:foo'" - self.assertEqual( - ret["hash"], "Fg==", "hash not match") - rs.Client().delete(bucket, key) - - -if __name__ == "__main__": - if not is_travis: - unittest.main() diff --git a/qiniu/test/rpc_test.py b/qiniu/test/rpc_test.py deleted file mode 100644 index ae326484..00000000 --- a/qiniu/test/rpc_test.py +++ /dev/null @@ -1,188 +0,0 @@ -# -*- coding: utf-8 -*- -import StringIO -import unittest - -from qiniu import rpc -from qiniu import conf - - -def round_tripper(client, method, path, body, header={}): - pass - - -class ClsTestClient(rpc.Client): - - def round_tripper(self, method, path, body, header={}): - round_tripper(self, method, path, body, header) - return super(ClsTestClient, self).round_tripper(method, path, body, header) - -client = ClsTestClient(conf.RS_HOST) - - -class TestClient(unittest.TestCase): - - def test_call(self): - global round_tripper - - def tripper(client, method, path, body, header={}): - self.assertEqual(path, "/hello") - assert body is None - - round_tripper = tripper - client.call("/hello") - - def test_call_with(self): - global round_tripper - - def tripper(client, method, path, body, header={}): - self.assertEqual(body, "body") - - round_tripper = tripper - client.call_with("/hello", "body") - - def test_call_with_multipart(self): - global round_tripper - - def tripper(client, method, path, body, header={}): - target_type = "multipart/form-data" - self.assertTrue( - header["Content-Type"].startswith(target_type)) - start_index = header["Content-Type"].find("boundary") - boundary = header["Content-Type"][start_index + 9:] - dispostion = 'Content-Disposition: form-data; name="auth"' - tpl = "--%s\r\n%s\r\n\r\n%s\r\n--%s--\r\n" % (boundary, dispostion, - "auth_string", boundary) - self.assertEqual(len(tpl), header["Content-Length"]) - self.assertEqual(len(tpl), body.length()) - - round_tripper = tripper - client.call_with_multipart("/hello", fields={"auth": "auth_string"}) - - def test_call_with_form(self): - global round_tripper - - def tripper(client, method, path, body, header={}): - self.assertEqual(body, "action=a&op=a&op=b") - target_type = "application/x-www-form-urlencoded" - self.assertEqual(header["Content-Type"], target_type) - self.assertEqual(header["Content-Length"], len(body)) - - round_tripper = tripper - client.call_with_form("/hello", dict(op=["a", "b"], action="a")) - - def test_call_after_call_with_form(self): - # test case for https://github.com/qiniu/python-sdk/issues/112 - global round_tripper - - def tripper(client, method, path, body, header={}): - pass - - round_tripper = tripper - client.call_with_form("/hello", dict(op=["a", "b"], action="a")) - client.call("/hello") - - -class TestMultiReader(unittest.TestCase): - - def test_multi_reader1(self): - a = StringIO.StringIO('你好') - b = StringIO.StringIO('abcdefg') - c = StringIO.StringIO(u'悲剧') - mr = rpc.MultiReader([a, b, c]) - data = mr.read() - assert data.index('悲剧') > data.index('abcdefg') - - def test_multi_reader2(self): - a = StringIO.StringIO('你好') - b = StringIO.StringIO('abcdefg') - c = StringIO.StringIO(u'悲剧') - mr = rpc.MultiReader([a, b, c]) - data = mr.read(8) - assert len(data) is 8 - - -def encode_multipart_formdata2(fields, files): - if files is None: - files = [] - if fields is None: - fields = [] - - BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$' - CRLF = '\r\n' - L = [] - for (key, value) in fields: - L.append('--' + BOUNDARY) - L.append('Content-Disposition: form-data; name="%s"' % key) - L.append('') - L.append(value) - for (key, filename, value) in files: - L.append('--' + BOUNDARY) - disposition = "Content-Disposition: form-data;" - L.append('%s name="%s"; filename="%s"' % (disposition, key, filename)) - L.append('Content-Type: application/octet-stream') - L.append('') - L.append(value) - L.append('--' + BOUNDARY + '--') - L.append('') - body = CRLF.join(L) - content_type = 'multipart/form-data; boundary=%s' % BOUNDARY - return content_type, body - - -class TestEncodeMultipartFormdata(unittest.TestCase): - - def test_encode(self): - fields = {'a': '1', 'b': '2'} - files = [ - { - 'filename': 'key1', - 'data': 'data1', - 'mime_type': 'application/octet-stream', - }, - { - 'filename': 'key2', - 'data': 'data2', - 'mime_type': 'application/octet-stream', - } - ] - content_type, mr = rpc.Client( - 'localhost').encode_multipart_formdata(fields, files) - t, b = encode_multipart_formdata2( - [('a', '1'), ('b', '2')], - [('file', 'key1', 'data1'), ('file', 'key2', 'data2')] - ) - assert t == content_type - assert len(b) == mr.length() - - def test_unicode(self): - def test1(): - files = [{'filename': '你好', 'data': '你好', 'mime_type': ''}] - _, body = rpc.Client( - 'localhost').encode_multipart_formdata(None, files) - return len(body.read()) - - def test2(): - files = [{'filename': u'你好', 'data': '你好', 'mime_type': ''}] - _, body = rpc.Client( - 'localhost').encode_multipart_formdata(None, files) - return len(body.read()) - - def test3(): - files = [{'filename': '你好', 'data': u'你好', 'mime_type': ''}] - _, body = rpc.Client( - 'localhost').encode_multipart_formdata(None, files) - return len(body.read()) - - def test4(): - files = [{'filename': u'你好', 'data': u'你好', 'mime_type': ''}] - _, body = rpc.Client( - 'localhost').encode_multipart_formdata(None, files) - return len(body.read()) - - assert test1() == test2() - assert test2() == test3() - assert test3() == test4() - - -if __name__ == "__main__": - unittest.main() diff --git a/qiniu/test/rsf_test.py b/qiniu/test/rsf_test.py deleted file mode 100644 index 7b7573d0..00000000 --- a/qiniu/test/rsf_test.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -import unittest -from qiniu import rsf -from qiniu import conf - -import os -conf.ACCESS_KEY = os.getenv("QINIU_ACCESS_KEY") -conf.SECRET_KEY = os.getenv("QINIU_SECRET_KEY") -bucket_name = os.getenv("QINIU_TEST_BUCKET") - - -class TestRsf(unittest.TestCase): - - def test_list_prefix(self): - c = rsf.Client() - ret, err = c.list_prefix(bucket_name, limit=4) - self.assertEqual(err is rsf.EOF or err is None, True) - assert len(ret.get('items')) == 4 - - -if __name__ == "__main__": - unittest.main() diff --git a/setup.py b/setup.py deleted file mode 100644 index 179ef86c..00000000 --- a/setup.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -try: - import setuptools - setup = setuptools.setup -except ImportError: - setuptools = None - from distutils.core import setup - -PACKAGE = 'qiniu' -NAME = 'qiniu' -DESCRIPTION = 'Qiniu Resource Storage SDK for Python 2.X.' -LONG_DESCRIPTION = 'see:\nhttps://github.com/qiniu/python-sdk\n' -AUTHOR = 'Shanghai Qiniu Information Technologies Co., Ltd.' -AUTHOR_EMAIL = 'sdk@qiniu.com' -MAINTAINER_EMAIL = 'support@qiniu.com' -URL = 'https://github.com/qiniu/python-sdk' -VERSION = __import__(PACKAGE).__version__ - - -setup( - name=NAME, - version=VERSION, - description=DESCRIPTION, - long_description=LONG_DESCRIPTION, - author=AUTHOR, - author_email=AUTHOR_EMAIL, - maintainer_email=MAINTAINER_EMAIL, - license='MIT', - url=URL, - packages=['qiniu', 'qiniu.test', 'qiniu.auth', - 'qiniu.rs', 'qiniu.rs.test'], - platforms='any', - classifiers=[ - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', - 'Topic :: Software Development :: Libraries :: Python Modules' - ], - test_suite='nose.collector' -) diff --git a/test-env.sh b/test-env.sh deleted file mode 100644 index cb40c24b..00000000 --- a/test-env.sh +++ /dev/null @@ -1,4 +0,0 @@ -export QINIU_ACCESS_KEY="" -export QINIU_SECRET_KEY="" -export QINIU_TEST_BUCKET="" -export QINIU_TEST_DOMAIN="" From d1c96c2bc9cbd8ce2a0df39cfafd9bfb16611e64 Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 13 Nov 2014 18:01:32 +0800 Subject: [PATCH 070/478] 7.0 --- CHANGELOG.md | 11 + CONTRIBUTING.md | 30 +++ LICENSE | 22 ++ README.md | 69 +++++++ qiniu/__init__.py | 23 +++ qiniu/auth.py | 143 +++++++++++++ qiniu/compat.py | 77 +++++++ qiniu/config.py | 36 ++++ qiniu/http.py | 120 +++++++++++ qiniu/main.py | 28 +++ qiniu/services/__init__.py | 0 qiniu/services/processing/__init__.py | 0 qiniu/services/processing/cmd.py | 22 ++ qiniu/services/processing/pfop.py | 26 +++ qiniu/services/storage/__init__.py | 0 qiniu/services/storage/bucket.py | 138 +++++++++++++ qiniu/services/storage/uploader.py | 122 +++++++++++ qiniu/utils.py | 71 +++++++ setup.py | 45 +++++ test_qiniu.py | 279 ++++++++++++++++++++++++++ 20 files changed, 1262 insertions(+) create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 qiniu/__init__.py create mode 100644 qiniu/auth.py create mode 100644 qiniu/compat.py create mode 100644 qiniu/config.py create mode 100644 qiniu/http.py create mode 100755 qiniu/main.py create mode 100644 qiniu/services/__init__.py create mode 100644 qiniu/services/processing/__init__.py create mode 100644 qiniu/services/processing/cmd.py create mode 100644 qiniu/services/processing/pfop.py create mode 100644 qiniu/services/storage/__init__.py create mode 100644 qiniu/services/storage/bucket.py create mode 100644 qiniu/services/storage/uploader.py create mode 100644 qiniu/utils.py create mode 100644 setup.py create mode 100644 test_qiniu.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..041e8217 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +#Changelog + + +## 7.0.0 (2014-11-13) + +### 增加 +* 简化上传接口 +* 自动选择断点续上传还是直传 +* 重构代码，内部结构更清晰，便于更换不同的http实现 +* 同时支持python 2.x 和 3.x +* 增加pfop支持 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..a4b79902 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,30 @@ +# 贡献代码指南 + +我们非常欢迎大家来贡献代码，我们会向贡献者致以最诚挚的敬意。 + +一般可以通过在Github上提交[Pull Request](https://github.com/qiniu/python-sdk)来贡献代码。 + +## Pull Request要求 + +- **代码规范** 遵从pep8，pythonic。 + +- **代码格式** 提交前 请按 pep8 进行格式化。 + +- **必须添加测试！** - 如果没有测试（单元测试、集成测试都可以），那么提交的补丁是不会通过的。 + +- **记得更新文档** - 保证`README.md`以及其他相关文档及时更新，和代码的变更保持一致性。 + +- **考虑我们的发布周期** - 我们的版本号会服从[SemVer v2.0.0](http://semver.org/)，我们绝对不会随意变更对外的API。 + +- **创建feature分支** - 最好不要从你的master分支提交 pull request。 + +- **一个feature提交一个pull请求** - 如果你的代码变更了多个操作，那就提交多个pull请求吧。 + +- **清晰的commit历史** - 保证你的pull请求的每次commit操作都是有意义的。如果你开发中需要执行多次的即时commit操作，那么请把它们放到一起再提交pull请求。 + +## 运行测试 + +``` bash +py.test + +``` diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..ba646be9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2014 Qiniu, Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/README.md b/README.md new file mode 100644 index 00000000..a16de536 --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# Qiniu Python SDK +[![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE.md) +[![Build Status](https://travis-ci.org/qiniu/python-sdk.svg)](https://travis-ci.org/qiniu/python-sdk) +[![Latest Stable Version](https://badge.fury.io/co/Qiniu.png)](https://github.com/qiniu/python-sdk/releases) + +## 安装 + +通过pip + +```bash +pip install qiniu +``` + +## 运行环境 + +| Qiniu SDK版本 | Python 版本 | +|:--------------------:|:---------------------------:| +| 7.x | 2.6, 2.7, 3.3, 3.4 | +| 6.x | 2.6, 2.7 | + +## 使用方法 + +```python +import qiniu +... + q = qiniu.Auth(access_key, secret_key) + key = 'hello' + data = 'hello qiniu!' + token = q.upload_token(bucket_name) + ret, info = put_data(token, key, data) + if ret is not None: + print('All is OK') + else: + print(info) # error message in info +... +``` + +## 测试 + +``` bash +py.test +``` + +## 常见问题 + +- 第二个参数info保留了请求响应的信息，失败情况下ret 为none, 将info可以打印出来，提交给我们。 +- API 的使用 demo 可以参考 [单元测试](https://github.com/qiniu/python-sdk/blob/master/test_qiniu.py)。 + +## 代码贡献 + +详情参考[代码提交指南](https://github.com/qiniu/python-sdk/blob/master/CONTRIBUTING.md)。 + +## 贡献记录 + +- [所有贡献者](https://github.com/qiniu/python-sdk/contributors) + +## 联系我们 + +- 如果需要帮助，请提交工单（在portal右侧点击咨询和建议提交工单，或者直接向 support@qiniu.com 发送邮件） +- 如果有什么问题，可以到问答社区提问，[问答社区](http://qiniu.segmentfault.com/) +- 更详细的文档，见[官方文档站](http://developer.qiniu.com/) +- 如果发现了bug， 欢迎提交 [issue](https://github.com/qiniu/python-sdk/issues) +- 如果有功能需求，欢迎提交 [issue](https://github.com/qiniu/python-sdk/issues) +- 如果要提交代码，欢迎提交 pull request +- 欢迎关注我们的[微信](http://www.qiniu.com/#weixin) [微博](http://weibo.com/qiniutek)，及时获取动态信息。 + +## 代码许可 + +The MIT License (MIT).详情见 [License文件](https://github.com/qiniu/python-sdk/blob/master/LICENSE). diff --git a/qiniu/__init__.py b/qiniu/__init__.py new file mode 100644 index 00000000..1c09a096 --- /dev/null +++ b/qiniu/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +''' +Qiniu Resource Storage SDK for Python +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For detailed document, please see: + +''' + +# flake8: noqa + +__version__ = '7.0.0' + +from .auth import Auth + +from .config import set_default + +from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, build_batch_delete +from .services.storage.uploader import put_data, put_file, put_stream +from .services.processing.pfop import PersistentFop +from .services.processing.cmd import build_op, pipe_cmd, op_save + +from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry diff --git a/qiniu/auth.py b/qiniu/auth.py new file mode 100644 index 00000000..769497c3 --- /dev/null +++ b/qiniu/auth.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- + +import hmac +import time +from hashlib import sha1 + +from requests.auth import AuthBase + +from .compat import urlparse, json, b +from .utils import urlsafe_base64_encode + + +_policy_fields = set([ + 'callbackUrl', + 'callbackBody', + 'callbackHost', + + 'returnUrl', + 'returnBody', + + 'endUser', + 'saveKey', + 'insertOnly', + + 'detectMime', + 'mimeLimit', + 'fsizeLimit', + + 'persistentOps', + 'persistentNotifyUrl', + 'persistentPipeline', +]) + +_deprecated_policy_fields = set([ + 'asyncOps' +]) + + +class Auth(object): + + def __init__(self, access_key, secret_key): + self.__checkKey(access_key, secret_key) + self.__access_key, self.__secret_key = access_key, secret_key + self.__secret_key = b(self.__secret_key) + + def __token(self, data): + data = b(data) + hashed = hmac.new(self.__secret_key, data, sha1) + return urlsafe_base64_encode(hashed.digest()) + + def token(self, data): + return '{0}:{1}'.format(self.__access_key, self.__token(data)) + + def token_with_data(self, data): + data = urlsafe_base64_encode(data) + return '{0}:{1}:{2}'.format(self.__access_key, self.__token(data), data) + + def token_of_request(self, url, body=None, content_type=None): + parsed_url = urlparse(url) + query = parsed_url.query + path = parsed_url.path + data = path + if query != '': + data = ''.join([data, '?', query]) + data = ''.join([data, "\n"]) + + if body: + mimes = [ + 'application/x-www-form-urlencoded', + ] + if content_type in mimes: + data += body + + return '{0}:{1}'.format(self.__access_key, self.__token(data)) + + @staticmethod + def __checkKey(access_key, secret_key): + if not (access_key and secret_key): + raise ValueError('invalid key') + + def private_download_url(self, url, expires=3600): + ''' + * return private url + ''' + + deadline = int(time.time()) + expires + if '?' in url: + url += '&' + else: + url += '?' + url = '{0}e={1}'.format(url, str(deadline)) + + token = self.token(url) + return '{0}&token={1}'.format(url, token) + + def upload_token(self, bucket, key=None, expires=3600, policy=None, strict_policy=True): + if bucket is None or bucket == '': + raise ValueError('invalid bucket name') + + scope = bucket + if key is not None: + scope = '{0}:{1}'.format(bucket, key) + + args = dict( + scope=scope, + deadline=int(time.time()) + expires, + ) + + if policy is not None: + self.__copy_policy(policy, args, strict_policy) + + return self.__upload_token(args) + + def __upload_token(self, policy): + data = json.dumps(policy, separators=(',', ':')) + return self.token_with_data(data) + + def verify_callback(self, origin_authorization, url, body): + token = self.token_of_request(url, body, 'application/x-www-form-urlencoded') + authorization = 'QBox {0}'.format(token) + return origin_authorization == authorization + + @staticmethod + def __copy_policy(policy, to, strict_policy): + for k, v in policy.items(): + if k in _deprecated_policy_fields: + raise ValueError(k + ' has deprecated') + if (not strict_policy) or k in _policy_fields: + to[k] = v + + +class RequestsAuth(AuthBase): + def __init__(self, auth): + self.auth = auth + + def __call__(self, r): + token = None + if r.body is not None and r.headers['Content-Type'] == 'application/x-www-form-urlencoded': + token = self.auth.token_of_request(r.url, r.body, 'application/x-www-form-urlencoded') + else: + token = self.auth.token_of_request(r.url) + r.headers['Authorization'] = 'QBox {0}'.format(token) + return r diff --git a/qiniu/compat.py b/qiniu/compat.py new file mode 100644 index 00000000..448d87b7 --- /dev/null +++ b/qiniu/compat.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +""" +pythoncompat +""" + +import sys + +try: + import simplejson as json +except (ImportError, SyntaxError): + # simplejson does not support Python 3.2, it thows a SyntaxError + # because of u'...' Unicode literals. + import json # noqa + + +# ------- +# Pythons +# ------- + +_ver = sys.version_info + +#: Python 2.x? +is_py2 = (_ver[0] == 2) + +#: Python 3.x? +is_py3 = (_ver[0] == 3) + + +# --------- +# Specifics +# --------- + +if is_py2: + from urlparse import urlparse # noqa + import StringIO + StringIO = BytesIO = StringIO.StringIO + + builtin_str = str + bytes = str + str = unicode # noqa + basestring = basestring # noqa + numeric_types = (int, long, float) # noqa + + def b(s): + return s + + def s(b): + return b + + def u(s): + return unicode(s, 'unicode_escape') # noqa + +elif is_py3: + from urllib.parse import urlparse # noqa + import io + StringIO = io.StringIO + BytesIO = io.BytesIO + + builtin_str = str + str = str + bytes = bytes + basestring = (str, bytes) + numeric_types = (int, float) + + def b(s): + if isinstance(s, str): + return s.encode('utf-8') + return s + + def s(b): + if isinstance(b, bytes): + b = b.decode('utf-8') + return b + + def u(s): + return s diff --git a/qiniu/config.py b/qiniu/config.py new file mode 100644 index 00000000..323a5b6f --- /dev/null +++ b/qiniu/config.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +RS_HOST = 'rs.qbox.me' +IO_HOST = 'iovip.qbox.me' +RSF_HOST = 'rsf.qbox.me' +API_HOST = 'api.qiniu.com' + +UPAUTO_HOST = 'up.qiniu.com' +UPDX_HOST = 'updx.qiniu.com' +UPLT_HOST = 'uplt.qiniu.com' +UPBACKUP_HOST = 'upload.qiniu.com' + +_config = { + 'default_up_host': UPAUTO_HOST, + 'connection_timeout': 30, + 'connection_retries': 3, + 'connection_pool': 10, + +} +_BLOCK_SIZE = 1024 * 1024 * 4 + + +def get_default(key): + return _config[key] + + +def set_default( + default_up_host=None, connection_retries=None, connection_pool=None, connection_timeout=None): + if default_up_host: + _config['default_up_host'] = default_up_host + if connection_retries: + _config['connection_retries'] = connection_retries + if connection_pool: + _config['connection_pool'] = connection_pool + if connection_timeout: + _config['connection_timeout'] = connection_timeout diff --git a/qiniu/http.py b/qiniu/http.py new file mode 100644 index 00000000..19050339 --- /dev/null +++ b/qiniu/http.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +import platform + +import requests +from requests.auth import AuthBase + +from qiniu import config +from .auth import RequestsAuth +from . import __version__ + + +_sys_info = '{0}; {1}'.format(platform.system(), platform.machine()) +_python_ver = platform.python_version() + +USER_AGENT = 'QiniuPython/{0} ({1}; ) Python/{2}'.format(__version__, _sys_info, _python_ver) + +_session = None +_headers = {'User-Agent': USER_AGENT} + + +def __return_wrapper(resp): + if resp.status_code != 200: + return None, ResponseInfo(resp) + ret = resp.json() if resp.text != '' else {} + return ret, ResponseInfo(resp) + + +def _init(): + session = requests.Session() + adapter = requests.adapters.HTTPAdapter( + pool_connections=config.get_default('connection_pool'), pool_maxsize=config.get_default('connection_pool'), + max_retries=config.get_default('connection_retries')) + session.mount('http://', adapter) + global _session + _session = session + + +def _post(url, data, files, auth): + if _session is None: + _init() + try: + r = _session.post( + url, data=data, files=files, auth=auth, headers=_headers, timeout=config.get_default('connection_timeout')) + except Exception as e: + return None, ResponseInfo(None, e) + return __return_wrapper(r) + + +def _get(url, params, auth): + try: + r = requests.get( + url, params=params, auth=RequestsAuth(auth), + timeout=config.get_default('connection_timeout'), headers=_headers) + except Exception as e: + return None, ResponseInfo(None, e) + return __return_wrapper(r) + + +class _TokenAuth(AuthBase): + def __init__(self, token): + self.token = token + + def __call__(self, r): + r.headers['Authorization'] = 'UpToken {0}'.format(self.token) + return r + + +def _post_with_token(url, data, token): + return _post(url, data, None, _TokenAuth(token)) + + +def _post_file(url, data, files): + return _post(url, data, files, None) + + +def _post_with_auth(url, data, auth): + return _post(url, data, None, RequestsAuth(auth)) + + +class ResponseInfo(object): + def __init__(self, response, exception=None): + self.__response = response + self.exception = exception + if response is None: + self.status_code = -1 + self.text_body = None + self.req_id = None + self.x_log = None + self.error = str(exception) + else: + self.status_code = response.status_code + self.text_body = response.text + self.req_id = response.headers['X-Reqid'] + self.x_log = response.headers['X-Log'] + if self.status_code >= 400: + ret = response.json() if response.text != '' else None + if ret is None or ret['error'] is None: + self.error = 'unknown' + else: + self.error = ret['error'] + + def ok(self): + self.status_code == 200 + + def need_retry(self): + if self.__response is None: + return True + code = self.status_code + if (code // 100 == 5 and code != 579) or code == 996: + return True + return False + + def connect_failed(self): + return self.__response is None + + def __str__(self): + return ', '.join(['%s:%s' % item for item in self.__dict__.items()]) + + def __repr__(self): + return self.__str__() diff --git a/qiniu/main.py b/qiniu/main.py new file mode 100755 index 00000000..10562fb6 --- /dev/null +++ b/qiniu/main.py @@ -0,0 +1,28 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse + +from qiniu import etag + + +def main(): + parser = argparse.ArgumentParser(prog='qiniu') + sub_parsers = parser.add_subparsers() + + parser_etag = sub_parsers.add_parser( + 'etag', description='calculate the etag of the file', help='etag [file...]') + parser_etag.add_argument( + 'etag_files', metavar='N', nargs='+', help='the file list for calculate') + + args = parser.parse_args() + + if args.etag_files: + r = [etag(file) for file in args.etag_files] + if len(r) == 1: + print(r[0]) + else: + print(' '.join(r)) + +if __name__ == '__main__': + main() diff --git a/qiniu/services/__init__.py b/qiniu/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qiniu/services/processing/__init__.py b/qiniu/services/processing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qiniu/services/processing/cmd.py b/qiniu/services/processing/cmd.py new file mode 100644 index 00000000..6feaba74 --- /dev/null +++ b/qiniu/services/processing/cmd.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +from qiniu.utils import entry + + +def build_op(cmd, first_arg, **kwargs): + op = [cmd] + if first_arg is not None: + op.append(first_arg) + + for k, v in kwargs.items(): + op.append('{0}/{1}'.format(k, v)) + + return '/'.join(op) + + +def pipe_cmd(*cmds): + return '|'.join(cmds) + + +def op_save(op, bucket, key): + return pipe_cmd(op, 'saveas/' + entry(bucket, key)) diff --git a/qiniu/services/processing/pfop.py b/qiniu/services/processing/pfop.py new file mode 100644 index 00000000..e5429b4d --- /dev/null +++ b/qiniu/services/processing/pfop.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +from qiniu import config +from qiniu import http + + +class PersistentFop(object): + + def __init__(self, auth, bucket, pipeline=None, notify_url=None): + self.auth = auth + self.bucket = bucket + self.pipeline = pipeline + self.notify_url = notify_url + + def execute(self, key, fops, force=None): + ops = ';'.join(fops) + data = {'bucket': self.bucket, 'key': key, 'fops': ops} + if self.pipeline: + data['pipeline'] = self.pipeline + if self.notify_url: + data['notifyURL'] = self.notify_url + if force == 1: + data['force'] = 1 + + url = 'http://{0}/pfop'.format(config.API_HOST) + return http._post_with_auth(url, data, self.auth) diff --git a/qiniu/services/storage/__init__.py b/qiniu/services/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py new file mode 100644 index 00000000..5fe847de --- /dev/null +++ b/qiniu/services/storage/bucket.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- + +from qiniu import config +from qiniu.utils import urlsafe_base64_encode, entry +from qiniu import http + + +class BucketManager(object): + + def __init__(self, auth): + self.auth = auth + + def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): + """前缀查询: + * bucket => str + * prefix => str + * marker => str + * limit => int + * delimiter => str + * return ret => {'items': items, 'marker': markerOut}, err => str + + 1. 首次请求 marker = None + 2. 无论 err 值如何，均应该先看 ret.get('items') 是否有内容 + 3. 如果后续没有更多数据，err 返回 EOF，markerOut 返回 None（但不通过该特征来判断是否结束） + """ + options = { + 'bucket': bucket, + } + if marker is not None: + options['marker'] = marker + if limit is not None: + options['limit'] = limit + if prefix is not None: + options['prefix'] = prefix + if delimiter is not None: + options['delimiter'] = delimiter + + url = 'http://{0}/list'.format(config.RSF_HOST) + ret, info = self.__get(url, options) + + eof = False + if ret and not ret.get('marker'): + eof = True + + return ret, eof, info + + def stat(self, bucket, key): + resource = entry(bucket, key) + return self.__rs_do('stat', resource) + + def delete(self, bucket, key): + resource = entry(bucket, key) + return self.__rs_do('delete', resource) + + def rename(self, bucket, key, key_to): + return self.move(bucket, key, bucket, key_to) + + def move(self, bucket, key, bucket_to, key_to): + resource = entry(bucket, key) + to = entry(bucket_to, key_to) + return self.__rs_do('move', resource, to) + + def copy(self, bucket, key, bucket_to, key_to): + resource = entry(bucket, key) + to = entry(bucket_to, key_to) + return self.__rs_do('copy', resource, to) + + def fetch(self, url, bucket, key): + resource = urlsafe_base64_encode(url) + to = entry(bucket, key) + return self.__io_do('fetch', resource, 'to/{0}'.format(to)) + + def prefetch(self, bucket, key): + resource = entry(bucket, key) + return self.__io_do('prefetch', resource) + + def change_mime(self, bucket, key, mime): + resource = entry(bucket, key) + encode_mime = urlsafe_base64_encode(mime) + return self.__rs_do('chgm', resource, 'mime/{0}'.format(encode_mime)) + + def batch(self, operations): + url = 'http://{0}/batch'.format(config.RS_HOST) + return self.__post(url, dict(op=operations)) + + def buckets(self): + return self.__rs_do('buckets') + + def __rs_do(self, operation, *args): + return self.__server_do(config.RS_HOST, operation, *args) + + def __io_do(self, operation, *args): + return self.__server_do(config.IO_HOST, operation, *args) + + def __server_do(self, host, operation, *args): + cmd = _build_op(operation, *args) + url = 'http://{0}/{1}'.format(host, cmd) + return self.__post(url) + + def __post(self, url, data=None): + return http._post_with_auth(url, data, self.auth) + + def __get(self, url, params=None): + return http._get(url, params, self.auth) + + +def _build_op(*args): + return '/'.join(args) + + +def build_batch_copy(source_bucket, key_pairs, target_bucket): + return _two_key_batch('copy', source_bucket, key_pairs, target_bucket) + + +def build_batch_rename(bucket, key_pairs): + return build_batch_move(bucket, key_pairs, bucket) + + +def build_batch_move(source_bucket, key_pairs, target_bucket): + return _two_key_batch('move', source_bucket, key_pairs, target_bucket) + + +def build_batch_delete(bucket, keys): + return _one_key_batch('delete', bucket, keys) + + +def build_batch_stat(bucket, keys): + return _one_key_batch('stat', bucket, keys) + + +def _one_key_batch(operation, bucket, keys): + return [_build_op(operation, entry(bucket, key)) for key in keys] + + +def _two_key_batch(operation, source_bucket, key_pairs, target_bucket): + if target_bucket is None: + target_bucket = source_bucket + return [_build_op(operation, entry(source_bucket, k), entry(target_bucket, v)) for k, v in key_pairs.items()] diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py new file mode 100644 index 00000000..54d1d9a5 --- /dev/null +++ b/qiniu/services/storage/uploader.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- + +import os + +from qiniu import config +from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter +from qiniu import http + + +def put_data( + up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None): + ''' put data to Qiniu + If key is None, the server will generate one. + data may be str or read()able object. + ''' + crc = crc32(data) if check_crc else None + return _form_put(up_token, key, data, params, mime_type, crc, False, progress_handler) + + +def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None): + ret = {} + size = os.stat(file_path).st_size + with open(file_path, 'rb') as input_stream: + if size > config._BLOCK_SIZE * 2: + ret, info = put_stream(up_token, key, input_stream, size, params, mime_type, progress_handler) + else: + crc = file_crc32(file_path) if check_crc else None + ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, True, progress_handler) + return ret, info + + +def _form_put(up_token, key, data, params, mime_type, crc32, is_file=False, progress_handler=None): + fields = {} + if params: + for k, v in params.items(): + fields[k] = str(v) + if crc32: + fields['crc32'] = crc32 + if key is not None: + fields['key'] = key + fields['token'] = up_token + url = 'http://' + config.get_default('default_up_host') + '/' + name = key if key else 'filename' + + r, info = http._post_file(url, data=fields, files={'file': (name, data, mime_type)}) + if r is None and info.need_retry(): + if info.connect_failed: + url = 'http://' + config.UPBACKUP_HOST + '/' + if is_file: + data.seek(0) + r, info = http._post_file(url, data=fields, files={'file': (name, data, mime_type)}) + + return r, info + + +def put_stream(up_token, key, input_stream, data_size, params=None, mime_type=None, progress_handler=None): + task = _Resume(up_token, key, input_stream, data_size, params, mime_type, progress_handler) + return task.upload() + + +class _Resume(object): + + def __init__(self, up_token, key, input_stream, data_size, params, mime_type, progress_handler): + self.up_token = up_token + self.key = key + self.input_stream = input_stream + self.size = data_size + self.params = params + self.mime_type = mime_type + self.progress_handler = progress_handler + + def upload(self): + self.blockStatus = [] + host = config.get_default('default_up_host') + for block in _file_iter(self.input_stream, config._BLOCK_SIZE): + length = len(block) + crc = crc32(block) + ret, info = self.make_block(block, length, host) + if ret is None and not info.need_retry: + return ret, info + if info.connect_failed: + host = config.UPBACKUP_HOST + if info.need_retry or crc != ret['crc32']: + ret, info = self.make_block(block, length, host) + if ret is None or crc != ret['crc32']: + return ret, info + + self.blockStatus.append(ret) + if(callable(self.progress_handler)): + self.progress_handler(((len(self.blockStatus) - 1) * config._BLOCK_SIZE)+length, self.size) + return self.make_file(host) + + def make_block(self, block, block_size, host): + url = self.block_url(host, block_size) + return self.post(url, block) + + def block_url(self, host, size): + return 'http://{0}/mkblk/{1}'.format(host, size) + + def file_url(self, host): + url = ['http://{0}/mkfile/{1}'.format(host, self.size)] + + if self.mime_type: + url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type))) + + if self.key is not None: + url.append('key/{0}'.format(urlsafe_base64_encode(self.key))) + + if self.params: + for k, v in self.params.items(): + url.append('{0}/{1}'.format(k, urlsafe_base64_encode(v))) + + url = '/'.join(url) + return url + + def make_file(self, host): + url = self.file_url(host) + body = ','.join([status['ctx'] for status in self.blockStatus]) + return self.post(url, body) + + def post(self, url, data): + return http._post_with_token(url, data, self.up_token) diff --git a/qiniu/utils.py b/qiniu/utils.py new file mode 100644 index 00000000..1fb1351d --- /dev/null +++ b/qiniu/utils.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +from hashlib import sha1 +from base64 import urlsafe_b64encode, urlsafe_b64decode + +from .config import _BLOCK_SIZE + +from .compat import b, s + +try: + import zlib + binascii = zlib +except ImportError: + zlib = None + import binascii + + +def urlsafe_base64_encode(data): + ret = urlsafe_b64encode(b(data)) + return s(ret) + + +def urlsafe_base64_decode(data): + ret = urlsafe_b64decode(s(data)) + return ret + + +def file_crc32(filePath): + crc = 0 + with open(filePath, 'rb') as f: + for block in _file_iter(f, _BLOCK_SIZE): + crc = binascii.crc32(block, crc) & 0xFFFFFFFF + return crc + + +def crc32(data): + return binascii.crc32(b(data)) & 0xffffffff + + +def _file_iter(input_stream, size): + d = input_stream.read(size) + while d: + yield d + d = input_stream.read(size) + + +def _sha1(data): + h = sha1() + h.update(data) + return h.digest() + + +def _etag(input_stream): + array = [_sha1(block) for block in _file_iter(input_stream, _BLOCK_SIZE)] + if len(array) == 1: + data = array[0] + prefix = b('\x16') + else: + s = b('').join(array) + data = _sha1(s) + prefix = b('\x96') + return urlsafe_base64_encode(prefix + data) + + +def etag(filePath): + with open(filePath, 'rb') as f: + return _etag(f) + + +def entry(bucket, key): + return urlsafe_base64_encode('{0}:{1}'.format(bucket, key)) diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..3763dcc7 --- /dev/null +++ b/setup.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +try: + import setuptools + setup = setuptools.setup +except ImportError: + setuptools = None + from distutils.core import setup + + +setup( + name='qiniu', + version=__import__('qiniu').__version__, + description='Qiniu Resource Storage SDK', + long_description='see:\nhttps://github.com/qiniu/python-sdk\n', + author='Shanghai Qiniu Information Technologies Co., Ltd.', + author_email='sdk@qiniu.com', + maintainer_email='support@qiniu.com', + license='MIT', + url='https://github.com/qiniu/python-sdk', + packages=['qiniu', 'qiniu.services', 'qiniu.services.storage', 'qiniu.services.processing'], + platforms='any', + classifiers=[ + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', + 'Topic :: Software Development :: Libraries :: Python Modules' + ], + install_requires=['requests'], + + entry_points={ + 'console_scripts': [ + 'qiniupy = qiniu.main:main', + ], + } +) diff --git a/test_qiniu.py b/test_qiniu.py new file mode 100644 index 00000000..84f39dca --- /dev/null +++ b/test_qiniu.py @@ -0,0 +1,279 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import os +import string +import random +import tempfile + +import unittest +import pytest + +from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save +from qiniu import put_data, put_file, put_stream +from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, build_batch_delete +from qiniu import urlsafe_base64_encode, urlsafe_base64_decode + +from qiniu.compat import is_py2, b + +from qiniu.services.storage.uploader import _form_put + +import qiniu.config + +if is_py2: + import sys + reload(sys) + sys.setdefaultencoding('utf-8') + +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') +bucket_name = os.getenv('QINIU_TEST_BUCKET') + +dummy_access_key = 'abcdefghklmnopq' +dummy_secret_key = '1234567890' +dummy_auth = Auth(dummy_access_key, dummy_secret_key) + + +def rand_string(length): + lib = string.ascii_uppercase + return ''.join([random.choice(lib) for i in range(0, length)]) + + +def create_temp_file(size): + t = tempfile.mktemp() + f = open(t, 'wb') + f.seek(size-1) + f.write(b('0')) + f.close() + return t + + +def remove_temp_file(file): + try: + os.remove(file) + except OSError: + pass + + +class UtilsTest(unittest.TestCase): + + def test_urlsafe(self): + a = '你好\x96' + u = urlsafe_base64_encode(a) + assert b(a) == urlsafe_base64_decode(u) + + +class AuthTestCase(unittest.TestCase): + + def test_token(self): + token = dummy_auth.token('test') + assert token == 'abcdefghklmnopq:mSNBTR7uS2crJsyFr2Amwv1LaYg=' + + def test_token_with_data(self): + token = dummy_auth.token_with_data('test') + assert token == 'abcdefghklmnopq:-jP8eEV9v48MkYiBGs81aDxl60E=:dGVzdA==' + + def test_noKey(self): + with pytest.raises(ValueError): + Auth(None, None).token('nokey') + with pytest.raises(ValueError): + Auth('', '').token('nokey') + + def test_token_of_request(self): + token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', '') + assert token == 'abcdefghklmnopq:cFyRVoWrE3IugPIMP5YJFTO-O-Y=' + token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', 'application/x-www-form-urlencoded') + assert token == 'abcdefghklmnopq:svWRNcacOE-YMsc70nuIYdaa1e4=' + + def test_deprecatedPolicy(self): + with pytest.raises(ValueError): + dummy_auth.upload_token('1', None, policy={'asyncOps': 1}) + + +class BucketTestCase(unittest.TestCase): + q = Auth(access_key, secret_key) + bucket = BucketManager(q) + + def test_list(self): + ret, eof, _ = self.bucket.list(bucket_name, limit=4) + assert eof is False + assert len(ret.get('items')) == 4 + ret, eof, _ = self.bucket.list(bucket_name, limit=100) + assert eof is True + + def test_buckets(self): + ret, _ = self.bucket.buckets() + assert bucket_name in ret + + def test_pefetch(self): + ret, _ = self.bucket.prefetch(bucket_name, 'python-sdk.html') + assert ret == {} + + def test_fetch(self): + ret, _ = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, 'fetch.html') + assert ret == {} + + def test_stat(self): + ret, _ = self.bucket.stat(bucket_name, 'python-sdk.html') + assert 'hash' in ret + + def test_delete(self): + ret, info = self.bucket.delete(bucket_name, 'del') + + def test_rename(self): + key = 'renameto'+rand_string(8) + self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + key2 = key + 'move' + ret, _ = self.bucket.rename(bucket_name, key, key2) + assert ret == {} + ret, _ = self.bucket.delete(bucket_name, key2) + assert ret == {} + + def test_copy(self): + key = 'copyto'+rand_string(8) + ret, _ = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + + assert ret == {} + ret, _ = self.bucket.delete(bucket_name, key) + assert ret == {} + + def test_batch_copy(self): + key = 'copyto'+rand_string(8) + ops = build_batch_copy(bucket_name, {'copyfrom': key}, bucket_name) + ret, _ = self.bucket.batch(ops) + assert ret[0]['code'] == 200 + ops = build_batch_delete(bucket_name, [key]) + ret, _ = self.bucket.batch(ops) + assert ret[0]['code'] == 200 + + def test_batch_move(self): + key = 'moveto'+rand_string(8) + self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + key2 = key2 = key + 'move' + ops = build_batch_move(bucket_name, {key: key2}, bucket_name) + ret, _ = self.bucket.batch(ops) + assert ret[0]['code'] == 200 + ret, _ = self.bucket.delete(bucket_name, key2) + assert ret == {} + + def test_batch_rename(self): + key = 'rename'+rand_string(8) + self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + key2 = key2 = key + 'rename' + ops = build_batch_move(bucket_name, {key: key2}, bucket_name) + ret, _ = self.bucket.batch(ops) + assert ret[0]['code'] == 200 + ret, _ = self.bucket.delete(bucket_name, key2) + assert ret == {} + + def test_batch_stat(self): + ops = build_batch_stat(bucket_name, ['python-sdk.html']) + ret, _ = self.bucket.batch(ops) + assert ret[0]['code'] == 200 + + +class UploaderTestCase(unittest.TestCase): + + mime_type = "text/plain" + params = {'x:a': 'a'} + q = Auth(access_key, secret_key) + + def test_put(self): + key = 'a\\b\\c"你好' + data = 'hello bubby!' + token = self.q.upload_token(bucket_name) + ret, info = put_data(token, key, data) + assert ret['key'] == key + + key = '' + data = 'hello bubby!' + token = self.q.upload_token(bucket_name, key) + ret, _ = put_data(token, key, data, check_crc=True) + assert ret['key'] == key + + def test_putfile(self): + localfile = __file__ + key = 'test_file' + + token = self.q.upload_token(bucket_name, key) + ret, _ = put_file(token, key, localfile, mime_type=self.mime_type, check_crc=True) + assert ret['key'] == key + assert ret['hash'] == etag(localfile) + + def test_putInvalidCrc(self): + key = 'test_invalid' + data = 'hello bubby!' + crc32 = 'wrong crc32' + token = self.q.upload_token(bucket_name) + ret, info = _form_put(token, key, data, None, None, crc32=crc32) + assert ret is None + assert info.status_code == 400 + + def test_putWithoutKey(self): + key = None + data = 'hello bubby!' + token = self.q.upload_token(bucket_name) + ret, _ = put_data(token, key, data) + assert ret['hash'] == ret['key'] + + data = 'hello bubby!' + token = self.q.upload_token(bucket_name, 'nokey2') + + ret, info = put_data(token, None, data) + + def test_retry(self): + key = 'retry' + data = 'hello retry!' + set_default(default_up_host='a') + token = self.q.upload_token(bucket_name) + ret, _ = put_data(token, key, data) + assert ret['key'] == key + qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + + +class ResumableUploaderTestCase(unittest.TestCase): + + mime_type = "text/plain" + params = {'x:a': 'a'} + q = Auth(access_key, secret_key) + + def test_putfile(self): + localfile = __file__ + key = 'test_file_r' + size = os.stat(localfile).st_size + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, size, self.params, self.mime_type) + print(info) + assert ret['key'] == key + + def test_big_file(self): + key = 'big' + token = self.q.upload_token(bucket_name, key) + localfile = create_temp_file(4 * 1024 * 1024 + 1) + progress_handler = lambda progress, total: progress + qiniu.set_default(default_up_host='a') + ret, info = put_file(token, key, localfile, self.params, self.mime_type, progress_handler=progress_handler) + assert ret['key'] == key + qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + remove_temp_file(localfile) + + def test_retry(self): + localfile = __file__ + key = 'test_file_r_retry' + qiniu.set_default(default_up_host='a') + token = self.q.upload_token(bucket_name, key) + ret, info = put_file(token, key, localfile, self.params, self.mime_type) + assert ret['key'] == key + qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + + +class MediaTestCase(unittest.TestCase): + def test_pfop(self): + q = Auth(access_key, secret_key) + pfop = PersistentFop(q, 'testres', 'sdktest') + op = op_save('avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', 'pythonsdk', 'pfoptest') + ret, info = pfop.execute('sintel_trailer.mp4', op, 1) + assert ret['persistentId'] is not None + +if __name__ == '__main__': + unittest.main() From d9dd06aa6b964439a117d7b57756e06f322c6ba8 Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 13 Nov 2014 18:04:33 +0800 Subject: [PATCH 071/478] travis --- .travis.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3e8d4cee..6ef3fd16 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,20 +2,19 @@ language: python python: - "2.6" - "2.7" + - "3.4" install: - - "pip install coverage --use-mirrors" - - "pip install pep8 --use-mirrors" - - "pip install pyflakes --use-mirrors" + - "pip install flake8 --use-mirrors" + - "pip install pytest --use-mirrors" + - "pip install pytest-cov --use-mirrors" + - "pip install requests --use-mirrors" before_script: - - "pep8 --max-line-length=160 ." - - "pyflakes ." - - export QINIU_ACCESS_KEY="X0XpjFmLMTJpHB_ESHjeolCtipk-1U3Ok7LVTdoN" - - export QINIU_SECRET_KEY="wenlwkU1AYwNBf7Q9cCoG4VT_GYyrHE9AS_R2u81" - - export QINIU_TEST_BUCKET="pysdk" - - export QINIU_TEST_DOMAIN="pysdk.qiniudn.com" + - "flake8 --show-source --show-pep8 --max-line-length=160 ." + - export QINIU_ACCESS_KEY="QWYn5TFQsLLU1pL5MFEmX3s5DmHdUThav9WyOWOm" + - export QINIU_SECRET_KEY="Bxckh6FA-Fbs9Yt3i3cbKVK22UPBmAOHJcL95pGz" + - export QINIU_TEST_BUCKET="pythonsdk" + - export QINIU_TEST_DOMAIN="pythonsdk.qiniudn.com" - export QINIU_TEST_ENV="travis" - export PYTHONPATH="$PYTHONPATH:." script: - - python setup.py nosetests - - python docs/gist/demo.py - - python docs/gist/conf.py + - py.test From f3a12e73d401121a44f926e421832487a01f9483 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Thu, 13 Nov 2014 21:09:44 +0800 Subject: [PATCH 072/478] [ci skip] badge --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a16de536..31dea771 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # Qiniu Python SDK [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE.md) [![Build Status](https://travis-ci.org/qiniu/python-sdk.svg)](https://travis-ci.org/qiniu/python-sdk) -[![Latest Stable Version](https://badge.fury.io/co/Qiniu.png)](https://github.com/qiniu/python-sdk/releases) +[![Latest Stable Version](https://img.shields.io/pypi/v/qiniu.svg)](https://pypi.python.org/pypi/qiniu) +[![Download Times](https://img.shields.io/pypi/dm/qiniu.svg)](https://pypi.python.org/pypi/qiniu) ## 安装 From 48de032fda984496727d123c49fd448e44757bea Mon Sep 17 00:00:00 2001 From: Bai Long Date: Thu, 13 Nov 2014 21:26:50 +0800 Subject: [PATCH 073/478] [ci skip] --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 31dea771..1854c0c9 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ 通过pip ```bash -pip install qiniu +$ pip install qiniu ``` ## 运行环境 @@ -21,6 +21,7 @@ pip install qiniu ## 使用方法 +### 上传 ```python import qiniu ... @@ -36,10 +37,16 @@ import qiniu ... ``` +### 命令行工具 +安装完后附带有命令行工具，可以计算etag +```bash +$ qiniupy etag yourfile +``` + ## 测试 ``` bash -py.test +$ py.test ``` ## 常见问题 From 8952ca45f5279fdf83f1ae9c8d293dce0400b0ba Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 13 Nov 2014 21:38:01 +0800 Subject: [PATCH 074/478] [ci skip] scrutinizer --- .scrutinizer.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .scrutinizer.yml diff --git a/.scrutinizer.yml b/.scrutinizer.yml new file mode 100644 index 00000000..135c31d8 --- /dev/null +++ b/.scrutinizer.yml @@ -0,0 +1,3 @@ +tools: + external_code_coverage: + timeout: 12000 # Timeout in seconds. From 313568e809607a878d590b5c998adf177f652f7f Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 13 Nov 2014 21:54:22 +0800 Subject: [PATCH 075/478] scrutinizer --- .travis.yml | 2 ++ test_qiniu.py | 76 +++++++++++++++++++++++++++++++++++---------------- 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ef3fd16..873cf9b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ install: - "pip install pytest --use-mirrors" - "pip install pytest-cov --use-mirrors" - "pip install requests --use-mirrors" + - "pip install scrutinizer-ocular --use-mirrors" before_script: - "flake8 --show-source --show-pep8 --max-line-length=160 ." - export QINIU_ACCESS_KEY="QWYn5TFQsLLU1pL5MFEmX3s5DmHdUThav9WyOWOm" @@ -18,3 +19,4 @@ before_script: - export PYTHONPATH="$PYTHONPATH:." script: - py.test + - ocular --data-file .coverage --config-file .coveragerc diff --git a/test_qiniu.py b/test_qiniu.py index 84f39dca..8f8bfd53 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -94,55 +94,69 @@ class BucketTestCase(unittest.TestCase): bucket = BucketManager(q) def test_list(self): - ret, eof, _ = self.bucket.list(bucket_name, limit=4) + ret, eof, info = self.bucket.list(bucket_name, limit=4) + print(info) assert eof is False assert len(ret.get('items')) == 4 - ret, eof, _ = self.bucket.list(bucket_name, limit=100) + ret, eof, info = self.bucket.list(bucket_name, limit=100) + print(info) assert eof is True def test_buckets(self): - ret, _ = self.bucket.buckets() + ret, info = self.bucket.buckets() + print(info) assert bucket_name in ret def test_pefetch(self): - ret, _ = self.bucket.prefetch(bucket_name, 'python-sdk.html') + ret, info = self.bucket.prefetch(bucket_name, 'python-sdk.html') + print(info) assert ret == {} def test_fetch(self): - ret, _ = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, 'fetch.html') + ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, 'fetch.html') + print(info) assert ret == {} def test_stat(self): - ret, _ = self.bucket.stat(bucket_name, 'python-sdk.html') + ret, info = self.bucket.stat(bucket_name, 'python-sdk.html') + print(info) assert 'hash' in ret def test_delete(self): ret, info = self.bucket.delete(bucket_name, 'del') + print(info) + assert ret is None + assert info.status_code == 612 def test_rename(self): key = 'renameto'+rand_string(8) self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) key2 = key + 'move' - ret, _ = self.bucket.rename(bucket_name, key, key2) + ret, info = self.bucket.rename(bucket_name, key, key2) + print(info) assert ret == {} - ret, _ = self.bucket.delete(bucket_name, key2) + ret, info = self.bucket.delete(bucket_name, key2) + print(info) assert ret == {} def test_copy(self): key = 'copyto'+rand_string(8) - ret, _ = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) - + ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + print(info) assert ret == {} - ret, _ = self.bucket.delete(bucket_name, key) + ret, info = self.bucket.delete(bucket_name, key) + print(info) assert ret == {} def test_batch_copy(self): key = 'copyto'+rand_string(8) ops = build_batch_copy(bucket_name, {'copyfrom': key}, bucket_name) - ret, _ = self.bucket.batch(ops) + ret, info = self.bucket.batch(ops) + print(info) assert ret[0]['code'] == 200 ops = build_batch_delete(bucket_name, [key]) - ret, _ = self.bucket.batch(ops) + ret, info = self.bucket.batch(ops) + print(info) assert ret[0]['code'] == 200 def test_batch_move(self): @@ -150,9 +164,11 @@ def test_batch_move(self): self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) key2 = key2 = key + 'move' ops = build_batch_move(bucket_name, {key: key2}, bucket_name) - ret, _ = self.bucket.batch(ops) + ret, info = self.bucket.batch(ops) + print(info) assert ret[0]['code'] == 200 - ret, _ = self.bucket.delete(bucket_name, key2) + ret, info = self.bucket.delete(bucket_name, key2) + print(info) assert ret == {} def test_batch_rename(self): @@ -160,14 +176,17 @@ def test_batch_rename(self): self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) key2 = key2 = key + 'rename' ops = build_batch_move(bucket_name, {key: key2}, bucket_name) - ret, _ = self.bucket.batch(ops) + ret, info = self.bucket.batch(ops) + print(info) assert ret[0]['code'] == 200 - ret, _ = self.bucket.delete(bucket_name, key2) + ret, info = self.bucket.delete(bucket_name, key2) + print(info) assert ret == {} def test_batch_stat(self): ops = build_batch_stat(bucket_name, ['python-sdk.html']) - ret, _ = self.bucket.batch(ops) + ret, info = self.bucket.batch(ops) + print(info) assert ret[0]['code'] == 200 @@ -182,12 +201,14 @@ def test_put(self): data = 'hello bubby!' token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) + print(info) assert ret['key'] == key key = '' data = 'hello bubby!' token = self.q.upload_token(bucket_name, key) - ret, _ = put_data(token, key, data, check_crc=True) + ret, info = put_data(token, key, data, check_crc=True) + print(info) assert ret['key'] == key def test_putfile(self): @@ -195,7 +216,8 @@ def test_putfile(self): key = 'test_file' token = self.q.upload_token(bucket_name, key) - ret, _ = put_file(token, key, localfile, mime_type=self.mime_type, check_crc=True) + ret, info = put_file(token, key, localfile, mime_type=self.mime_type, check_crc=True) + print(info) assert ret['key'] == key assert ret['hash'] == etag(localfile) @@ -205,6 +227,7 @@ def test_putInvalidCrc(self): crc32 = 'wrong crc32' token = self.q.upload_token(bucket_name) ret, info = _form_put(token, key, data, None, None, crc32=crc32) + print(info) assert ret is None assert info.status_code == 400 @@ -212,20 +235,24 @@ def test_putWithoutKey(self): key = None data = 'hello bubby!' token = self.q.upload_token(bucket_name) - ret, _ = put_data(token, key, data) + ret, info = put_data(token, key, data) + print(info) assert ret['hash'] == ret['key'] data = 'hello bubby!' token = self.q.upload_token(bucket_name, 'nokey2') - ret, info = put_data(token, None, data) + print(info) + assert ret is None + assert info.status_code == 401 # key not match def test_retry(self): key = 'retry' data = 'hello retry!' set_default(default_up_host='a') token = self.q.upload_token(bucket_name) - ret, _ = put_data(token, key, data) + ret, info = put_data(token, key, data) + print(info) assert ret['key'] == key qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) @@ -253,6 +280,7 @@ def test_big_file(self): progress_handler = lambda progress, total: progress qiniu.set_default(default_up_host='a') ret, info = put_file(token, key, localfile, self.params, self.mime_type, progress_handler=progress_handler) + print(info) assert ret['key'] == key qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) remove_temp_file(localfile) @@ -263,6 +291,7 @@ def test_retry(self): qiniu.set_default(default_up_host='a') token = self.q.upload_token(bucket_name, key) ret, info = put_file(token, key, localfile, self.params, self.mime_type) + print(info) assert ret['key'] == key qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) @@ -273,6 +302,7 @@ def test_pfop(self): pfop = PersistentFop(q, 'testres', 'sdktest') op = op_save('avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', 'pythonsdk', 'pfoptest') ret, info = pfop.execute('sintel_trailer.mp4', op, 1) + print(info) assert ret['persistentId'] is not None if __name__ == '__main__': From 86bf4d14ab34c49ace42d439f7d1ccd3223982ae Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 13 Nov 2014 22:13:49 +0800 Subject: [PATCH 076/478] coverage --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 873cf9b8..b4d43aa9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,5 +18,5 @@ before_script: - export QINIU_TEST_ENV="travis" - export PYTHONPATH="$PYTHONPATH:." script: - - py.test - - ocular --data-file .coverage --config-file .coveragerc + - py.test --cov qiniu + - ocular --data-file .coverage From 9583a671f0ffd93772f5c0f668f4266f5ecdccf4 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Thu, 13 Nov 2014 22:15:53 +0800 Subject: [PATCH 077/478] [ci skip] --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1854c0c9..99e15b03 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ [![Build Status](https://travis-ci.org/qiniu/python-sdk.svg)](https://travis-ci.org/qiniu/python-sdk) [![Latest Stable Version](https://img.shields.io/pypi/v/qiniu.svg)](https://pypi.python.org/pypi/qiniu) [![Download Times](https://img.shields.io/pypi/dm/qiniu.svg)](https://pypi.python.org/pypi/qiniu) - +[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/qiniu/python-sdk/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/qiniu/python-sdk/?branch=master) +[![Code Coverage](https://scrutinizer-ci.com/g/qiniu/python-sdk/badges/coverage.png?b=master)](https://scrutinizer-ci.com/g/qiniu/python-sdk/?branch=master) ## 安装 通过pip From c9a848decb6c0fe47cbceff534111fc2f43863cf Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 13 Nov 2014 23:01:12 +0800 Subject: [PATCH 078/478] scrutinizer quality --- .scrutinizer.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 135c31d8..b37be4cb 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -1,3 +1,8 @@ +checks: + python: + code_rating: true + duplicate_code: true + tools: external_code_coverage: timeout: 12000 # Timeout in seconds. From 7b0cf20c4acd4066697086d718c1cf42c39518b3 Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 14 Nov 2014 08:19:32 +0800 Subject: [PATCH 079/478] code quality scrutinizer --- .scrutinizer.yml | 8 ++++++++ .travis.yml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index b37be4cb..47c6b8e9 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -1,8 +1,16 @@ checks: + php: + code_rating: true + duplication: true + python: code_rating: true duplicate_code: true + ruby: + code_rating: true + duplicate_code: true + tools: external_code_coverage: timeout: 12000 # Timeout in seconds. diff --git a/.travis.yml b/.travis.yml index b4d43aa9..f1b885e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ install: - "pip install requests --use-mirrors" - "pip install scrutinizer-ocular --use-mirrors" before_script: - - "flake8 --show-source --show-pep8 --max-line-length=160 ." - export QINIU_ACCESS_KEY="QWYn5TFQsLLU1pL5MFEmX3s5DmHdUThav9WyOWOm" - export QINIU_SECRET_KEY="Bxckh6FA-Fbs9Yt3i3cbKVK22UPBmAOHJcL95pGz" - export QINIU_TEST_BUCKET="pythonsdk" @@ -18,5 +17,6 @@ before_script: - export QINIU_TEST_ENV="travis" - export PYTHONPATH="$PYTHONPATH:." script: + - flake8 --show-source --show-pep8 --max-line-length=160 . - py.test --cov qiniu - ocular --data-file .coverage From 4534030ac1efd84164ad440c1d43c3a6bff89b13 Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 14 Nov 2014 08:49:55 +0800 Subject: [PATCH 080/478] scrutinizer --- .scrutinizer.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 47c6b8e9..eb9b64b5 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -1,15 +1,9 @@ -checks: - php: - code_rating: true - duplication: true +checks: python: code_rating: true duplicate_code: true - - ruby: - code_rating: true - duplicate_code: true + variables_redefined_outer_name: true tools: external_code_coverage: From 06d2909f012fc9da28c661e1fea4d4b38030eba3 Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 14 Nov 2014 13:05:07 +0800 Subject: [PATCH 081/478] verify callback test --- qiniu/compat.py | 32 +++++++++++++++--------------- qiniu/services/storage/uploader.py | 6 +++--- qiniu/utils.py | 4 ++-- test_qiniu.py | 14 ++++++++++++- 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/qiniu/compat.py b/qiniu/compat.py index 448d87b7..63f22b25 100644 --- a/qiniu/compat.py +++ b/qiniu/compat.py @@ -42,14 +42,14 @@ basestring = basestring # noqa numeric_types = (int, long, float) # noqa - def b(s): - return s + def b(data): + return data - def s(b): - return b + def s(data): + return data - def u(s): - return unicode(s, 'unicode_escape') # noqa + def u(data): + return unicode(data, 'unicode_escape') # noqa elif is_py3: from urllib.parse import urlparse # noqa @@ -63,15 +63,15 @@ def u(s): basestring = (str, bytes) numeric_types = (int, float) - def b(s): - if isinstance(s, str): - return s.encode('utf-8') - return s + def b(data): + if isinstance(data, str): + return data.encode('utf-8') + return data - def s(b): - if isinstance(b, bytes): - b = b.decode('utf-8') - return b + def s(data): + if isinstance(data, bytes): + data = data.decode('utf-8') + return data - def u(s): - return s + def u(data): + return data diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 54d1d9a5..390184ed 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -29,13 +29,13 @@ def put_file(up_token, key, file_path, params=None, mime_type='application/octet return ret, info -def _form_put(up_token, key, data, params, mime_type, crc32, is_file=False, progress_handler=None): +def _form_put(up_token, key, data, params, mime_type, crc, is_file=False, progress_handler=None): fields = {} if params: for k, v in params.items(): fields[k] = str(v) - if crc32: - fields['crc32'] = crc32 + if crc: + fields['crc32'] = crc if key is not None: fields['key'] = key fields['token'] = up_token diff --git a/qiniu/utils.py b/qiniu/utils.py index 1fb1351d..14bf1936 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -56,8 +56,8 @@ def _etag(input_stream): data = array[0] prefix = b('\x16') else: - s = b('').join(array) - data = _sha1(s) + sha1_str = b('').join(array) + data = _sha1(sha1_str) prefix = b('\x96') return urlsafe_base64_encode(prefix + data) diff --git a/test_qiniu.py b/test_qiniu.py index 8f8bfd53..57d484d2 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -88,6 +88,18 @@ def test_deprecatedPolicy(self): with pytest.raises(ValueError): dummy_auth.upload_token('1', None, policy={'asyncOps': 1}) + def test_token_of_request(self): + token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', '') + assert token == 'abcdefghklmnopq:cFyRVoWrE3IugPIMP5YJFTO-O-Y=' + token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', 'application/x-www-form-urlencoded') + assert token == 'abcdefghklmnopq:svWRNcacOE-YMsc70nuIYdaa1e4=' + + def test_verify_callback(self): + body = 'name=sunflower.jpg&hash=Fn6qeQi4VDLQ347NiRm-RlQx_4O2&location=Shanghai&price=1500.00&uid=123' + url = 'test.qiniu.com/callback' + ok = dummy_auth.verify_callback('QBox abcdefghklmnopq:ZWyeM5ljWMRFwuPTPOwQ4RwSto4=', url, body) + assert ok + class BucketTestCase(unittest.TestCase): q = Auth(access_key, secret_key) @@ -226,7 +238,7 @@ def test_putInvalidCrc(self): data = 'hello bubby!' crc32 = 'wrong crc32' token = self.q.upload_token(bucket_name) - ret, info = _form_put(token, key, data, None, None, crc32=crc32) + ret, info = _form_put(token, key, data, None, None, crc=crc32) print(info) assert ret is None assert info.status_code == 400 From f7cc25c6e905e9b011a84d17aeb21ecbbf166730 Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 14 Nov 2014 13:26:06 +0800 Subject: [PATCH 082/478] change mime test --- test_qiniu.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test_qiniu.py b/test_qiniu.py index 57d484d2..2635306d 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -160,6 +160,11 @@ def test_copy(self): print(info) assert ret == {} + def test_change_mime(self): + ret, info = self.bucket.change_mime(bucket_name, 'python-sdk.html', 'text/html') + print(info) + assert ret == {} + def test_batch_copy(self): key = 'copyto'+rand_string(8) ops = build_batch_copy(bucket_name, {'copyfrom': key}, bucket_name) From 3847b1f52efa36e1ebd78f40ea4defa68001fb81 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Fri, 14 Nov 2014 13:33:05 +0800 Subject: [PATCH 083/478] [ci skip] --- CHANGELOG.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 041e8217..1d766d7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ ### 增加 * 简化上传接口 * 自动选择断点续上传还是直传 -* 重构代码，内部结构更清晰，便于更换不同的http实现 +* 重构代码，接口和内部结构更清晰 * 同时支持python 2.x 和 3.x -* 增加pfop支持 +* 支持pfop +* 支持verify callback +* 改变mime +* 代码覆盖度报告 +* policy改为dict, 便于灵活增加，并加入过期字段检查 +* 文件列表支持目录形式 From 9ed345709a271cd02dc363be3993a9460ff9206f Mon Sep 17 00:00:00 2001 From: Bai Long Date: Sun, 16 Nov 2014 22:15:12 +0800 Subject: [PATCH 084/478] [ci skip] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 99e15b03..dd27ba3e 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ $ py.test - 第二个参数info保留了请求响应的信息，失败情况下ret 为none, 将info可以打印出来，提交给我们。 - API 的使用 demo 可以参考 [单元测试](https://github.com/qiniu/python-sdk/blob/master/test_qiniu.py)。 - +- 如果碰到`ImportError: No module named requests.auth` 请安装 `requests` 。 ## 代码贡献 详情参考[代码提交指南](https://github.com/qiniu/python-sdk/blob/master/CONTRIBUTING.md)。 From 225c24915539a4929dbacd205fd80eb97feff329 Mon Sep 17 00:00:00 2001 From: longbai Date: Mon, 17 Nov 2014 19:32:13 +0800 Subject: [PATCH 085/478] fixed test prefetch func name --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 2635306d..b87d8b15 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -119,7 +119,7 @@ def test_buckets(self): print(info) assert bucket_name in ret - def test_pefetch(self): + def test_prefetch(self): ret, info = self.bucket.prefetch(bucket_name, 'python-sdk.html') print(info) assert ret == {} From 3f34bde97597d8daa8340161007005cc5edf6667 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Mon, 17 Nov 2014 23:27:24 +0800 Subject: [PATCH 086/478] [ci skip] --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index dd27ba3e..2dc1ea19 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ $ py.test - 第二个参数info保留了请求响应的信息，失败情况下ret 为none, 将info可以打印出来，提交给我们。 - API 的使用 demo 可以参考 [单元测试](https://github.com/qiniu/python-sdk/blob/master/test_qiniu.py)。 - 如果碰到`ImportError: No module named requests.auth` 请安装 `requests` 。 + ## 代码贡献 详情参考[代码提交指南](https://github.com/qiniu/python-sdk/blob/master/CONTRIBUTING.md)。 From 3a34e9d7fc4a7736437e1d732ae8babe2be555f7 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Thu, 20 Nov 2014 11:52:13 +0800 Subject: [PATCH 087/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9batch=20move=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 8f8bfd53..d2c072f7 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -162,7 +162,7 @@ def test_batch_copy(self): def test_batch_move(self): key = 'moveto'+rand_string(8) self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) - key2 = key2 = key + 'move' + key2 = key + 'move' ops = build_batch_move(bucket_name, {key: key2}, bucket_name) ret, info = self.bucket.batch(ops) print(info) From 3e9dada9b71971619fd16051ebe72d4b5eddbf68 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Thu, 20 Nov 2014 17:57:54 +0800 Subject: [PATCH 088/478] fix pfop test --- test_qiniu.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index d2c072f7..4bc9d406 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -301,7 +301,8 @@ def test_pfop(self): q = Auth(access_key, secret_key) pfop = PersistentFop(q, 'testres', 'sdktest') op = op_save('avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', 'pythonsdk', 'pfoptest') - ret, info = pfop.execute('sintel_trailer.mp4', op, 1) + ops.append(op); + ret, info = pfop.execute('sintel_trailer.mp4', ops, 1) print(info) assert ret['persistentId'] is not None From 2a94349612a2e0e5bbd54ca5272f9938a29c0d37 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Thu, 20 Nov 2014 18:01:31 +0800 Subject: [PATCH 089/478] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E5=88=86=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 4bc9d406..ce0c749d 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -301,7 +301,7 @@ def test_pfop(self): q = Auth(access_key, secret_key) pfop = PersistentFop(q, 'testres', 'sdktest') op = op_save('avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', 'pythonsdk', 'pfoptest') - ops.append(op); + ops.append(op) ret, info = pfop.execute('sintel_trailer.mp4', ops, 1) print(info) assert ret['persistentId'] is not None From 528d0feea5d2dac94fc989f78ebdde547181cf10 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Thu, 20 Nov 2014 18:12:47 +0800 Subject: [PATCH 090/478] =?UTF-8?q?=E7=94=B3=E6=98=8Eops?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_qiniu.py b/test_qiniu.py index ce0c749d..ed62a09b 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -301,6 +301,7 @@ def test_pfop(self): q = Auth(access_key, secret_key) pfop = PersistentFop(q, 'testres', 'sdktest') op = op_save('avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', 'pythonsdk', 'pfoptest') + ops = [] ops.append(op) ret, info = pfop.execute('sintel_trailer.mp4', ops, 1) print(info) From 56f3946c51cac872f12352698caa4c28b1832edf Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Sat, 22 Nov 2014 02:21:37 +0800 Subject: [PATCH 091/478] update pfop test --- test_qiniu.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index ed62a09b..baec7060 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -4,6 +4,7 @@ import string import random import tempfile +import json import unittest import pytest @@ -305,7 +306,7 @@ def test_pfop(self): ops.append(op) ret, info = pfop.execute('sintel_trailer.mp4', ops, 1) print(info) - assert ret['persistentId'] is not None + print json.dumps(ret) if __name__ == '__main__': unittest.main() From 35be934d0f347518d76712baa19e9028d2b42f50 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Mon, 24 Nov 2014 22:46:46 +0800 Subject: [PATCH 092/478] add private url test --- test_qiniu.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index baec7060..b504b9d0 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -4,7 +4,7 @@ import string import random import tempfile -import json +import urllib import unittest import pytest @@ -296,6 +296,16 @@ def test_retry(self): assert ret['key'] == key qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) +class DownloadTestCase(unittest.TestCase): + def test_private_url(self): + q = Auth(access_key, secret_key) + bucket = 'test_private_bucket' + key = 'test_private_key' + key = key.encode('utf8') + base_url = 'http://%s/%s' % (bucket, urllib.quote(key)) + private_url = self.q.private_download_url(baseurl, expires=3600) + print(private_url) + class MediaTestCase(unittest.TestCase): def test_pfop(self): @@ -306,7 +316,7 @@ def test_pfop(self): ops.append(op) ret, info = pfop.execute('sintel_trailer.mp4', ops, 1) print(info) - print json.dumps(ret) + assert ret[0]['persistentId'] is not None if __name__ == '__main__': unittest.main() From 03671a4602011feed6465106e20fadbc882f1331 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Mon, 24 Nov 2014 22:49:45 +0800 Subject: [PATCH 093/478] =?UTF-8?q?fix=20=E8=AF=AF=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index b504b9d0..1d5d9588 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -316,7 +316,7 @@ def test_pfop(self): ops.append(op) ret, info = pfop.execute('sintel_trailer.mp4', ops, 1) print(info) - assert ret[0]['persistentId'] is not None + assert ret['persistentId'] is not None if __name__ == '__main__': unittest.main() From 7ce1ae47ed95ba742af13209abe14e034779fd28 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Mon, 24 Nov 2014 23:17:49 +0800 Subject: [PATCH 094/478] update test --- test_qiniu.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 1d5d9588..ccb66c01 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -297,8 +297,10 @@ def test_retry(self): qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) class DownloadTestCase(unittest.TestCase): + + q = Auth(access_key, secret_key) + def test_private_url(self): - q = Auth(access_key, secret_key) bucket = 'test_private_bucket' key = 'test_private_key' key = key.encode('utf8') From f7c22fc0690dc1c324704c511f2f3d15d2e62798 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Mon, 24 Nov 2014 23:30:19 +0800 Subject: [PATCH 095/478] update test --- test_qiniu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index ccb66c01..e0a05f91 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -297,15 +297,15 @@ def test_retry(self): qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) class DownloadTestCase(unittest.TestCase): - + q = Auth(access_key, secret_key) def test_private_url(self): bucket = 'test_private_bucket' key = 'test_private_key' key = key.encode('utf8') - base_url = 'http://%s/%s' % (bucket, urllib.quote(key)) - private_url = self.q.private_download_url(baseurl, expires=3600) + base_url = 'http://%s/%s' % (bucket, key) + private_url = self.q.private_download_url(base_url, expires=3600) print(private_url) From fc601af77cf189f59d40dce1a370c4880c193ffe Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Mon, 24 Nov 2014 23:52:00 +0800 Subject: [PATCH 096/478] update test --- test_qiniu.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index e0a05f91..a556e4d0 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -4,7 +4,6 @@ import string import random import tempfile -import urllib import unittest import pytest From f9367cf3755a3649ffa31538478fa7c38e4efcb7 Mon Sep 17 00:00:00 2001 From: longbai Date: Tue, 25 Nov 2014 20:47:33 +0800 Subject: [PATCH 097/478] setup version --- setup.py | 32 ++++++++++++++++++++++++++++++-- test_qiniu.py | 6 ------ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 3763dcc7..0c78b7b9 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import io +import os +import re + try: import setuptools setup = setuptools.setup @@ -9,9 +13,33 @@ from distutils.core import setup +packages = [ + 'qiniu', + 'qiniu.services', + 'qiniu.services.storage', + 'qiniu.services.processing', +] + + +def read(*names, **kwargs): + return io.open( + os.path.join(os.path.dirname(__file__), *names), + encoding=kwargs.get("encoding", "utf8") + ).read() + + +def find_version(*file_paths): + version_file = read(*file_paths) + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", + version_file, re.M) + if version_match: + return version_match.group(1) + raise RuntimeError("Unable to find version string.") + + setup( name='qiniu', - version=__import__('qiniu').__version__, + version=find_version("qiniu/__init__.py"), description='Qiniu Resource Storage SDK', long_description='see:\nhttps://github.com/qiniu/python-sdk\n', author='Shanghai Qiniu Information Technologies Co., Ltd.', @@ -19,8 +47,8 @@ maintainer_email='support@qiniu.com', license='MIT', url='https://github.com/qiniu/python-sdk', - packages=['qiniu', 'qiniu.services', 'qiniu.services.storage', 'qiniu.services.processing'], platforms='any', + packages=packages, classifiers=[ 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', diff --git a/test_qiniu.py b/test_qiniu.py index cb59364f..6a04f98e 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -88,12 +88,6 @@ def test_deprecatedPolicy(self): with pytest.raises(ValueError): dummy_auth.upload_token('1', None, policy={'asyncOps': 1}) - def test_token_of_request(self): - token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', '') - assert token == 'abcdefghklmnopq:cFyRVoWrE3IugPIMP5YJFTO-O-Y=' - token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', 'application/x-www-form-urlencoded') - assert token == 'abcdefghklmnopq:svWRNcacOE-YMsc70nuIYdaa1e4=' - def test_verify_callback(self): body = 'name=sunflower.jpg&hash=Fn6qeQi4VDLQ347NiRm-RlQx_4O2&location=Shanghai&price=1500.00&uid=123' url = 'test.qiniu.com/callback' From 72cfed9a3834d832af8725bcec21a437f1b153d7 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Tue, 25 Nov 2014 21:02:03 +0800 Subject: [PATCH 098/478] [ci skip] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d766d7c..c41a5f43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ #Changelog +## 7.0.1 (2014-11-) +### 增加 +* setup.py从文件中读取版本号，而不是用导入方式 +* 补充及修正了一些单元测试 ## 7.0.0 (2014-11-13) From 0fc7f1c6aef360aeda75950f7c9c8fabdbf59dd4 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Tue, 25 Nov 2014 21:03:08 +0800 Subject: [PATCH 099/478] update version [ci skip] --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 1c09a096..d92e3979 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.0.0' +__version__ = '7.0.1' from .auth import Auth From 44d1fd4f4dce286bdb27f75ae0bb36715cfa580a Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 26 Nov 2014 10:56:27 +0800 Subject: [PATCH 100/478] fix --- test_qiniu.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index a556e4d0..31676b0c 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -302,7 +302,6 @@ class DownloadTestCase(unittest.TestCase): def test_private_url(self): bucket = 'test_private_bucket' key = 'test_private_key' - key = key.encode('utf8') base_url = 'http://%s/%s' % (bucket, key) private_url = self.q.private_download_url(base_url, expires=3600) print(private_url) From 213a366b08f68eb4997c784dda5732d20da98b70 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 26 Nov 2014 11:13:04 +0800 Subject: [PATCH 101/478] fix timeout --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 31676b0c..78915b9f 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -304,7 +304,7 @@ def test_private_url(self): key = 'test_private_key' base_url = 'http://%s/%s' % (bucket, key) private_url = self.q.private_download_url(base_url, expires=3600) - print(private_url) + print(private_url) class MediaTestCase(unittest.TestCase): From 110be1be0871621412f12e7ae90179a922b594f9 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 26 Nov 2014 15:36:59 +0800 Subject: [PATCH 102/478] fix --- test_qiniu.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_qiniu.py b/test_qiniu.py index 78915b9f..1360a444 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -4,6 +4,7 @@ import string import random import tempfile +import requests import unittest import pytest @@ -305,6 +306,8 @@ def test_private_url(self): base_url = 'http://%s/%s' % (bucket, key) private_url = self.q.private_download_url(base_url, expires=3600) print(private_url) + r = requests.get(private_url) + assert r.status_code == 200 class MediaTestCase(unittest.TestCase): From 1ed0a39bce00fcd0592593c9bb5153ffd0268970 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 26 Nov 2014 15:51:47 +0800 Subject: [PATCH 103/478] fix timeout --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 1360a444..0cc3fb27 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # flake8: noqa -import os +import os import string import random import tempfile From 05ec7e7ff0aead6ed18ce7d646e150e914e8a15d Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 26 Nov 2014 16:00:24 +0800 Subject: [PATCH 104/478] fix timeout again --- test_qiniu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index 0cc3fb27..c9611d30 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -301,9 +301,9 @@ class DownloadTestCase(unittest.TestCase): q = Auth(access_key, secret_key) def test_private_url(self): - bucket = 'test_private_bucket' - key = 'test_private_key' - base_url = 'http://%s/%s' % (bucket, key) + private_bucket = 'private-res' + private_key = 'gogopher.jpg' + base_url = 'http://%s/%s' % (private_bucket+'.qiniudn.com', private_key) private_url = self.q.private_download_url(base_url, expires=3600) print(private_url) r = requests.get(private_url) From 7acdb8011906b1d180ec8ae47c9c57b0fc4b30be Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 26 Nov 2014 16:21:10 +0800 Subject: [PATCH 105/478] fix timeout again --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index c9611d30..232aa781 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # flake8: noqa -import os +import os import string import random import tempfile From a3c6799d25b5ad15ec708de4073b65d7613a5f77 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Wed, 26 Nov 2014 16:27:21 +0800 Subject: [PATCH 106/478] [ci skip] --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c41a5f43..879d22ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ #Changelog -## 7.0.1 (2014-11-) +## 7.0.1 (2014-11-26) ### 增加 * setup.py从文件中读取版本号，而不是用导入方式 * 补充及修正了一些单元测试 From 284f6e2806c90a7772752f75efa6042e2e4d857e Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Thu, 27 Nov 2014 03:24:06 +0800 Subject: [PATCH 107/478] add demo --- demo/batch_demo.py | 54 ++++++++++++++++++++++++++ demo/bucket_demo.py | 86 +++++++++++++++++++++++++++++++++++++++++ demo/p_download_demo.py | 25 ++++++++++++ demo/pfop_demo.py | 25 ++++++++++++ demo/r_upload_demo.py | 49 +++++++++++++++++++++++ demo/upload_demo.py | 45 +++++++++++++++++++++ 6 files changed, 284 insertions(+) create mode 100644 demo/batch_demo.py create mode 100644 demo/bucket_demo.py create mode 100644 demo/p_download_demo.py create mode 100644 demo/pfop_demo.py create mode 100644 demo/r_upload_demo.py create mode 100644 demo/upload_demo.py diff --git a/demo/batch_demo.py b/demo/batch_demo.py new file mode 100644 index 00000000..ab21feb6 --- /dev/null +++ b/demo/batch_demo.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import os + +from qiniu import Auth +from qiniu import BucketManager, build_batch_rename +from qiniu.compat import is_py2 + +if is_py2: + import sys + reload(sys) + sys.setdefaultencoding('utf-8') + +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') +bucket_name = os.getenv('QINIU_TEST_BUCKET') + +q = Auth(access_key, secret_key) +bucket = BucketManager(q) + +# batch stat +from qiniu import build_batch_stat + +ops = build_batch_stat(bucket_name, ['python-sdk.html', 'python-sdk.html']) +ret, info = bucket.batch(ops) +print(info) +assert ret[0]['code'] == 200 + +# # batch copy +# from qiniu import build_batch_copy + +# key = 'copyto' +# ops = build_batch_copy(bucket_name, {'copyfrom': key}, bucket_name) +# ret, info = bucket.batch(ops) +# print(info) +# assert ret[0]['code'] == 200 + +# # batch move +# from qiniu import build_batch_move + +# key = 'moveto' +# key2 = key + 'move' +# ops = build_batch_move(bucket_name, {key: key2}, bucket_name) +# ret, info = bucket.batch(ops) +# print(info) +# assert ret[0]['code'] == 200 + +# batch delete +# from qiniu import build_batch_delete + +# ops = build_batch_delete(bucket_name, ['python-sdk.html']) +# ret, info = self.bucket.batch(ops) +# print(info) +# assert ret[0]['code'] == 612 \ No newline at end of file diff --git a/demo/bucket_demo.py b/demo/bucket_demo.py new file mode 100644 index 00000000..e3c3b96c --- /dev/null +++ b/demo/bucket_demo.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import os + +from qiniu import Auth +from qiniu import BucketManager +from qiniu.compat import is_py2 + +if is_py2: + import sys + reload(sys) + sys.setdefaultencoding('utf-8') + +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') +bucket_name = os.getenv('QINIU_TEST_BUCKET') + +q = Auth(access_key, secret_key) +bucket = BucketManager(q) + + +# stat +ret, info = bucket.stat(bucket_name, 'python-sdk.html') +print(info) +assert 'hash' in ret + +# # copy +# key = 'copyto' +# ret, info = bucket.copy(bucket_name, 'copyfrom', bucket_name, key) +# print(info) +# assert ret == {} + +# # move +# key = 'renameto' +# key2 = key + 'move' +# ret, info = bucket.move(bucket_name, key, bucket_name, key2) +# print(info) +# assert ret == {} + +# # delete +# ret, info = bucket.delete(bucket_name, 'del') +# print(info) +# assert ret is None +# assert info.status_code == 612 + + +# # prefetch + +# ret, info = bucket.prefetch(bucket_name, 'python-sdk.html') +# print(info) +# assert ret == {} + +# # fetch +# ret, info = bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, 'fetch.html') +# print(info) +# assert ret == {} + + +# # list + +# ret, eof, info = bucket.list(bucket_name, limit=4) +# print(info) +# assert eof is False +# assert len(ret.get('items')) == 4 +# ret, eof, info = bucket.list(bucket_name, limit=100) +# print(info) +# assert eof is True + +# # list all + +# def list_all(bucket_name, bucket=None, prefix=None, limit=None): +# if bucket is None: +# bucket = BucketManager(q) +# marker = None +# eof = False +# while eof is False: +# ret, eof, info = bucket.list(bucket_name, prefix=prefix, marker=marker, limit=limit) +# marker = ret.get('marker', None) +# for item in ret['items']: +# print(item['key']) +# pass +# if eof is not True: +# # 错误处理 +# pass + +# list_all(bucket_name, bucket, 't', 10) \ No newline at end of file diff --git a/demo/p_download_demo.py b/demo/p_download_demo.py new file mode 100644 index 00000000..bd4b4985 --- /dev/null +++ b/demo/p_download_demo.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import os +import requests + +from qiniu import Auth +from qiniu.compat import is_py2 + +if is_py2: + import sys + reload(sys) + sys.setdefaultencoding('utf-8') + +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') + +q = Auth(access_key, secret_key) + +bucket = 'private-res' +key = 'gogopher.jpg' +base_url = 'http://%s/%s' % (bucket + '.qiniudn.com', key) +private_url = q.private_download_url(base_url, expires=3600) +print(private_url) +r = requests.get(private_url) +assert r.status_code == 200 \ No newline at end of file diff --git a/demo/pfop_demo.py b/demo/pfop_demo.py new file mode 100644 index 00000000..24ef83a3 --- /dev/null +++ b/demo/pfop_demo.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import os + +from qiniu import Auth, PersistentFop, build_op, op_save +from qiniu.compat import is_py2 + +if is_py2: + import sys + reload(sys) + sys.setdefaultencoding('utf-8') + +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') +bucket_name = os.getenv('QINIU_TEST_BUCKET') + +q = Auth(access_key, secret_key) + +pfop = PersistentFop(q, 'testres', 'sdktest') +op = op_save('avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', 'pythonsdk', 'pfoptest') +ops = [] +ops.append(op) +ret, info = pfop.execute('sintel_trailer.mp4', ops, 1) +print(info) +assert ret['persistentId'] is not None \ No newline at end of file diff --git a/demo/r_upload_demo.py b/demo/r_upload_demo.py new file mode 100644 index 00000000..9cac66a8 --- /dev/null +++ b/demo/r_upload_demo.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import os +import tempfile + +from qiniu import Auth +from qiniu import put_file +from qiniu.compat import is_py2, b + +import qiniu.config + +if is_py2: + import sys + reload(sys) + sys.setdefaultencoding('utf-8') + +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') +bucket_name = os.getenv('QINIU_TEST_BUCKET') + +def create_temp_file(size): + t = tempfile.mktemp() + f = open(t, 'wb') + f.seek(size-1) + f.write(b('0')) + f.close() + return t + +def remove_temp_file(file): + try: + os.remove(file) + except OSError: + pass + +q = Auth(access_key, secret_key) + +mime_type = "text/plain" +params = {'x:a': 'a'} + +key = 'big' +token = q.upload_token(bucket_name, key) +localfile = create_temp_file(4 * 1024 * 1024 + 1) +progress_handler = lambda progress, total: progress +qiniu.set_default(default_up_host='a') +ret, info = put_file(token, key, localfile, params, mime_type, progress_handler=progress_handler) +print(info) +assert ret['key'] == key +qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) +remove_temp_file(localfile) \ No newline at end of file diff --git a/demo/upload_demo.py b/demo/upload_demo.py new file mode 100644 index 00000000..15ec95a5 --- /dev/null +++ b/demo/upload_demo.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import os + +from qiniu import Auth, etag +from qiniu import put_data, put_file +from qiniu.compat import is_py2 + +if is_py2: + import sys + reload(sys) + sys.setdefaultencoding('utf-8') + +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') +bucket_name = os.getenv('QINIU_TEST_BUCKET') + +q = Auth(access_key, secret_key) + +# 上传流 +key = 'a\\b\\c"你好' +data = 'hello bubby!' +token = q.upload_token(bucket_name) +ret, info = put_data(token, key, data) +print(info) +assert ret['key'] == key + +key = '' +data = 'hello bubby!' +token = q.upload_token(bucket_name, key) +ret, info = put_data(token, key, data, check_crc=True) +print(info) +assert ret['key'] == key + +# 上传文件 +localfile = __file__ +key = 'test_file' +mime_type = "text/plain" +params = {'x:a': 'a'} + +token = q.upload_token(bucket_name, key) +ret, info = put_file(token, key, localfile, params=params, mime_type=mime_type, check_crc=True) +print(info) +assert ret['key'] == key +assert ret['hash'] == etag(localfile) \ No newline at end of file From 892a5ab97f06e06a82525d4c3e90a2abe52df432 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Mon, 1 Dec 2014 00:48:10 +0800 Subject: [PATCH 108/478] python3 adaption --- qiniu/main.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/qiniu/main.py b/qiniu/main.py index 10562fb6..adf5b14a 100755 --- a/qiniu/main.py +++ b/qiniu/main.py @@ -17,7 +17,15 @@ def main(): args = parser.parse_args() - if args.etag_files: + try: + etag_files = args.etag_files + + except AttributeError: + # In Python-3* `main.py` (without arguments) raises + # AttributeError. I have not found any standard way to display same + # error message as in Python-2*. + parser.error('too few arguments') + else: r = [etag(file) for file in args.etag_files] if len(r) == 1: print(r[0]) From 7e5daea09569be9243338f0f39d5aedac7235667 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Mon, 1 Dec 2014 01:05:29 +0800 Subject: [PATCH 109/478] python3 adaption --- demo/batch_demo.py | 54 -------------------------- demo/bucket_demo.py | 86 ----------------------------------------- demo/p_download_demo.py | 25 ------------ demo/pfop_demo.py | 25 ------------ demo/r_upload_demo.py | 49 ----------------------- demo/upload_demo.py | 45 --------------------- 6 files changed, 284 deletions(-) delete mode 100644 demo/batch_demo.py delete mode 100644 demo/bucket_demo.py delete mode 100644 demo/p_download_demo.py delete mode 100644 demo/pfop_demo.py delete mode 100644 demo/r_upload_demo.py delete mode 100644 demo/upload_demo.py diff --git a/demo/batch_demo.py b/demo/batch_demo.py deleted file mode 100644 index ab21feb6..00000000 --- a/demo/batch_demo.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa -import os - -from qiniu import Auth -from qiniu import BucketManager, build_batch_rename -from qiniu.compat import is_py2 - -if is_py2: - import sys - reload(sys) - sys.setdefaultencoding('utf-8') - -access_key = os.getenv('QINIU_ACCESS_KEY') -secret_key = os.getenv('QINIU_SECRET_KEY') -bucket_name = os.getenv('QINIU_TEST_BUCKET') - -q = Auth(access_key, secret_key) -bucket = BucketManager(q) - -# batch stat -from qiniu import build_batch_stat - -ops = build_batch_stat(bucket_name, ['python-sdk.html', 'python-sdk.html']) -ret, info = bucket.batch(ops) -print(info) -assert ret[0]['code'] == 200 - -# # batch copy -# from qiniu import build_batch_copy - -# key = 'copyto' -# ops = build_batch_copy(bucket_name, {'copyfrom': key}, bucket_name) -# ret, info = bucket.batch(ops) -# print(info) -# assert ret[0]['code'] == 200 - -# # batch move -# from qiniu import build_batch_move - -# key = 'moveto' -# key2 = key + 'move' -# ops = build_batch_move(bucket_name, {key: key2}, bucket_name) -# ret, info = bucket.batch(ops) -# print(info) -# assert ret[0]['code'] == 200 - -# batch delete -# from qiniu import build_batch_delete - -# ops = build_batch_delete(bucket_name, ['python-sdk.html']) -# ret, info = self.bucket.batch(ops) -# print(info) -# assert ret[0]['code'] == 612 \ No newline at end of file diff --git a/demo/bucket_demo.py b/demo/bucket_demo.py deleted file mode 100644 index e3c3b96c..00000000 --- a/demo/bucket_demo.py +++ /dev/null @@ -1,86 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa -import os - -from qiniu import Auth -from qiniu import BucketManager -from qiniu.compat import is_py2 - -if is_py2: - import sys - reload(sys) - sys.setdefaultencoding('utf-8') - -access_key = os.getenv('QINIU_ACCESS_KEY') -secret_key = os.getenv('QINIU_SECRET_KEY') -bucket_name = os.getenv('QINIU_TEST_BUCKET') - -q = Auth(access_key, secret_key) -bucket = BucketManager(q) - - -# stat -ret, info = bucket.stat(bucket_name, 'python-sdk.html') -print(info) -assert 'hash' in ret - -# # copy -# key = 'copyto' -# ret, info = bucket.copy(bucket_name, 'copyfrom', bucket_name, key) -# print(info) -# assert ret == {} - -# # move -# key = 'renameto' -# key2 = key + 'move' -# ret, info = bucket.move(bucket_name, key, bucket_name, key2) -# print(info) -# assert ret == {} - -# # delete -# ret, info = bucket.delete(bucket_name, 'del') -# print(info) -# assert ret is None -# assert info.status_code == 612 - - -# # prefetch - -# ret, info = bucket.prefetch(bucket_name, 'python-sdk.html') -# print(info) -# assert ret == {} - -# # fetch -# ret, info = bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, 'fetch.html') -# print(info) -# assert ret == {} - - -# # list - -# ret, eof, info = bucket.list(bucket_name, limit=4) -# print(info) -# assert eof is False -# assert len(ret.get('items')) == 4 -# ret, eof, info = bucket.list(bucket_name, limit=100) -# print(info) -# assert eof is True - -# # list all - -# def list_all(bucket_name, bucket=None, prefix=None, limit=None): -# if bucket is None: -# bucket = BucketManager(q) -# marker = None -# eof = False -# while eof is False: -# ret, eof, info = bucket.list(bucket_name, prefix=prefix, marker=marker, limit=limit) -# marker = ret.get('marker', None) -# for item in ret['items']: -# print(item['key']) -# pass -# if eof is not True: -# # 错误处理 -# pass - -# list_all(bucket_name, bucket, 't', 10) \ No newline at end of file diff --git a/demo/p_download_demo.py b/demo/p_download_demo.py deleted file mode 100644 index bd4b4985..00000000 --- a/demo/p_download_demo.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa -import os -import requests - -from qiniu import Auth -from qiniu.compat import is_py2 - -if is_py2: - import sys - reload(sys) - sys.setdefaultencoding('utf-8') - -access_key = os.getenv('QINIU_ACCESS_KEY') -secret_key = os.getenv('QINIU_SECRET_KEY') - -q = Auth(access_key, secret_key) - -bucket = 'private-res' -key = 'gogopher.jpg' -base_url = 'http://%s/%s' % (bucket + '.qiniudn.com', key) -private_url = q.private_download_url(base_url, expires=3600) -print(private_url) -r = requests.get(private_url) -assert r.status_code == 200 \ No newline at end of file diff --git a/demo/pfop_demo.py b/demo/pfop_demo.py deleted file mode 100644 index 24ef83a3..00000000 --- a/demo/pfop_demo.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa -import os - -from qiniu import Auth, PersistentFop, build_op, op_save -from qiniu.compat import is_py2 - -if is_py2: - import sys - reload(sys) - sys.setdefaultencoding('utf-8') - -access_key = os.getenv('QINIU_ACCESS_KEY') -secret_key = os.getenv('QINIU_SECRET_KEY') -bucket_name = os.getenv('QINIU_TEST_BUCKET') - -q = Auth(access_key, secret_key) - -pfop = PersistentFop(q, 'testres', 'sdktest') -op = op_save('avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', 'pythonsdk', 'pfoptest') -ops = [] -ops.append(op) -ret, info = pfop.execute('sintel_trailer.mp4', ops, 1) -print(info) -assert ret['persistentId'] is not None \ No newline at end of file diff --git a/demo/r_upload_demo.py b/demo/r_upload_demo.py deleted file mode 100644 index 9cac66a8..00000000 --- a/demo/r_upload_demo.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa -import os -import tempfile - -from qiniu import Auth -from qiniu import put_file -from qiniu.compat import is_py2, b - -import qiniu.config - -if is_py2: - import sys - reload(sys) - sys.setdefaultencoding('utf-8') - -access_key = os.getenv('QINIU_ACCESS_KEY') -secret_key = os.getenv('QINIU_SECRET_KEY') -bucket_name = os.getenv('QINIU_TEST_BUCKET') - -def create_temp_file(size): - t = tempfile.mktemp() - f = open(t, 'wb') - f.seek(size-1) - f.write(b('0')) - f.close() - return t - -def remove_temp_file(file): - try: - os.remove(file) - except OSError: - pass - -q = Auth(access_key, secret_key) - -mime_type = "text/plain" -params = {'x:a': 'a'} - -key = 'big' -token = q.upload_token(bucket_name, key) -localfile = create_temp_file(4 * 1024 * 1024 + 1) -progress_handler = lambda progress, total: progress -qiniu.set_default(default_up_host='a') -ret, info = put_file(token, key, localfile, params, mime_type, progress_handler=progress_handler) -print(info) -assert ret['key'] == key -qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) -remove_temp_file(localfile) \ No newline at end of file diff --git a/demo/upload_demo.py b/demo/upload_demo.py deleted file mode 100644 index 15ec95a5..00000000 --- a/demo/upload_demo.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa -import os - -from qiniu import Auth, etag -from qiniu import put_data, put_file -from qiniu.compat import is_py2 - -if is_py2: - import sys - reload(sys) - sys.setdefaultencoding('utf-8') - -access_key = os.getenv('QINIU_ACCESS_KEY') -secret_key = os.getenv('QINIU_SECRET_KEY') -bucket_name = os.getenv('QINIU_TEST_BUCKET') - -q = Auth(access_key, secret_key) - -# 上传流 -key = 'a\\b\\c"你好' -data = 'hello bubby!' -token = q.upload_token(bucket_name) -ret, info = put_data(token, key, data) -print(info) -assert ret['key'] == key - -key = '' -data = 'hello bubby!' -token = q.upload_token(bucket_name, key) -ret, info = put_data(token, key, data, check_crc=True) -print(info) -assert ret['key'] == key - -# 上传文件 -localfile = __file__ -key = 'test_file' -mime_type = "text/plain" -params = {'x:a': 'a'} - -token = q.upload_token(bucket_name, key) -ret, info = put_file(token, key, localfile, params=params, mime_type=mime_type, check_crc=True) -print(info) -assert ret['key'] == key -assert ret['hash'] == etag(localfile) \ No newline at end of file From 426eeb48a0696050ac139044f9ed949d3ba8f62a Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Mon, 1 Dec 2014 01:25:34 +0800 Subject: [PATCH 110/478] fix --- qiniu/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/qiniu/main.py b/qiniu/main.py index adf5b14a..2eb77ec1 100755 --- a/qiniu/main.py +++ b/qiniu/main.py @@ -1,9 +1,9 @@ -#! /usr/bin/env python # -*- coding: utf-8 -*- +# flake8: noqa import argparse -from qiniu import etag +from qiniu.utils import etag def main(): @@ -21,12 +21,12 @@ def main(): etag_files = args.etag_files except AttributeError: - # In Python-3* `main.py` (without arguments) raises - # AttributeError. I have not found any standard way to display same + # In Python-3* `main.py` (without arguments) raises + # AttributeError. I have not found any standard way to display same # error message as in Python-2*. parser.error('too few arguments') else: - r = [etag(file) for file in args.etag_files] + r = [etag(file) for file in etag_files] if len(r) == 1: print(r[0]) else: From c01ff281ade3ed8d1e1ed9b2bcc5e551eaa3230f Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Mon, 1 Dec 2014 01:36:12 +0800 Subject: [PATCH 111/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E5=BC=95=E7=94=A8=E5=92=8C=E9=87=8D=E8=AF=95ci=20timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/main.py b/qiniu/main.py index 2eb77ec1..4ede51d9 100755 --- a/qiniu/main.py +++ b/qiniu/main.py @@ -1,9 +1,9 @@ +#! /usr/bin/env python # -*- coding: utf-8 -*- -# flake8: noqa import argparse -from qiniu.utils import etag +from qiniu import etag def main(): From 84627c1cfe29cb226914e958354012b71278da2b Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Mon, 1 Dec 2014 01:45:17 +0800 Subject: [PATCH 112/478] fix timeout again --- qiniu/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/main.py b/qiniu/main.py index 4ede51d9..a320ca0c 100755 --- a/qiniu/main.py +++ b/qiniu/main.py @@ -34,3 +34,4 @@ def main(): if __name__ == '__main__': main() + From 30dd124dba244b5fb3caca5de6e17018b88dbdfd Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Mon, 1 Dec 2014 01:49:46 +0800 Subject: [PATCH 113/478] fix flake8 --- qiniu/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qiniu/main.py b/qiniu/main.py index a320ca0c..91561e7b 100755 --- a/qiniu/main.py +++ b/qiniu/main.py @@ -33,5 +33,4 @@ def main(): print(' '.join(r)) if __name__ == '__main__': - main() - + main() \ No newline at end of file From 960710828f4e3971666efd16c6a2cf6c47ee8e89 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Mon, 1 Dec 2014 02:02:33 +0800 Subject: [PATCH 114/478] fix flake8 --- qiniu/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/main.py b/qiniu/main.py index 91561e7b..4ede51d9 100755 --- a/qiniu/main.py +++ b/qiniu/main.py @@ -33,4 +33,4 @@ def main(): print(' '.join(r)) if __name__ == '__main__': - main() \ No newline at end of file + main() From bdc06760701543d8a5e24293565db6cc384ee329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=5B=E5=B8=8C=E5=93=81=5D=E9=99=88=E6=9E=97=E6=B3=B3?= Date: Tue, 2 Dec 2014 14:01:35 +0800 Subject: [PATCH 115/478] fix readme bug --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2dc1ea19..9bf8d6a7 100644 --- a/README.md +++ b/README.md @@ -25,12 +25,14 @@ $ pip install qiniu ### 上传 ```python import qiniu +import qiniu.services.storage.uploader as uploader + ... q = qiniu.Auth(access_key, secret_key) key = 'hello' data = 'hello qiniu!' token = q.upload_token(bucket_name) - ret, info = put_data(token, key, data) + ret, info = uploader.put_data(token, key, data) if ret is not None: print('All is OK') else: From 3713973f4339515582f1f31069e9ab0527cb8fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=5B=E5=B8=8C=E5=93=81=5D=E9=99=88=E6=9E=97=E6=B3=B3?= Date: Tue, 2 Dec 2014 14:57:29 +0800 Subject: [PATCH 116/478] fix bugs --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 9bf8d6a7..59b436dd 100644 --- a/README.md +++ b/README.md @@ -25,14 +25,13 @@ $ pip install qiniu ### 上传 ```python import qiniu -import qiniu.services.storage.uploader as uploader ... q = qiniu.Auth(access_key, secret_key) key = 'hello' data = 'hello qiniu!' token = q.upload_token(bucket_name) - ret, info = uploader.put_data(token, key, data) + ret, info = qiniu.put_data(token, key, data) if ret is not None: print('All is OK') else: From 136682e949fc9381dd89ac48be34cf218f507ba0 Mon Sep 17 00:00:00 2001 From: Jemy Date: Tue, 23 Dec 2014 09:54:51 +0800 Subject: [PATCH 117/478] Add support for none auth get requests. --- qiniu/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/http.py b/qiniu/http.py index 19050339..1c9fecca 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -49,10 +49,10 @@ def _post(url, data, files, auth): def _get(url, params, auth): try: r = requests.get( - url, params=params, auth=RequestsAuth(auth), + url, params=params, auth=RequestsAuth(auth) if auth is not None else None, timeout=config.get_default('connection_timeout'), headers=_headers) except Exception as e: - return None, ResponseInfo(None, e) + return None, ResponseInfo(None, e) return __return_wrapper(r) From 05868aaff6bff5689e937ceb81df77333ed1bbcf Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 24 Dec 2014 14:30:06 +0800 Subject: [PATCH 118/478] fix --- qiniu/main.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/qiniu/main.py b/qiniu/main.py index 4ede51d9..bd5ff522 100755 --- a/qiniu/main.py +++ b/qiniu/main.py @@ -21,11 +21,10 @@ def main(): etag_files = args.etag_files except AttributeError: - # In Python-3* `main.py` (without arguments) raises - # AttributeError. I have not found any standard way to display same - # error message as in Python-2*. - parser.error('too few arguments') - else: + etag_files = None + + + if etag_files r = [etag(file) for file in etag_files] if len(r) == 1: print(r[0]) From 8a6a847ee96c1081c864d4f5ed72c1923af65e56 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 24 Dec 2014 23:15:28 +0800 Subject: [PATCH 119/478] fix format --- qiniu/main.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/qiniu/main.py b/qiniu/main.py index bd5ff522..b3916a2d 100755 --- a/qiniu/main.py +++ b/qiniu/main.py @@ -23,13 +23,12 @@ def main(): except AttributeError: etag_files = None - - if etag_files - r = [etag(file) for file in etag_files] - if len(r) == 1: - print(r[0]) - else: - print(' '.join(r)) +if etag_files + r = [etag(file) for file in etag_files] + if len(r) == 1: + print(r[0]) + else: + print(' '.join(r)) if __name__ == '__main__': main() From a7c8a0e500c8a942566a092dd73c17a3d6632351 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 24 Dec 2014 23:45:44 +0800 Subject: [PATCH 120/478] fix format --- qiniu/main.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/qiniu/main.py b/qiniu/main.py index b3916a2d..c7dac25a 100755 --- a/qiniu/main.py +++ b/qiniu/main.py @@ -22,13 +22,13 @@ def main(): except AttributeError: etag_files = None - -if etag_files - r = [etag(file) for file in etag_files] - if len(r) == 1: - print(r[0]) - else: - print(' '.join(r)) + + if etag_files: + r = [etag(file) for file in etag_files] + if len(r) == 1: + print(r[0]) + else: + print(' '.join(r)) if __name__ == '__main__': main() From 8ecbb2483afc1f7df79b1423770a944a9d1d7024 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Thu, 25 Dec 2014 00:33:51 +0800 Subject: [PATCH 121/478] [ci skip] --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 879d22ef..aecf1905 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ #Changelog +## 7.0.2 (2014-12-24) +### 修正 +* 内部http get当没有auth会出错 +* python3下的qiniupy 没有参数时 arg parse会抛异常 + ## 7.0.1 (2014-11-26) ### 增加 * setup.py从文件中读取版本号，而不是用导入方式 From d8fe2167c66c4bbd58c1776827473fe4ac986895 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Thu, 25 Dec 2014 00:34:34 +0800 Subject: [PATCH 122/478] update version [ci skip] --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index d92e3979..c11ea48c 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.0.1' +__version__ = '7.0.2' from .auth import Auth From 8721adba6a65d8a1c23fffa5d61984c58e87d099 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Thu, 25 Dec 2014 01:22:23 +0800 Subject: [PATCH 123/478] add callback feature --- qiniu/auth.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index 769497c3..8df392b4 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -14,6 +14,8 @@ 'callbackUrl', 'callbackBody', 'callbackHost', + 'callbackBodyType', + 'callbackFetchKey', 'returnUrl', 'returnBody', @@ -67,6 +69,7 @@ def token_of_request(self, url, body=None, content_type=None): if body: mimes = [ 'application/x-www-form-urlencoded', + 'application/json' ] if content_type in mimes: data += body @@ -115,8 +118,8 @@ def __upload_token(self, policy): data = json.dumps(policy, separators=(',', ':')) return self.token_with_data(data) - def verify_callback(self, origin_authorization, url, body): - token = self.token_of_request(url, body, 'application/x-www-form-urlencoded') + def verify_callback(self, origin_authorization, url, body, content_type='application/x-www-form-urlencoded'): + token = self.token_of_request(url, body, content_type) authorization = 'QBox {0}'.format(token) return origin_authorization == authorization From 5df62f47358b6e0e07de542659d2083886c91f81 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Thu, 25 Dec 2014 10:23:38 +0800 Subject: [PATCH 124/478] [ci skip] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aecf1905..33a16499 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### 修正 * 内部http get当没有auth会出错 * python3下的qiniupy 没有参数时 arg parse会抛异常 +* 增加callback policy ## 7.0.1 (2014-11-26) ### 增加 From 6a8d599f959326ffecc8a03ea8a3d97be8d1f801 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Tue, 3 Feb 2015 15:02:25 +0800 Subject: [PATCH 125/478] [ci skip] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 59b436dd..5aaa4ded 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Qiniu Python SDK -[![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE.md) +[![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE) [![Build Status](https://travis-ci.org/qiniu/python-sdk.svg)](https://travis-ci.org/qiniu/python-sdk) [![Latest Stable Version](https://img.shields.io/pypi/v/qiniu.svg)](https://pypi.python.org/pypi/qiniu) [![Download Times](https://img.shields.io/pypi/dm/qiniu.svg)](https://pypi.python.org/pypi/qiniu) From e52927cfde06b0e9fb149ed7938e8846da2a1ecd Mon Sep 17 00:00:00 2001 From: Bai Long Date: Tue, 3 Feb 2015 19:57:33 +0800 Subject: [PATCH 126/478] [ci skip] --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index f531c328..c18cc05d 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -186,7 +186,7 @@ def test_batch_move(self): def test_batch_rename(self): key = 'rename'+rand_string(8) self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) - key2 = key2 = key + 'rename' + key2 = key + 'rename' ops = build_batch_move(bucket_name, {key: key2}, bucket_name) ret, info = self.bucket.batch(ops) print(info) From 87ba61057d10da86c2678e86e13c172e8525c505 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Tue, 3 Feb 2015 19:58:32 +0800 Subject: [PATCH 127/478] [ci skip] --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5aaa4ded..b1462260 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# Qiniu Python SDK +# Qiniu Resource Storage SDK for Python + +[![@qiniu on weibo](http://img.shields.io/badge/weibo-%40qiniutek-blue.svg)](http://weibo.com/qiniutek) [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE) [![Build Status](https://travis-ci.org/qiniu/python-sdk.svg)](https://travis-ci.org/qiniu/python-sdk) [![Latest Stable Version](https://img.shields.io/pypi/v/qiniu.svg)](https://pypi.python.org/pypi/qiniu) From 84f261986990ff4bb5dd9aaad0345d7b6e6d51b3 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 4 Feb 2015 02:15:24 +0800 Subject: [PATCH 128/478] add comment for 7.x --- qiniu/auth.py | 94 ++++++++++++++++++++++-------- qiniu/config.py | 26 ++++----- qiniu/http.py | 13 +++++ qiniu/services/processing/pfop.py | 23 ++++++++ qiniu/services/storage/bucket.py | 40 ++++++++++--- qiniu/services/storage/uploader.py | 53 +++++++++++++++-- qiniu/utils.py | 83 ++++++++++++++++++++++++++ 7 files changed, 285 insertions(+), 47 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index 8df392b4..9bd5d6f5 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -10,27 +10,29 @@ from .utils import urlsafe_base64_encode +# 上传策略，参数规格详见 +# http://developer.qiniu.com/docs/v6/api/reference/security/put-policy.html _policy_fields = set([ - 'callbackUrl', - 'callbackBody', - 'callbackHost', - 'callbackBodyType', - 'callbackFetchKey', - - 'returnUrl', - 'returnBody', - - 'endUser', - 'saveKey', - 'insertOnly', - - 'detectMime', - 'mimeLimit', - 'fsizeLimit', - - 'persistentOps', - 'persistentNotifyUrl', - 'persistentPipeline', + 'callbackUrl', # 回调URL + 'callbackBody', # 回调Body + 'callbackHost', # 回调URL指定的Host + 'callbackBodyType', # 回调Body的Content-Type + 'callbackFetchKey', # 回调FetchKey模式开关 + + 'returnUrl', # 上传端的303跳转URL + 'returnBody', # 上传端简单反馈获取的Body + + 'endUser', # 回调时上传端标识 + 'saveKey', # 自定义资源名 + 'insertOnly', # 插入模式开关 + + 'detectMime', # MimeType侦测开关 + 'mimeLimit', # MimeType限制 + 'fsizeLimit', # 上传文件大小限制 + + 'persistentOps', # 持久化处理操作 + 'persistentNotifyUrl', # 持久化处理结果通知URL + 'persistentPipeline', # 持久化处理独享队列 ]) _deprecated_policy_fields = set([ @@ -39,8 +41,17 @@ class Auth(object): + """七牛安全机制类 + + 该类主要内容是七牛上传凭证、下载凭证、管理凭证三种凭证的签名接口的实现，以及回调验证。 + + Attributes: + __access_key: 账号密钥对中的accessKey，详见 https://portal.qiniu.com/setting/key + __secret_key: 账号密钥对重的secretKey，详见 https://portal.qiniu.com/setting/key + """ def __init__(self, access_key, secret_key): + """初始化Auth类""" self.__checkKey(access_key, secret_key) self.__access_key, self.__secret_key = access_key, secret_key self.__secret_key = b(self.__secret_key) @@ -58,6 +69,16 @@ def token_with_data(self, data): return '{0}:{1}:{2}'.format(self.__access_key, self.__token(data), data) def token_of_request(self, url, body=None, content_type=None): + """带请求体的签名（本质上是管理凭证的签名） + + Args: + url: 待签名请求的url + body: 待签名请求的body + content_type: 待签名请求的body的Content-Type + + Returns: + 管理凭证 + """ parsed_url = urlparse(url) query = parsed_url.query path = parsed_url.path @@ -82,10 +103,15 @@ def __checkKey(access_key, secret_key): raise ValueError('invalid key') def private_download_url(self, url, expires=3600): - ''' - * return private url - ''' + """生成私有资源下载链接 + + Args: + url: 私有空间资源的原始URL + expires: 下载凭证有效期，默认为3600s + Returns: + 私有资源的下载链接 + """ deadline = int(time.time()) + expires if '?' in url: url += '&' @@ -97,6 +123,17 @@ def private_download_url(self, url, expires=3600): return '{0}&token={1}'.format(url, token) def upload_token(self, bucket, key=None, expires=3600, policy=None, strict_policy=True): + """生成上传凭证 + + Args: + bucket: 上传的空间名 + key: 上传的文件名，默认为空 + expires: 上传凭证的过期时间，默认为3600s + policy: 上传策略，默认为空 + + Returns: + 上传凭证 + """ if bucket is None or bucket == '': raise ValueError('invalid bucket name') @@ -119,6 +156,17 @@ def __upload_token(self, policy): return self.token_with_data(data) def verify_callback(self, origin_authorization, url, body, content_type='application/x-www-form-urlencoded'): + """回调验证 + + Args: + origin_authorization: 回调时请求Header中的Authorization字段 + url: 回调请求的url + body: 回调请求的body + content_type: 回调请求body的Content-Type + + Returns: + 返回true表示验证成功，返回false表示验证失败 + """ token = self.token_of_request(url, body, content_type) authorization = 'QBox {0}'.format(token) return origin_authorization == authorization diff --git a/qiniu/config.py b/qiniu/config.py index 323a5b6f..39b7ea2c 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -1,23 +1,23 @@ # -*- coding: utf-8 -*- -RS_HOST = 'rs.qbox.me' -IO_HOST = 'iovip.qbox.me' -RSF_HOST = 'rsf.qbox.me' -API_HOST = 'api.qiniu.com' +RS_HOST = 'rs.qbox.me' # 管理操作Host +IO_HOST = 'iovip.qbox.me' # 七牛源站Host +RSF_HOST = 'rsf.qbox.me' # 列举操作Host +API_HOST = 'api.qiniu.com' # 数据处理操作Host -UPAUTO_HOST = 'up.qiniu.com' -UPDX_HOST = 'updx.qiniu.com' -UPLT_HOST = 'uplt.qiniu.com' -UPBACKUP_HOST = 'upload.qiniu.com' +UPAUTO_HOST = 'up.qiniu.com' # 默认上传Host +UPDX_HOST = 'updx.qiniu.com' # 电信上传Host +UPLT_HOST = 'uplt.qiniu.com' # 移动上传Host +UPBACKUP_HOST = 'upload.qiniu.com' # 备用上传Host _config = { - 'default_up_host': UPAUTO_HOST, - 'connection_timeout': 30, - 'connection_retries': 3, - 'connection_pool': 10, + 'default_up_host': UPAUTO_HOST, # 设置为默认上传Host + 'connection_timeout': 30, # 链接超时为时间为30s + 'connection_retries': 3, # 链接重试次数为3次 + 'connection_pool': 10, # 链接池个数为10 } -_BLOCK_SIZE = 1024 * 1024 * 4 +_BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 def get_default(key): diff --git a/qiniu/http.py b/qiniu/http.py index 1c9fecca..6fc898ca 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -78,7 +78,20 @@ def _post_with_auth(url, data, auth): class ResponseInfo(object): + """七牛HTTP请求返回信息类 + + 该类主要是用于获取和解析对七牛发起各种请求后的响应包的header和body。 + + Attributes: + status_code: 整数变量，响应状态码 + text_body: 字符串变量，响应的body + req_id: 字符串变量，七牛HTTP扩展字段，参考 http://developer.qiniu.com/docs/v6/api/reference/extended-headers.html + x_log: 字符串变量，七牛HTTP扩展字段，参考 http://developer.qiniu.com/docs/v6/api/reference/extended-headers.html + error: 字符串变量，响应的错误内容 + """ + def __init__(self, response, exception=None): + """用响应包和异常信息初始化ResponseInfo类""" self.__response = response self.exception = exception if response is None: diff --git a/qiniu/services/processing/pfop.py b/qiniu/services/processing/pfop.py index e5429b4d..2bbebd52 100644 --- a/qiniu/services/processing/pfop.py +++ b/qiniu/services/processing/pfop.py @@ -5,14 +5,37 @@ class PersistentFop(object): + """持久化处理类 + + 该类用于主动触发异步持久化操作，具体规格参考： + http://developer.qiniu.com/docs/v6/api/reference/fop/pfop/pfop.html + + Attributes: + auth: 账号管理密钥对，Auth对象 + bucket: 操作资源所在空间 + pipeline: 多媒体处理队列，详见 https://portal.qiniu.com/mps/pipeline + notify_url: 持久化处理结果通知URL + """ def __init__(self, auth, bucket, pipeline=None, notify_url=None): + """初始化持久化处理类""" self.auth = auth self.bucket = bucket self.pipeline = pipeline self.notify_url = notify_url def execute(self, key, fops, force=None): + """执行持久化处理: + + Args: + key: 待处理的源文件 + fops: 处理详细操作，规格详见 http://developer.qiniu.com/docs/v6/api/reference/fop/ + force: 强制执行持久化处理开关 + + Returns: + 一个json串，返回持久化处理的persistentId，类似{"persistentId": 5476bedf7823de4068253bae}; + 一个包含响应头部信息的字符串。 + """ ops = ';'.join(fops) data = {'bucket': self.bucket, 'key': key, 'fops': ops} if self.pipeline: diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 5fe847de..abaa6370 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -6,22 +6,38 @@ class BucketManager(object): + """空间管理类 + + 主要涉及了空间资源管理及批量操作接口的实现，具体的接口规格可以参考： + http://developer.qiniu.com/docs/v6/api/reference/rs/ + + Attributes: + auth: 账号管理密钥对，Auth对象 + """ def __init__(self, auth): self.auth = auth def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): """前缀查询: - * bucket => str - * prefix => str - * marker => str - * limit => int - * delimiter => str - * return ret => {'items': items, 'marker': markerOut}, err => str 1. 首次请求 marker = None 2. 无论 err 值如何，均应该先看 ret.get('items') 是否有内容 - 3. 如果后续没有更多数据，err 返回 EOF，markerOut 返回 None（但不通过该特征来判断是否结束） + 3. 如果后续没有更多数据，err 返回 EOF，marker 返回 None（但不通过该特征来判断是否结束） + 具体规格参考: + http://developer.qiniu.com/docs/v6/api/reference/rs/list.html + + Args: + bucket: 空间名 + prefix: 列举前缀 + marker: 列举标识符 + limit: 单次列举个数限制 + delimiter: 指定目录分隔符 + + Returns: + 一个json串，内容详见list接口返回的items。 + 一个包含响应头部信息的字符串。 + 一个EOF信息。 """ options = { 'bucket': bucket, @@ -45,45 +61,55 @@ def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): return ret, eof, info def stat(self, bucket, key): + """获取文件信息""" resource = entry(bucket, key) return self.__rs_do('stat', resource) def delete(self, bucket, key): + """删除文件""" resource = entry(bucket, key) return self.__rs_do('delete', resource) def rename(self, bucket, key, key_to): + """重命名文件""" return self.move(bucket, key, bucket, key_to) def move(self, bucket, key, bucket_to, key_to): + """移动文件""" resource = entry(bucket, key) to = entry(bucket_to, key_to) return self.__rs_do('move', resource, to) def copy(self, bucket, key, bucket_to, key_to): + """复制文件""" resource = entry(bucket, key) to = entry(bucket_to, key_to) return self.__rs_do('copy', resource, to) def fetch(self, url, bucket, key): + """抓取文件""" resource = urlsafe_base64_encode(url) to = entry(bucket, key) return self.__io_do('fetch', resource, 'to/{0}'.format(to)) def prefetch(self, bucket, key): + """镜像回源预取文件""" resource = entry(bucket, key) return self.__io_do('prefetch', resource) def change_mime(self, bucket, key, mime): + """修改文件mimeType""" resource = entry(bucket, key) encode_mime = urlsafe_base64_encode(mime) return self.__rs_do('chgm', resource, 'mime/{0}'.format(encode_mime)) def batch(self, operations): + """批量操作""" url = 'http://{0}/batch'.format(config.RS_HOST) return self.__post(url, dict(op=operations)) def buckets(self): + """获取所有空间名""" return self.__rs_do('buckets') def __rs_do(self, operation, *args): diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 390184ed..00dd19f5 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -9,15 +9,41 @@ def put_data( up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None): - ''' put data to Qiniu - If key is None, the server will generate one. - data may be str or read()able object. - ''' + """上传二进制流到七牛 + + Args: + up_token: 上传凭证 + key: 上传文件名 + data: 上传二进制流 + params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + mime_type: 上传数据的mimeType + check_crc: 是否校验crc32 + progress_handler: 上传进度 + + Returns: + 一个json串，类似 {"hash": "", "key": ""}; + 一个包含响应头部信息的字符串。 + """ crc = crc32(data) if check_crc else None return _form_put(up_token, key, data, params, mime_type, crc, False, progress_handler) def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None): + """上传文件到七牛 + + Args: + up_token: 上传凭证 + key: 上传文件名 + file_path: 上传文件的路径 + params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + mime_type: 上传数据的mimeType + check_crc: 是否校验crc32 + progress_handler: 上传进度 + + Returns: + 一个json串，类似 {"hash": "", "key": ""}; + 一个包含响应头部信息的字符串。 + """ ret = {} size = os.stat(file_path).st_size with open(file_path, 'rb') as input_stream: @@ -59,8 +85,24 @@ def put_stream(up_token, key, input_stream, data_size, params=None, mime_type=No class _Resume(object): + """断点续上传类 + + 该类主要实现了断点续上传中的分块上传，以及相应地创建块和创建文件过程，详细规格参考： + http://developer.qiniu.com/docs/v6/api/reference/up/mkblk.html + http://developer.qiniu.com/docs/v6/api/reference/up/mkfile.html + + Attributes: + up_token: 上传凭证 + key: 上传文件名 + input_stream: 上传二进制流 + data_size: 上传流大小 + params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + mime_type: 上传数据的mimeType + progress_handler: 上传进度 + """ def __init__(self, up_token, key, input_stream, data_size, params, mime_type, progress_handler): + """初始化断点续上传""" self.up_token = up_token self.key = key self.input_stream = input_stream @@ -70,6 +112,7 @@ def __init__(self, up_token, key, input_stream, data_size, params, mime_type, pr self.progress_handler = progress_handler def upload(self): + """上传操作""" self.blockStatus = [] host = config.get_default('default_up_host') for block in _file_iter(self.input_stream, config._BLOCK_SIZE): @@ -91,6 +134,7 @@ def upload(self): return self.make_file(host) def make_block(self, block, block_size, host): + """创建块""" url = self.block_url(host, block_size) return self.post(url, block) @@ -114,6 +158,7 @@ def file_url(self, host): return url def make_file(self, host): + """创建文件""" url = self.file_url(host) body = ','.join([status['ctx'] for status in self.blockStatus]) return self.post(url, body) diff --git a/qiniu/utils.py b/qiniu/utils.py index 14bf1936..78eddcba 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -16,16 +16,45 @@ def urlsafe_base64_encode(data): + """urlsafe的base64编码: + + 对提供的数据进行urlsafe的base64编码。规格参考： + http://developer.qiniu.com/docs/v6/api/overview/appendix.html#urlsafe-base64 + + Args: + data: 待编码的数据，一般为字符串 + + Returns: + 编码后的字符串 + """ ret = urlsafe_b64encode(b(data)) return s(ret) def urlsafe_base64_decode(data): + """urlsafe的base64解码: + + 对提供的urlsafe的base64编码的数据进行解码 + + Args: + data: 待解码的数据，一般为字符串 + + Returns: + 解码后的字符串。 + """ ret = urlsafe_b64decode(s(data)) return ret def file_crc32(filePath): + """计算文件的crc32检验码: + + Args: + filePath: 待计算校验码的文件路径 + + Returns: + 文件内容的crc32校验码。 + """ crc = 0 with open(filePath, 'rb') as f: for block in _file_iter(f, _BLOCK_SIZE): @@ -34,10 +63,27 @@ def file_crc32(filePath): def crc32(data): + """计算输入流的crc32检验码: + + Args: + data: 待计算校验码的字符流 + + Returns: + 输入流的crc32校验码。 + """ return binascii.crc32(b(data)) & 0xffffffff def _file_iter(input_stream, size): + """读取输入流: + + Args: + input_stream: 待读取文件的二进制流 + size: 二进制流的大小 + + Raises: + IOError: 文件流读取失败 + """ d = input_stream.read(size) while d: yield d @@ -45,12 +91,30 @@ def _file_iter(input_stream, size): def _sha1(data): + """单块计算hash: + + Args: + data: 待计算hash的数据 + + Returns: + 输入数据计算的hash值 + """ h = sha1() h.update(data) return h.digest() def _etag(input_stream): + """计算输入流的etag: + + etag规格参考 http://developer.qiniu.com/docs/v6/api/overview/appendix.html#qiniu-etag + + Args: + input_stream: 待计算etag的二进制流 + + Returns: + 输入流的etag值 + """ array = [_sha1(block) for block in _file_iter(input_stream, _BLOCK_SIZE)] if len(array) == 1: data = array[0] @@ -63,9 +127,28 @@ def _etag(input_stream): def etag(filePath): + """计算文件的etag: + + Args: + filePath: 待计算etag的文件路径 + + Returns: + 输入文件的etag值 + """ with open(filePath, 'rb') as f: return _etag(f) def entry(bucket, key): + """计算七牛API中的数据格式: + + entry规格参考 http://developer.qiniu.com/docs/v6/api/reference/data-formats.html + + Args: + bucket: 待操作的空间名 + key: 待操作的文件名 + + Returns: + 符合七牛API规格的数据格式 + """ return urlsafe_base64_encode('{0}:{1}'.format(bucket, key)) From 10c3690688fbb92797784bf952d023f7f1700a16 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 4 Feb 2015 14:40:54 +0800 Subject: [PATCH 129/478] fix some desc --- qiniu/services/processing/pfop.py | 4 +- qiniu/services/storage/bucket.py | 154 ++++++++++++++++++++++++++--- qiniu/services/storage/uploader.py | 8 +- 3 files changed, 148 insertions(+), 18 deletions(-) diff --git a/qiniu/services/processing/pfop.py b/qiniu/services/processing/pfop.py index 2bbebd52..1ac32054 100644 --- a/qiniu/services/processing/pfop.py +++ b/qiniu/services/processing/pfop.py @@ -33,8 +33,8 @@ def execute(self, key, fops, force=None): force: 强制执行持久化处理开关 Returns: - 一个json串，返回持久化处理的persistentId，类似{"persistentId": 5476bedf7823de4068253bae}; - 一个包含响应头部信息的字符串。 + 一个dict变量，返回持久化处理的persistentId，类似{"persistentId": 5476bedf7823de4068253bae}; + 一个ReponseInfo对象 """ ops = ';'.join(fops) data = {'bucket': self.bucket, 'key': key, 'fops': ops} diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index abaa6370..c6d4e5e3 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -35,8 +35,8 @@ def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): delimiter: 指定目录分隔符 Returns: - 一个json串，内容详见list接口返回的items。 - 一个包含响应头部信息的字符串。 + 一个dict变量，类似 {"hash": "", "key": ""} + 一个ReponseInfo对象 一个EOF信息。 """ options = { @@ -61,55 +61,185 @@ def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): return ret, eof, info def stat(self, bucket, key): - """获取文件信息""" + """获取文件信息: + + 获取资源的元信息，但不返回文件内容，具体规格参考： + http://developer.qiniu.com/docs/v6/api/reference/rs/stat.html + + Args: + bucket: 待获取信息资源所在的空间 + key: 待获取资源的文件名 + + Returns: + 一个dict变量，类似： + { + "fsize": 5122935, + "hash": "ljfockr0lOil_bZfyaI2ZY78HWoH", + "mimeType": "application/octet-stream", + "putTime": 13603956734587420 + } + 一个ReponseInfo对象 + """ resource = entry(bucket, key) return self.__rs_do('stat', resource) def delete(self, bucket, key): - """删除文件""" + """删除文件: + + 删除指定资源，具体规格参考： + http://developer.qiniu.com/docs/v6/api/reference/rs/delete.html + + Args: + bucket: 待获取信息资源所在的空间 + key: 待获取资源的文件名 + + Returns: + 一个dict变量，成功返回NULL，失败返回{"error": ""} + 一个ReponseInfo对象 + """ resource = entry(bucket, key) return self.__rs_do('delete', resource) def rename(self, bucket, key, key_to): - """重命名文件""" + """重命名文件: + + 给资源进行重命名，本质为move操作。 + + Args: + bucket: 待操作资源所在空间 + key: 待操作资源文件名 + key_to: 目标资源文件名 + + Returns: + 一个dict变量，成功返回NULL，失败返回{"error": ""} + 一个ReponseInfo对象 + """ return self.move(bucket, key, bucket, key_to) def move(self, bucket, key, bucket_to, key_to): - """移动文件""" + """移动文件: + + 将资源从一个空间到另一个空间，具体规格参考： + http://developer.qiniu.com/docs/v6/api/reference/rs/move.html + + Args: + bucket: 待操作资源所在空间 + bucket_to: 目标资源空间名 + key: 待操作资源文件名 + key_to: 目标资源文件名 + + Returns: + 一个dict变量，成功返回NULL，失败返回{"error": ""} + 一个ReponseInfo对象 + """ resource = entry(bucket, key) to = entry(bucket_to, key_to) return self.__rs_do('move', resource, to) def copy(self, bucket, key, bucket_to, key_to): - """复制文件""" + """复制文件: + + 将指定资源复制为新命名资源，具体规格参考： + http://developer.qiniu.com/docs/v6/api/reference/rs/copy.html + + Args: + bucket: 待操作资源所在空间 + bucket_to: 目标资源空间名 + key: 待操作资源文件名 + key_to: 目标资源文件名 + + Returns: + 一个dict变量，成功返回NULL，失败返回{"error": ""} + 一个ReponseInfo对象 + """ resource = entry(bucket, key) to = entry(bucket_to, key_to) return self.__rs_do('copy', resource, to) def fetch(self, url, bucket, key): - """抓取文件""" + """抓取文件: + 从指定URL抓取资源，并将该资源存储到指定空间中，具体规格参考： + http://developer.qiniu.com/docs/v6/api/reference/rs/fetch.html + + Args: + url: 指定的URL + bucket: 目标资源空间 + key: 目标资源文件名 + + Returns: + 一个dict变量，成功返回NULL，失败返回{"error": ""} + 一个ReponseInfo对象 + """ resource = urlsafe_base64_encode(url) to = entry(bucket, key) return self.__io_do('fetch', resource, 'to/{0}'.format(to)) def prefetch(self, bucket, key): - """镜像回源预取文件""" + """镜像回源预取文件: + + 从镜像源站抓取资源到空间中，如果空间中已经存在，则覆盖该资源，具体规格参考 + http://developer.qiniu.com/docs/v6/api/reference/rs/prefetch.html + + Args: + bucket: 待获取资源所在的空间 + key: 代获取资源文件名 + + Returns: + 一个dict变量，成功返回NULL，失败返回{"error": ""} + 一个ReponseInfo对象 + """ resource = entry(bucket, key) return self.__io_do('prefetch', resource) def change_mime(self, bucket, key, mime): - """修改文件mimeType""" + """修改文件mimeType: + + 主动修改指定资源的文件类型，具体规格参考： + http://developer.qiniu.com/docs/v6/api/reference/rs/chgm.html + + Args: + bucket: 待操作资源所在空间 + key: 待操作资源文件名 + mime: 待操作文件目标mimeType + """ resource = entry(bucket, key) encode_mime = urlsafe_base64_encode(mime) return self.__rs_do('chgm', resource, 'mime/{0}'.format(encode_mime)) def batch(self, operations): - """批量操作""" + """批量操作: + + 在单次请求中进行多个资源管理操作，具体规格参考： + http://developer.qiniu.com/docs/v6/api/reference/rs/batch.html + + Args: + operations: 资源管理操作数组，可通过 + + Returns: + 一个dict变量，返回结果类似： + [ + { "code": , "data": }, + { "code": }, + { "code": }, + { "code": }, + { "code": , "data": { "error": "" } }, + ... + ] + 一个ReponseInfo对象 + """ url = 'http://{0}/batch'.format(config.RS_HOST) return self.__post(url, dict(op=operations)) def buckets(self): - """获取所有空间名""" + """获取所有空间名: + + 获取指定账号下所有的空间名。 + + Returns: + 一个dict变量，类似： + [ , , ... ] + 一个ReponseInfo对象 + """ return self.__rs_do('buckets') def __rs_do(self, operation, *args): diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 00dd19f5..5c3d528c 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -21,8 +21,8 @@ def put_data( progress_handler: 上传进度 Returns: - 一个json串，类似 {"hash": "", "key": ""}; - 一个包含响应头部信息的字符串。 + 一个dict变量，类似 {"hash": "", "key": ""} + 一个ReponseInfo对象 """ crc = crc32(data) if check_crc else None return _form_put(up_token, key, data, params, mime_type, crc, False, progress_handler) @@ -41,8 +41,8 @@ def put_file(up_token, key, file_path, params=None, mime_type='application/octet progress_handler: 上传进度 Returns: - 一个json串，类似 {"hash": "", "key": ""}; - 一个包含响应头部信息的字符串。 + 一个dict变量，类似 {"hash": "", "key": ""} + 一个ReponseInfo对象 """ ret = {} size = os.stat(file_path).st_size From 37fed6671093cb4eacceb90d86247200f4eee12d Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 4 Feb 2015 15:02:05 +0800 Subject: [PATCH 130/478] fix format --- qiniu/services/storage/bucket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index c6d4e5e3..32990860 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -91,7 +91,7 @@ def delete(self, bucket, key): Args: bucket: 待获取信息资源所在的空间 - key: 待获取资源的文件名 + key: 待获取资源的文件名 Returns: 一个dict变量，成功返回NULL，失败返回{"error": ""} @@ -162,7 +162,7 @@ def fetch(self, url, bucket, key): http://developer.qiniu.com/docs/v6/api/reference/rs/fetch.html Args: - url: 指定的URL + url: 指定的URL bucket: 目标资源空间 key: 目标资源文件名 From 14b0edd0580a36d447cfd13f2bc169bc13b34cf4 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 11 Mar 2015 13:12:55 +0800 Subject: [PATCH 131/478] add default config for io/rs/api host --- qiniu/config.py | 19 +++++++++++++++++-- qiniu/services/processing/pfop.py | 2 +- qiniu/services/storage/bucket.py | 8 ++++---- test_qiniu.py | 5 +++-- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/qiniu/config.py b/qiniu/config.py index 39b7ea2c..ed957348 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -7,11 +7,16 @@ UPAUTO_HOST = 'up.qiniu.com' # 默认上传Host UPDX_HOST = 'updx.qiniu.com' # 电信上传Host -UPLT_HOST = 'uplt.qiniu.com' # 移动上传Host +UPLT_HOST = 'uplt.qiniu.com' # 联通上传Host +UPYD_HOST = 'upyd.qiniu.com' # 移动上传Host UPBACKUP_HOST = 'upload.qiniu.com' # 备用上传Host _config = { 'default_up_host': UPAUTO_HOST, # 设置为默认上传Host + 'default_rs_host': RS_HOST, + 'default_io_host': IO_HOST, + 'default_rsf_host': RSF_HOST, + 'default_api_host': API_HOST, 'connection_timeout': 30, # 链接超时为时间为30s 'connection_retries': 3, # 链接重试次数为3次 'connection_pool': 10, # 链接池个数为10 @@ -25,9 +30,19 @@ def get_default(key): def set_default( - default_up_host=None, connection_retries=None, connection_pool=None, connection_timeout=None): + default_up_host=None, connection_retries=None, connection_pool=None, + connection_timeout=None, default_rs_host=None, default_io_host=None, + default_rsf_host=None, default_api_host=None): if default_up_host: _config['default_up_host'] = default_up_host + if default_rs_host: + _config['default_rs_host'] = default_rs_host + if default_io_host: + _config['default_io_host'] = default_io_host + if default_rsf_host: + _config['default_rsf_host'] = default_rsf_host + if default_api_host: + _config['default_api_host'] = default_api_host if connection_retries: _config['connection_retries'] = connection_retries if connection_pool: diff --git a/qiniu/services/processing/pfop.py b/qiniu/services/processing/pfop.py index 1ac32054..f24579e6 100644 --- a/qiniu/services/processing/pfop.py +++ b/qiniu/services/processing/pfop.py @@ -45,5 +45,5 @@ def execute(self, key, fops, force=None): if force == 1: data['force'] = 1 - url = 'http://{0}/pfop'.format(config.API_HOST) + url = 'http://{0}/pfop'.format(config.get_default('default_api_host')) return http._post_with_auth(url, data, self.auth) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 32990860..8bc30037 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -51,7 +51,7 @@ def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): if delimiter is not None: options['delimiter'] = delimiter - url = 'http://{0}/list'.format(config.RSF_HOST) + url = 'http://{0}/list'.format(config.get_default('default_rsf_host')) ret, info = self.__get(url, options) eof = False @@ -227,7 +227,7 @@ def batch(self, operations): ] 一个ReponseInfo对象 """ - url = 'http://{0}/batch'.format(config.RS_HOST) + url = 'http://{0}/batch'.format(config.get_default('default_rs_host')) return self.__post(url, dict(op=operations)) def buckets(self): @@ -243,10 +243,10 @@ def buckets(self): return self.__rs_do('buckets') def __rs_do(self, operation, *args): - return self.__server_do(config.RS_HOST, operation, *args) + return self.__server_do(config.get_default('default_rs_host'), operation, *args) def __io_do(self, operation, *args): - return self.__server_do(config.IO_HOST, operation, *args) + return self.__server_do(config.get_default('default_io_host'), operation, *args) def __server_do(self, host, operation, *args): cmd = _build_op(operation, *args) diff --git a/test_qiniu.py b/test_qiniu.py index c18cc05d..4bccaa46 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # flake8: noqa -import os +import os import string import random import tempfile @@ -307,6 +307,7 @@ def test_retry(self): assert ret['key'] == key qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + class DownloadTestCase(unittest.TestCase): q = Auth(access_key, secret_key) @@ -316,7 +317,7 @@ def test_private_url(self): private_key = 'gogopher.jpg' base_url = 'http://%s/%s' % (private_bucket+'.qiniudn.com', private_key) private_url = self.q.private_download_url(base_url, expires=3600) - print(private_url) + print(private_url) r = requests.get(private_url) assert r.status_code == 200 From 61091405cd282f633fc599b58d11020c03a3a95b Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 11 Mar 2015 13:14:17 +0800 Subject: [PATCH 132/478] version --- CHANGELOG.md | 4 ++++ qiniu/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33a16499..a48367db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ #Changelog +## 7.0.3 (2014-03-11) +### 增加 +* 可以配置 io/rs/api/rsf host + ## 7.0.2 (2014-12-24) ### 修正 * 内部http get当没有auth会出错 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index c11ea48c..824a0f37 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.0.2' +__version__ = '7.0.3' from .auth import Auth From f7545db417dfc0dcb11e040ac53ac3e9ed1a7562 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Fri, 20 Mar 2015 23:18:41 +0800 Subject: [PATCH 133/478] add return to responseinfo --- qiniu/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/http.py b/qiniu/http.py index 6fc898ca..20e880be 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -113,7 +113,7 @@ def __init__(self, response, exception=None): self.error = ret['error'] def ok(self): - self.status_code == 200 + return self.status_code == 200 def need_retry(self): if self.__response is None: From cb8d84580feab200a47a3392e654eef249794631 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Tue, 21 Apr 2015 23:18:08 +0800 Subject: [PATCH 134/478] Reponse => Response --- qiniu/services/processing/pfop.py | 2 +- qiniu/services/storage/bucket.py | 20 ++++++++++---------- qiniu/services/storage/uploader.py | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/qiniu/services/processing/pfop.py b/qiniu/services/processing/pfop.py index f24579e6..a787cb7e 100644 --- a/qiniu/services/processing/pfop.py +++ b/qiniu/services/processing/pfop.py @@ -34,7 +34,7 @@ def execute(self, key, fops, force=None): Returns: 一个dict变量，返回持久化处理的persistentId，类似{"persistentId": 5476bedf7823de4068253bae}; - 一个ReponseInfo对象 + 一个ResponseInfo对象 """ ops = ';'.join(fops) data = {'bucket': self.bucket, 'key': key, 'fops': ops} diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 8bc30037..a73b98f5 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -36,7 +36,7 @@ def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): Returns: 一个dict变量，类似 {"hash": "", "key": ""} - 一个ReponseInfo对象 + 一个ResponseInfo对象 一个EOF信息。 """ options = { @@ -78,7 +78,7 @@ def stat(self, bucket, key): "mimeType": "application/octet-stream", "putTime": 13603956734587420 } - 一个ReponseInfo对象 + 一个ResponseInfo对象 """ resource = entry(bucket, key) return self.__rs_do('stat', resource) @@ -95,7 +95,7 @@ def delete(self, bucket, key): Returns: 一个dict变量，成功返回NULL，失败返回{"error": ""} - 一个ReponseInfo对象 + 一个ResponseInfo对象 """ resource = entry(bucket, key) return self.__rs_do('delete', resource) @@ -112,7 +112,7 @@ def rename(self, bucket, key, key_to): Returns: 一个dict变量，成功返回NULL，失败返回{"error": ""} - 一个ReponseInfo对象 + 一个ResponseInfo对象 """ return self.move(bucket, key, bucket, key_to) @@ -130,7 +130,7 @@ def move(self, bucket, key, bucket_to, key_to): Returns: 一个dict变量，成功返回NULL，失败返回{"error": ""} - 一个ReponseInfo对象 + 一个ResponseInfo对象 """ resource = entry(bucket, key) to = entry(bucket_to, key_to) @@ -150,7 +150,7 @@ def copy(self, bucket, key, bucket_to, key_to): Returns: 一个dict变量，成功返回NULL，失败返回{"error": ""} - 一个ReponseInfo对象 + 一个ResponseInfo对象 """ resource = entry(bucket, key) to = entry(bucket_to, key_to) @@ -168,7 +168,7 @@ def fetch(self, url, bucket, key): Returns: 一个dict变量，成功返回NULL，失败返回{"error": ""} - 一个ReponseInfo对象 + 一个ResponseInfo对象 """ resource = urlsafe_base64_encode(url) to = entry(bucket, key) @@ -186,7 +186,7 @@ def prefetch(self, bucket, key): Returns: 一个dict变量，成功返回NULL，失败返回{"error": ""} - 一个ReponseInfo对象 + 一个ResponseInfo对象 """ resource = entry(bucket, key) return self.__io_do('prefetch', resource) @@ -225,7 +225,7 @@ def batch(self, operations): { "code": , "data": { "error": "" } }, ... ] - 一个ReponseInfo对象 + 一个ResponseInfo对象 """ url = 'http://{0}/batch'.format(config.get_default('default_rs_host')) return self.__post(url, dict(op=operations)) @@ -238,7 +238,7 @@ def buckets(self): Returns: 一个dict变量，类似： [ , , ... ] - 一个ReponseInfo对象 + 一个ResponseInfo对象 """ return self.__rs_do('buckets') diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 5c3d528c..81019189 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -22,7 +22,7 @@ def put_data( Returns: 一个dict变量，类似 {"hash": "", "key": ""} - 一个ReponseInfo对象 + 一个ResponseInfo对象 """ crc = crc32(data) if check_crc else None return _form_put(up_token, key, data, params, mime_type, crc, False, progress_handler) @@ -42,7 +42,7 @@ def put_file(up_token, key, file_path, params=None, mime_type='application/octet Returns: 一个dict变量，类似 {"hash": "", "key": ""} - 一个ReponseInfo对象 + 一个ResponseInfo对象 """ ret = {} size = os.stat(file_path).st_size From e609b191bd1f8b11605133caf7eb4ad5ef3c3a0c Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Tue, 21 Apr 2015 23:27:34 +0800 Subject: [PATCH 135/478] =?UTF-8?q?=E5=8F=AA=E6=9C=89application/x-www-for?= =?UTF-8?q?m-urlencoded=E7=9A=84=E5=9B=9E=E8=B0=83=E5=B8=A6body=E7=AD=BE?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/auth.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index 9bd5d6f5..4864499d 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -89,8 +89,7 @@ def token_of_request(self, url, body=None, content_type=None): if body: mimes = [ - 'application/x-www-form-urlencoded', - 'application/json' + 'application/x-www-form-urlencoded' ] if content_type in mimes: data += body From bd474dcdae4c188ce4f6cd1556f2b35c16ad4ec2 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 22 Apr 2015 13:25:00 +0800 Subject: [PATCH 136/478] update qiniu_test --- test_qiniu.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index 4bccaa46..b8bf73e4 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -122,7 +122,8 @@ def test_prefetch(self): def test_fetch(self): ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, 'fetch.html') print(info) - assert ret == {} + assert ret['key'] == 'fetch.html' + assert ret['hash'] == 'FhwVT7vs6xqs1nu_vEdo_4x4qBMB' def test_stat(self): ret, info = self.bucket.stat(bucket_name, 'python-sdk.html') @@ -256,7 +257,7 @@ def test_putWithoutKey(self): ret, info = put_data(token, None, data) print(info) assert ret is None - assert info.status_code == 401 # key not match + assert info.status_code == 403 # key not match def test_retry(self): key = 'retry' From e66f738104d5119205ac75a3170cd62f7d4154cf Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Fri, 24 Apr 2015 01:09:27 +0800 Subject: [PATCH 137/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9put=5Fdata=E9=87=8D?= =?UTF-8?q?=E8=AF=95=E4=B8=BA=E4=B8=8A=E4=BC=A0=E4=B8=BA=E7=A9=BA=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- test_qiniu.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 81019189..46cc2572 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -72,7 +72,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, is_file=False, progre if r is None and info.need_retry(): if info.connect_failed: url = 'http://' + config.UPBACKUP_HOST + '/' - if is_file: + if hasattr(data, 'seek'): data.seek(0) r, info = http._post_file(url, data=fields, files={'file': (name, data, mime_type)}) diff --git a/test_qiniu.py b/test_qiniu.py index b8bf73e4..ef23ebeb 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -267,6 +267,7 @@ def test_retry(self): ret, info = put_data(token, key, data) print(info) assert ret['key'] == key + assert ret['hash'] == 'FlYu0iBR1WpvYi4whKXiBuQpyLLk' qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) @@ -306,6 +307,7 @@ def test_retry(self): ret, info = put_file(token, key, localfile, self.params, self.mime_type) print(info) assert ret['key'] == key + assert ret['hash'] == etag(localfile) qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) From d0a7387eccb6782ef77b50985251e168d4f69edb Mon Sep 17 00:00:00 2001 From: Bai Long Date: Fri, 24 Apr 2015 13:29:23 +0800 Subject: [PATCH 138/478] [ci skip] docker --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index f1b885e3..b701a2ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,6 @@ +# using docker container +sudo: false + language: python python: - "2.6" From f4ce21e81717cf7a1fa676bd89aa861f1db2a44b Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Wed, 29 Apr 2015 23:43:00 +0800 Subject: [PATCH 139/478] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E9=87=8D=E8=AF=95=E4=B8=BA=E7=A9=BA=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 6 +++++- test_qiniu.py | 24 +++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 46cc2572..8c07291d 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -72,8 +72,12 @@ def _form_put(up_token, key, data, params, mime_type, crc, is_file=False, progre if r is None and info.need_retry(): if info.connect_failed: url = 'http://' + config.UPBACKUP_HOST + '/' - if hasattr(data, 'seek'): + if hasattr(data, 'read') is False: + pass + elif hasattr(data, 'seek'): data.seek(0) + else: + return r, info r, info = http._post_file(url, data=fields, files={'file': (name, data, mime_type)}) return r, info diff --git a/test_qiniu.py b/test_qiniu.py index ef23ebeb..a4987fde 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -259,7 +259,7 @@ def test_putWithoutKey(self): assert ret is None assert info.status_code == 403 # key not match - def test_retry(self): + def test_withoutRead_withoutSeek_retry(self): key = 'retry' data = 'hello retry!' set_default(default_up_host='a') @@ -270,6 +270,28 @@ def test_retry(self): assert ret['hash'] == 'FlYu0iBR1WpvYi4whKXiBuQpyLLk' qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + def test_hasRead_hasSeek_retry(self): + key = 'withReadAndSeek_retry' + data = StringIO.StringIO('hello retry again!') + set_default(default_up_host='a') + token = self.q.upload_token(bucket_name) + ret, info = put_data(token, key, data) + print(info) + assert ret['key'] == key + assert ret['hash'] == 'FuEbdt6JP2BqwQJi7PezYhmuVYOo' + qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + + def test_hasRead_withoutSeek_retry(self): + key = 'withReadAndWithoutSeek_retry' + import urllib2 + data = urllib2.urlopen('http://pythonsdk.qiniudn.com/python-sdk.html') + set_default(default_up_host='a') + token = self.q.upload_token(bucket_name) + ret, info = put_data(token, key, data) + print(info) + assert ret == None + qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + class ResumableUploaderTestCase(unittest.TestCase): From ae6b32f36328b1e71a18f40997a795e0e1593727 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Thu, 30 Apr 2015 01:35:18 +0800 Subject: [PATCH 140/478] =?UTF-8?q?=E4=BF=AE=E6=AD=A3python=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E5=85=BC=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index a4987fde..dd0dc414 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -14,7 +14,7 @@ from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, build_batch_delete from qiniu import urlsafe_base64_encode, urlsafe_base64_decode -from qiniu.compat import is_py2, b +from qiniu.compat import is_py2, is_py3, b from qiniu.services.storage.uploader import _form_put @@ -22,8 +22,13 @@ if is_py2: import sys + import StringIO reload(sys) sys.setdefaultencoding('utf-8') + StringIO = StringIO.StringIO +elif is_py3: + import io + StringIO = io.StringIO access_key = os.getenv('QINIU_ACCESS_KEY') secret_key = os.getenv('QINIU_SECRET_KEY') @@ -272,7 +277,7 @@ def test_withoutRead_withoutSeek_retry(self): def test_hasRead_hasSeek_retry(self): key = 'withReadAndSeek_retry' - data = StringIO.StringIO('hello retry again!') + data = StringIO('hello retry again!') set_default(default_up_host='a') token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) @@ -283,8 +288,7 @@ def test_hasRead_hasSeek_retry(self): def test_hasRead_withoutSeek_retry(self): key = 'withReadAndWithoutSeek_retry' - import urllib2 - data = urllib2.urlopen('http://pythonsdk.qiniudn.com/python-sdk.html') + data = ReadWithoutSeek('I only have read attribute!') set_default(default_up_host='a') token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) @@ -358,5 +362,14 @@ def test_pfop(self): print(info) assert ret['persistentId'] is not None + +class ReadWithoutSeek(object): + def __init__(self, str): + self.str = str + pass + + def read(self): + print(self.str) + if __name__ == '__main__': unittest.main() From a66739e1b582e9a84685503e2d6cb360b2bfa695 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Sun, 3 May 2015 23:34:51 +0800 Subject: [PATCH 141/478] =?UTF-8?q?fix=20urllib=E5=9C=A8python3=E7=9A=84?= =?UTF-8?q?=E5=85=BC=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/http.py | 4 ++-- qiniu/services/storage/uploader.py | 8 ++++---- test_qiniu.py | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/qiniu/http.py b/qiniu/http.py index 20e880be..38c566de 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -103,8 +103,8 @@ def __init__(self, response, exception=None): else: self.status_code = response.status_code self.text_body = response.text - self.req_id = response.headers['X-Reqid'] - self.x_log = response.headers['X-Log'] + self.req_id = response.headers.get('X-Reqid') + self.x_log = response.headers.get('X-Log') if self.status_code >= 400: ret = response.json() if response.text != '' else None if ret is None or ret['error'] is None: diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 8c07291d..42942f17 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -25,7 +25,7 @@ def put_data( 一个ResponseInfo对象 """ crc = crc32(data) if check_crc else None - return _form_put(up_token, key, data, params, mime_type, crc, False, progress_handler) + return _form_put(up_token, key, data, params, mime_type, crc, progress_handler) def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None): @@ -51,11 +51,11 @@ def put_file(up_token, key, file_path, params=None, mime_type='application/octet ret, info = put_stream(up_token, key, input_stream, size, params, mime_type, progress_handler) else: crc = file_crc32(file_path) if check_crc else None - ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, True, progress_handler) + ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler) return ret, info -def _form_put(up_token, key, data, params, mime_type, crc, is_file=False, progress_handler=None): +def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None): fields = {} if params: for k, v in params.items(): @@ -74,7 +74,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, is_file=False, progre url = 'http://' + config.UPBACKUP_HOST + '/' if hasattr(data, 'read') is False: pass - elif hasattr(data, 'seek'): + elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()): data.seek(0) else: return r, info diff --git a/test_qiniu.py b/test_qiniu.py index dd0dc414..fab9d6ea 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -23,12 +23,16 @@ if is_py2: import sys import StringIO + import urllib reload(sys) sys.setdefaultencoding('utf-8') StringIO = StringIO.StringIO + urlopen = urllib.urlopen elif is_py3: import io + import urllib StringIO = io.StringIO + urlopen = urllib.request.urlopen access_key = os.getenv('QINIU_ACCESS_KEY') secret_key = os.getenv('QINIU_SECRET_KEY') @@ -296,6 +300,16 @@ def test_hasRead_withoutSeek_retry(self): assert ret == None qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + def test_hasRead_WithoutSeek_retry2(self): + key = 'withReadAndWithoutSeek_retry2' + data = urlopen("http://www.qiniu.com") + set_default(default_up_host='a') + token = self.q.upload_token(bucket_name) + ret, info = put_data(token, key, data) + print(info) + assert ret == None + qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + class ResumableUploaderTestCase(unittest.TestCase): From 453671d6c1f1c685dd016796b494edb45a1c6a76 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Mon, 4 May 2015 20:30:54 +0800 Subject: [PATCH 142/478] [ci skip] changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a48367db..43067027 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ #Changelog +## 7.0.4 (2014-05-04) +### 修正 +* 上传重试为空文件 +* 回调应该只对form data 签名。 + + ## 7.0.3 (2014-03-11) ### 增加 * 可以配置 io/rs/api/rsf host From f93667141b0befbf798b5a0cdf48ad66edce7882 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Mon, 4 May 2015 20:31:34 +0800 Subject: [PATCH 143/478] [ci skip]version num --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 824a0f37..03bf72ea 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.0.3' +__version__ = '7.0.4' from .auth import Auth From d182fa9af09cb6c85c620a469d76dd33017dfeae Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Fri, 22 May 2015 22:03:05 +0800 Subject: [PATCH 144/478] update entry construction --- qiniu/utils.py | 5 ++++- test_qiniu.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/qiniu/utils.py b/qiniu/utils.py index 78eddcba..1b992fa9 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -151,4 +151,7 @@ def entry(bucket, key): Returns: 符合七牛API规格的数据格式 """ - return urlsafe_base64_encode('{0}:{1}'.format(bucket, key)) + if key is None: + return urlsafe_base64_encode('{0}'.format(bucket)) + else: + return urlsafe_base64_encode('{0}:{1}'.format(bucket, key)) diff --git a/test_qiniu.py b/test_qiniu.py index b8bf73e4..99655f69 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -125,6 +125,12 @@ def test_fetch(self): assert ret['key'] == 'fetch.html' assert ret['hash'] == 'FhwVT7vs6xqs1nu_vEdo_4x4qBMB' + def test_fetch_without_key(self): + ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name) + print(info) + assert ret['key'] == 'FhwVT7vs6xqs1nu_vEdo_4x4qBMB' + assert ret['hash'] == 'FhwVT7vs6xqs1nu_vEdo_4x4qBMB' + def test_stat(self): ret, info = self.bucket.stat(bucket_name, 'python-sdk.html') print(info) From 2d67ae898e68f17480033efe170381c39f7b9445 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Fri, 22 May 2015 22:11:07 +0800 Subject: [PATCH 145/478] fix --- qiniu/services/storage/bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index a73b98f5..86e73569 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -156,7 +156,7 @@ def copy(self, bucket, key, bucket_to, key_to): to = entry(bucket_to, key_to) return self.__rs_do('copy', resource, to) - def fetch(self, url, bucket, key): + def fetch(self, url, bucket, key=None): """抓取文件: 从指定URL抓取资源，并将该资源存储到指定空间中，具体规格参考： http://developer.qiniu.com/docs/v6/api/reference/rs/fetch.html From 5f4e4607029a75c8d886c7c98fbe065c48edc433 Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 25 Jun 2015 23:08:51 +0800 Subject: [PATCH 146/478] add zone --- CHANGELOG.md | 12 +++++++--- qiniu/__init__.py | 4 ++-- qiniu/auth.py | 4 ++-- qiniu/config.py | 36 +++++++++++++++++++++--------- qiniu/services/storage/uploader.py | 4 ++-- test_qiniu.py | 36 +++++++++++++++--------------- 6 files changed, 58 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43067027..bb2b671b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,18 @@ #Changelog -## 7.0.4 (2014-05-04) +## 7.0.5 (2015-06-25) +### 变更 +* 配置up_host 改为配置zone + +### 增加 +* fectch 支持不指定key + +## 7.0.4 (2015-05-04) ### 修正 * 上传重试为空文件 * 回调应该只对form data 签名。 - -## 7.0.3 (2014-03-11) +## 7.0.3 (2015-03-11) ### 增加 * 可以配置 io/rs/api/rsf host diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 03bf72ea..c430216e 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,11 +9,11 @@ # flake8: noqa -__version__ = '7.0.4' +__version__ = '7.0.5' from .auth import Auth -from .config import set_default +from .config import set_default, Zone from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, build_batch_delete from .services.storage.uploader import put_data, put_file, put_stream diff --git a/qiniu/auth.py b/qiniu/auth.py index 4864499d..4d0d0d40 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -53,8 +53,8 @@ class Auth(object): def __init__(self, access_key, secret_key): """初始化Auth类""" self.__checkKey(access_key, secret_key) - self.__access_key, self.__secret_key = access_key, secret_key - self.__secret_key = b(self.__secret_key) + self.__access_key = access_key + self.__secret_key = b(secret_key) def __token(self, data): data = b(data) diff --git a/qiniu/config.py b/qiniu/config.py index ed957348..9f162d61 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -5,14 +5,29 @@ RSF_HOST = 'rsf.qbox.me' # 列举操作Host API_HOST = 'api.qiniu.com' # 数据处理操作Host -UPAUTO_HOST = 'up.qiniu.com' # 默认上传Host -UPDX_HOST = 'updx.qiniu.com' # 电信上传Host -UPLT_HOST = 'uplt.qiniu.com' # 联通上传Host -UPYD_HOST = 'upyd.qiniu.com' # 移动上传Host -UPBACKUP_HOST = 'upload.qiniu.com' # 备用上传Host +_BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 + + +class Zone(object): + """七牛上传区域类 + + 该类主要内容上传区域地址。 + + Attributes: + up_host: 首选上传地址 + up_host_backup: 备用上传地址 + """ + def __init__(self, up_host, up_host_backup): + """初始化Zone类""" + self.up_host, self.up_host_backup = up_host, up_host_backup + + +zone0 = Zone('up.qiniu.com', 'upload.qiniu.com') +zone1 = Zone('up-z1.qiniu.com', 'upload-z1.qiniu.com') _config = { - 'default_up_host': UPAUTO_HOST, # 设置为默认上传Host + 'default_up_host': zone0.up_host, # 设置为默认上传Host + 'default_up_host_backup': zone0.up_host_backup, 'default_rs_host': RS_HOST, 'default_io_host': IO_HOST, 'default_rsf_host': RSF_HOST, @@ -20,9 +35,7 @@ 'connection_timeout': 30, # 链接超时为时间为30s 'connection_retries': 3, # 链接重试次数为3次 'connection_pool': 10, # 链接池个数为10 - } -_BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 def get_default(key): @@ -30,11 +43,12 @@ def get_default(key): def set_default( - default_up_host=None, connection_retries=None, connection_pool=None, + default_zone=None, connection_retries=None, connection_pool=None, connection_timeout=None, default_rs_host=None, default_io_host=None, default_rsf_host=None, default_api_host=None): - if default_up_host: - _config['default_up_host'] = default_up_host + if default_zone: + _config['default_up_host'] = default_zone.up_host + _config['default_up_host_backup'] = default_zone.up_host_backup if default_rs_host: _config['default_rs_host'] = default_rs_host if default_io_host: diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 42942f17..6938a8dd 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -71,7 +71,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None r, info = http._post_file(url, data=fields, files={'file': (name, data, mime_type)}) if r is None and info.need_retry(): if info.connect_failed: - url = 'http://' + config.UPBACKUP_HOST + '/' + url = 'http://' + config.get_default('default_up_host_backup') + '/' if hasattr(data, 'read') is False: pass elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()): @@ -126,7 +126,7 @@ def upload(self): if ret is None and not info.need_retry: return ret, info if info.connect_failed: - host = config.UPBACKUP_HOST + host = config.get_default('default_up_host_backup') if info.need_retry or crc != ret['crc32']: ret, info = self.make_block(block, length, host) if ret is None or crc != ret['crc32']: diff --git a/test_qiniu.py b/test_qiniu.py index 32a4b1ed..583533de 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -9,7 +9,7 @@ import unittest import pytest -from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save +from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone from qiniu import put_data, put_file, put_stream from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, build_batch_delete from qiniu import urlsafe_base64_encode, urlsafe_base64_decode @@ -132,13 +132,13 @@ def test_fetch(self): ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, 'fetch.html') print(info) assert ret['key'] == 'fetch.html' - assert ret['hash'] == 'FhwVT7vs6xqs1nu_vEdo_4x4qBMB' + assert 'hash' in ret def test_fetch_without_key(self): ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name) print(info) - assert ret['key'] == 'FhwVT7vs6xqs1nu_vEdo_4x4qBMB' - assert ret['hash'] == 'FhwVT7vs6xqs1nu_vEdo_4x4qBMB' + assert ret['key'] == ret['hash'] + assert 'hash' in ret def test_stat(self): ret, info = self.bucket.stat(bucket_name, 'python-sdk.html') @@ -277,44 +277,44 @@ def test_putWithoutKey(self): def test_withoutRead_withoutSeek_retry(self): key = 'retry' data = 'hello retry!' - set_default(default_up_host='a') + set_default(default_zone=Zone('a', 'upload.qiniu.com')) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) assert ret['key'] == key assert ret['hash'] == 'FlYu0iBR1WpvYi4whKXiBuQpyLLk' - qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + qiniu.set_default(default_zone=qiniu.config.zone0) def test_hasRead_hasSeek_retry(self): key = 'withReadAndSeek_retry' data = StringIO('hello retry again!') - set_default(default_up_host='a') + set_default(default_zone=Zone('a', 'upload.qiniu.com')) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) assert ret['key'] == key assert ret['hash'] == 'FuEbdt6JP2BqwQJi7PezYhmuVYOo' - qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + qiniu.set_default(default_zone=qiniu.config.zone0) def test_hasRead_withoutSeek_retry(self): key = 'withReadAndWithoutSeek_retry' data = ReadWithoutSeek('I only have read attribute!') - set_default(default_up_host='a') + set_default(default_zone=Zone('a', 'upload.qiniu.com')) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) - assert ret == None - qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + assert ret is None + qiniu.set_default(default_zone=qiniu.config.zone0) def test_hasRead_WithoutSeek_retry2(self): key = 'withReadAndWithoutSeek_retry2' data = urlopen("http://www.qiniu.com") - set_default(default_up_host='a') + set_default(default_zone=Zone('a', 'upload.qiniu.com')) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) - assert ret == None - qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + assert ret is None + qiniu.set_default(default_zone=qiniu.config.zone0) class ResumableUploaderTestCase(unittest.TestCase): @@ -338,23 +338,23 @@ def test_big_file(self): token = self.q.upload_token(bucket_name, key) localfile = create_temp_file(4 * 1024 * 1024 + 1) progress_handler = lambda progress, total: progress - qiniu.set_default(default_up_host='a') + qiniu.set_default(default_zone=Zone('a', 'upload.qiniu.com')) ret, info = put_file(token, key, localfile, self.params, self.mime_type, progress_handler=progress_handler) print(info) assert ret['key'] == key - qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + qiniu.set_default(default_zone=qiniu.config.zone0) remove_temp_file(localfile) def test_retry(self): localfile = __file__ key = 'test_file_r_retry' - qiniu.set_default(default_up_host='a') + qiniu.set_default(default_zone=Zone('a', 'upload.qiniu.com')) token = self.q.upload_token(bucket_name, key) ret, info = put_file(token, key, localfile, self.params, self.mime_type) print(info) assert ret['key'] == key assert ret['hash'] == etag(localfile) - qiniu.set_default(default_up_host=qiniu.config.UPAUTO_HOST) + qiniu.set_default(default_zone=qiniu.config.zone0) class DownloadTestCase(unittest.TestCase): From 076ceb432011d0b602a317aaf4125f75dc3eaced Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Sun, 5 Jul 2015 14:09:24 +0800 Subject: [PATCH 147/478] Support resumable upload. --- .../storage/upload_progress_recorder.py | 46 ++++++++++++++++ qiniu/services/storage/uploader.py | 54 ++++++++++++++++--- qiniu/utils.py | 3 +- 3 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 qiniu/services/storage/upload_progress_recorder.py diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py new file mode 100644 index 00000000..16e86efa --- /dev/null +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +import json +import os +import tempfile + +class UploadProgressRecorder(object): + """持久化上传记录类 + + 该类默认保存每个文件的上传记录到文件系统中，用于断点续传 + 上传记录为json格式： + { + "size": file_size, + "offset": upload_offset, + "modify_time": file_modify_time, + "contexts": contexts + } + + Attributes: + record_folder: 保存上传记录的目录 + """ + def __init__(self, record_folder=tempfile.gettempdir()): + self.record_folder = record_folder + + + def get_upload_record(self, key): + upload_record_file_path = os.path.join(self.record_folder, key) + if not os.path.isfile(upload_record_file_path): + return None + with open(upload_record_file_path, 'r') as f: + json_data = json.load(f) + return json_data + + + def set_upload_record(self, key, data): + upload_record_file_path = os.path.join(self.record_folder, key) + folder = os.path.dirname(upload_record_file_path) + if not os.path.exists(folder): + os.makedirs(folder) + with open(upload_record_file_path, 'w') as f: + json.dump(data, f) + + + def delete_upload_record(self, key): + record_file_path = os.path.join(self.record_folder, key) + os.remove(record_file_path) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 6938a8dd..40449bda 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -5,6 +5,7 @@ from qiniu import config from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter from qiniu import http +from upload_progress_recorder import UploadProgressRecorder def put_data( @@ -28,7 +29,10 @@ def put_data( return _form_put(up_token, key, data, params, mime_type, crc, progress_handler) -def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None): +def put_file(up_token, key, file_path, params=None, + mime_type='application/octet-stream', check_crc=False, + progress_handler=None, upload_progress_recorder=None): + """上传文件到七牛 Args: @@ -48,7 +52,10 @@ def put_file(up_token, key, file_path, params=None, mime_type='application/octet size = os.stat(file_path).st_size with open(file_path, 'rb') as input_stream: if size > config._BLOCK_SIZE * 2: - ret, info = put_stream(up_token, key, input_stream, size, params, mime_type, progress_handler) + ret, info = put_stream(up_token, key, input_stream, size, params, + mime_type, progress_handler, + upload_progress_recorder=upload_progress_recorder, + modify_time=(int)(os.path.getmtime(file_path))) else: crc = file_crc32(file_path) if check_crc else None ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler) @@ -83,15 +90,18 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None return r, info -def put_stream(up_token, key, input_stream, data_size, params=None, mime_type=None, progress_handler=None): - task = _Resume(up_token, key, input_stream, data_size, params, mime_type, progress_handler) +def put_stream(up_token, key, input_stream, data_size, params=None, + mime_type=None, progress_handler=None, + upload_progress_recorder=None, modify_time=None): + task = _Resume(up_token, key, input_stream, data_size, params, mime_type, + progress_handler, upload_progress_recorder, modify_time) return task.upload() class _Resume(object): """断点续上传类 - 该类主要实现了断点续上传中的分块上传，以及相应地创建块和创建文件过程，详细规格参考： + 该类主要实现了分块上传，断点续上，以及相应地创建块和创建文件过程，详细规格参考： http://developer.qiniu.com/docs/v6/api/reference/up/mkblk.html http://developer.qiniu.com/docs/v6/api/reference/up/mkfile.html @@ -103,9 +113,12 @@ class _Resume(object): params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar mime_type: 上传数据的mimeType progress_handler: 上传进度 + upload_progress_recorder: 记录上传进度，用于断点续传 + modify_time: 上传文件修改日期 """ - def __init__(self, up_token, key, input_stream, data_size, params, mime_type, progress_handler): + def __init__(self, up_token, key, input_stream, data_size, params, mime_type, + progress_handler, upload_progress_recorder, modify_time): """初始化断点续上传""" self.up_token = up_token self.key = key @@ -114,12 +127,37 @@ def __init__(self, up_token, key, input_stream, data_size, params, mime_type, pr self.params = params self.mime_type = mime_type self.progress_handler = progress_handler + self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() + self.modify_time = modify_time + + def record_upload_progress(self, offset): + record_data = { + 'size': self.size, + 'offset': offset, + 'contexts': [block['ctx'] for block in self.blockStatus] + } + if self.modify_time: + record_data['modify_time'] = self.modify_time + self.upload_progress_recorder.set_upload_record(self.key, record_data) + + def recovery_from_record(self): + record = self.upload_progress_recorder.get_upload_record(self.key) + if not record: + return 0 + + if not record['modify_time'] or record['size'] != self.size or \ + record['modify_time'] != self.modify_time: + return 0 + + self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] + return record['offset'] def upload(self): """上传操作""" self.blockStatus = [] host = config.get_default('default_up_host') - for block in _file_iter(self.input_stream, config._BLOCK_SIZE): + offset = self.recovery_from_record() + for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset): length = len(block) crc = crc32(block) ret, info = self.make_block(block, length, host) @@ -133,6 +171,8 @@ def upload(self): return ret, info self.blockStatus.append(ret) + offset += length + self.record_upload_progress(offset) if(callable(self.progress_handler)): self.progress_handler(((len(self.blockStatus) - 1) * config._BLOCK_SIZE)+length, self.size) return self.make_file(host) diff --git a/qiniu/utils.py b/qiniu/utils.py index 1b992fa9..04a88f03 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -74,7 +74,7 @@ def crc32(data): return binascii.crc32(b(data)) & 0xffffffff -def _file_iter(input_stream, size): +def _file_iter(input_stream, size, offset=0): """读取输入流: Args: @@ -84,6 +84,7 @@ def _file_iter(input_stream, size): Raises: IOError: 文件流读取失败 """ + input_stream.seek(offset) d = input_stream.read(size) while d: yield d From d9327965792c95a6fa9bc04d3db29b4007f2949d Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Mon, 6 Jul 2015 15:09:26 +0800 Subject: [PATCH 148/478] Fix pep8 check error. --- .../storage/upload_progress_recorder.py | 76 +++++++++---------- qiniu/services/storage/uploader.py | 9 ++- 2 files changed, 42 insertions(+), 43 deletions(-) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index 16e86efa..bc3c811d 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -4,43 +4,41 @@ import os import tempfile + class UploadProgressRecorder(object): - """持久化上传记录类 - - 该类默认保存每个文件的上传记录到文件系统中，用于断点续传 - 上传记录为json格式： - { - "size": file_size, - "offset": upload_offset, - "modify_time": file_modify_time, - "contexts": contexts - } - - Attributes: - record_folder: 保存上传记录的目录 - """ - def __init__(self, record_folder=tempfile.gettempdir()): - self.record_folder = record_folder - - - def get_upload_record(self, key): - upload_record_file_path = os.path.join(self.record_folder, key) - if not os.path.isfile(upload_record_file_path): - return None - with open(upload_record_file_path, 'r') as f: - json_data = json.load(f) - return json_data - - - def set_upload_record(self, key, data): - upload_record_file_path = os.path.join(self.record_folder, key) - folder = os.path.dirname(upload_record_file_path) - if not os.path.exists(folder): - os.makedirs(folder) - with open(upload_record_file_path, 'w') as f: - json.dump(data, f) - - - def delete_upload_record(self, key): - record_file_path = os.path.join(self.record_folder, key) - os.remove(record_file_path) + """持久化上传记录类 + + 该类默认保存每个文件的上传记录到文件系统中，用于断点续传 + 上传记录为json格式： + { + "size": file_size, + "offset": upload_offset, + "modify_time": file_modify_time, + "contexts": contexts + } + + Attributes: + record_folder: 保存上传记录的目录 + """ + def __init__(self, record_folder=tempfile.gettempdir()): + self.record_folder = record_folder + + def get_upload_record(self, key): + upload_record_file_path = os.path.join(self.record_folder, key) + if not os.path.isfile(upload_record_file_path): + return None + with open(upload_record_file_path, 'r') as f: + json_data = json.load(f) + return json_data + + def set_upload_record(self, key, data): + upload_record_file_path = os.path.join(self.record_folder, key) + folder = os.path.dirname(upload_record_file_path) + if not os.path.exists(folder): + os.makedirs(folder) + with open(upload_record_file_path, 'w') as f: + json.dump(data, f) + + def delete_upload_record(self, key): + record_file_path = os.path.join(self.record_folder, key) + os.remove(record_file_path) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 40449bda..7d216ad9 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -5,7 +5,7 @@ from qiniu import config from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter from qiniu import http -from upload_progress_recorder import UploadProgressRecorder +from .upload_progress_recorder import UploadProgressRecorder def put_data( @@ -31,7 +31,8 @@ def put_data( def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, - progress_handler=None, upload_progress_recorder=None): + progress_handler=None, upload_progress_recorder=None, + cancel_upload_signal=None): """上传文件到七牛 @@ -146,10 +147,10 @@ def recovery_from_record(self): return 0 if not record['modify_time'] or record['size'] != self.size or \ - record['modify_time'] != self.modify_time: + record['modify_time'] != self.modify_time: return 0 - self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] + self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] return record['offset'] def upload(self): From 1e0b2ee879cd2cca5e086d0565e9781ac397f113 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Mon, 6 Jul 2015 19:11:22 +0800 Subject: [PATCH 149/478] Generate record file name with base64. --- .../services/storage/upload_progress_recorder.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index bc3c811d..ee57c220 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import base64 import json import os import tempfile @@ -24,7 +25,9 @@ def __init__(self, record_folder=tempfile.gettempdir()): self.record_folder = record_folder def get_upload_record(self, key): - upload_record_file_path = os.path.join(self.record_folder, key) + record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') + upload_record_file_path = os.path.join(self.record_folder, + record_file_name) if not os.path.isfile(upload_record_file_path): return None with open(upload_record_file_path, 'r') as f: @@ -32,13 +35,14 @@ def get_upload_record(self, key): return json_data def set_upload_record(self, key, data): - upload_record_file_path = os.path.join(self.record_folder, key) - folder = os.path.dirname(upload_record_file_path) - if not os.path.exists(folder): - os.makedirs(folder) + record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') + upload_record_file_path = os.path.join(self.record_folder, + record_file_name) with open(upload_record_file_path, 'w') as f: json.dump(data, f) def delete_upload_record(self, key): - record_file_path = os.path.join(self.record_folder, key) + record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') + record_file_path = os.path.join(self.record_folder, + record_file_name) os.remove(record_file_path) From 26e0a1e2093bf966f6f21a1304a01edbc3a3aa09 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Fri, 10 Jul 2015 16:02:14 +0800 Subject: [PATCH 150/478] Correct put_file arguments. --- qiniu/services/storage/uploader.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 7d216ad9..2f83f85c 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -31,9 +31,7 @@ def put_data( def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, - progress_handler=None, upload_progress_recorder=None, - cancel_upload_signal=None): - + progress_handler=None, upload_progress_recorder=None): """上传文件到七牛 Args: @@ -44,6 +42,7 @@ def put_file(up_token, key, file_path, params=None, mime_type: 上传数据的mimeType check_crc: 是否校验crc32 progress_handler: 上传进度 + upload_progress_recorder: 记录上传进度，用于断点续传 Returns: 一个dict变量，类似 {"hash": "", "key": ""} From 5341b99926710fbb9085386cb910954202cc6831 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Thu, 16 Jul 2015 19:09:15 +0800 Subject: [PATCH 151/478] fix bugs --- qiniu/services/storage/uploader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 2f83f85c..7d9159cd 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -161,11 +161,11 @@ def upload(self): length = len(block) crc = crc32(block) ret, info = self.make_block(block, length, host) - if ret is None and not info.need_retry: + if ret is None and not info.need_retry(): return ret, info - if info.connect_failed: + if info.connect_failed(): host = config.get_default('default_up_host_backup') - if info.need_retry or crc != ret['crc32']: + if info.need_retry() or crc != ret['crc32']: ret, info = self.make_block(block, length, host) if ret is None or crc != ret['crc32']: return ret, info From aca73ad409820babe0eabbc2556a1f1e3659b807 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Thu, 16 Jul 2015 19:14:38 +0800 Subject: [PATCH 152/478] add etag_stream --- qiniu/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/utils.py b/qiniu/utils.py index 04a88f03..b54b0a03 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -105,7 +105,7 @@ def _sha1(data): return h.digest() -def _etag(input_stream): +def etag_stream(input_stream): """计算输入流的etag: etag规格参考 http://developer.qiniu.com/docs/v6/api/overview/appendix.html#qiniu-etag @@ -137,7 +137,7 @@ def etag(filePath): 输入文件的etag值 """ with open(filePath, 'rb') as f: - return _etag(f) + return etag_stream(f) def entry(bucket, key): From feeeacb336dce4f61e64bd15ef01c3715edd2eb0 Mon Sep 17 00:00:00 2001 From: longbai Date: Tue, 15 Sep 2015 00:13:19 +0800 Subject: [PATCH 153/478] 3.5 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b701a2ef..365ec8ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "2.6" - "2.7" - "3.4" + - "3.5" install: - "pip install flake8 --use-mirrors" - "pip install pytest --use-mirrors" From f51f821688e04505233888c89986ab3d6cd1afd1 Mon Sep 17 00:00:00 2001 From: songfei Date: Thu, 29 Oct 2015 16:46:27 +0800 Subject: [PATCH 154/478] add fsizeMin --- qiniu/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/auth.py b/qiniu/auth.py index 4d0d0d40..2b587c9d 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -29,6 +29,7 @@ 'detectMime', # MimeType侦测开关 'mimeLimit', # MimeType限制 'fsizeLimit', # 上传文件大小限制 + 'fsizeMin' , #上传文件最少字节数 'persistentOps', # 持久化处理操作 'persistentNotifyUrl', # 持久化处理结果通知URL From 512fd0dd7f64eec0b14755282ef409ad48796e88 Mon Sep 17 00:00:00 2001 From: songfei Date: Thu, 29 Oct 2015 17:20:37 +0800 Subject: [PATCH 155/478] add fsizeMin --- qiniu/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index 2b587c9d..81a04ffb 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -29,7 +29,7 @@ 'detectMime', # MimeType侦测开关 'mimeLimit', # MimeType限制 'fsizeLimit', # 上传文件大小限制 - 'fsizeMin' , #上传文件最少字节数 + 'fsizeMin', # 上传文件最少字节数 'persistentOps', # 持久化处理操作 'persistentNotifyUrl', # 持久化处理结果通知URL From ed5dfc18c1143697bc369d1ae7f777a82c1a45e3 Mon Sep 17 00:00:00 2001 From: songfei Date: Thu, 29 Oct 2015 21:02:34 +0800 Subject: [PATCH 156/478] change test_prefetch assert --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 583533de..d6123fba 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -126,7 +126,7 @@ def test_buckets(self): def test_prefetch(self): ret, info = self.bucket.prefetch(bucket_name, 'python-sdk.html') print(info) - assert ret == {} + assert ret['key'] == 'python-sdk.html' def test_fetch(self): ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, 'fetch.html') From 4dbaa759012ed67de4deb5a1ea0c6f6e1658cf81 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 18 Nov 2015 14:41:05 +0800 Subject: [PATCH 157/478] remove pip deprecated use mirror --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 365ec8ce..338ae507 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,11 @@ python: - "3.4" - "3.5" install: - - "pip install flake8 --use-mirrors" - - "pip install pytest --use-mirrors" - - "pip install pytest-cov --use-mirrors" - - "pip install requests --use-mirrors" - - "pip install scrutinizer-ocular --use-mirrors" + - "pip install flake8" + - "pip install pytest" + - "pip install pytest-cov" + - "pip install requests" + - "pip install scrutinizer-ocular" before_script: - export QINIU_ACCESS_KEY="QWYn5TFQsLLU1pL5MFEmX3s5DmHdUThav9WyOWOm" - export QINIU_SECRET_KEY="Bxckh6FA-Fbs9Yt3i3cbKVK22UPBmAOHJcL95pGz" From 77de4fd06d2fef0a372ea8f4abec6edf49e0e617 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 18 Nov 2015 15:02:27 +0800 Subject: [PATCH 158/478] readme [ci skip] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b1462260..f321d235 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ $ pip install qiniu | Qiniu SDK版本 | Python 版本 | |:--------------------:|:---------------------------:| -| 7.x | 2.6, 2.7, 3.3, 3.4 | +| 7.x | 2.6, 2.7, 3.3, 3.4, 3.5| | 6.x | 2.6, 2.7 | ## 使用方法 From d6872dc6e43b53300645ee90fd12a565ce230278 Mon Sep 17 00:00:00 2001 From: longbai Date: Sat, 5 Dec 2015 23:04:58 +0800 Subject: [PATCH 159/478] fixed unicode string in 2.x --- CHANGELOG.md | 11 +++++++++++ qiniu/__init__.py | 2 +- qiniu/compat.py | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb2b671b..5b0d69dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ #Changelog +## 7.0.6 (2015-12-05) +### 修正 +* 2.x unicode 问题 by @hunter007 +* 上传重试判断 + +### 增加 +* fsizeMin 上传策略 +* 断点上传记录 by @hokein +* 计算stream etag +* 3.5 ci 支持 + ## 7.0.5 (2015-06-25) ### 变更 * 配置up_host 改为配置zone diff --git a/qiniu/__init__.py b/qiniu/__init__.py index c430216e..e6f5bf45 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.0.5' +__version__ = '7.0.6' from .auth import Auth diff --git a/qiniu/compat.py b/qiniu/compat.py index 63f22b25..3333a58c 100644 --- a/qiniu/compat.py +++ b/qiniu/compat.py @@ -43,10 +43,10 @@ numeric_types = (int, long, float) # noqa def b(data): - return data + return bytes(data) def s(data): - return data + return bytes(data) def u(data): return unicode(data, 'unicode_escape') # noqa From bb6c8f326b9089fb0a959cf644febd5de637fea1 Mon Sep 17 00:00:00 2001 From: longbai Date: Sat, 5 Dec 2015 23:58:18 +0800 Subject: [PATCH 160/478] fixed dns hijack --- qiniu/http.py | 10 ++++++---- qiniu/services/storage/uploader.py | 2 +- test_qiniu.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/qiniu/http.py b/qiniu/http.py index 38c566de..0f53a6d8 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -19,7 +19,7 @@ def __return_wrapper(resp): - if resp.status_code != 200: + if resp.status_code != 200 or resp.headers.get('X-Reqid') is None: return None, ResponseInfo(resp) ret = resp.json() if resp.text != '' else {} return ret, ResponseInfo(resp) @@ -111,12 +111,14 @@ def __init__(self, response, exception=None): self.error = 'unknown' else: self.error = ret['error'] + if self.req_id is None and self.status_code == 200: + self.error = 'server is not qiniu' def ok(self): - return self.status_code == 200 + return self.status_code == 200 and self.req_id is not None def need_retry(self): - if self.__response is None: + if self.__response is None or self.req_id is None: return True code = self.status_code if (code // 100 == 5 and code != 579) or code == 996: @@ -124,7 +126,7 @@ def need_retry(self): return False def connect_failed(self): - return self.__response is None + return self.__response is None or self.req_id is None def __str__(self): return ', '.join(['%s:%s' % item for item in self.__dict__.items()]) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 7d9159cd..8c8c32c0 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -128,7 +128,7 @@ def __init__(self, up_token, key, input_stream, data_size, params, mime_type, self.mime_type = mime_type self.progress_handler = progress_handler self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() - self.modify_time = modify_time + self.modify_time = modify_time or time.time() def record_upload_progress(self, offset): record_data = { diff --git a/test_qiniu.py b/test_qiniu.py index d6123fba..12dcc866 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -323,7 +323,7 @@ class ResumableUploaderTestCase(unittest.TestCase): params = {'x:a': 'a'} q = Auth(access_key, secret_key) - def test_putfile(self): + def test_put_stream(self): localfile = __file__ key = 'test_file_r' size = os.stat(localfile).st_size From 5591d34f8d0095ac103b383de473dfe03c160bc5 Mon Sep 17 00:00:00 2001 From: longbai Date: Sat, 5 Dec 2015 23:59:37 +0800 Subject: [PATCH 161/478] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b0d69dd..e7300c1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### 修正 * 2.x unicode 问题 by @hunter007 * 上传重试判断 +* 上传时 dns劫持处理 ### 增加 * fsizeMin 上传策略 From 3e15b7f7f9487cdf4717e9792a03b8a36b8031e4 Mon Sep 17 00:00:00 2001 From: longbai Date: Mon, 7 Dec 2015 10:08:10 +0800 Subject: [PATCH 162/478] fixed import --- qiniu/services/storage/uploader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 8c8c32c0..5bc690c8 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import os +import time from qiniu import config from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter From f4c774e7880ff1a12804bb171e17b5ecad5727cb Mon Sep 17 00:00:00 2001 From: longbai Date: Mon, 7 Dec 2015 10:23:37 +0800 Subject: [PATCH 163/478] fixed key not exist error --- qiniu/services/storage/uploader.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 5bc690c8..22301922 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -131,6 +131,9 @@ def __init__(self, up_token, key, input_stream, data_size, params, mime_type, self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() self.modify_time = modify_time or time.time() + print(self.modify_time) + print(modify_time) + def record_upload_progress(self, offset): record_data = { 'size': self.size, @@ -139,6 +142,8 @@ def record_upload_progress(self, offset): } if self.modify_time: record_data['modify_time'] = self.modify_time + + print(record_data) self.upload_progress_recorder.set_upload_record(self.key, record_data) def recovery_from_record(self): @@ -146,8 +151,11 @@ def recovery_from_record(self): if not record: return 0 - if not record['modify_time'] or record['size'] != self.size or \ - record['modify_time'] != self.modify_time: + try: + if not record['modify_time'] or record['size'] != self.size or \ + record['modify_time'] != self.modify_time: + return 0 + except KeyError: return 0 self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] From 2ae54189c39c4054db973795a48b5d6f4ffc70a5 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Mon, 7 Dec 2015 14:14:37 +0800 Subject: [PATCH 164/478] [ci skip] version --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0c78b7b9..4ab32884 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ def find_version(*file_paths): 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 'Topic :: Software Development :: Libraries :: Python Modules' ], From 468544c29071e07dc0a8923d8924b5ec43f529e4 Mon Sep 17 00:00:00 2001 From: Robert LU Date: Tue, 8 Mar 2016 17:00:38 +0800 Subject: [PATCH 165/478] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=A4=A7=E4=BA=8E4M?= =?UTF-8?q?=E7=9A=84=E6=96=87=E4=BB=B6hash=E8=AE=A1=E7=AE=97=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/utils.py b/qiniu/utils.py index b54b0a03..dbf0d5ce 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -119,11 +119,11 @@ def etag_stream(input_stream): array = [_sha1(block) for block in _file_iter(input_stream, _BLOCK_SIZE)] if len(array) == 1: data = array[0] - prefix = b('\x16') + prefix = b'\x16' else: sha1_str = b('').join(array) data = _sha1(sha1_str) - prefix = b('\x96') + prefix = b'\x96' return urlsafe_base64_encode(prefix + data) From 5740a0a97b9da333f9eabd0ade2f9f079ef71738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=B2=81=E4=B8=A5=E6=B3=A2?= Date: Tue, 8 Mar 2016 17:44:11 +0800 Subject: [PATCH 166/478] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=95=BF=E5=BA=A6?= =?UTF-8?q?=E4=B8=BA0=E7=9A=84=E6=96=87=E4=BB=B6=E7=9A=84hash=E8=AE=A1?= =?UTF-8?q?=E7=AE=97=E9=94=99=E8=AF=AF=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qiniu/utils.py b/qiniu/utils.py index dbf0d5ce..bfc9cc93 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -117,6 +117,8 @@ def etag_stream(input_stream): 输入流的etag值 """ array = [_sha1(block) for block in _file_iter(input_stream, _BLOCK_SIZE)] + if len(array) == 0: + array = [_sha1(b'')] if len(array) == 1: data = array[0] prefix = b'\x16' From 9ee1f455fb64cd1a5985a91da24acbcc72535ea8 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Thu, 17 Mar 2016 13:43:53 +0800 Subject: [PATCH 167/478] add fname --- .../storage/upload_progress_recorder.py | 12 +- qiniu/services/storage/uploader.py | 230 ++++++++++++++++++ 2 files changed, 239 insertions(+), 3 deletions(-) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index ee57c220..5360f9dc 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -4,6 +4,7 @@ import json import os import tempfile +from qiniu.utils import urlsafe_base64_encode class UploadProgressRecorder(object): @@ -24,7 +25,10 @@ class UploadProgressRecorder(object): def __init__(self, record_folder=tempfile.gettempdir()): self.record_folder = record_folder - def get_upload_record(self, key): + def get_upload_record(self,file_name, key): + + key = '{0}/{1}'.format(key,urlsafe_base64_encode(file_name)) + record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') upload_record_file_path = os.path.join(self.record_folder, record_file_name) @@ -34,14 +38,16 @@ def get_upload_record(self, key): json_data = json.load(f) return json_data - def set_upload_record(self, key, data): + def set_upload_record(self,file_name, key, data): + key = '{0}/{1}'.format(key,urlsafe_base64_encode(file_name)) record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') upload_record_file_path = os.path.join(self.record_folder, record_file_name) with open(upload_record_file_path, 'w') as f: json.dump(data, f) - def delete_upload_record(self, key): + def delete_upload_record(self, file_name, key): + key = '{0}/{1}'.format(key,urlsafe_base64_encode(file_name)) record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') record_file_path = os.path.join(self.record_folder, record_file_name) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 22301922..f98c763c 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -9,6 +9,236 @@ from .upload_progress_recorder import UploadProgressRecorder +def put_data( + up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None): + """上传二进制流到七牛 + + Args: + up_token: 上传凭证 + key: 上传文件名 + data: 上传二进制流 + params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + mime_type: 上传数据的mimeType + check_crc: 是否校验crc32 + progress_handler: 上传进度 + + Returns: + 一个dict变量，类似 {"hash": "", "key": ""} + 一个ResponseInfo对象 + """ + crc = crc32(data) if check_crc else None + return _form_put(up_token, key, data, params, mime_type, crc, progress_handler) + + +def put_file(up_token, key, file_path, params=None, + mime_type='application/octet-stream', check_crc=False, + progress_handler=None, upload_progress_recorder=None): + """上传文件到七牛 + + Args: + up_token: 上传凭证 + key: 上传文件名 + file_path: 上传文件的路径 + params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + mime_type: 上传数据的mimeType + check_crc: 是否校验crc32 + progress_handler: 上传进度 + upload_progress_recorder: 记录上传进度，用于断点续传 + + Returns: + 一个dict变量，类似 {"hash": "", "key": ""} + 一个ResponseInfo对象 + """ + ret = {} + size = os.stat(file_path).st_size + # fname = os.path.basename(file_path) + with open(file_path, 'rb') as input_stream: + if size > config._BLOCK_SIZE * 2: + ret, info = put_stream(up_token, key, input_stream, size, params, + mime_type, progress_handler, + upload_progress_recorder=upload_progress_recorder, + modify_time=(int)(os.path.getmtime(file_path)), file_name=os.path.basename(file_path)) + else: + crc = file_crc32(file_path) if check_crc else None + ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler,file_name=os.path.basename(file_path)) + #ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler) + return ret, info + + +def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None,file_name=None): + fields = {} + if params: + for k, v in params.items(): + fields[k] = str(v) + if crc: + fields['crc32'] = crc + if key is not None: + fields['key'] = key + + fields['token'] = up_token + url = 'http://' + config.get_default('default_up_host') + '/' + #name = key if key else file_name + + fname = file_name + + + + r, info = http._post_file(url, data=fields, files={'file': (fname,data, mime_type)}) + if r is None and info.need_retry(): + if info.connect_failed: + url = 'http://' + config.get_default('default_up_host_backup') + '/' + if hasattr(data, 'read') is False: + pass + elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()): + data.seek(0) + else: + return r, info + r, info = http._post_file(url, data=fields, files={'file': ( fname,data, mime_type)}) + + return r, info + + +def put_stream(up_token, key, input_stream, data_size, params=None, + mime_type=None, progress_handler=None, + upload_progress_recorder=None, modify_time=None, file_name=None): + task = _Resume(up_token, key, input_stream, data_size, params, mime_type, + progress_handler, upload_progress_recorder, modify_time, file_name) + return task.upload() + + +class _Resume(object): + """断点续上传类 + + 该类主要实现了分块上传，断点续上，以及相应地创建块和创建文件过程，详细规格参考： + http://developer.qiniu.com/docs/v6/api/reference/up/mkblk.html + http://developer.qiniu.com/docs/v6/api/reference/up/mkfile.html + + Attributes: + up_token: 上传凭证 + key: 上传文件名 + input_stream: 上传二进制流 + data_size: 上传流大小 + params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + mime_type: 上传数据的mimeType + progress_handler: 上传进度 + upload_progress_recorder: 记录上传进度，用于断点续传 + modify_time: 上传文件修改日期 + """ + + def __init__(self, up_token, key, input_stream, data_size, params, mime_type, + progress_handler, upload_progress_recorder, modify_time, file_name): + """初始化断点续上传""" + self.up_token = up_token + self.key = key + self.input_stream = input_stream + self.size = data_size + self.params = params + self.mime_type = mime_type + self.progress_handler = progress_handler + self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() + self.modify_time = modify_time or time.time() + self.file_name = file_name + + + #print(self.modify_time) + #print(modify_time) + + def record_upload_progress(self, offset): + record_data = { + 'size': self.size, + 'offset': offset, + 'contexts': [block['ctx'] for block in self.blockStatus] + } + if self.modify_time: + record_data['modify_time'] = self.modify_time + #print(record_data) + self.upload_progress_recorder.set_upload_record(self.file_name,self.key, record_data) + + def recovery_from_record(self): + record = self.upload_progress_recorder.get_upload_record(self.file_name,self.key) + if not record: + return 0 + + try: + if not record['modify_time'] or record['size'] != self.size or \ + record['modify_time'] != self.modify_time: + return 0 + except KeyError: + return 0 + self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] + return record['offset'] + + def upload(self): + """上传操作""" + self.blockStatus = [] + host = config.get_default('default_up_host') + offset = self.recovery_from_record() + for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset): + length = len(block) + crc = crc32(block) + ret, info = self.make_block(block, length, host) + if ret is None and not info.need_retry(): + return ret, info + if info.connect_failed(): + host = config.get_default('default_up_host_backup') + if info.need_retry() or crc != ret['crc32']: + ret, info = self.make_block(block, length, host) + if ret is None or crc != ret['crc32']: + return ret, info + self.blockStatus.append(ret) + offset += length + self.record_upload_progress(offset) + if(callable(self.progress_handler)): + self.progress_handler(((len(self.blockStatus) - 1) * config._BLOCK_SIZE)+length, self.size) + return self.make_file(host) + + def make_block(self, block, block_size, host): + """创建块""" + url = self.block_url(host, block_size) + return self.post(url, block) + + def block_url(self, host, size): + return 'http://{0}/mkblk/{1}'.format(host, size) + + def file_url(self, host): + url = ['http://{0}/mkfile/{1}'.format(host, self.size)] + + if self.mime_type: + url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type))) + + if self.key is not None: + url.append('key/{0}'.format(urlsafe_base64_encode(self.key))) + + if self.file_name is not None: + url.append('fname/{0}'.format(urlsafe_base64_encode(self.file_name))) + + if self.params: + for k, v in self.params.items(): + url.append('{0}/{1}'.format(k, urlsafe_base64_encode(v))) + pass + url = '/'.join(url) + print url + return url + + def make_file(self, host): + """创建文件""" + url = self.file_url(host) + body = ','.join([status['ctx'] for status in self.blockStatus]) + return self.post(url, body) + + def post(self, url, data): + return http._post_with_token(url, data, self.up_token) +# -*- coding: utf-8 -*- + +import os +import time + +from qiniu import config +from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter +from qiniu import http +from .upload_progress_recorder import UploadProgressRecorder + + def put_data( up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None): """上传二进制流到七牛 From 615fec8326e548c17b9f0b76595b00486c776f22 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Thu, 17 Mar 2016 15:28:53 +0800 Subject: [PATCH 168/478] add fname --- .../storage/upload_progress_recorder.py | 10 +- qiniu/services/storage/uploader.py | 248 +----------------- 2 files changed, 17 insertions(+), 241 deletions(-) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index 5360f9dc..2a88b152 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -25,9 +25,9 @@ class UploadProgressRecorder(object): def __init__(self, record_folder=tempfile.gettempdir()): self.record_folder = record_folder - def get_upload_record(self,file_name, key): + def get_upload_record(self, file_name, key): - key = '{0}/{1}'.format(key,urlsafe_base64_encode(file_name)) + key = '{0}/{1}'.format(key, urlsafe_base64_encode(file_name)) record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') upload_record_file_path = os.path.join(self.record_folder, @@ -38,8 +38,8 @@ def get_upload_record(self,file_name, key): json_data = json.load(f) return json_data - def set_upload_record(self,file_name, key, data): - key = '{0}/{1}'.format(key,urlsafe_base64_encode(file_name)) + def set_upload_record(self, file_name, key, data): + key = '{0}/{1}'.format(key, urlsafe_base64_encode(file_name)) record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') upload_record_file_path = os.path.join(self.record_folder, record_file_name) @@ -47,7 +47,7 @@ def set_upload_record(self,file_name, key, data): json.dump(data, f) def delete_upload_record(self, file_name, key): - key = '{0}/{1}'.format(key,urlsafe_base64_encode(file_name)) + key = '{0}/{1}'.format(key, urlsafe_base64_encode(file_name)) record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') record_file_path = os.path.join(self.record_folder, record_file_name) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index f98c763c..52e39edc 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -60,12 +60,12 @@ def put_file(up_token, key, file_path, params=None, modify_time=(int)(os.path.getmtime(file_path)), file_name=os.path.basename(file_path)) else: crc = file_crc32(file_path) if check_crc else None - ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler,file_name=os.path.basename(file_path)) - #ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler) + ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler, file_name=os.path.basename(file_path)) + # ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler) return ret, info -def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None,file_name=None): +def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None, file_name=None): fields = {} if params: for k, v in params.items(): @@ -77,13 +77,11 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None fields['token'] = up_token url = 'http://' + config.get_default('default_up_host') + '/' - #name = key if key else file_name + # name = key if key else file_name fname = file_name - - - r, info = http._post_file(url, data=fields, files={'file': (fname,data, mime_type)}) + r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) if r is None and info.need_retry(): if info.connect_failed: url = 'http://' + config.get_default('default_up_host_backup') + '/' @@ -93,7 +91,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None data.seek(0) else: return r, info - r, info = http._post_file(url, data=fields, files={'file': ( fname,data, mime_type)}) + r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) return r, info @@ -138,10 +136,8 @@ def __init__(self, up_token, key, input_stream, data_size, params, mime_type, self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() self.modify_time = modify_time or time.time() self.file_name = file_name - - - #print(self.modify_time) - #print(modify_time) + # print(self.modify_time) + # print(modify_time) def record_upload_progress(self, offset): record_data = { @@ -151,11 +147,11 @@ def record_upload_progress(self, offset): } if self.modify_time: record_data['modify_time'] = self.modify_time - #print(record_data) - self.upload_progress_recorder.set_upload_record(self.file_name,self.key, record_data) + # print(record_data) + self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) def recovery_from_record(self): - record = self.upload_progress_recorder.get_upload_record(self.file_name,self.key) + record = self.upload_progress_recorder.get_upload_record(self.file_name, self.key) if not record: return 0 @@ -217,227 +213,7 @@ def file_url(self, host): url.append('{0}/{1}'.format(k, urlsafe_base64_encode(v))) pass url = '/'.join(url) - print url - return url - - def make_file(self, host): - """创建文件""" - url = self.file_url(host) - body = ','.join([status['ctx'] for status in self.blockStatus]) - return self.post(url, body) - - def post(self, url, data): - return http._post_with_token(url, data, self.up_token) -# -*- coding: utf-8 -*- - -import os -import time - -from qiniu import config -from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter -from qiniu import http -from .upload_progress_recorder import UploadProgressRecorder - - -def put_data( - up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None): - """上传二进制流到七牛 - - Args: - up_token: 上传凭证 - key: 上传文件名 - data: 上传二进制流 - params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar - mime_type: 上传数据的mimeType - check_crc: 是否校验crc32 - progress_handler: 上传进度 - - Returns: - 一个dict变量，类似 {"hash": "", "key": ""} - 一个ResponseInfo对象 - """ - crc = crc32(data) if check_crc else None - return _form_put(up_token, key, data, params, mime_type, crc, progress_handler) - - -def put_file(up_token, key, file_path, params=None, - mime_type='application/octet-stream', check_crc=False, - progress_handler=None, upload_progress_recorder=None): - """上传文件到七牛 - - Args: - up_token: 上传凭证 - key: 上传文件名 - file_path: 上传文件的路径 - params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar - mime_type: 上传数据的mimeType - check_crc: 是否校验crc32 - progress_handler: 上传进度 - upload_progress_recorder: 记录上传进度，用于断点续传 - - Returns: - 一个dict变量，类似 {"hash": "", "key": ""} - 一个ResponseInfo对象 - """ - ret = {} - size = os.stat(file_path).st_size - with open(file_path, 'rb') as input_stream: - if size > config._BLOCK_SIZE * 2: - ret, info = put_stream(up_token, key, input_stream, size, params, - mime_type, progress_handler, - upload_progress_recorder=upload_progress_recorder, - modify_time=(int)(os.path.getmtime(file_path))) - else: - crc = file_crc32(file_path) if check_crc else None - ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler) - return ret, info - - -def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None): - fields = {} - if params: - for k, v in params.items(): - fields[k] = str(v) - if crc: - fields['crc32'] = crc - if key is not None: - fields['key'] = key - fields['token'] = up_token - url = 'http://' + config.get_default('default_up_host') + '/' - name = key if key else 'filename' - - r, info = http._post_file(url, data=fields, files={'file': (name, data, mime_type)}) - if r is None and info.need_retry(): - if info.connect_failed: - url = 'http://' + config.get_default('default_up_host_backup') + '/' - if hasattr(data, 'read') is False: - pass - elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()): - data.seek(0) - else: - return r, info - r, info = http._post_file(url, data=fields, files={'file': (name, data, mime_type)}) - - return r, info - - -def put_stream(up_token, key, input_stream, data_size, params=None, - mime_type=None, progress_handler=None, - upload_progress_recorder=None, modify_time=None): - task = _Resume(up_token, key, input_stream, data_size, params, mime_type, - progress_handler, upload_progress_recorder, modify_time) - return task.upload() - - -class _Resume(object): - """断点续上传类 - - 该类主要实现了分块上传，断点续上，以及相应地创建块和创建文件过程，详细规格参考： - http://developer.qiniu.com/docs/v6/api/reference/up/mkblk.html - http://developer.qiniu.com/docs/v6/api/reference/up/mkfile.html - - Attributes: - up_token: 上传凭证 - key: 上传文件名 - input_stream: 上传二进制流 - data_size: 上传流大小 - params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar - mime_type: 上传数据的mimeType - progress_handler: 上传进度 - upload_progress_recorder: 记录上传进度，用于断点续传 - modify_time: 上传文件修改日期 - """ - - def __init__(self, up_token, key, input_stream, data_size, params, mime_type, - progress_handler, upload_progress_recorder, modify_time): - """初始化断点续上传""" - self.up_token = up_token - self.key = key - self.input_stream = input_stream - self.size = data_size - self.params = params - self.mime_type = mime_type - self.progress_handler = progress_handler - self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() - self.modify_time = modify_time or time.time() - - print(self.modify_time) - print(modify_time) - - def record_upload_progress(self, offset): - record_data = { - 'size': self.size, - 'offset': offset, - 'contexts': [block['ctx'] for block in self.blockStatus] - } - if self.modify_time: - record_data['modify_time'] = self.modify_time - - print(record_data) - self.upload_progress_recorder.set_upload_record(self.key, record_data) - - def recovery_from_record(self): - record = self.upload_progress_recorder.get_upload_record(self.key) - if not record: - return 0 - - try: - if not record['modify_time'] or record['size'] != self.size or \ - record['modify_time'] != self.modify_time: - return 0 - except KeyError: - return 0 - - self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] - return record['offset'] - - def upload(self): - """上传操作""" - self.blockStatus = [] - host = config.get_default('default_up_host') - offset = self.recovery_from_record() - for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset): - length = len(block) - crc = crc32(block) - ret, info = self.make_block(block, length, host) - if ret is None and not info.need_retry(): - return ret, info - if info.connect_failed(): - host = config.get_default('default_up_host_backup') - if info.need_retry() or crc != ret['crc32']: - ret, info = self.make_block(block, length, host) - if ret is None or crc != ret['crc32']: - return ret, info - - self.blockStatus.append(ret) - offset += length - self.record_upload_progress(offset) - if(callable(self.progress_handler)): - self.progress_handler(((len(self.blockStatus) - 1) * config._BLOCK_SIZE)+length, self.size) - return self.make_file(host) - - def make_block(self, block, block_size, host): - """创建块""" - url = self.block_url(host, block_size) - return self.post(url, block) - - def block_url(self, host, size): - return 'http://{0}/mkblk/{1}'.format(host, size) - - def file_url(self, host): - url = ['http://{0}/mkfile/{1}'.format(host, self.size)] - - if self.mime_type: - url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type))) - - if self.key is not None: - url.append('key/{0}'.format(urlsafe_base64_encode(self.key))) - - if self.params: - for k, v in self.params.items(): - url.append('{0}/{1}'.format(k, urlsafe_base64_encode(v))) - - url = '/'.join(url) + # print url return url def make_file(self, host): From 93f11aaffc9e56ac592df73ba874442604f96cb2 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Thu, 17 Mar 2016 20:13:53 +0800 Subject: [PATCH 169/478] add fname --- qiniu/services/storage/uploader.py | 11 ++++++----- test_qiniu.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 52e39edc..c7571a72 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -53,14 +53,15 @@ def put_file(up_token, key, file_path, params=None, size = os.stat(file_path).st_size # fname = os.path.basename(file_path) with open(file_path, 'rb') as input_stream: + file_name=os.path.basename(file_path) if size > config._BLOCK_SIZE * 2: - ret, info = put_stream(up_token, key, input_stream, size, params, + ret, info = put_stream(up_token, key, input_stream, file_name, size, params, mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, - modify_time=(int)(os.path.getmtime(file_path)), file_name=os.path.basename(file_path)) + modify_time=(int)(os.path.getmtime(file_path))) else: crc = file_crc32(file_path) if check_crc else None - ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler, file_name=os.path.basename(file_path)) + ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler, file_name) # ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler) return ret, info @@ -96,9 +97,9 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None return r, info -def put_stream(up_token, key, input_stream, data_size, params=None, +def put_stream(up_token, key, input_stream, file_name, data_size, params=None, mime_type=None, progress_handler=None, - upload_progress_recorder=None, modify_time=None, file_name=None): + upload_progress_recorder=None, modify_time=None): task = _Resume(up_token, key, input_stream, data_size, params, mime_type, progress_handler, upload_progress_recorder, modify_time, file_name) return task.upload() diff --git a/test_qiniu.py b/test_qiniu.py index 12dcc866..c1b07fca 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -329,7 +329,7 @@ def test_put_stream(self): size = os.stat(localfile).st_size with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, size, self.params, self.mime_type) + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, self.params, self.mime_type) print(info) assert ret['key'] == key From bae220a6125698088d5f7bb64f3653987d9f1e72 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Thu, 17 Mar 2016 20:17:01 +0800 Subject: [PATCH 170/478] add fname --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index c7571a72..e4240896 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -53,7 +53,7 @@ def put_file(up_token, key, file_path, params=None, size = os.stat(file_path).st_size # fname = os.path.basename(file_path) with open(file_path, 'rb') as input_stream: - file_name=os.path.basename(file_path) + file_name = os.path.basename(file_path) if size > config._BLOCK_SIZE * 2: ret, info = put_stream(up_token, key, input_stream, file_name, size, params, mime_type, progress_handler, From 1161f318db19c7eb631938e5bcf451004147b868 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Mon, 11 Apr 2016 18:52:55 +0800 Subject: [PATCH 171/478] add fname --- --- qiniu/services/storage/upload_progress_recorder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index 2a88b152..3aac18eb 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -27,7 +27,7 @@ def __init__(self, record_folder=tempfile.gettempdir()): def get_upload_record(self, file_name, key): - key = '{0}/{1}'.format(key, urlsafe_base64_encode(file_name)) + key = '{0}/{1}'.format(key, file_name) record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') upload_record_file_path = os.path.join(self.record_folder, @@ -39,7 +39,7 @@ def get_upload_record(self, file_name, key): return json_data def set_upload_record(self, file_name, key, data): - key = '{0}/{1}'.format(key, urlsafe_base64_encode(file_name)) + key = '{0}/{1}'.format(key, file_name) record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') upload_record_file_path = os.path.join(self.record_folder, record_file_name) @@ -47,7 +47,7 @@ def set_upload_record(self, file_name, key, data): json.dump(data, f) def delete_upload_record(self, file_name, key): - key = '{0}/{1}'.format(key, urlsafe_base64_encode(file_name)) + key = '{0}/{1}'.format(key, file_name) record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') record_file_path = os.path.join(self.record_folder, record_file_name) From 6e75fb7bf77f8dbb28e90b0703a25258d6d2a29e Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Mon, 11 Apr 2016 19:11:20 +0800 Subject: [PATCH 172/478] add fname --- qiniu/services/storage/upload_progress_recorder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index 3aac18eb..4e777883 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -4,7 +4,6 @@ import json import os import tempfile -from qiniu.utils import urlsafe_base64_encode class UploadProgressRecorder(object): From 8ab59bb679d8f81739cd1db7337a211421fd8066 Mon Sep 17 00:00:00 2001 From: clouddxy Date: Fri, 29 Apr 2016 13:36:28 +0800 Subject: [PATCH 173/478] add some demos --- examples/copy.py | 24 ++++++++++++++++++++++++ examples/delete.py | 22 ++++++++++++++++++++++ examples/download.py | 23 +++++++++++++++++++++++ examples/fops.py | 29 +++++++++++++++++++++++++++++ examples/move.py | 24 ++++++++++++++++++++++++ examples/stat.py | 22 ++++++++++++++++++++++ examples/upload.py | 29 +++++++++++++++++++++++++++++ examples/upload_callback.py | 29 +++++++++++++++++++++++++++++ examples/upload_pfops.py | 0 9 files changed, 202 insertions(+) create mode 100644 examples/copy.py create mode 100644 examples/delete.py create mode 100644 examples/download.py create mode 100644 examples/fops.py create mode 100644 examples/move.py create mode 100644 examples/stat.py create mode 100644 examples/upload.py create mode 100644 examples/upload_callback.py create mode 100644 examples/upload_pfops.py diff --git a/examples/copy.py b/examples/copy.py new file mode 100644 index 00000000..9855d7d8 --- /dev/null +++ b/examples/copy.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth +from qiniu import BucketManager + +access_key = 'Access_Key' +secret_key = 'Secret_Key' + +#初始化Auth状态 +q = Auth(access_key, secret_key) + +#初始化BucketManager +bucket = BucketManager(q) + +#你要测试的空间， 并且这个key在你空间中存在 +bucket_name = 'Bucket_Name' +key = 'python-logo.png' + +#将文件从文件key 复制到文件key2。 可以在不同bucket复制 +key2 = 'python-logo2.png' + +ret, info = bucket.copy(bucket_name, key, bucket_name, key2) +print(info) +assert ret == {} \ No newline at end of file diff --git a/examples/delete.py b/examples/delete.py new file mode 100644 index 00000000..407f3583 --- /dev/null +++ b/examples/delete.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth +from qiniu import BucketManager + +access_key = 'Access_Key' +secret_key = 'Secret_Key' + +#初始化Auth状态 +q = Auth(access_key, secret_key) + +#初始化BucketManager +bucket = BucketManager(q) + +#你要测试的空间， 并且这个key在你空间中存在 +bucket_name = 'Bucket_Name' +key = 'python-logo.png' + +#删除bucket_name 中的文件 key +ret, info = bucket.delete(bucket_name, key) +print(info) +assert ret == {} \ No newline at end of file diff --git a/examples/download.py b/examples/download.py new file mode 100644 index 00000000..dce0c8b0 --- /dev/null +++ b/examples/download.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import requests + +from qiniu import Auth + +access_key = 'AK' +secret_key = 'SK' + +q = Auth(access_key, secret_key) + +#有两种方式构造base_url的形式 +base_url = 'http://%s/%s' % (bucket_domain, key) + +#或者直接输入url的方式下载 +base_url = 'http://domain/key' + +#可以设置token过期时间 +private_url = q.private_download_url(base_url, expires=3600) + +print(private_url) +r = requests.get(private_url) +assert r.status_code == 200 \ No newline at end of file diff --git a/examples/fops.py b/examples/fops.py new file mode 100644 index 00000000..114ba5bd --- /dev/null +++ b/examples/fops.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth, PersistentFop, build_op, op_save, urlsafe_base64_encode + +#对已经上传到七牛的视频发起异步转码操作 +access_key = 'Access_Key' +secret_key = 'Secret_Key' +q = Auth(access_key, secret_key) + +#要转码的文件所在的空间和文件名。 +bucket = 'Bucket_Name' +key = '1.mp4' + +#转码是使用的队列名称。 +pipeline = 'mpsdemo' + +#要进行转码的转码操作。 +fops = 'avthumb/mp4/s/640x360/vb/1.25m' + +#可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 +saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') +fops = fops+'|saveas/'+saveas_key + +pfop = PersistentFop(q, bucket, pipeline) +ops = [] +ops.append(fops) +ret, info = pfop.execute(key, ops, 1) +print(info) +assert ret['persistentId'] is not None \ No newline at end of file diff --git a/examples/move.py b/examples/move.py new file mode 100644 index 00000000..d363853a --- /dev/null +++ b/examples/move.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth +from qiniu import BucketManager + +access_key = 'Access_Key' +secret_key = 'Secret_Key' + +#初始化Auth状态 +q = Auth(access_key, secret_key) + +#初始化BucketManager +bucket = BucketManager(q) + +#你要测试的空间， 并且这个key在你空间中存在 +bucket_name = 'Bucket_Name' +key = 'python-logo.png' + +#将文件从文件key 移动到文件key2，可以实现文件的重命名 可以在不同bucket移动 +key2 = 'python-logo2.png' + +ret, info = bucket.move(bucket_name, key, bucket_name, key2) +print(info) +assert ret == {} \ No newline at end of file diff --git a/examples/stat.py b/examples/stat.py new file mode 100644 index 00000000..66e89927 --- /dev/null +++ b/examples/stat.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth +from qiniu import BucketManager + +access_key = 'Access_Key' +secret_key = 'Secret_Key' + +#初始化Auth状态 +q = Auth(access_key, secret_key) + +#初始化BucketManager +bucket = BucketManager(q) + +#你要测试的空间， 并且这个key在你空间中存在 +bucket_name = 'Bucket_Name' +key = 'python-logo.png' + +#获取文件的状态信息 +ret, info = bucket.stat(bucket_name, key) +print(info) +assert 'hash' in ret \ No newline at end of file diff --git a/examples/upload.py b/examples/upload.py new file mode 100644 index 00000000..e8a80423 --- /dev/null +++ b/examples/upload.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth, put_file, etag, urlsafe_base64_encode +import qiniu.config + +#需要填写你的 Access Key 和 Secret Key +access_key = 'Access_Key' +secret_key = 'Secret_Key' + +#构建鉴权对象 +q = Auth(access_key, secret_key) + +#要上传的空间 +bucket_name = 'Bucket_Name' + +#上传到七牛后保存的文件名 +key = 'my-python-logo.png'; + +#生成上传 Token，可以指定过期时间等 +token = q.upload_token(bucket_name, key, 3600) + +#要上传文件的本地路径 +localfile = './sync/bbb.jpg' + +ret, info = put_file(token, key, localfile) +print(info) +assert ret['key'] == key +assert ret['hash'] == etag(localfile) \ No newline at end of file diff --git a/examples/upload_callback.py b/examples/upload_callback.py new file mode 100644 index 00000000..33cd9083 --- /dev/null +++ b/examples/upload_callback.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth, put_file, etag, +import qiniu.config + +access_key = 'Access_Key' +secret_key = 'Secret_Key' + +q = Auth(access_key, secret_key) + +bucket_name = 'Bucket_Name' + +key = 'my-python-logo.png'; + +#上传文件到七牛后， 七牛将文件名和文件大小回调给业务服务器。 +policy={ + 'callbackUrl':'http://your.domain.com/callback.php', + 'callbackBody':'filename=$(fname)&filesize=$(fsize)' + } + +token = q.upload_token(bucket_name, key, 3600, policy) + +localfile = './sync/bbb.jpg' + +ret, info = put_file(token, key, localfile) +print(info) +assert ret['key'] == key +assert ret['hash'] == etag(localfile) \ No newline at end of file diff --git a/examples/upload_pfops.py b/examples/upload_pfops.py new file mode 100644 index 00000000..e69de29b From bfec66d88ff3a1b6e9d679c30af6485c073ad686 Mon Sep 17 00:00:00 2001 From: Lifu Mao Date: Sat, 30 Apr 2016 22:29:42 +0800 Subject: [PATCH 174/478] add etag unittest --- demos/etag.py | 0 demos/r_up.py | 0 demos/shiwei.py | 0 mkzip.py | 0 test_qiniu.py | 18 ++++++++++++++++++ 5 files changed, 18 insertions(+) create mode 100644 demos/etag.py create mode 100644 demos/r_up.py create mode 100644 demos/shiwei.py create mode 100644 mkzip.py diff --git a/demos/etag.py b/demos/etag.py new file mode 100644 index 00000000..e69de29b diff --git a/demos/r_up.py b/demos/r_up.py new file mode 100644 index 00000000..e69de29b diff --git a/demos/shiwei.py b/demos/shiwei.py new file mode 100644 index 00000000..e69de29b diff --git a/mkzip.py b/mkzip.py new file mode 100644 index 00000000..e69de29b diff --git a/test_qiniu.py b/test_qiniu.py index c1b07fca..5069c973 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -383,6 +383,24 @@ def test_pfop(self): assert ret['persistentId'] is not None +class EtagTestCase(unittest.TestCase): + def test_zero_size(self): + open("x", 'a').close() + hash = etag("x") + assert hash == 'Fto5o-5ea0sNMlW_75VgGJCv2AcJ' + remove_temp_file("x") + def test_small_size(self): + localfile = create_temp_file(1024 * 1024) + hash = etag(localfile) + assert hash == 'FnlAdmDasGTQOIgrU1QIZaGDv_1D' + remove_temp_file(localfile) + def test_large_size(self): + localfile = create_temp_file(4 * 1024 * 1024 + 1) + hash = etag(localfile) + assert hash == 'ljF323utglY3GI6AvLgawSJ4_dgk' + remove_temp_file(localfile) + + class ReadWithoutSeek(object): def __init__(self, str): self.str = str From 8a00a8054a01d7ec9cc88ccc77cbac8cedee0e04 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Sat, 30 Apr 2016 22:34:52 +0800 Subject: [PATCH 175/478] Delete etag.py --- demos/etag.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 demos/etag.py diff --git a/demos/etag.py b/demos/etag.py deleted file mode 100644 index e69de29b..00000000 From ff21aefe7cb2fd9827f4e8117ca6e378016ca085 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Sat, 30 Apr 2016 22:35:02 +0800 Subject: [PATCH 176/478] Delete r_up.py --- demos/r_up.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 demos/r_up.py diff --git a/demos/r_up.py b/demos/r_up.py deleted file mode 100644 index e69de29b..00000000 From 5469e25615f34fc2c70ddfbe2c981fefc2db356e Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Sat, 30 Apr 2016 22:35:08 +0800 Subject: [PATCH 177/478] Delete shiwei.py --- demos/shiwei.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 demos/shiwei.py diff --git a/demos/shiwei.py b/demos/shiwei.py deleted file mode 100644 index e69de29b..00000000 From e2618879919734721de64a72a9a9e8723599c914 Mon Sep 17 00:00:00 2001 From: forrest-mao Date: Sat, 30 Apr 2016 22:35:57 +0800 Subject: [PATCH 178/478] Delete mkzip.py --- mkzip.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 mkzip.py diff --git a/mkzip.py b/mkzip.py deleted file mode 100644 index e69de29b..00000000 From 28fd5e7a50ff5078295ff24a71b03e217de2b7b3 Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 6 May 2016 00:37:32 +0800 Subject: [PATCH 179/478] travis release --- .travis.yml | 45 +++++++++++++++++++++++++-------------------- CHANGELOG.md | 9 +++++++++ qiniu/__init__.py | 2 +- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index 338ae507..2eb7703a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,31 @@ -# using docker container sudo: false - language: python python: - - "2.6" - - "2.7" - - "3.4" - - "3.5" +- '2.6' +- '2.7' +- '3.4' +- '3.5' install: - - "pip install flake8" - - "pip install pytest" - - "pip install pytest-cov" - - "pip install requests" - - "pip install scrutinizer-ocular" +- pip install flake8 +- pip install pytest +- pip install pytest-cov +- pip install requests +- pip install scrutinizer-ocular before_script: - - export QINIU_ACCESS_KEY="QWYn5TFQsLLU1pL5MFEmX3s5DmHdUThav9WyOWOm" - - export QINIU_SECRET_KEY="Bxckh6FA-Fbs9Yt3i3cbKVK22UPBmAOHJcL95pGz" - - export QINIU_TEST_BUCKET="pythonsdk" - - export QINIU_TEST_DOMAIN="pythonsdk.qiniudn.com" - - export QINIU_TEST_ENV="travis" - - export PYTHONPATH="$PYTHONPATH:." +- export QINIU_ACCESS_KEY="QWYn5TFQsLLU1pL5MFEmX3s5DmHdUThav9WyOWOm" +- export QINIU_SECRET_KEY="Bxckh6FA-Fbs9Yt3i3cbKVK22UPBmAOHJcL95pGz" +- export QINIU_TEST_BUCKET="pythonsdk" +- export QINIU_TEST_DOMAIN="pythonsdk.qiniudn.com" +- export QINIU_TEST_ENV="travis" +- export PYTHONPATH="$PYTHONPATH:." script: - - flake8 --show-source --show-pep8 --max-line-length=160 . - - py.test --cov qiniu - - ocular --data-file .coverage +- flake8 --show-source --show-pep8 --max-line-length=160 . +- py.test --cov qiniu +- ocular --data-file .coverage +deploy: + provider: pypi + user: qiniusdk + password: + secure: N2u9xzhncbziIhoDdpaCcr7D3lW/N7AOIZDpx+M5QW0lPqIXkZDioOTZ7b4QNwx/XFMu6tdeK79A2Wg7T9/8VfEWDd2bYL7a1J7spoFJi9k3HVHHiFBmg7vXr1OGn3D51xqsrq3Kh9uRP150a5CA2qxYabKb6b6dn5QhOTPhfFY= + on: + tags: true diff --git a/CHANGELOG.md b/CHANGELOG.md index e7300c1c..e55a56f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ #Changelog +## 7.0.7 (2016-05-05) +### 修正 +* 修复大于4M的文件hash计算错误的问题 +* add fname + +### 增加 +* 一些demo +* travis 直接发布 + ## 7.0.6 (2015-12-05) ### 修正 * 2.x unicode 问题 by @hunter007 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index e6f5bf45..b9bd2ce9 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.0.6' +__version__ = '7.0.7' from .auth import Auth From 850601080df25f0f2148804a291266fdf2ae955c Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Tue, 5 Jul 2016 19:17:02 +0800 Subject: [PATCH 180/478] add force & example & deleteafterdays --- examples/batch.py | 24 +++++++++++++++ examples/fetch.py | 16 ++++++++++ examples/list.py | 26 ++++++++++++++++ qiniu/auth.py | 1 + qiniu/services/storage/bucket.py | 30 ++++++++++--------- test_qiniu.py | 51 ++++++++++++++++++++++++++++++++ 6 files changed, 134 insertions(+), 14 deletions(-) create mode 100644 examples/batch.py create mode 100644 examples/fetch.py create mode 100644 examples/list.py diff --git a/examples/batch.py b/examples/batch.py new file mode 100644 index 00000000..96914418 --- /dev/null +++ b/examples/batch.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth +from qiniu import BucketManager,build_batch_rename +# from qiniu import build_batch_copy, +# from qiniu import build_batch_move,build_batch_rename +access_key = 'access_key' +secret_key = 'secret_key' + +# 初始化Auth状态 +q = Auth(access_key, secret_key) + +# 初始化BucketManager +bucket = BucketManager(q) +keys = {'123.jpg':'123.jpg'} + +# ops = build_batch_copy( 'teest', keys, 'teest',force='true') +# ops = build_batch_move('teest', keys, 'teest', force='true') +ops = build_batch_rename('teest', keys,force='true') + +ret, info = bucket.batch(ops) +print(ret) +print(info) +assert ret == {} diff --git a/examples/fetch.py b/examples/fetch.py new file mode 100644 index 00000000..87ed2904 --- /dev/null +++ b/examples/fetch.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +access_key = 'access_key' +secret_key = 'secret_key' +bucket_name = 'xxx' +q = Auth(access_key, secret_key) +bucket = BucketManager(q) +url = 'http://7xr875.com1.z0.glb.clouddn.com/test.jpg' +key = 'test.jpg' +ret, info = bucket.fetch( url, bucket_name, key) +print(info) +assert ret['key'] == key \ No newline at end of file diff --git a/examples/list.py b/examples/list.py new file mode 100644 index 00000000..9e9af70e --- /dev/null +++ b/examples/list.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth +from qiniu import BucketManager + +access_key = 'access_key' +secret_key = 'secret_key' + +q = Auth(access_key, secret_key) +bucket = BucketManager(q) + +bucket_name = 'dontdelete' +# 前缀 +prefix = None +# 列举条目 +limit = 10 +# 列举出除'/'的所有文件以及以'/'为分隔的所有前缀 +delimiter = None +# 标记 +marker = None + +ret, eof, info = bucket.list(bucket_name, prefix, marker, limit, delimiter) + +print(info) + +assert len(ret.get('items')) is not None \ No newline at end of file diff --git a/qiniu/auth.py b/qiniu/auth.py index 81a04ffb..436f8b0e 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -34,6 +34,7 @@ 'persistentOps', # 持久化处理操作 'persistentNotifyUrl', # 持久化处理结果通知URL 'persistentPipeline', # 持久化处理独享队列 + 'deleteAfterDays', # 文件多少天后自动删除 ]) _deprecated_policy_fields = set([ diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 86e73569..5a0124c1 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -100,7 +100,7 @@ def delete(self, bucket, key): resource = entry(bucket, key) return self.__rs_do('delete', resource) - def rename(self, bucket, key, key_to): + def rename(self, bucket, key, key_to, force='false'): """重命名文件: 给资源进行重命名，本质为move操作。 @@ -114,9 +114,9 @@ def rename(self, bucket, key, key_to): 一个dict变量，成功返回NULL，失败返回{"error": ""} 一个ResponseInfo对象 """ - return self.move(bucket, key, bucket, key_to) + return self.move(bucket, key, bucket, key_to, force) - def move(self, bucket, key, bucket_to, key_to): + def move(self, bucket, key, bucket_to, key_to, force='false'): """移动文件: 将资源从一个空间到另一个空间，具体规格参考： @@ -134,9 +134,10 @@ def move(self, bucket, key, bucket_to, key_to): """ resource = entry(bucket, key) to = entry(bucket_to, key_to) - return self.__rs_do('move', resource, to) + return self.__rs_do('move', resource, to, 'force/{0}'.format(force)) - def copy(self, bucket, key, bucket_to, key_to): + + def copy(self, bucket, key, bucket_to, key_to, force='false'): """复制文件: 将指定资源复制为新命名资源，具体规格参考： @@ -154,7 +155,8 @@ def copy(self, bucket, key, bucket_to, key_to): """ resource = entry(bucket, key) to = entry(bucket_to, key_to) - return self.__rs_do('copy', resource, to) + return self.__rs_do('copy', resource, to, 'force/{0}'.format(force)) + def fetch(self, url, bucket, key=None): """抓取文件: @@ -264,16 +266,16 @@ def _build_op(*args): return '/'.join(args) -def build_batch_copy(source_bucket, key_pairs, target_bucket): - return _two_key_batch('copy', source_bucket, key_pairs, target_bucket) +def build_batch_copy(source_bucket, key_pairs, target_bucket, force='false'): + return _two_key_batch('copy', source_bucket, key_pairs, target_bucket, force) -def build_batch_rename(bucket, key_pairs): - return build_batch_move(bucket, key_pairs, bucket) +def build_batch_rename(bucket, key_pairs, force='false'): + return build_batch_move(bucket, key_pairs, bucket, force) -def build_batch_move(source_bucket, key_pairs, target_bucket): - return _two_key_batch('move', source_bucket, key_pairs, target_bucket) +def build_batch_move(source_bucket, key_pairs, target_bucket, force='false'): + return _two_key_batch('move', source_bucket, key_pairs, target_bucket, force) def build_batch_delete(bucket, keys): @@ -288,7 +290,7 @@ def _one_key_batch(operation, bucket, keys): return [_build_op(operation, entry(bucket, key)) for key in keys] -def _two_key_batch(operation, source_bucket, key_pairs, target_bucket): +def _two_key_batch(operation, source_bucket, key_pairs, target_bucket, force='false'): if target_bucket is None: target_bucket = source_bucket - return [_build_op(operation, entry(source_bucket, k), entry(target_bucket, v)) for k, v in key_pairs.items()] + return [_build_op(operation, entry(source_bucket, k), entry(target_bucket, v), 'force/{0}'.format(force)) for k, v in key_pairs.items()] diff --git a/test_qiniu.py b/test_qiniu.py index 5069c973..fab636af 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -176,6 +176,33 @@ def test_change_mime(self): print(info) assert ret == {} + def test_copy(self): + key = 'copyto'+rand_string(8) + ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + print(info) + assert ret == {} + ret, info = self.bucket.delete(bucket_name, key) + print(info) + assert ret == {} + + def test_copy_force(self): + key = 'copyto'+rand_string(8) + self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key,) + ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + print(info) + assert ret == {} + ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key, force='true') + print(info) + assert info.status_code == 200 + ret, info = self.bucket.delete(bucket_name, key) + print(info) + assert ret == {} + + def test_change_mime(self): + ret, info = self.bucket.change_mime(bucket_name, 'python-sdk.html', 'text/html') + print(info) + assert ret == {} + def test_batch_copy(self): key = 'copyto'+rand_string(8) ops = build_batch_copy(bucket_name, {'copyfrom': key}, bucket_name) @@ -187,6 +214,12 @@ def test_batch_copy(self): print(info) assert ret[0]['code'] == 200 + def test_batch_copy_force(self): + ops = build_batch_copy(bucket_name, {'copyfrom': 'copyfrom'}, bucket_name, force='true') + ret, info = self.bucket.batch(ops) + print(info) + assert ret[0]['code'] == 200 + def test_batch_move(self): key = 'moveto'+rand_string(8) self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) @@ -199,6 +232,15 @@ def test_batch_move(self): print(info) assert ret == {} + def test_batch_move_force(self): + ret,info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, 'copyfrom', force='true') + print(info) + assert info.status_code == 200 + ops = build_batch_move(bucket_name, {'copyfrom':'copyfrom'}, bucket_name,force='true') + ret, info = self.bucket.batch(ops) + print(info) + assert ret[0]['code'] == 200 + def test_batch_rename(self): key = 'rename'+rand_string(8) self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) @@ -211,6 +253,15 @@ def test_batch_rename(self): print(info) assert ret == {} + def test_batch_rename_force(self): + ret,info = self.bucket.rename(bucket_name, 'copyfrom', 'copyfrom', force='true') + print(info) + assert info.status_code == 200 + ops = build_batch_rename(bucket_name, {'copyfrom':'copyfrom'}, force='true') + ret, info = self.bucket.batch(ops) + print(info) + assert ret[0]['code'] == 200 + def test_batch_stat(self): ops = build_batch_stat(bucket_name, ['python-sdk.html']) ret, info = self.bucket.batch(ops) From 869ab7f0f4f9728a78e3308923cfd0c0eb81a8b5 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Tue, 5 Jul 2016 19:43:01 +0800 Subject: [PATCH 181/478] add force --- examples/batch.py | 1 + examples/fetch.py | 2 +- examples/list.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/batch.py b/examples/batch.py index 96914418..699afffb 100644 --- a/examples/batch.py +++ b/examples/batch.py @@ -22,3 +22,4 @@ print(ret) print(info) assert ret == {} + diff --git a/examples/fetch.py b/examples/fetch.py index 87ed2904..fa25ec81 100644 --- a/examples/fetch.py +++ b/examples/fetch.py @@ -13,4 +13,4 @@ key = 'test.jpg' ret, info = bucket.fetch( url, bucket_name, key) print(info) -assert ret['key'] == key \ No newline at end of file +assert ret['key'] == key diff --git a/examples/list.py b/examples/list.py index 9e9af70e..3b48e2a8 100644 --- a/examples/list.py +++ b/examples/list.py @@ -23,4 +23,4 @@ print(info) -assert len(ret.get('items')) is not None \ No newline at end of file +assert len(ret.get('items')) is not None From 0954ca75ce05b57b061a97b3f575a0326bf653d8 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Tue, 5 Jul 2016 20:09:19 +0800 Subject: [PATCH 182/478] add force &eg --- qiniu/services/storage/bucket.py | 2 -- test_qiniu.py | 10 +--------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 5a0124c1..48046cb4 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -136,7 +136,6 @@ def move(self, bucket, key, bucket_to, key_to, force='false'): to = entry(bucket_to, key_to) return self.__rs_do('move', resource, to, 'force/{0}'.format(force)) - def copy(self, bucket, key, bucket_to, key_to, force='false'): """复制文件: @@ -157,7 +156,6 @@ def copy(self, bucket, key, bucket_to, key_to, force='false'): to = entry(bucket_to, key_to) return self.__rs_do('copy', resource, to, 'force/{0}'.format(force)) - def fetch(self, url, bucket, key=None): """抓取文件: 从指定URL抓取资源，并将该资源存储到指定空间中，具体规格参考： diff --git a/test_qiniu.py b/test_qiniu.py index fab636af..410d8d5c 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -186,17 +186,9 @@ def test_copy(self): assert ret == {} def test_copy_force(self): - key = 'copyto'+rand_string(8) - self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key,) - ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) - print(info) - assert ret == {} - ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key, force='true') + ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, 'copyfrom', force='true') print(info) assert info.status_code == 200 - ret, info = self.bucket.delete(bucket_name, key) - print(info) - assert ret == {} def test_change_mime(self): ret, info = self.bucket.change_mime(bucket_name, 'python-sdk.html', 'text/html') From 26961e45c9972f808ac5254cab345800714a85bb Mon Sep 17 00:00:00 2001 From: loyachen Date: Fri, 8 Jul 2016 16:35:31 +0800 Subject: [PATCH 183/478] add upload_pfops.py --- examples/batch.py | 7 +++--- examples/{copy.py => copy_to.py} | 7 +++--- examples/delete.py | 7 +++--- examples/download.py | 8 +++---- examples/fetch.py | 12 +++++++--- examples/fops.py | 14 +++++------ examples/list.py | 6 ++--- examples/{move.py => move_to.py} | 8 +++---- examples/stat.py | 6 ++--- examples/upload.py | 6 ++--- examples/upload_callback.py | 6 ++--- examples/upload_pfops.py | 40 ++++++++++++++++++++++++++++++++ 12 files changed, 88 insertions(+), 39 deletions(-) rename examples/{copy.py => copy_to.py} (87%) rename examples/{move.py => move_to.py} (78%) diff --git a/examples/batch.py b/examples/batch.py index 699afffb..4f653661 100644 --- a/examples/batch.py +++ b/examples/batch.py @@ -1,11 +1,13 @@ # -*- coding: utf-8 -*- # flake8: noqa + from qiniu import Auth from qiniu import BucketManager,build_batch_rename # from qiniu import build_batch_copy, # from qiniu import build_batch_move,build_batch_rename -access_key = 'access_key' -secret_key = 'secret_key' + +access_key = '...' +secret_key = '...' # 初始化Auth状态 q = Auth(access_key, secret_key) @@ -22,4 +24,3 @@ print(ret) print(info) assert ret == {} - diff --git a/examples/copy.py b/examples/copy_to.py similarity index 87% rename from examples/copy.py rename to examples/copy_to.py index 9855d7d8..bab02f7a 100644 --- a/examples/copy.py +++ b/examples/copy_to.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- # flake8: noqa + from qiniu import Auth from qiniu import BucketManager -access_key = 'Access_Key' -secret_key = 'Secret_Key' +access_key = '...' +secret_key = '...' #初始化Auth状态 q = Auth(access_key, secret_key) @@ -21,4 +22,4 @@ ret, info = bucket.copy(bucket_name, key, bucket_name, key2) print(info) -assert ret == {} \ No newline at end of file +assert ret == {} diff --git a/examples/delete.py b/examples/delete.py index 407f3583..ab5570a2 100644 --- a/examples/delete.py +++ b/examples/delete.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- # flake8: noqa + from qiniu import Auth from qiniu import BucketManager -access_key = 'Access_Key' -secret_key = 'Secret_Key' +access_key = '...' +secret_key = '...' #初始化Auth状态 q = Auth(access_key, secret_key) @@ -19,4 +20,4 @@ #删除bucket_name 中的文件 key ret, info = bucket.delete(bucket_name, key) print(info) -assert ret == {} \ No newline at end of file +assert ret == {} diff --git a/examples/download.py b/examples/download.py index dce0c8b0..a3895a7d 100644 --- a/examples/download.py +++ b/examples/download.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- # flake8: noqa -import requests +import requests from qiniu import Auth -access_key = 'AK' -secret_key = 'SK' +access_key = '...' +secret_key = '...' q = Auth(access_key, secret_key) @@ -20,4 +20,4 @@ print(private_url) r = requests.get(private_url) -assert r.status_code == 200 \ No newline at end of file +assert r.status_code == 200 diff --git a/examples/fetch.py b/examples/fetch.py index fa25ec81..bc19d7fb 100644 --- a/examples/fetch.py +++ b/examples/fetch.py @@ -4,13 +4,19 @@ from qiniu import Auth from qiniu import BucketManager -access_key = 'access_key' -secret_key = 'secret_key' -bucket_name = 'xxx' +access_key = '...' +secret_key = '...' + +bucket_name = 'Bucket_Name' + q = Auth(access_key, secret_key) + bucket = BucketManager(q) + url = 'http://7xr875.com1.z0.glb.clouddn.com/test.jpg' + key = 'test.jpg' + ret, info = bucket.fetch( url, bucket_name, key) print(info) assert ret['key'] == key diff --git a/examples/fops.py b/examples/fops.py index 114ba5bd..74cfe5df 100644 --- a/examples/fops.py +++ b/examples/fops.py @@ -2,17 +2,17 @@ # flake8: noqa from qiniu import Auth, PersistentFop, build_op, op_save, urlsafe_base64_encode -#对已经上传到七牛的视频发起异步转码操作 -access_key = 'Access_Key' -secret_key = 'Secret_Key' +#对已经上传到七牛的视频发起异步转码操作 +access_key = '...' +secret_key = '...' q = Auth(access_key, secret_key) #要转码的文件所在的空间和文件名。 -bucket = 'Bucket_Name' +bucket_name = 'Bucket_Name' key = '1.mp4' #转码是使用的队列名称。 -pipeline = 'mpsdemo' +pipeline = 'your_pipeline' #要进行转码的转码操作。 fops = 'avthumb/mp4/s/640x360/vb/1.25m' @@ -21,9 +21,9 @@ saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') fops = fops+'|saveas/'+saveas_key -pfop = PersistentFop(q, bucket, pipeline) +pfop = PersistentFop(q, bucket_name, pipeline) ops = [] ops.append(fops) ret, info = pfop.execute(key, ops, 1) print(info) -assert ret['persistentId'] is not None \ No newline at end of file +assert ret['persistentId'] is not None diff --git a/examples/list.py b/examples/list.py index 3b48e2a8..b90f870f 100644 --- a/examples/list.py +++ b/examples/list.py @@ -3,13 +3,13 @@ from qiniu import Auth from qiniu import BucketManager -access_key = 'access_key' -secret_key = 'secret_key' +access_key = '...' +secret_key = '...' q = Auth(access_key, secret_key) bucket = BucketManager(q) -bucket_name = 'dontdelete' +bucket_name = 'Bucket_Name' # 前缀 prefix = None # 列举条目 diff --git a/examples/move.py b/examples/move_to.py similarity index 78% rename from examples/move.py rename to examples/move_to.py index d363853a..df4d4f46 100644 --- a/examples/move.py +++ b/examples/move_to.py @@ -3,8 +3,8 @@ from qiniu import Auth from qiniu import BucketManager -access_key = 'Access_Key' -secret_key = 'Secret_Key' +access_key = '...' +secret_key = '...' #初始化Auth状态 q = Auth(access_key, secret_key) @@ -19,6 +19,6 @@ #将文件从文件key 移动到文件key2，可以实现文件的重命名 可以在不同bucket移动 key2 = 'python-logo2.png' -ret, info = bucket.move(bucket_name, key, bucket_name, key2) +ret, info = bucket.move(bucket_name, key, bucket, key2) print(info) -assert ret == {} \ No newline at end of file +assert ret == {} diff --git a/examples/stat.py b/examples/stat.py index 66e89927..ca07e125 100644 --- a/examples/stat.py +++ b/examples/stat.py @@ -3,8 +3,8 @@ from qiniu import Auth from qiniu import BucketManager -access_key = 'Access_Key' -secret_key = 'Secret_Key' +access_key = '...' +secret_key = '...' #初始化Auth状态 q = Auth(access_key, secret_key) @@ -19,4 +19,4 @@ #获取文件的状态信息 ret, info = bucket.stat(bucket_name, key) print(info) -assert 'hash' in ret \ No newline at end of file +assert 'hash' in ret diff --git a/examples/upload.py b/examples/upload.py index e8a80423..a8920de9 100644 --- a/examples/upload.py +++ b/examples/upload.py @@ -5,8 +5,8 @@ import qiniu.config #需要填写你的 Access Key 和 Secret Key -access_key = 'Access_Key' -secret_key = 'Secret_Key' +access_key = '...' +secret_key = '...' #构建鉴权对象 q = Auth(access_key, secret_key) @@ -26,4 +26,4 @@ ret, info = put_file(token, key, localfile) print(info) assert ret['key'] == key -assert ret['hash'] == etag(localfile) \ No newline at end of file +assert ret['hash'] == etag(localfile) diff --git a/examples/upload_callback.py b/examples/upload_callback.py index 33cd9083..26e864f5 100644 --- a/examples/upload_callback.py +++ b/examples/upload_callback.py @@ -4,8 +4,8 @@ from qiniu import Auth, put_file, etag, import qiniu.config -access_key = 'Access_Key' -secret_key = 'Secret_Key' +access_key = '...' +secret_key = ...' q = Auth(access_key, secret_key) @@ -26,4 +26,4 @@ ret, info = put_file(token, key, localfile) print(info) assert ret['key'] == key -assert ret['hash'] == etag(localfile) \ No newline at end of file +assert ret['hash'] == etag(localfile) diff --git a/examples/upload_pfops.py b/examples/upload_pfops.py index e69de29b..6f04b1b6 100644 --- a/examples/upload_pfops.py +++ b/examples/upload_pfops.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth, put_file, etag, urlsafe_base64_encode +import qiniu.config + +# access_key = '...' +# secret_key = '...' + +#初始化Auth状态 +q = Auth(access_key, secret_key) + +#你要测试的空间， 并且这个key在你空间中存在 +bucket_name = 'Bucket_Name' +key = 'python_video.flv' + +# 指定转码使用的队列名称 +pipeline = 'your_pipeline' + +# 设置转码参数（以视频转码为例） +fops = 'avthumb/mp4/vcodec/libx264' + +# 通过添加'|saveas'参数，指定处理后的文件保存的bucket和key，不指定默认保存在当前空间，bucket_saved为目标bucket，bucket_saved为目标key +saveas_key = urlsafe_base64_encode('bucket_saved:bucket_saved')# + +fops = fops+'|saveas/'+saveas_key + +# 在上传策略中指定fobs和pipeline +policy={ + 'persistentOps':fops, + 'persistentPipeline':pipeline + } + +token = q.upload_token(bucket_name, key, 3600, policy) + +localfile = './python_video.flv' + +ret, info = put_file(token, key, localfile) +print(info) +assert ret['key'] == key +assert ret['hash'] == etag(localfile) From 388c9d362925b656c4cba857042a834d27adcb76 Mon Sep 17 00:00:00 2001 From: loyachen Date: Fri, 8 Jul 2016 16:57:37 +0800 Subject: [PATCH 184/478] add upload_pfops.py --- examples/upload_pfops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/upload_pfops.py b/examples/upload_pfops.py index 6f04b1b6..4a144912 100644 --- a/examples/upload_pfops.py +++ b/examples/upload_pfops.py @@ -6,10 +6,10 @@ # access_key = '...' # secret_key = '...' -#初始化Auth状态 +# 初始化Auth状态 q = Auth(access_key, secret_key) -#你要测试的空间， 并且这个key在你空间中存在 +# 你要测试的空间， 并且这个key在你空间中存在 bucket_name = 'Bucket_Name' key = 'python_video.flv' From 389fc07564dfb4a4667b15b7377c2845d8477fa1 Mon Sep 17 00:00:00 2001 From: Lifu Mao Date: Fri, 8 Jul 2016 23:20:28 +0800 Subject: [PATCH 185/478] fix put_data 400 --- qiniu/services/storage/uploader.py | 6 +++-- test_qiniu.py | 42 ++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index e4240896..57c0bf03 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -10,7 +10,7 @@ def put_data( - up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None): + up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, fname=None): """上传二进制流到七牛 Args: @@ -27,7 +27,7 @@ def put_data( 一个ResponseInfo对象 """ crc = crc32(data) if check_crc else None - return _form_put(up_token, key, data, params, mime_type, crc, progress_handler) + return _form_put(up_token, key, data, params, mime_type, crc, progress_handler, fname) def put_file(up_token, key, file_path, params=None, @@ -81,6 +81,8 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None # name = key if key else file_name fname = file_name + if not fname or not fname.strip(): + fname = 'file_name' r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) if r is None and info.need_retry(): diff --git a/test_qiniu.py b/test_qiniu.py index 410d8d5c..df028436 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -64,10 +64,14 @@ def remove_temp_file(file): pass +def is_travis(): + return os.environ['QINIU_TEST_ENV'] == 'travis' + + class UtilsTest(unittest.TestCase): def test_urlsafe(self): - a = '你好\x96' + a = 'hello\x96' u = urlsafe_base64_encode(a) assert b(a) == urlsafe_base64_decode(u) @@ -268,13 +272,14 @@ class UploaderTestCase(unittest.TestCase): q = Auth(access_key, secret_key) def test_put(self): - key = 'a\\b\\c"你好' + key = 'a\\b\\c"hello' data = 'hello bubby!' token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) assert ret['key'] == key + def test_put_crc(self): key = '' data = 'hello bubby!' token = self.q.upload_token(bucket_name, key) @@ -359,6 +364,39 @@ def test_hasRead_WithoutSeek_retry2(self): assert ret is None qiniu.set_default(default_zone=qiniu.config.zone0) + def test_putData_without_fname(self): + if is_travis(): + return + localfile = create_temp_file(30 * 1024 * 1024) + key = 'test_putData_without_fname' + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name) + ret, info = put_data(token, key, input_stream) + print(info) + assert ret is not None + + def test_putData_without_fname1(self): + if is_travis(): + return + localfile = create_temp_file(30 * 1024 * 1024) + key = 'test_putData_without_fname1' + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name) + ret, info = put_data(token, key, input_stream, self.params, self.mime_type, False, None, "") + print(info) + assert ret is not None + + def test_putData_without_fname2(self): + if is_travis(): + return + localfile = create_temp_file(30 * 1024 * 1024) + key = 'test_putData_without_fname2' + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name) + ret, info = put_data(token, key, input_stream, self.params, self.mime_type, False, None, " ") + print(info) + assert ret is not None + class ResumableUploaderTestCase(unittest.TestCase): From d8976821fff225f0fbfc9aa1cce6fb2b04d9a283 Mon Sep 17 00:00:00 2001 From: clouddxy Date: Thu, 14 Jul 2016 13:31:38 +0800 Subject: [PATCH 186/478] add_some_demos [ci skip] --- examples/pfop_ vframe.py | 29 +++++++++++++++++++++++++++++ examples/pfop_ watermark.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 examples/pfop_ vframe.py create mode 100644 examples/pfop_ watermark.py diff --git a/examples/pfop_ vframe.py b/examples/pfop_ vframe.py new file mode 100644 index 00000000..6c1c2ba5 --- /dev/null +++ b/examples/pfop_ vframe.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth, PersistentFop, build_op, op_save, urlsafe_base64_encode + +#对已经上传到七牛的视频发起异步转码操作 +access_key = 'Access_Key' +secret_key = 'Secret_Key' +q = Auth(access_key, secret_key) + +#要转码的文件所在的空间和文件名。 +bucket = 'Bucket_Name' +key = '1.mp4' + +#转码是使用的队列名称。 +pipeline = 'mpsdemo' + +#要进行视频截图操作。 +fops = 'vframe/jpg/offset/1/w/480/h/360/rotate/90' + +#可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 +saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') +fops = fops+'|saveas/'+saveas_key + +pfop = PersistentFop(q, bucket, pipeline) +ops = [] +ops.append(fops) +ret, info = pfop.execute(key, ops, 1) +print(info) +assert ret['persistentId'] is not None \ No newline at end of file diff --git a/examples/pfop_ watermark.py b/examples/pfop_ watermark.py new file mode 100644 index 00000000..dd1a97d4 --- /dev/null +++ b/examples/pfop_ watermark.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth, PersistentFop, build_op, op_save, urlsafe_base64_encode + +#对已经上传到七牛的视频发起异步转码操作 +access_key = 'Access_Key' +secret_key = 'Secret_Key' +q = Auth(access_key, secret_key) + +#要转码的文件所在的空间和文件名。 +bucket = 'Bucket_Name' +key = '1.mp4' + +#转码是使用的队列名称。 +pipeline = 'mpsdemo' + +#需要添加水印的图片UrlSafeBase64,可以参考http://developer.qiniu.com/code/v6/api/dora-api/av/video-watermark.html +base64URL = urlsafe_base64_encode('http://developer.qiniu.com/resource/logo-2.jpg'); + +#视频水印参数 +fops = 'avthumb/mp4/'+base64URL + +#可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 +saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') +fops = fops+'|saveas/'+saveas_key + +pfop = PersistentFop(q, bucket, pipeline) +ops = [] +ops.append(fops) +ret, info = pfop.execute(key, ops, 1) +print(info) +assert ret['persistentId'] \ No newline at end of file From a2790aa236b0ea1fbbcb0b2fd181d23f34ae7310 Mon Sep 17 00:00:00 2001 From: Lifu Mao Date: Thu, 4 Aug 2016 09:19:15 +0800 Subject: [PATCH 187/478] update --- CHANGELOG.md | 9 +++++++++ qiniu/host_mgr.py | 0 2 files changed, 9 insertions(+) create mode 100644 qiniu/host_mgr.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e55a56f5..334fb81b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ #Changelog +## 7.0.8 (2016-07-05) +### 修正 +* 修复表单上传大于20M文件的400错误 + +### 增加 +* copy 和 move 操作增加 force 字段，允许强制覆盖 copy 和 move +* 增加上传策略 deleteAfterDays 字段 +* 一些 demo + ## 7.0.7 (2016-05-05) ### 修正 * 修复大于4M的文件hash计算错误的问题 diff --git a/qiniu/host_mgr.py b/qiniu/host_mgr.py new file mode 100644 index 00000000..e69de29b From fca51f416c776f324447d0dd484b88de9943cf65 Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 4 Aug 2016 12:17:26 +0800 Subject: [PATCH 188/478] version update --- qiniu/__init__.py | 2 +- qiniu/host_mgr.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 qiniu/host_mgr.py diff --git a/qiniu/__init__.py b/qiniu/__init__.py index b9bd2ce9..1a927774 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.0.7' +__version__ = '7.0.8' from .auth import Auth diff --git a/qiniu/host_mgr.py b/qiniu/host_mgr.py deleted file mode 100644 index e69de29b..00000000 From 916d4e9b577eed38b725f56fc6b7bc6f5e5018cc Mon Sep 17 00:00:00 2001 From: longbai Date: Thu, 4 Aug 2016 12:36:26 +0800 Subject: [PATCH 189/478] trip pep8 option for flake8 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2eb7703a..e693bfb6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ before_script: - export QINIU_TEST_ENV="travis" - export PYTHONPATH="$PYTHONPATH:." script: -- flake8 --show-source --show-pep8 --max-line-length=160 . +- flake8 --show-source --max-line-length=160 . - py.test --cov qiniu - ocular --data-file .coverage deploy: From 2354506961fcc04bdfd41c3b61fb8ae15c9c5bc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wang=20D=C3=A0p=C3=A9ng?= Date: Mon, 15 Aug 2016 16:37:50 +0800 Subject: [PATCH 190/478] Add sdk usage doc link --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f321d235..b200edbb 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,9 @@ import qiniu else: print(info) # error message in info ... + ``` +更多参见SDK使用指南： http://developer.qiniu.com/code/v7/sdk/python.html ### 命令行工具 安装完后附带有命令行工具，可以计算etag From 94c247dfc988468bda8b142b4ba9f88f1a13b04d Mon Sep 17 00:00:00 2001 From: Lifu Mao Date: Sun, 9 Oct 2016 06:55:42 +0800 Subject: [PATCH 191/478] support multi-zone --- qiniu/__init__.py | 3 +- qiniu/auth.py | 3 + qiniu/config.py | 37 ++------ qiniu/http.py | 6 +- qiniu/services/processing/pfop.py | 2 +- qiniu/services/storage/bucket.py | 24 ++++-- qiniu/services/storage/uploader.py | 12 +-- qiniu/zone.py | 133 +++++++++++++++++++++++++++++ 8 files changed, 171 insertions(+), 49 deletions(-) create mode 100644 qiniu/zone.py diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 1a927774..8f213b88 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -13,7 +13,8 @@ from .auth import Auth -from .config import set_default, Zone +from .config import set_default +from .zone import Zone from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, build_batch_delete from .services.storage.uploader import put_data, put_file, put_stream diff --git a/qiniu/auth.py b/qiniu/auth.py index 436f8b0e..876a53ac 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -58,6 +58,9 @@ def __init__(self, access_key, secret_key): self.__access_key = access_key self.__secret_key = b(secret_key) + def get_access_key(self): + return self.__access_key + def __token(self, data): data = b(data) hashed = hmac.new(self.__secret_key, data, sha1) diff --git a/qiniu/config.py b/qiniu/config.py index 9f162d61..5431d174 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -1,35 +1,17 @@ # -*- coding: utf-8 -*- -RS_HOST = 'rs.qbox.me' # 管理操作Host -IO_HOST = 'iovip.qbox.me' # 七牛源站Host -RSF_HOST = 'rsf.qbox.me' # 列举操作Host -API_HOST = 'api.qiniu.com' # 数据处理操作Host +from qiniu import zone -_BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 - - -class Zone(object): - """七牛上传区域类 - - 该类主要内容上传区域地址。 - - Attributes: - up_host: 首选上传地址 - up_host_backup: 备用上传地址 - """ - def __init__(self, up_host, up_host_backup): - """初始化Zone类""" - self.up_host, self.up_host_backup = up_host, up_host_backup +RS_HOST = 'http://rs.qbox.me' # 管理操作Host +RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host +API_HOST = 'http://api.qiniu.com' # 数据处理操作Host +_BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 -zone0 = Zone('up.qiniu.com', 'upload.qiniu.com') -zone1 = Zone('up-z1.qiniu.com', 'upload-z1.qiniu.com') _config = { - 'default_up_host': zone0.up_host, # 设置为默认上传Host - 'default_up_host_backup': zone0.up_host_backup, + 'default_zone': zone.Zone(), 'default_rs_host': RS_HOST, - 'default_io_host': IO_HOST, 'default_rsf_host': RSF_HOST, 'default_api_host': API_HOST, 'connection_timeout': 30, # 链接超时为时间为30s @@ -44,15 +26,12 @@ def get_default(key): def set_default( default_zone=None, connection_retries=None, connection_pool=None, - connection_timeout=None, default_rs_host=None, default_io_host=None, + connection_timeout=None, default_rs_host=None, default_rsf_host=None, default_api_host=None): if default_zone: - _config['default_up_host'] = default_zone.up_host - _config['default_up_host_backup'] = default_zone.up_host_backup + _config['default_zone'] = default_zone if default_rs_host: _config['default_rs_host'] = default_rs_host - if default_io_host: - _config['default_io_host'] = default_io_host if default_rsf_host: _config['default_rsf_host'] = default_rsf_host if default_api_host: diff --git a/qiniu/http.py b/qiniu/http.py index 0f53a6d8..011db8e3 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -5,7 +5,7 @@ from requests.auth import AuthBase from qiniu import config -from .auth import RequestsAuth +import qiniu.auth from . import __version__ @@ -49,7 +49,7 @@ def _post(url, data, files, auth): def _get(url, params, auth): try: r = requests.get( - url, params=params, auth=RequestsAuth(auth) if auth is not None else None, + url, params=params, auth=qiniu.auth.RequestsAuth(auth) if auth is not None else None, timeout=config.get_default('connection_timeout'), headers=_headers) except Exception as e: return None, ResponseInfo(None, e) @@ -74,7 +74,7 @@ def _post_file(url, data, files): def _post_with_auth(url, data, auth): - return _post(url, data, None, RequestsAuth(auth)) + return _post(url, data, None, qiniu.auth.RequestsAuth(auth)) class ResponseInfo(object): diff --git a/qiniu/services/processing/pfop.py b/qiniu/services/processing/pfop.py index a787cb7e..c8f1830c 100644 --- a/qiniu/services/processing/pfop.py +++ b/qiniu/services/processing/pfop.py @@ -45,5 +45,5 @@ def execute(self, key, fops, force=None): if force == 1: data['force'] = 1 - url = 'http://{0}/pfop'.format(config.get_default('default_api_host')) + url = '{0}/pfop'.format(config.get_default('default_api_host')) return http._post_with_auth(url, data, self.auth) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 48046cb4..43f03d39 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- from qiniu import config -from qiniu.utils import urlsafe_base64_encode, entry from qiniu import http +from qiniu.utils import urlsafe_base64_encode, entry class BucketManager(object): @@ -15,8 +15,12 @@ class BucketManager(object): auth: 账号管理密钥对，Auth对象 """ - def __init__(self, auth): + def __init__(self, auth, zone=None): self.auth = auth + if(zone is None): + self.zone = config.get_default('default_zone') + else: + self.zone = zone def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): """前缀查询: @@ -51,7 +55,7 @@ def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): if delimiter is not None: options['delimiter'] = delimiter - url = 'http://{0}/list'.format(config.get_default('default_rsf_host')) + url = '{0}/list'.format(config.get_default('default_rsf_host')) ret, info = self.__get(url, options) eof = False @@ -172,7 +176,7 @@ def fetch(self, url, bucket, key=None): """ resource = urlsafe_base64_encode(url) to = entry(bucket, key) - return self.__io_do('fetch', resource, 'to/{0}'.format(to)) + return self.__io_do(bucket, 'fetch', resource, 'to/{0}'.format(to)) def prefetch(self, bucket, key): """镜像回源预取文件: @@ -189,7 +193,7 @@ def prefetch(self, bucket, key): 一个ResponseInfo对象 """ resource = entry(bucket, key) - return self.__io_do('prefetch', resource) + return self.__io_do(bucket, 'prefetch', resource) def change_mime(self, bucket, key, mime): """修改文件mimeType: @@ -227,7 +231,7 @@ def batch(self, operations): ] 一个ResponseInfo对象 """ - url = 'http://{0}/batch'.format(config.get_default('default_rs_host')) + url = '{0}/batch'.format(config.get_default('default_rs_host')) return self.__post(url, dict(op=operations)) def buckets(self): @@ -245,12 +249,14 @@ def buckets(self): def __rs_do(self, operation, *args): return self.__server_do(config.get_default('default_rs_host'), operation, *args) - def __io_do(self, operation, *args): - return self.__server_do(config.get_default('default_io_host'), operation, *args) + def __io_do(self, bucket, operation, *args): + ak = self.auth.get_access_key() + io_host = self.zone.get_io_host(ak, bucket) + return self.__server_do(io_host, operation, *args) def __server_do(self, host, operation, *args): cmd = _build_op(operation, *args) - url = 'http://{0}/{1}'.format(host, cmd) + url = '{0}/{1}'.format(host, cmd) return self.__post(url) def __post(self, url, data=None): diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 57c0bf03..44cbcfeb 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -77,7 +77,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None fields['key'] = key fields['token'] = up_token - url = 'http://' + config.get_default('default_up_host') + '/' + url = config.get_default('default_zone').get_up_host_by_token(up_token) + '/' # name = key if key else file_name fname = file_name @@ -87,7 +87,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) if r is None and info.need_retry(): if info.connect_failed: - url = 'http://' + config.get_default('default_up_host_backup') + '/' + url = config.get_default('default_zone').get_up_host_backup_by_token(up_token) + '/' if hasattr(data, 'read') is False: pass elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()): @@ -170,7 +170,7 @@ def recovery_from_record(self): def upload(self): """上传操作""" self.blockStatus = [] - host = config.get_default('default_up_host') + host = config.get_default('default_zone').get_up_host_by_token(self.up_token) offset = self.recovery_from_record() for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset): length = len(block) @@ -179,7 +179,7 @@ def upload(self): if ret is None and not info.need_retry(): return ret, info if info.connect_failed(): - host = config.get_default('default_up_host_backup') + host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token) if info.need_retry() or crc != ret['crc32']: ret, info = self.make_block(block, length, host) if ret is None or crc != ret['crc32']: @@ -197,10 +197,10 @@ def make_block(self, block, block_size, host): return self.post(url, block) def block_url(self, host, size): - return 'http://{0}/mkblk/{1}'.format(host, size) + return '{0}/mkblk/{1}'.format(host, size) def file_url(self, host): - url = ['http://{0}/mkfile/{1}'.format(host, self.size)] + url = ['{0}/mkfile/{1}'.format(host, self.size)] if self.mime_type: url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type))) diff --git a/qiniu/zone.py b/qiniu/zone.py new file mode 100644 index 00000000..4e7c328b --- /dev/null +++ b/qiniu/zone.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- + +import os +import time + +from qiniu import http +from qiniu import compat, utils + +UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host + + +class Zone(object): + """七牛上传区域类 + + 该类主要内容上传区域地址。 + + Attributes: + up_host: 首选上传地址 + up_host_backup: 备用上传地址 + """ + def __init__(self, up_host=None, up_host_backup=None, io_host=None, host_cache={}, scheme="http"): + """初始化Zone类""" + self.up_host, self.up_host_backup, self.io_host = up_host, up_host_backup, io_host + self.host_cache = host_cache + self.scheme = scheme + + def get_up_host_by_token(self, up_token): + ak, bucket = self.unmarshal_up_token(up_token) + up_hosts = self.get_up_host(ak, bucket) + return up_hosts[0] + + def get_up_host_backup_by_token(self, up_token): + ak, bucket = self.unmarshal_up_token(up_token) + up_hosts = self.get_up_host(ak, bucket) + + if (len(up_hosts) <= 1): + up_host = up_hosts[0] + else: + up_host = up_hosts[1] + return up_host + + def get_io_host(self, ak, bucket): + bucket_hosts = self.get_bucket_hosts(ak, bucket) + io_hosts = bucket_hosts['ioHosts'] + return io_hosts[0] + + def get_up_host(self, ak, bucket): + bucket_hosts = self.get_bucket_hosts(ak, bucket) + up_hosts = bucket_hosts['upHosts'] + return up_hosts + + def unmarshal_up_token(self, up_token): + token = up_token.split(':') + if(len(token) != 3): + raise ValueError('invalid up_token') + + ak = token[0] + policy = compat.json.loads(str(utils.urlsafe_base64_decode(token[2]), "utf-8")) + + scope = policy["scope"] + bucket = scope + if(':' in scope): + bucket = scope.split(':')[0] + + return ak, bucket + + def get_bucket_hosts(self, ak, bucket): + key = self.scheme + ":" + ak + ":" + bucket + + bucket_hosts = self.get_bucket_hosts_to_cache(key) + if(len(bucket_hosts) > 0): + return bucket_hosts + + hosts = compat.json.loads(self.bucket_hosts(ak, bucket)) + + scheme_hosts = hosts[self.scheme] + bucket_hosts = { + 'upHosts': scheme_hosts['up'], + 'ioHosts': scheme_hosts['io'], + 'deadline': int(time.time()) + hosts['ttl'] + } + + self.set_bucket_hosts_to_cache(key, bucket_hosts) + + # hosts = self.bucket_hosts(ak, bucket) + # self.up_host = compat.json.loads(hosts)[self.scheme]["up"][0] + # self.up_host_backup = compat.json.loads(hosts)[self.scheme]["up"][1] + # self.io_host = compat.json.loads(hosts)[self.scheme]["io"][0] + return bucket_hosts + + def get_bucket_hosts_to_cache(self, key): + ret = [] + if(len(self.host_cache) == 0): + self.host_cache_from_file() + + if(not (key in self.host_cache)): + return ret + + if(self.host_cache[key]['deadline'] > time.time()): + ret = self.host_cache[key] + + return ret + + def set_bucket_hosts_to_cache(self, key, val): + self.host_cache[key] = val + self.host_cache_to_file() + return + + def host_cache_from_file(self): + path = self.host_cache_file_path() + if not os.path.isfile(path): + return None + with open(path, 'r') as f: + bucket_hosts = compat.json.load(f) + self.host_cache = bucket_hosts + f.close() + return + + def host_cache_to_file(self): + path = self.host_cache_file_path() + with open(path, 'w') as f: + compat.json.dump(self.host_cache, f) + f.close() + + def host_cache_file_path(self): + home = os.getenv("HOME") + return home + "/.qiniu_pythonsdk_hostscache2.json" + + def bucket_hosts(self, ak, bucket): + url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) + ret, info = http._get(url, None, None) + data = compat.json.dumps(ret, separators=(',', ':')) + return data From 82ef91d2cd64db69b9e8cb879283c4608bfd6364 Mon Sep 17 00:00:00 2001 From: Lifu Mao Date: Sun, 9 Oct 2016 08:46:35 +0800 Subject: [PATCH 192/478] fix ci and unitest --- qiniu/utils.py | 4 ++-- qiniu/zone.py | 18 ++++++++---------- test_qiniu.py | 20 +++++++------------- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/qiniu/utils.py b/qiniu/utils.py index bfc9cc93..e4316e09 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -3,8 +3,6 @@ from hashlib import sha1 from base64 import urlsafe_b64encode, urlsafe_b64decode -from .config import _BLOCK_SIZE - from .compat import b, s try: @@ -14,6 +12,8 @@ zlib = None import binascii +_BLOCK_SIZE = 1024 * 1024 * 4 + def urlsafe_base64_encode(data): """urlsafe的base64编码: diff --git a/qiniu/zone.py b/qiniu/zone.py index 4e7c328b..5e71c7e1 100644 --- a/qiniu/zone.py +++ b/qiniu/zone.py @@ -2,9 +2,10 @@ import os import time +import requests -from qiniu import http -from qiniu import compat, utils +from qiniu import compat +from qiniu import utils UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host @@ -55,7 +56,7 @@ def unmarshal_up_token(self, up_token): raise ValueError('invalid up_token') ak = token[0] - policy = compat.json.loads(str(utils.urlsafe_base64_decode(token[2]), "utf-8")) + policy = compat.json.loads(compat.s(utils.urlsafe_base64_decode(token[2]))) scope = policy["scope"] bucket = scope @@ -82,10 +83,6 @@ def get_bucket_hosts(self, ak, bucket): self.set_bucket_hosts_to_cache(key, bucket_hosts) - # hosts = self.bucket_hosts(ak, bucket) - # self.up_host = compat.json.loads(hosts)[self.scheme]["up"][0] - # self.up_host_backup = compat.json.loads(hosts)[self.scheme]["up"][1] - # self.io_host = compat.json.loads(hosts)[self.scheme]["io"][0] return bucket_hosts def get_bucket_hosts_to_cache(self, key): @@ -124,10 +121,11 @@ def host_cache_to_file(self): def host_cache_file_path(self): home = os.getenv("HOME") - return home + "/.qiniu_pythonsdk_hostscache2.json" + return home + "/.qiniu_pythonsdk_hostscache.json" def bucket_hosts(self, ak, bucket): url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) - ret, info = http._get(url, None, None) - data = compat.json.dumps(ret, separators=(',', ':')) + ret = requests.get(url) + # ret, info = http._get(url, None, None) + data = compat.json.dumps(ret.json(), separators=(',', ':')) return data diff --git a/test_qiniu.py b/test_qiniu.py index df028436..b9598366 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -325,44 +325,40 @@ def test_putWithoutKey(self): def test_withoutRead_withoutSeek_retry(self): key = 'retry' data = 'hello retry!' - set_default(default_zone=Zone('a', 'upload.qiniu.com')) + set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) assert ret['key'] == key assert ret['hash'] == 'FlYu0iBR1WpvYi4whKXiBuQpyLLk' - qiniu.set_default(default_zone=qiniu.config.zone0) def test_hasRead_hasSeek_retry(self): key = 'withReadAndSeek_retry' data = StringIO('hello retry again!') - set_default(default_zone=Zone('a', 'upload.qiniu.com')) + set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) assert ret['key'] == key assert ret['hash'] == 'FuEbdt6JP2BqwQJi7PezYhmuVYOo' - qiniu.set_default(default_zone=qiniu.config.zone0) def test_hasRead_withoutSeek_retry(self): key = 'withReadAndWithoutSeek_retry' data = ReadWithoutSeek('I only have read attribute!') - set_default(default_zone=Zone('a', 'upload.qiniu.com')) + set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) assert ret is None - qiniu.set_default(default_zone=qiniu.config.zone0) def test_hasRead_WithoutSeek_retry2(self): key = 'withReadAndWithoutSeek_retry2' data = urlopen("http://www.qiniu.com") - set_default(default_zone=Zone('a', 'upload.qiniu.com')) + set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) - assert ret is None - qiniu.set_default(default_zone=qiniu.config.zone0) + assert ret is not None def test_putData_without_fname(self): if is_travis(): @@ -419,23 +415,21 @@ def test_big_file(self): token = self.q.upload_token(bucket_name, key) localfile = create_temp_file(4 * 1024 * 1024 + 1) progress_handler = lambda progress, total: progress - qiniu.set_default(default_zone=Zone('a', 'upload.qiniu.com')) + qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) ret, info = put_file(token, key, localfile, self.params, self.mime_type, progress_handler=progress_handler) print(info) assert ret['key'] == key - qiniu.set_default(default_zone=qiniu.config.zone0) remove_temp_file(localfile) def test_retry(self): localfile = __file__ key = 'test_file_r_retry' - qiniu.set_default(default_zone=Zone('a', 'upload.qiniu.com')) + qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) token = self.q.upload_token(bucket_name, key) ret, info = put_file(token, key, localfile, self.params, self.mime_type) print(info) assert ret['key'] == key assert ret['hash'] == etag(localfile) - qiniu.set_default(default_zone=qiniu.config.zone0) class DownloadTestCase(unittest.TestCase): From 485f7de0e907b404d13c5912ba392dd727b97523 Mon Sep 17 00:00:00 2001 From: Lifu Mao Date: Sun, 9 Oct 2016 11:23:43 +0800 Subject: [PATCH 193/478] add changelog --- CHANGELOG.md | 5 +++++ qiniu/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 334fb81b..3feb2c7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ #Changelog +## 7.0.9 (2016-10-09) +### 增加 +* 多机房接口调用支持 + + ## 7.0.8 (2016-07-05) ### 修正 * 修复表单上传大于20M文件的400错误 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 8f213b88..a290f2d4 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.0.8' +__version__ = '7.0.9' from .auth import Auth From 78a286bef3294ab72887ececb5bc6b125c92c804 Mon Sep 17 00:00:00 2001 From: Qin Xiao Date: Tue, 22 Nov 2016 19:42:00 +0800 Subject: [PATCH 194/478] add kirk sdk skeleton --- examples/kirk/README.md | 5 ++ examples/kirk/list_apps.py | 18 ++++ examples/kirk/list_services.py | 26 ++++++ examples/kirk/list_stacks.py | 23 ++++++ qiniu/__init__.py | 3 +- qiniu/auth.py | 82 ++++++++++++++++++ qiniu/config.py | 1 - qiniu/http.py | 12 ++- qiniu/services/kirk/__init__.py | 0 qiniu/services/kirk/app.py | 142 ++++++++++++++++++++++++++++++++ qiniu/services/kirk/config.py | 21 +++++ qiniu/services/kirk/qcos_api.py | 65 +++++++++++++++ setup.py | 1 + 13 files changed, 396 insertions(+), 3 deletions(-) create mode 100644 examples/kirk/README.md create mode 100644 examples/kirk/list_apps.py create mode 100644 examples/kirk/list_services.py create mode 100644 examples/kirk/list_stacks.py create mode 100644 qiniu/services/kirk/__init__.py create mode 100644 qiniu/services/kirk/app.py create mode 100644 qiniu/services/kirk/config.py create mode 100644 qiniu/services/kirk/qcos_api.py diff --git a/examples/kirk/README.md b/examples/kirk/README.md new file mode 100644 index 00000000..23e5a852 --- /dev/null +++ b/examples/kirk/README.md @@ -0,0 +1,5 @@ +# Examples + +``` +$ python list_apps.py +``` diff --git a/examples/kirk/list_apps.py b/examples/kirk/list_apps.py new file mode 100644 index 00000000..2062b8b1 --- /dev/null +++ b/examples/kirk/list_apps.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +import sys +from qiniu import QiniuMacAuth +from qiniu import AccountClient + +access_key = sys.argv[1] +secret_key = sys.argv[2] + +acc_client = AccountClient(QiniuMacAuth(access_key, secret_key)) + +ret, info = acc_client.list_apps() + +print(ret) +print(info) + +assert len(ret) is not None diff --git a/examples/kirk/list_services.py b/examples/kirk/list_services.py new file mode 100644 index 00000000..9ec683fb --- /dev/null +++ b/examples/kirk/list_services.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +import sys +from qiniu import QiniuMacAuth +from qiniu import AccountClient + +access_key = sys.argv[1] +secret_key = sys.argv[2] + +acc_client = AccountClient(QiniuMacAuth(access_key, secret_key)) +apps, info = acc_client.list_apps() + +for app in apps: + if app.get('runMode') == 'Private': + uri = app.get('uri') + qcos = acc_client.get_qcos_client(uri) + if qcos != None: + stacks, info = qcos.list_stacks() + for stack in stacks: + stack_name = stack.get('name') + services, info = qcos.list_services(stack_name) + print("list_services of '%s : %s':"%(uri, stack_name)) + print(services) + print(info) + assert len(services) is not None diff --git a/examples/kirk/list_stacks.py b/examples/kirk/list_stacks.py new file mode 100644 index 00000000..67815603 --- /dev/null +++ b/examples/kirk/list_stacks.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +import sys +from qiniu import QiniuMacAuth +from qiniu import AccountClient + +access_key = sys.argv[1] +secret_key = sys.argv[2] + +acc_client = AccountClient(QiniuMacAuth(access_key, secret_key)) +apps, info = acc_client.list_apps() + +for app in apps: + if app.get('runMode') == 'Private': + uri = app.get('uri') + qcos = acc_client.get_qcos_client(uri) + if qcos != None: + stacks, info = qcos.list_stacks() + print("list_stacks of '%s':"%uri) + print(stacks) + print(info) + assert len(stacks) is not None diff --git a/qiniu/__init__.py b/qiniu/__init__.py index a290f2d4..a807abcb 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -11,7 +11,7 @@ __version__ = '7.0.9' -from .auth import Auth +from .auth import Auth, QiniuMacAuth from .config import set_default from .zone import Zone @@ -20,5 +20,6 @@ from .services.storage.uploader import put_data, put_file, put_stream from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save +from .services.kirk.app import AccountClient from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry diff --git a/qiniu/auth.py b/qiniu/auth.py index 876a53ac..04f92e4d 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -196,3 +196,85 @@ def __call__(self, r): token = self.auth.token_of_request(r.url) r.headers['Authorization'] = 'QBox {0}'.format(token) return r + +class QiniuMacAuth(object): + """ + Sign Requests + + Attributes: + __access_key + __secret_key + + http://kirk-docs.qiniu.com/apidocs/#TOC_325b437b89e8465e62e958cccc25c63f + """ + + def __init__(self, access_key, secret_key): + self.qiniu_header_prefix = "X-Qiniu-" + self.__checkKey(access_key, secret_key) + self.__access_key = access_key + self.__secret_key = b(secret_key) + + def __token(self, data): + data = b(data) + hashed = hmac.new(self.__secret_key, data, sha1) + return urlsafe_base64_encode(hashed.digest()) + + def token_of_request(self, method, host, url, qheaders, content_type=None, body=None): + """ + + Host: + Content-Type: + [ Headers] + + [] #这里的 只有在 存在且不为 application/octet-stream 时才签进去。 + + """ + parsed_url = urlparse(url) + netloc = parsed_url.netloc + path = parsed_url.path + query = parsed_url.query + + if not host: + host = netloc + + path_with_query = path + if query != '': + path_with_query = ''.join([path_with_query, '?', query]) + data = ''.join(["%s %s"%(method, path_with_query) , "\n", "Host: %s"%host, "\n"]) + + if content_type: + data += "Content-Type: %s"%s(content_type) + "\n" + + data += qheaders + data += "\n" + + if content_type and content_type != "application/octet-stream" and body: + data += body + + return '{0}:{1}'.format(self.__access_key, self.__token(data)) + + def qiniu_headers(self, headers): + res = "" + for key in headers: + if key.startswith(self.qiniu_header_prefix): + res += key+": %s\n"%s(headers.get(key)) + return res + + @staticmethod + def __checkKey(access_key, secret_key): + if not (access_key and secret_key): + raise ValueError('QiniuMacAuthSign : Invalid key') + +class QiniuMacRequestsAuth(AuthBase): + def __init__(self, auth): + self.auth = auth + + def __call__(self, r): + token = self.auth.token_of_request( + r.method, r.headers.get('Host', None), + r.url, self.auth.qiniu_headers(r.headers), + r.headers.get('Content-Type', None), + r.body + ) + r.headers['Authorization'] = 'Qiniu {0}'.format(token) + return r diff --git a/qiniu/config.py b/qiniu/config.py index 5431d174..9b827962 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -8,7 +8,6 @@ _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 - _config = { 'default_zone': zone.Zone(), 'default_rs_host': RS_HOST, diff --git a/qiniu/http.py b/qiniu/http.py index 011db8e3..2a413b33 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -72,10 +72,20 @@ def _post_with_token(url, data, token): def _post_file(url, data, files): return _post(url, data, files, None) - def _post_with_auth(url, data, auth): return _post(url, data, None, qiniu.auth.RequestsAuth(auth)) +def _post_with_qiniu_mac(url, data, auth): + return _post(url, data, None, qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None) + +def _get_with_qiniu_mac(url, params, auth): + try: + r = requests.get( + url, params=params, auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, + timeout=config.get_default('connection_timeout'), headers=_headers) + except Exception as e: + return None, ResponseInfo(None, e) + return __return_wrapper(r) class ResponseInfo(object): """七牛HTTP请求返回信息类 diff --git a/qiniu/services/kirk/__init__.py b/qiniu/services/kirk/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qiniu/services/kirk/app.py b/qiniu/services/kirk/app.py new file mode 100644 index 00000000..11b6400c --- /dev/null +++ b/qiniu/services/kirk/app.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- + +from qiniu import config, http, QiniuMacAuth +from .config import KIRK_HOST +from .qcos_api import QcosClient + +class AccountClient(object): + """客户端入口 + + 使用账号密钥生成账号客户端，可以进一步： + 1、获取和操作账号数据 + 2、获得部署的应用的客户端 + + 属性： + auth: 账号管理密钥对，QiniuMacAuth对象 + host: API host，在『内网模式』下使用时，auth=None，会自动使用 apiproxy 服务 + + 接口： + get_qcos_client(app_uri) + create_qcos_client(app_uri) + get_app_keys(app_uri) + get_valid_app_auth(app_uri) + get_account_info() + get_app_region_products(app_uri) + get_region_products(region) + list_regions() + list_apps() + + """ + + def __init__(self, auth, host=None): + self.auth = auth + self.qcos_clients = {} + if (auth is None): + self.host = KIRK_HOST['APPPROXY'] + else: + self.host = KIRK_HOST['APPGLOBAL'] + acc, info = self.get_account_info() + self.uri = acc.get('name') + + def get_qcos_client(self, app_uri): + """获得资源管理客户端 + 缓存，但不是线程安全的 + """ + + client = self.qcos_clients.get(app_uri) + if (client is None): + client = self.create_qcos_client(app_uri) + self.qcos_clients[app_uri] = client + + return client + + def create_qcos_client(self, app_uri): + """创建资源管理客户端 + + """ + + if (self.auth is None): + return QcosClient(None) + + products = self.get_app_region_products(app_uri) + auth = self.get_valid_app_auth(app_uri) + + if products is None or auth is None: + return None + + return QcosClient(auth, products.get('api')) + + def get_app_keys(self, app_uri): + """获得账号下应用的密钥 + + """ + + url = '{0}/v3/apps/{1}/keys'.format(self.host, app_uri) + return http._get_with_qiniu_mac(url, None, self.auth) + + def get_valid_app_auth(self, app_uri): + """获得账号下可用的应用的密钥 + + """ + + ret, retInfo = self.get_app_keys(app_uri) + + if ret is None: + return None + + for k in ret: + if (k.get('state') == 'enabled'): + return QiniuMacAuth(k.get('ak'), k.get('sk')) + + return None + + def get_account_info(self): + """获得当前账号的信息 + + """ + + url = '{0}/v3/info'.format(self.host) + return http._get_with_qiniu_mac(url, None, self.auth) + + def get_app_region_products(self, app_uri): + """获得指定应用所在区域的产品信息 + + """ + apps, retInfo = self.list_apps() + if apps is None: + return None + + for app in apps: + if (app.get('uri') == app_uri): + return self.get_region_products(app.get('region')) + + return + + def get_region_products(self, region): + """获得指定区域的产品信息 + + """ + + regions, retInfo = self.list_regions() + if regions is None: + return None + + for r in regions: + if r.get('name') == region: + return r.get('products') + + def list_regions(self): + """获得账号可见的区域的信息 + + """ + + url = '{0}/v3/regions'.format(self.host) + return http._get_with_qiniu_mac(url, None, self.auth) + + def list_apps(self): + """获得当前账号的应用列表 + + """ + + url = '{0}/v3/apps'.format(self.host) + return http._get_with_qiniu_mac(url, None, self.auth) diff --git a/qiniu/services/kirk/config.py b/qiniu/services/kirk/config.py new file mode 100644 index 00000000..7a117b4f --- /dev/null +++ b/qiniu/services/kirk/config.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +KIRK_HOST = { + 'APPGLOBAL': "https://app-api.qiniu.com", # 公有云 APP API + 'APPPROXY': "http://app.qcos.qiniu", # 内网 APP API + 'APIPROXY': "http://api.qcos.qiniu", # 内网 API +} + +CONTAINER_UINT_TYPE = { + '1U1G': '单核(CPU)，1GB(内存)', + '1U2G': '单核(CPU)，2GB(内存)', + '1U4G': '单核(CPU)，4GB(内存)', + '1U8G': '单核(CPU)，8GB(内存)', + '2U2G': '双核(CPU)，2GB(内存)', + '2U4G': '双核(CPU)，4GB(内存)', + '2U8G': '双核(CPU)，8GB(内存)', + '2U16G': '双核(CPU)，16GB(内存)', + '4U8G': '四核(CPU)，8GB(内存)', + '4U16G': '四核(CPU)，16GB(内存)', + '8U16G': '八核(CPU)，16GB(内存)', +} diff --git a/qiniu/services/kirk/qcos_api.py b/qiniu/services/kirk/qcos_api.py new file mode 100644 index 00000000..810cb129 --- /dev/null +++ b/qiniu/services/kirk/qcos_api.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +from qiniu import config, http +from .config import KIRK_HOST + +class QcosClient(object): + """资源管理客户端 + + 使用应用密钥生成资源管理客户端，可以进一步： + 1、部署服务和容器，获得信息 + 2、创建网络资源，获得信息 + + 属性： + auth: 应用密钥对，QiniuMacAuth对象 + host: API host，在『内网模式』下使用时，auth=None，会自动使用 apiproxy 服务，只能管理当前容器所在的应用资源。 + + 接口： + list_stacks() + # create_stack(args) + # delete_stack(stack) + + list_services(stack) + # create_service(stack, args) + # get_service_inspect(stack, service) + # update_service(stack, service, args) + # scale_service(stack, service, args) + # delete_service(stack, service) + + # list_containers(args) + # get_container_inspect(ip) + # start_container(ip) + # stop_container(ip) + # restart_container(ip) + + # list_aps() + # create_ap(args) + # search_ap(mode, args) + # get_ap(id) + # set_ap_port(id, port, args) + # delete_ap(id) + + """ + + def __init__(self, auth, host=None): + self.auth = auth + if (auth is None): + self.host = KIRK_HOST['APIPROXY'] + else: + self.host = host + + def list_stacks(self): + """获得服务组列表 + + """ + + url = '{0}/v3/stacks'.format(self.host) + return http._get_with_qiniu_mac(url, None, self.auth) + + def list_services(self, stack): + """获得服务列表 + + """ + + url = '{0}/v3/stacks/{1}/services'.format(self.host, stack) + return http._get_with_qiniu_mac(url, None, self.auth) diff --git a/setup.py b/setup.py index 4ab32884..9483eb9d 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ 'qiniu.services', 'qiniu.services.storage', 'qiniu.services.processing', + 'qiniu.services.kirk', ] From e6cb9f220c5448c49cc98aba5ee5f9c5e8bdfe80 Mon Sep 17 00:00:00 2001 From: pqx Date: Fri, 25 Nov 2016 21:50:52 +0800 Subject: [PATCH 195/478] =?UTF-8?q?=E5=88=9D=E6=AD=A5=E5=AE=8C=E6=88=90qco?= =?UTF-8?q?s=5Fapi?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/auth.py | 2 +- qiniu/http.py | 20 +- qiniu/services/kirk/app.py | 18 + qiniu/services/kirk/qcos_api.py | 660 ++++++++++++++++++++++++++++++-- test_kirk.py | 331 ++++++++++++++++ 5 files changed, 1006 insertions(+), 25 deletions(-) create mode 100644 test_kirk.py diff --git a/qiniu/auth.py b/qiniu/auth.py index 04f92e4d..a797945a 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -243,7 +243,7 @@ def token_of_request(self, method, host, url, qheaders, content_type=None, body= data = ''.join(["%s %s"%(method, path_with_query) , "\n", "Host: %s"%host, "\n"]) if content_type: - data += "Content-Type: %s"%s(content_type) + "\n" + data += "Content-Type: %s"%(content_type) + "\n" data += qheaders data += "\n" diff --git a/qiniu/http.py b/qiniu/http.py index 2a413b33..ad79178b 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -76,7 +76,15 @@ def _post_with_auth(url, data, auth): return _post(url, data, None, qiniu.auth.RequestsAuth(auth)) def _post_with_qiniu_mac(url, data, auth): - return _post(url, data, None, qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None) + + qn_auth = qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None + timeout=config.get_default('connection_timeout') + + try: + r = requests.post(url, json=data, auth=qn_auth, timeout=timeout, headers=_headers) + except Exception as e: + return None, ResponseInfo(None, e) + return __return_wrapper(r) def _get_with_qiniu_mac(url, params, auth): try: @@ -87,6 +95,16 @@ def _get_with_qiniu_mac(url, params, auth): return None, ResponseInfo(None, e) return __return_wrapper(r) +def _delete_with_qiniu_mac(url, params, auth): + try: + r = requests.delete( + url, params=params, auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, + timeout=config.get_default('connection_timeout'), headers=_headers) + except Exception as e: + return None, ResponseInfo(None, e) + return __return_wrapper(r) + + class ResponseInfo(object): """七牛HTTP请求返回信息类 diff --git a/qiniu/services/kirk/app.py b/qiniu/services/kirk/app.py index 11b6400c..d837fce9 100644 --- a/qiniu/services/kirk/app.py +++ b/qiniu/services/kirk/app.py @@ -25,6 +25,8 @@ class AccountClient(object): get_region_products(region) list_regions() list_apps() + create_app(args) + delete_app(app_uri) """ @@ -140,3 +142,19 @@ def list_apps(self): url = '{0}/v3/apps'.format(self.host) return http._get_with_qiniu_mac(url, None, self.auth) + + def create_app(self, args): + """创建应用 + + """ + + url = '{0}/v3/apps'.format(self.host) + return http._post_with_qiniu_mac(url, args, self.auth) + + def delete_app(self, app_uri): + """删除应用 + + """ + + url = '{0}/v3/apps/{1}'.format(self.host, app_uri) + return http._delete_with_qiniu_mac(url, None, self.auth) diff --git a/qiniu/services/kirk/qcos_api.py b/qiniu/services/kirk/qcos_api.py index 810cb129..29a8254c 100644 --- a/qiniu/services/kirk/qcos_api.py +++ b/qiniu/services/kirk/qcos_api.py @@ -16,34 +16,49 @@ class QcosClient(object): 接口： list_stacks() - # create_stack(args) - # delete_stack(stack) + create_stack(args) + delete_stack(stack) + get_stack(stack) + start_stack(stack) + stop_stack(stack) list_services(stack) - # create_service(stack, args) - # get_service_inspect(stack, service) - # update_service(stack, service, args) - # scale_service(stack, service, args) - # delete_service(stack, service) - - # list_containers(args) - # get_container_inspect(ip) - # start_container(ip) - # stop_container(ip) - # restart_container(ip) - - # list_aps() - # create_ap(args) - # search_ap(mode, args) - # get_ap(id) - # set_ap_port(id, port, args) - # delete_ap(id) + create_service(stack, args) + get_service_inspect(stack, service) + start_service(stack, service) + stop_service(stack, service) + update_service(stack, service, args) + scale_service(stack, service, args) + delete_service(stack, service) + create_service_volume(stack, service, volume, args) + extend_service_volume(stack, service, volume, args) + delete_service_volume(stack, service, volume) + list_containers(args) + get_container_inspect(ip) + start_container(ip) + stop_container(ip) + restart_container(ip) + + list_aps() + create_ap(args) + search_ap(mode, query) + get_ap(apid) + update_ap(apid, args) + set_ap_port(apid, port, args) + delete_ap(apid) + publish_ap(apid, args) + unpublish_ap(apid) + get_ap_port_healthcheck(apid, port) + set_ap_port_container(apid, port, args) + disable_ap_port(apid, port) + enable_ap_port(apid, port) + get_ap_providers() """ def __init__(self, auth, host=None): self.auth = auth - if (auth is None): + if auth is None: self.host = KIRK_HOST['APIPROXY'] else: self.host = host @@ -51,15 +66,614 @@ def __init__(self, auth, host=None): def list_stacks(self): """获得服务组列表 + 列出当前应用的所有服务组信息。 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回服务组列表[, , ...]，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 """ + url = '{0}/v3/stacks'.format(self.host) + return self.__get(url) + + def create_stack(self, args): + """创建服务组 + + 创建新一个指定名称的服务组，并创建其下的服务。 + + Args: + - args: 服务组描述，参考 http://kirk-docs.qiniu.com/apidocs/ + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ url = '{0}/v3/stacks'.format(self.host) - return http._get_with_qiniu_mac(url, None, self.auth) + return self.__post(url, args) + + def delete_stack(self, stack): + """删除服务组 + + 删除服务组内所有服务并销毁服务组。 + + Args: + - stack: 服务所属的服务组名称 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}'.format(self.host, stack) + return self.__delete(url) + + def get_stack(self, stack): + """获取服务组 + + 查看服务组的属性信息。 + + Args: + - stack: 服务所属的服务组名称 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回stack信息，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}'.format(self.host, stack) + return self.__get(url) + + def start_stack(self, stack): + """启动服务组 + + 启动服务组中的所有停止状态的服务。 + + Args: + - stack: 服务所属的服务组名称 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/start'.format(self.host, stack) + return self.__post(url) + + def stop_stack(self, stack): + """停止服务组 + + 停止服务组中所有运行状态的服务。 + + Args: + - stack: 服务所属的服务组名称 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/stop'.format(self.host, stack) + return self.__post(url) + def list_services(self, stack): """获得服务列表 + 列出指定名称的服务组内所有的服务, 返回一组详细的服务信息。 + + Args: + - stack: 服务所属的服务组名称 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回服务信息列表[, , ...]，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 """ + url = '{0}/v3/stacks/{1}/services'.format(self.host, stack) + return self.__get(url) + + def create_service(self, stack, args): + """创建服务 + + 创建一个服务，平台会异步地按模板分配资源并部署所有容器。 + Args: + - stack: 服务所属的服务组名称 + - args: 服务具体描述请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ url = '{0}/v3/stacks/{1}/services'.format(self.host, stack) - return http._get_with_qiniu_mac(url, None, self.auth) + return self.__post(url, args) + + def delete_service(self, stack, service): + """删除服务 + + 删除指定名称服务，并自动销毁服务已部署的所有容器和存储卷。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}'.format(self.host, stack, service) + return self.__delete(url) + + def get_service_inspect(self, stack, service): + """查看服务 + + 查看指定名称服务的属性。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回服务信息，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/inspect'.format(self.host, stack, service) + return self.__get(url) + + def start_service(self, stack, service): + """启动服务 + + 启动指定名称服务的所有容器。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/start'.format(self.host, stack, service) + return self.__post(url) + + def stop_service(self, stack, service): + """停止服务 + + 停止指定名称服务的所有容器。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/stop'.format(self.host, stack, service) + return self.__post(url) + + def update_service(self, stack, service, args): + """更新服务 + + 更新指定名称服务的配置如容器镜像等参数，容器被重新部署后生效。 + 如果指定manualUpdate参数，则需要额外调用 部署服务 接口并指定参数进行部署；处于人工升级模式的服务禁止执行其他修改操作。 + 如果不指定manualUpdate参数，平台会自动完成部署。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + - args: 服务具体描述请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}'.format(self.host, stack, service) + return self.__post(url, args) + + def scale_service(self, stack, service, args): + """扩容/缩容服务 + + 更新指定名称服务的配置如容器镜像等参数，容器被重新部署后生效。 + 如果指定manualUpdate参数，则需要额外调用 部署服务 接口并指定参数进行部署；处于人工升级模式的服务禁止执行其他修改操作。 + 如果不指定manualUpdate参数，平台会自动完成部署。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/scale'.format(self.host, stack, service) + return self.__post(url, args) + + def create_service_volume(self, stack, service, args): + """创建存储卷 + + 为指定名称的服务增加存储卷资源，并挂载到部署的容器中。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/volumes'.format(self.host, stack, service) + return self.__post(url, args) + + def extend_service_volume(self, stack, service, volume, args): + """扩容存储卷 + + 为指定名称的服务增加存储卷资源，并挂载到部署的容器中。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + - volume: 存储卷名 + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/volumes/{3}/extend'\ + .format(self.host, stack, service, volume) + return self.__post(url, args) + + def delete_service_volume(self, stack, service, volume): + """删除存储卷 + + 从部署的容器中移除挂载，并销毁指定服务下指定名称的存储卷, 并重新启动该容器。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + - volume: 存储卷名 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/volumes/{3}'.format(self.host, stack, service, volume) + return self.__delete(url) + + def list_containers(self, stack=None, service=None): + """列出容器列表 + + 列出应用内所有部署的容器, 返回一组容器IP。 + + Args: + - stack: 要列出容器的服务组名(可不填，表示默认列出所有) + - service: 要列出容器服务的服务名(可不填，表示默认列出所有) + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回容器的ip数组，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/containers'.format(self.host) + params = {} + if stack is not None: + params['stack'] = stack + if service is not None: + params['service'] = service + return self.__get(url, params or None) + + def get_container_inspect(self, ip): + """查看容器 + + 查看指定IP的容器，返回容器属性。 + + Args: + - ip: 容器ip + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回容器的信息，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/containers/{1}/inspect'.format(self.host, ip) + return self.__get(url) + + def start_container(self, ip): + """启动容器 + + 启动指定IP的容器。 + + Args: + - ip: 容器ip + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/containers/{1}/start'.format(self.host, ip) + return self.__post(url) + + def stop_container(self, ip): + """停止容器 + + 停止指定IP的容器。 + + Args: + - ip: 容器ip + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/containers/{1}/stop'.format(self.host, ip) + return self.__post(url) + + def restart_container(self, ip): + """重启容器 + + 重启指定IP的容器。 + + Args: + - ip: 容器ip + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/containers/{1}/restart'.format(self.host, ip) + return self.__post(url) + + def list_aps(self): + """列出接入点 + + 列出当前应用的所有接入点。 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回接入点列表，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps'.format(self.host) + return self.__get(url) + + def create_ap(self, args): + """申请接入点 + + 申请指定配置的接入点资源。 + + Args: + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回申请到的接入点信息，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps'.format(self.host) + return self.__post(url, args) + + def search_ap(self, mode, query): + """搜索接入点 + + 查看指定接入点的所有配置信息，包括所有监听端口的配置。 + + Args: + - mode: 搜索模式，可以是domain、ip、host + - query: 搜索文本 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回搜索结果，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/search?{1}={2}'.format(self.host, mode, query) + return self.__get(url) + + def get_ap(self, apid): + """查看接入点 + + 给出接入点的域名或IP，查看配置信息，包括所有监听端口的配置。 + + Args: + - apid: 接入点ID + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回接入点信息，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}'.format(self.host, apid) + return self.__get(url) + + def update_ap(self, apid, args): + """更新接入点 + + 更新指定接入点的配置，如带宽。 + + Args: + - apid: 接入点ID + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}'.format(self.host, apid) + return self.__post(url, args) + + def set_ap_port(self, apid, port, args): + """更新接入点端口配置 + + 更新接入点指定端口的配置。 + + Args: + - apid: 接入点ID + - port: 要设置的端口号 + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/{2}'.format(self.host, apid, port) + return self.__post(url, args) + + def delete_ap(self, apid): + """释放接入点 + + 销毁指定接入点资源。 + + Args: + - apid: 接入点ID + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}'.format(self.host, apid) + return self.__delete(url) + + def publish_ap(self, apid, args): + """绑定自定义域名 + + 绑定用户自定义的域名，仅对公网域名模式接入点生效。 + + Args: + - apid: 接入点ID + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/publish'.format(self.host, apid) + return self.__post(url, args) + + def unpublish_ap(self, apid, args): + """解绑自定义域名 + + 解绑用户自定义的域名，仅对公网域名模式接入点生效。 + + Args: + - apid: 接入点ID + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/unpublish'.format(self.host, apid) + return self.__post(url, args) + + def get_ap_port_healthcheck(self, apid, port): + """查看健康检查结果 + + 检查接入点的指定端口的后端健康状况。 + + Args: + - apid: 接入点ID + - port: 要设置的端口号 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回健康状况，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/{2}/healthcheck'.format(self.host, apid, port) + return self.__get(url) + + def set_ap_port_container(self, apid, port, args): + """调整后端实例配置 + + 调整接入点指定后端实例（容器）的配置，例如临时禁用流量等。 + + Args: + - apid: 接入点ID + - port: 要设置的端口号 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/{2}/setcontainer'.format(self.host, apid, port) + return self.__post(url, args) + + def disable_ap_port(self, apid, port): + """临时关闭接入点端口 + + 临时关闭接入点端口，仅对公网域名，公网ip有效。 + + Args: + - apid: 接入点ID + - port: 要设置的端口号 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/{2}/disable'.format(self.host, apid, port) + return self.__post(url) + + def enable_ap_port(self, apid, port): + """开启接入点端口 + + 开启临时关闭的接入点端口，仅对公网域名，公网ip有效。 + + Args: + - apid: 接入点ID + - port: 要设置的端口号 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/{2}/enable'.format(self.host, apid, port) + return self.__post(url) + + def get_ap_providers(self): + """列出入口提供商 + + 列出当前支持的入口提供商，仅对申请公网IP模式接入点有效。 + 注：公网IP供应商telecom=电信，unicom=联通，mobile=移动。 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回接入商列表，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/providers'.format(self.host) + return self.__get(url) + + def __post(self, url, data=None): + return http._post_with_qiniu_mac(url, data, self.auth) + + def __get(self, url, params=None): + return http._get_with_qiniu_mac(url, params, self.auth) + + def __delete(self, url): + return http._delete_with_qiniu_mac(url, None, self.auth) diff --git a/test_kirk.py b/test_kirk.py new file mode 100644 index 00000000..3e246b6c --- /dev/null +++ b/test_kirk.py @@ -0,0 +1,331 @@ +# -*- coding: utf-8 -*- +import os +import sys +import time +import logging +import pytest +from qiniu import auth +from qiniu.services import kirk + + +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') +qn_auth = auth.QiniuMacAuth(access_key, secret_key) +acc_client = kirk.app.AccountClient(qn_auth) +qcos_client = None +user_name = '' +app_uri = '' +app_name = 'appjust4test' +app_region = 'nq' + + +def setup_module(module): + acc_client = kirk.app.AccountClient(qn_auth) + user_info = acc_client.get_account_info()[0] + acc_client.create_app({'name': app_name, 'title': 'whatever', 'region': app_region}) + + module.user_name = user_info['name'] + module.app_uri = '{0}.{1}'.format(module.user_name, app_name) + module.qcos_client = acc_client.create_qcos_client(module.app_uri) + + +def teardown_module(module): + module.app_uri + acc_client.delete_app(module.app_uri) + + +class TestApp: + """应用测试用例""" + + def test_create_and_delete_app(self): + _name_create = 'appjust4testcreate' + _uri_create = '' + _args = {'name': _name_create, 'title': 'whatever', 'region': app_region} + + with Call(acc_client, 'create_app', _args) as r: + assert r[0] is not None + _uri_create = r[0]['uri'] + + with Call(acc_client, 'delete_app', _uri_create) as r: + assert r[0] == {} + + def test_get_app_keys(self): + with Call(acc_client, 'get_app_keys', app_uri) as r: + assert len(r[0]) > 0 + + def test_get_account_info(self): + with Call(acc_client, 'get_account_info') as r: + assert r[0] is not None + + +class TestStack: + """服务组测试用例""" + + _name = 'just4test' + _name_del = 'just4del' + _name_create = 'just4create' + + @classmethod + def setup_class(cls): + qcos_client.create_stack({'name': cls._name}) + qcos_client.create_stack({'name': cls._name_del}) + + @classmethod + def teardown_class(cls): + qcos_client.delete_stack(cls._name) + qcos_client.delete_stack(cls._name_create) + qcos_client.delete_stack(cls._name_del) + + def test_create_stack(self): + with Call(qcos_client, 'create_stack', {'name': self._name_create}) as r: + assert r[0] == {} + + def test_delete_stack(self): + with Call(qcos_client, 'delete_stack', self._name_del) as r: + assert r[0] == {} + + def test_list_stacks(self): + with Call(qcos_client, 'list_stacks') as r: + assert len(r) > 0 + assert self._name in [stack['name'] for stack in r[0]] + + def test_get_stack(self): + with Call(qcos_client, 'get_stack', self._name) as r: + assert r[0]['name'] == self._name + + def test_start_stack(self): + with Call(qcos_client, 'start_stack', self._name) as r: + assert r[0] == {} + + def test_stop_stack(self): + with Call(qcos_client, 'stop_stack', self._name) as r: + assert r[0] == {} + + +class TestService: + """服务测试用例""" + + _stack = 'just4test2' + _name = 'spaceship' + _name_del = 'spaceship4del' + _name_create = 'spaceship4create' + _image = 'library/nginx:stable' + _unit = '1U1G' + _spec = {'image': _image, 'unitType': _unit} + + @classmethod + def setup_class(cls): + qcos_client.delete_stack(cls._stack) + qcos_client.create_stack({'name': cls._stack}) + qcos_client.create_service(cls._stack, {'name': cls._name, 'spec': cls._spec}) + qcos_client.create_service(cls._stack, {'name': cls._name_del, 'spec': cls._spec}) + + _debug_info('waiting for services to setup ...') + time.sleep(10) + + @classmethod + def teardown_class(cls): + # 删除stack会清理所有相关服务 + qcos_client.delete_stack(cls._stack) + + def test_create_service(self): + service = {'name': self._name_create, 'spec': self._spec} + with Call(qcos_client, 'create_service', self._stack, service) as r: + assert r[0] == {} + + def test_delete_service(self): + with Call(qcos_client, 'delete_service', self._stack, self._name_del) as r: + assert r[0] == {} + + def test_list_services(self): + with Call(qcos_client, 'list_services', self._stack) as r: + assert len(r) > 0 + assert self._name in [service['name'] for service in r[0]] + + def test_get_service_inspect(self): + with Call(qcos_client, 'get_service_inspect', self._stack, self._name) as r: + assert r[0]['name'] == self._name + assert r[0]['spec']['unitType'] == self._unit + + def test_update_service(self): + data = {'spec': {'autoRestart': 'ON_FAILURE'}} + with Call(qcos_client, 'update_service', self._stack, self._name, data) as r: + assert r[0] == {} + + _debug_info('waiting for update services to ready ...') + time.sleep(10) + + def test_scale_service(self): + data = {'instanceNum': 2} + with Call(qcos_client, 'scale_service', self._stack, self._name, data) as r: + assert r[0] == {} + + _debug_info('waiting for scale services to ready ...') + time.sleep(10) + + +class TestContainer: + """容器测试用例""" + + _stack = 'just4test3' + _service = 'spaceship' + _spec = {'image': 'library/nginx:stable', 'unitType': '1U1G'} + # 为了方便测试，容器数量最少为2 + _instanceNum = 2 + + @classmethod + def setup_class(cls): + qcos_client.delete_stack(cls._stack) + qcos_client.create_stack({'name': cls._stack}) + qcos_client.create_service(cls._stack, {'name': cls._service, 'spec': cls._spec, 'instanceNum': cls._instanceNum}) + + _debug_info('waiting for containers to setup ...') + time.sleep(10) + + @classmethod + def teardown_class(cls): + qcos_client.delete_stack(cls._stack) + + def test_list_containers(self): + with Call(qcos_client, 'list_containers', self._stack, self._service) as r: + assert len(r[0]) > 0 + assert len(r[0]) <= self._instanceNum + + def test_get_container_inspect(self): + ips = qcos_client.list_containers(self._stack, self._service)[0] + # 查看第1个容器 + with Call(qcos_client, 'get_container_inspect', ips[0]) as r: + assert r[0]['ip'] == ips[0] + + def test_stop_and_strat_container(self): + ips = qcos_client.list_containers(self._stack, self._service)[0] + # 停止第2个容器 + with Call(qcos_client, 'stop_container', ips[1]) as r: + assert r[0] == {} + + _debug_info('waiting for containers to stop ...') + time.sleep(3) + + # 启动第2个容器 + with Call(qcos_client, 'start_container', ips[1]) as r: + assert r[0] == {} + + def test_restart_container(self): + ips = qcos_client.list_containers(self._stack, self._service)[0] + # 重启第1个容器 + with Call(qcos_client, 'restart_container', ips[0]) as r: + assert r[0] == {} + + +class TestAp: + """接入点测试用例""" + + _stack = 'just4test4' + _service = 'spaceship' + _spec = {'image': 'library/nginx:stable', 'unitType': '1U1G'} + # 为了方便测试，容器数量最少为2 + _instanceNum = 2 + _apid_domain = {} + _apid_ip = {} + _apid_ip_port = 8080 + _user_domain = 'just4test001.example.com' + + @classmethod + def setup_class(cls): + qcos_client.delete_stack(cls._stack) + qcos_client.create_stack({'name': cls._stack}) + qcos_client.create_service(cls._stack, {'name': cls._service, 'spec': cls._spec, 'instanceNum': cls._instanceNum}) + cls._ap_domain = qcos_client.create_ap({'type': 'DOMAIN', 'provider': 'Telecom', 'unitType': 'BW_10M', 'title': 'public1'})[0] + cls._ap_ip = qcos_client.create_ap({'type': 'PUBLIC_IP', 'provider': 'Telecom', 'unitType': 'BW_10M', 'title': 'public2'})[0] + qcos_client.set_ap_port(cls._ap_ip['apid'], cls._apid_ip_port, {'proto': 'http'}) + + + @classmethod + def teardown_class(cls): + qcos_client.delete_stack(cls._stack) + qcos_client.delete_ap(cls._ap_domain['apid']) + qcos_client.delete_ap(cls._ap_ip['apid']) + + def test_list_aps(self): + with Call(qcos_client, 'list_aps') as r: + assert len(r[0]) > 0 + assert self._ap_domain['apid'] in [ap['apid'] for ap in r[0]] + assert self._ap_domain['apid'] in [ap['apid'] for ap in r[0]] + + def test_create_and_delete_ap(self): + apid = 0 + ap = {'type': 'DOMAIN', 'provider': 'Telecom', 'unitType': 'BW_10M', 'title': 'public1'} + + with Call(qcos_client, 'create_ap', ap) as r: + assert r[0] is not None and r[0]['apid'] > 0 + apid = r[0]['apid'] + + with Call(qcos_client, 'delete_ap', apid) as r: + assert r[0] == {} + + def test_search_ap(self): + with Call(qcos_client, 'search_ap', 'ip', self._ap_ip['ip']) as r: + assert str(r[0]['apid']) == self._ap_ip['apid'] + + def test_get_ap(self): + with Call(qcos_client, 'get_ap', self._ap_ip['apid']) as r: + assert str(r[0]['apid']) == self._ap_ip['apid'] + + def test_update_ap(self): + with Call(qcos_client, 'update_ap', self._ap_ip['apid'], {}) as r: + assert r[0] == {} + + def test_set_ap_port(self): + with Call(qcos_client, 'set_ap_port', self._ap_ip['apid'], 80, {'proto': 'http'}) as r: + assert r[0] == {} + + def test_publish_ap(self): + domain = {'userDomain': self._user_domain} + with Call(qcos_client, 'publish_ap', self._ap_domain['apid'], domain) as r: + assert r[0] == {} + + def test_unpublish_ap(self): + domain = {'userDomain': self._user_domain} + with Call(qcos_client, 'unpublish_ap', self._ap_domain['apid'], domain) as r: + assert r[0] == {} + + def test_get_ap_port_healthcheck(self): + with Call(qcos_client, 'get_ap_port_healthcheck', self._ap_ip['apid'], self._apid_ip_port) as r: + assert r[0] is not None + + def test_disable_ap_port(self): + with Call(qcos_client, 'disable_ap_port', self._ap_ip['apid'], self._apid_ip_port) as r: + assert r[0] == {} + + def test_enable_ap_port(self): + with Call(qcos_client, 'enable_ap_port', self._ap_ip['apid'], self._apid_ip_port) as r: + assert r[0] == {} + + def test_get_ap_providers(self): + with Call(qcos_client, 'get_ap_providers') as r: + assert len(r[0]) > 0 + + +class Call(object): + def __init__(self, obj, method, *args): + self.context = (obj, method, args) + self.result = None + + def __enter__(self): + self.result = getattr(self.context[0], self.context[1])(*self.context[2]) + assert self.result is not None + return self.result + + def __exit__(self, type, value, traceback): + _debug_info('\033[94m%s.%s\x1b[0m: %s', self.context[0].__class__, self.context[1], self.result) + + +def _debug_info(*args): + logger = logging.getLogger(__name__) + logger.debug(*args) + + +if __name__ == '__main__': + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + pytest.main() + From a98200140d3082f2b42a6f29dbb8eb497296bb99 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Wed, 30 Nov 2016 16:45:19 +0800 Subject: [PATCH 196/478] fix --- examples/download.py | 2 ++ examples/upload_callback.py | 7 ++++--- examples/upload_pfops.py | 4 ++-- qiniu/zone.py | 12 ++++++------ 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/examples/download.py b/examples/download.py index a3895a7d..3ca0fa08 100644 --- a/examples/download.py +++ b/examples/download.py @@ -8,6 +8,8 @@ secret_key = '...' q = Auth(access_key, secret_key) +bucket_domain = "..." +key = "..." #有两种方式构造base_url的形式 base_url = 'http://%s/%s' % (bucket_domain, key) diff --git a/examples/upload_callback.py b/examples/upload_callback.py index 26e864f5..86d146d0 100644 --- a/examples/upload_callback.py +++ b/examples/upload_callback.py @@ -1,17 +1,17 @@ # -*- coding: utf-8 -*- # flake8: noqa -from qiniu import Auth, put_file, etag, +from qiniu import Auth, put_file, etag import qiniu.config access_key = '...' -secret_key = ...' +secret_key = '...' q = Auth(access_key, secret_key) bucket_name = 'Bucket_Name' -key = 'my-python-logo.png'; +key = 'my-python-logo.png' #上传文件到七牛后， 七牛将文件名和文件大小回调给业务服务器。 policy={ @@ -27,3 +27,4 @@ print(info) assert ret['key'] == key assert ret['hash'] == etag(localfile) + diff --git a/examples/upload_pfops.py b/examples/upload_pfops.py index 4a144912..b0e95493 100644 --- a/examples/upload_pfops.py +++ b/examples/upload_pfops.py @@ -3,8 +3,8 @@ from qiniu import Auth, put_file, etag, urlsafe_base64_encode import qiniu.config -# access_key = '...' -# secret_key = '...' +access_key = '...' +secret_key = '...' # 初始化Auth状态 q = Auth(access_key, secret_key) diff --git a/qiniu/zone.py b/qiniu/zone.py index 5e71c7e1..376cd73b 100644 --- a/qiniu/zone.py +++ b/qiniu/zone.py @@ -3,7 +3,7 @@ import os import time import requests - +import tempfile from qiniu import compat from qiniu import utils @@ -19,11 +19,12 @@ class Zone(object): up_host: 首选上传地址 up_host_backup: 备用上传地址 """ - def __init__(self, up_host=None, up_host_backup=None, io_host=None, host_cache={}, scheme="http"): + def __init__(self, up_host=None, up_host_backup=None, io_host=None, host_cache={}, scheme="http", home_dir=tempfile.gettempdir()): """初始化Zone类""" self.up_host, self.up_host_backup, self.io_host = up_host, up_host_backup, io_host self.host_cache = host_cache self.scheme = scheme + self.home_dir = home_dir def get_up_host_by_token(self, up_token): ak, bucket = self.unmarshal_up_token(up_token) @@ -113,16 +114,15 @@ def host_cache_from_file(self): f.close() return + def host_cache_file_path(self): + return os.path.join(self.home_dir, ".qiniu_pythonsdk_hostscache.json") + def host_cache_to_file(self): path = self.host_cache_file_path() with open(path, 'w') as f: compat.json.dump(self.host_cache, f) f.close() - def host_cache_file_path(self): - home = os.getenv("HOME") - return home + "/.qiniu_pythonsdk_hostscache.json" - def bucket_hosts(self, ak, bucket): url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) ret = requests.get(url) From 77f4215d07bb84f3c0bde4b611c8017b5ca560b1 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Wed, 30 Nov 2016 17:01:32 +0800 Subject: [PATCH 197/478] fix home --- qiniu/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/main.py b/qiniu/main.py index c7dac25a..7f8653e8 100755 --- a/qiniu/main.py +++ b/qiniu/main.py @@ -30,5 +30,6 @@ def main(): else: print(' '.join(r)) + if __name__ == '__main__': main() From bff2ae5109ef29f40961a76519e5e888735754bf Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Wed, 30 Nov 2016 18:19:03 +0800 Subject: [PATCH 198/478] fix home --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index b9598366..a1bd5773 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -355,7 +355,7 @@ def test_hasRead_WithoutSeek_retry2(self): key = 'withReadAndWithoutSeek_retry2' data = urlopen("http://www.qiniu.com") set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) - token = self.q.upload_token(bucket_name) + token = self.q.upload_token(bucket_name, key) ret, info = put_data(token, key, data) print(info) assert ret is not None From 352bcb125f7c88e9d8d70a2e48a3e04ace78ff76 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 30 Nov 2016 19:40:53 +0800 Subject: [PATCH 199/478] update version --- .travis.yml | 4 ++-- CHANGELOG.md | 5 ++++- qiniu/__init__.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e693bfb6..961aaa58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: false language: python python: -- '2.6' +- '2.6.9' - '2.7' - '3.4' - '3.5' @@ -19,7 +19,7 @@ before_script: - export QINIU_TEST_ENV="travis" - export PYTHONPATH="$PYTHONPATH:." script: -- flake8 --show-source --max-line-length=160 . +- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 .; fi - py.test --cov qiniu - ocular --data-file .coverage deploy: diff --git a/CHANGELOG.md b/CHANGELOG.md index 3feb2c7f..a83eed10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,13 @@ #Changelog +## 7.0.10 (2016-11-29) +### 修正 +* 去掉homedir + ## 7.0.9 (2016-10-09) ### 增加 * 多机房接口调用支持 - ## 7.0.8 (2016-07-05) ### 修正 * 修复表单上传大于20M文件的400错误 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index a290f2d4..9fefd58f 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.0.9' +__version__ = '7.0.10' from .auth import Auth From 618d322352347c0a9f5d17ba8bd44ae003796483 Mon Sep 17 00:00:00 2001 From: pqx Date: Thu, 1 Dec 2016 16:34:32 +0800 Subject: [PATCH 200/478] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BD=91=E7=BB=9C?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/kirk/qcos_api.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/qiniu/services/kirk/qcos_api.py b/qiniu/services/kirk/qcos_api.py index 29a8254c..c1174102 100644 --- a/qiniu/services/kirk/qcos_api.py +++ b/qiniu/services/kirk/qcos_api.py @@ -54,6 +54,7 @@ class QcosClient(object): disable_ap_port(apid, port) enable_ap_port(apid, port) get_ap_providers() + get_web_proxy(backend) """ def __init__(self, auth, host=None): @@ -669,6 +670,22 @@ def get_ap_providers(self): url = '{0}/v3/aps/providers'.format(self.host) return self.__get(url) + def get_web_proxy(self, backend): + """获取代理 + + 对内网地址获取一个外部可访问的代理地址 + + Args: + - backend: 后端地址，如："10.128.0.1:8080" + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回代理地址信息，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/webproxy'.format(self.host) + return self.__post(url, {'backend': backend}) + def __post(self, url, data=None): return http._post_with_qiniu_mac(url, data, self.auth) From 6129e105d80f1aa8a2a4e437d56c5ff5eda9477c Mon Sep 17 00:00:00 2001 From: pqx Date: Fri, 2 Dec 2016 11:24:59 +0800 Subject: [PATCH 201/478] =?UTF-8?q?=E5=AE=8C=E5=96=84docstring?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/kirk/app.py | 64 +++++++++++++++++++++++++++++++++ qiniu/services/kirk/qcos_api.py | 4 +-- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/qiniu/services/kirk/app.py b/qiniu/services/kirk/app.py index d837fce9..1e6191b0 100644 --- a/qiniu/services/kirk/app.py +++ b/qiniu/services/kirk/app.py @@ -71,6 +71,17 @@ def create_qcos_client(self, app_uri): def get_app_keys(self, app_uri): """获得账号下应用的密钥 + 列出指定应用的密钥，仅当访问者对指定应用有管理权限时有效： + 用户对创建的应用有管理权限。 + 用户对使用的第三方应用没有管理权限，第三方应用的运维方有管理权限。 + + Args: + - app_uri: 应用的完整标识 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回秘钥列表，失败返回None + - ResponseInfo 请求的Response信息 """ url = '{0}/v3/apps/{1}/keys'.format(self.host, app_uri) @@ -79,6 +90,15 @@ def get_app_keys(self, app_uri): def get_valid_app_auth(self, app_uri): """获得账号下可用的应用的密钥 + 列出指定应用的可用密钥 + + Args: + - app_uri: 应用的完整标识 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回可用秘钥列表，失败返回None + - ResponseInfo 请求的Response信息 """ ret, retInfo = self.get_app_keys(app_uri) @@ -95,6 +115,12 @@ def get_valid_app_auth(self, app_uri): def get_account_info(self): """获得当前账号的信息 + 查看当前请求方（请求鉴权使用的 AccessKey 的属主）的账号信息。 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回用户信息，失败返回None + - ResponseInfo 请求的Response信息 """ url = '{0}/v3/info'.format(self.host) @@ -103,6 +129,11 @@ def get_account_info(self): def get_app_region_products(self, app_uri): """获得指定应用所在区域的产品信息 + Args: + - app_uri: 应用的完整标识 + + Returns: + 返回产品信息列表，若失败则返回None """ apps, retInfo = self.list_apps() if apps is None: @@ -117,6 +148,11 @@ def get_app_region_products(self, app_uri): def get_region_products(self, region): """获得指定区域的产品信息 + Args: + - region: 区域，如："nq" + + Returns: + 返回该区域的产品信息，若失败则返回None """ regions, retInfo = self.list_regions() @@ -130,6 +166,12 @@ def get_region_products(self, region): def list_regions(self): """获得账号可见的区域的信息 + 列出当前用户所有可使用的区域。 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回区域列表，失败返回None + - ResponseInfo 请求的Response信息 """ url = '{0}/v3/regions'.format(self.host) @@ -138,6 +180,12 @@ def list_regions(self): def list_apps(self): """获得当前账号的应用列表 + 列出所属应用为当前请求方的应用列表。 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回应用列表，失败返回None + - ResponseInfo 请求的Response信息 """ url = '{0}/v3/apps'.format(self.host) @@ -146,6 +194,14 @@ def list_apps(self): def create_app(self, args): """创建应用 + 在指定区域创建一个新应用，所属应用为当前请求方。 + + Args: + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + - result 成功返回所创建的应用信息，若失败则返回None + - ResponseInfo 请求的Response信息 """ url = '{0}/v3/apps'.format(self.host) @@ -154,6 +210,14 @@ def create_app(self, args): def delete_app(self, app_uri): """删除应用 + 删除指定标识的应用，当前请求方对该应用应有删除权限。 + + Args: + - app_uri: 应用的完整标识 + + Returns: + - result 成功返回空dict{}，若失败则返回None + - ResponseInfo 请求的Response信息 """ url = '{0}/v3/apps/{1}'.format(self.host, app_uri) diff --git a/qiniu/services/kirk/qcos_api.py b/qiniu/services/kirk/qcos_api.py index c1174102..c062c715 100644 --- a/qiniu/services/kirk/qcos_api.py +++ b/qiniu/services/kirk/qcos_api.py @@ -671,9 +671,9 @@ def get_ap_providers(self): return self.__get(url) def get_web_proxy(self, backend): - """获取代理 + """获取一次性代理地址 - 对内网地址获取一个外部可访问的代理地址 + 对内网地址获取一个一次性的外部可访问的代理地址 Args: - backend: 后端地址，如："10.128.0.1:8080" From 57701643ba70c3d445f8a1860a3352d8eff84f34 Mon Sep 17 00:00:00 2001 From: pqx Date: Fri, 2 Dec 2016 20:05:22 +0800 Subject: [PATCH 202/478] fix code style --- test_kirk.py => manual_test_kirk.py | 11 +++++++---- qiniu/auth.py | 8 +++++--- qiniu/http.py | 6 +++++- qiniu/main.py | 1 + qiniu/services/kirk/app.py | 8 ++++---- qiniu/services/kirk/config.py | 6 +++--- qiniu/services/kirk/qcos_api.py | 10 ++++------ 7 files changed, 29 insertions(+), 21 deletions(-) rename test_kirk.py => manual_test_kirk.py (99%) diff --git a/test_kirk.py b/manual_test_kirk.py similarity index 99% rename from test_kirk.py rename to manual_test_kirk.py index 3e246b6c..d1ff20fd 100644 --- a/test_kirk.py +++ b/manual_test_kirk.py @@ -1,4 +1,9 @@ # -*- coding: utf-8 -*- +""" +======================= + 注意：必须手动运行 +======================= +""" import os import sys import time @@ -14,7 +19,7 @@ acc_client = kirk.app.AccountClient(qn_auth) qcos_client = None user_name = '' -app_uri = '' +app_uri = '' app_name = 'appjust4test' app_region = 'nq' @@ -23,7 +28,7 @@ def setup_module(module): acc_client = kirk.app.AccountClient(qn_auth) user_info = acc_client.get_account_info()[0] acc_client.create_app({'name': app_name, 'title': 'whatever', 'region': app_region}) - + module.user_name = user_info['name'] module.app_uri = '{0}.{1}'.format(module.user_name, app_name) module.qcos_client = acc_client.create_qcos_client(module.app_uri) @@ -238,7 +243,6 @@ def setup_class(cls): cls._ap_domain = qcos_client.create_ap({'type': 'DOMAIN', 'provider': 'Telecom', 'unitType': 'BW_10M', 'title': 'public1'})[0] cls._ap_ip = qcos_client.create_ap({'type': 'PUBLIC_IP', 'provider': 'Telecom', 'unitType': 'BW_10M', 'title': 'public2'})[0] qcos_client.set_ap_port(cls._ap_ip['apid'], cls._apid_ip_port, {'proto': 'http'}) - @classmethod def teardown_class(cls): @@ -328,4 +332,3 @@ def _debug_info(*args): if __name__ == '__main__': logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) pytest.main() - diff --git a/qiniu/auth.py b/qiniu/auth.py index a797945a..feef3bcc 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -197,6 +197,7 @@ def __call__(self, r): r.headers['Authorization'] = 'QBox {0}'.format(token) return r + class QiniuMacAuth(object): """ Sign Requests @@ -240,10 +241,10 @@ def token_of_request(self, method, host, url, qheaders, content_type=None, body= path_with_query = path if query != '': path_with_query = ''.join([path_with_query, '?', query]) - data = ''.join(["%s %s"%(method, path_with_query) , "\n", "Host: %s"%host, "\n"]) + data = ''.join(["%s %s" % (method, path_with_query), "\n", "Host: %s" % host, "\n"]) if content_type: - data += "Content-Type: %s"%(content_type) + "\n" + data += "Content-Type: %s" % (content_type) + "\n" data += qheaders data += "\n" @@ -257,7 +258,7 @@ def qiniu_headers(self, headers): res = "" for key in headers: if key.startswith(self.qiniu_header_prefix): - res += key+": %s\n"%s(headers.get(key)) + res += key+": %s\n" % (headers.get(key)) return res @staticmethod @@ -265,6 +266,7 @@ def __checkKey(access_key, secret_key): if not (access_key and secret_key): raise ValueError('QiniuMacAuthSign : Invalid key') + class QiniuMacRequestsAuth(AuthBase): def __init__(self, auth): self.auth = auth diff --git a/qiniu/http.py b/qiniu/http.py index ad79178b..0efb7ff7 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -72,13 +72,15 @@ def _post_with_token(url, data, token): def _post_file(url, data, files): return _post(url, data, files, None) + def _post_with_auth(url, data, auth): return _post(url, data, None, qiniu.auth.RequestsAuth(auth)) + def _post_with_qiniu_mac(url, data, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None - timeout=config.get_default('connection_timeout') + timeout = config.get_default('connection_timeout') try: r = requests.post(url, json=data, auth=qn_auth, timeout=timeout, headers=_headers) @@ -86,6 +88,7 @@ def _post_with_qiniu_mac(url, data, auth): return None, ResponseInfo(None, e) return __return_wrapper(r) + def _get_with_qiniu_mac(url, params, auth): try: r = requests.get( @@ -95,6 +98,7 @@ def _get_with_qiniu_mac(url, params, auth): return None, ResponseInfo(None, e) return __return_wrapper(r) + def _delete_with_qiniu_mac(url, params, auth): try: r = requests.delete( diff --git a/qiniu/main.py b/qiniu/main.py index c7dac25a..7f8653e8 100755 --- a/qiniu/main.py +++ b/qiniu/main.py @@ -30,5 +30,6 @@ def main(): else: print(' '.join(r)) + if __name__ == '__main__': main() diff --git a/qiniu/services/kirk/app.py b/qiniu/services/kirk/app.py index 1e6191b0..61b95635 100644 --- a/qiniu/services/kirk/app.py +++ b/qiniu/services/kirk/app.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- - -from qiniu import config, http, QiniuMacAuth +from qiniu import http, QiniuMacAuth from .config import KIRK_HOST from .qcos_api import QcosClient + class AccountClient(object): """客户端入口 @@ -191,7 +191,7 @@ def list_apps(self): url = '{0}/v3/apps'.format(self.host) return http._get_with_qiniu_mac(url, None, self.auth) - def create_app(self, args): + def create_app(self, args): """创建应用 在指定区域创建一个新应用，所属应用为当前请求方。 @@ -207,7 +207,7 @@ def create_app(self, args): url = '{0}/v3/apps'.format(self.host) return http._post_with_qiniu_mac(url, args, self.auth) - def delete_app(self, app_uri): + def delete_app(self, app_uri): """删除应用 删除指定标识的应用，当前请求方对该应用应有删除权限。 diff --git a/qiniu/services/kirk/config.py b/qiniu/services/kirk/config.py index 7a117b4f..045ed784 100644 --- a/qiniu/services/kirk/config.py +++ b/qiniu/services/kirk/config.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- KIRK_HOST = { - 'APPGLOBAL': "https://app-api.qiniu.com", # 公有云 APP API - 'APPPROXY': "http://app.qcos.qiniu", # 内网 APP API - 'APIPROXY': "http://api.qcos.qiniu", # 内网 API + 'APPGLOBAL': "https://app-api.qiniu.com", # 公有云 APP API + 'APPPROXY': "http://app.qcos.qiniu", # 内网 APP API + 'APIPROXY': "http://api.qcos.qiniu", # 内网 API } CONTAINER_UINT_TYPE = { diff --git a/qiniu/services/kirk/qcos_api.py b/qiniu/services/kirk/qcos_api.py index c062c715..250a4bf6 100644 --- a/qiniu/services/kirk/qcos_api.py +++ b/qiniu/services/kirk/qcos_api.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- - -from qiniu import config, http +from qiniu import http from .config import KIRK_HOST + class QcosClient(object): """资源管理客户端 @@ -157,7 +157,6 @@ def stop_stack(self, stack): url = '{0}/v3/stacks/{1}/stop'.format(self.host, stack) return self.__post(url) - def list_services(self, stack): """获得服务列表 @@ -333,8 +332,7 @@ def extend_service_volume(self, stack, service, volume, args): - result 成功返回空dict{}，失败返回{"error": ""} - ResponseInfo 请求的Response信息 """ - url = '{0}/v3/stacks/{1}/services/{2}/volumes/{3}/extend'\ - .format(self.host, stack, service, volume) + url = '{0}/v3/stacks/{1}/services/{2}/volumes/{3}/extend'.format(self.host, stack, service, volume) return self.__post(url, args) def delete_service_volume(self, stack, service, volume): @@ -529,7 +527,7 @@ def set_ap_port(self, apid, port, args): - apid: 接入点ID - port: 要设置的端口号 - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ - + Returns: 返回一个tuple对象，其格式为(, ) - result 成功返回空dict{}，失败返回{"error": ""} From 16e5c19f8025bbfa2717db741db0ddc64fbbdaa3 Mon Sep 17 00:00:00 2001 From: pqx Date: Thu, 8 Dec 2016 11:14:27 +0800 Subject: [PATCH 203/478] =?UTF-8?q?qiniu.services.kirk=E6=9B=B4=E6=94=B9?= =?UTF-8?q?=E4=B8=BAqiniu.services.compute?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- manual_test_kirk.py | 6 +++--- qiniu/__init__.py | 5 +++-- qiniu/services/{kirk => compute}/__init__.py | 0 qiniu/services/{kirk => compute}/app.py | 0 qiniu/services/{kirk => compute}/config.py | 0 qiniu/services/{kirk => compute}/qcos_api.py | 0 6 files changed, 6 insertions(+), 5 deletions(-) rename qiniu/services/{kirk => compute}/__init__.py (100%) rename qiniu/services/{kirk => compute}/app.py (100%) rename qiniu/services/{kirk => compute}/config.py (100%) rename qiniu/services/{kirk => compute}/qcos_api.py (100%) diff --git a/manual_test_kirk.py b/manual_test_kirk.py index d1ff20fd..e6791f30 100644 --- a/manual_test_kirk.py +++ b/manual_test_kirk.py @@ -10,13 +10,13 @@ import logging import pytest from qiniu import auth -from qiniu.services import kirk +from qiniu.services import compute access_key = os.getenv('QINIU_ACCESS_KEY') secret_key = os.getenv('QINIU_SECRET_KEY') qn_auth = auth.QiniuMacAuth(access_key, secret_key) -acc_client = kirk.app.AccountClient(qn_auth) +acc_client = compute.app.AccountClient(qn_auth) qcos_client = None user_name = '' app_uri = '' @@ -25,7 +25,7 @@ def setup_module(module): - acc_client = kirk.app.AccountClient(qn_auth) + acc_client = compute.app.AccountClient(qn_auth) user_info = acc_client.get_account_info()[0] acc_client.create_app({'name': app_name, 'title': 'whatever', 'region': app_region}) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 5e2a360a..e1921953 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.0.10' +__version__ = '7.1.0' from .auth import Auth, QiniuMacAuth @@ -20,6 +20,7 @@ from .services.storage.uploader import put_data, put_file, put_stream from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save -from .services.kirk.app import AccountClient +from .services.compute.app import AccountClient +from .services.compute.qcos_api import QcosClient from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry diff --git a/qiniu/services/kirk/__init__.py b/qiniu/services/compute/__init__.py similarity index 100% rename from qiniu/services/kirk/__init__.py rename to qiniu/services/compute/__init__.py diff --git a/qiniu/services/kirk/app.py b/qiniu/services/compute/app.py similarity index 100% rename from qiniu/services/kirk/app.py rename to qiniu/services/compute/app.py diff --git a/qiniu/services/kirk/config.py b/qiniu/services/compute/config.py similarity index 100% rename from qiniu/services/kirk/config.py rename to qiniu/services/compute/config.py diff --git a/qiniu/services/kirk/qcos_api.py b/qiniu/services/compute/qcos_api.py similarity index 100% rename from qiniu/services/kirk/qcos_api.py rename to qiniu/services/compute/qcos_api.py From 2007c793655ccae5c03a0bb6e92497dfc53643f7 Mon Sep 17 00:00:00 2001 From: pqx Date: Thu, 8 Dec 2016 11:15:09 +0800 Subject: [PATCH 204/478] =?UTF-8?q?=E6=9B=B4=E6=96=B0changelog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a83eed10..eb70367b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ #Changelog +## 7.1.0 (2016-12-08) +### 增加 +* 通用计算支持 + ## 7.0.10 (2016-11-29) ### 修正 * 去掉homedir From b359011536d98226b05f0218ea9de2291328ae9b Mon Sep 17 00:00:00 2001 From: pqx Date: Thu, 8 Dec 2016 12:33:33 +0800 Subject: [PATCH 205/478] fix version --- CHANGELOG.md | 2 +- qiniu/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb70367b..8d34a3fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ #Changelog -## 7.1.0 (2016-12-08) +## 7.1.1 (2016-12-08) ### 增加 * 通用计算支持 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index e1921953..eb0b4c69 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.1.0' +__version__ = '7.1.1' from .auth import Auth, QiniuMacAuth From 55d7b480bc84be094ed22edd53bc94e8b905f138 Mon Sep 17 00:00:00 2001 From: pqx Date: Thu, 8 Dec 2016 13:32:51 +0800 Subject: [PATCH 206/478] fix package set up --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9483eb9d..794e59b3 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ 'qiniu.services', 'qiniu.services.storage', 'qiniu.services.processing', - 'qiniu.services.kirk', + 'qiniu.services.compute', ] From f6eab73e8dcc32c5c70526df45e6bc716ae0af1b Mon Sep 17 00:00:00 2001 From: pqx Date: Thu, 8 Dec 2016 13:36:35 +0800 Subject: [PATCH 207/478] Revert "fix version" This reverts commit b359011536d98226b05f0218ea9de2291328ae9b. --- CHANGELOG.md | 2 +- qiniu/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d34a3fe..eb70367b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ #Changelog -## 7.1.1 (2016-12-08) +## 7.1.0 (2016-12-08) ### 增加 * 通用计算支持 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index eb0b4c69..e1921953 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.1.1' +__version__ = '7.1.0' from .auth import Auth, QiniuMacAuth From 313a7b91541bff688c8162af7903c0e57a7ecb06 Mon Sep 17 00:00:00 2001 From: Jemy Date: Sun, 8 Jan 2017 17:55:22 +0800 Subject: [PATCH 208/478] add cdn related functions to python sdk --- examples/cdn_manager.py | 121 ++++++++++++++++++++ qiniu/__init__.py | 4 +- qiniu/http.py | 15 ++- qiniu/services/cdn/__init__.py | 0 qiniu/services/cdn/manager.py | 194 +++++++++++++++++++++++++++++++++ setup.py | 1 + 6 files changed, 330 insertions(+), 5 deletions(-) create mode 100644 examples/cdn_manager.py create mode 100644 qiniu/services/cdn/__init__.py create mode 100644 qiniu/services/cdn/manager.py diff --git a/examples/cdn_manager.py b/examples/cdn_manager.py new file mode 100644 index 00000000..56dfc8b6 --- /dev/null +++ b/examples/cdn_manager.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +import qiniu +from qiniu import CdnManager +from qiniu import create_timestamp_anti_leech_url +import time + + +# 演示函数调用结果 +def print_result(result): + if result[0] is not None: + print(type(result[0])) + print(result[0]) + + print(type(result[1])) + print(result[1]) + + +# 账户ak，sk +access_key = '...' +secret_key = '...' + +auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) +cdn_manager = CdnManager(auth) + +urls = [ + 'http://if-pbl.qiniudn.com/qiniu.jpg', + 'http://if-pbl.qiniudn.com/qiniu2.jpg' +] + +# 注意链接最后的斜杠表示目录 +dirs = [ + 'http://if-pbl.qiniudn.com/test1/', + 'http://if-pbl.qiniudn.com/test2/' +] +"""刷新文件，目录""" + +# 刷新链接 +print('刷新文件') +refresh_url_result = cdn_manager.refresh_urls(urls) +print_result(refresh_url_result) + +# 刷新目录需要联系七牛技术支持开通权限 +print('刷新目录') +refresh_dir_result = cdn_manager.refresh_dirs(dirs) +print_result(refresh_dir_result) + +# 同时刷新文件和目录 +print('刷新文件和目录') +refresh_all_result = cdn_manager.refresh_urls_and_dirs(urls, dirs) +print_result(refresh_all_result) + +"""预取文件""" + +# 预取文件链接 +print('预取文件链接') +prefetch_url_result = cdn_manager.prefetch_urls(urls) +print_result(prefetch_url_result) + +"""获取带宽和流量数据""" + +domains = ['if-pbl.qiniudn.com', 'qdisk.qiniudn.com'] + +start_date = '2017-01-01' +end_date = '2017-01-02' + +# 5min or hour or day +granularity = 'day' + +# 获取带宽数据 +print('获取带宽数据') +bandwidth_data = cdn_manager.get_bandwidth_data(domains, start_date, end_date, granularity) +print_result(bandwidth_data) + +# 获取流量数据 +print('获取流量数据') +flux_data = cdn_manager.get_flux_data(domains, start_date, end_date, granularity) +print_result(flux_data) + +"""获取日志文件下载地址列表""" +# 获取日志列表 +print('获取日志列表') +log_date = '2017-01-01' +log_data = cdn_manager.get_log_list_data(domains, log_date) +print_result(log_data) + +"""构建时间戳防盗链""" + +# 构建时间戳防盗链 +print('构建时间戳防盗链') + +# 时间戳防盗链密钥，后台获取 +encrypt_key = '...' + +# 原始文件名，必须是utf8编码 +test_file_name1 = '基本概括.mp4' +test_file_name2 = '2017/01/07/test.png' + +# 查询参数列表 +query_string = 'name=七牛&year=2017' + +# 带访问协议的域名 +host = 'http://video.example.com' + +# unix时间戳 +deadline = int(time.time()) + 3600 + +# 带查询参数，中文文件名 +signed_url1 = create_timestamp_anti_leech_url(host, test_file_name1, query_string, encrypt_key, deadline) +print(signed_url1) + +# 带查询参数，英文文件名 +signed_url2 = create_timestamp_anti_leech_url(host, test_file_name2, query_string, encrypt_key, deadline) +print(signed_url2) + +# 不带查询参数，中文文件名 +signed_url3 = create_timestamp_anti_leech_url(host, test_file_name1, '', encrypt_key, deadline) +print(signed_url3) + +# 不带查询参数，英文文件名 +signed_url4 = create_timestamp_anti_leech_url(host, test_file_name2, '', encrypt_key, deadline) +print(signed_url4) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index e1921953..69bf6d93 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -16,8 +16,10 @@ from .config import set_default from .zone import Zone -from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, build_batch_delete +from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ + build_batch_stat, build_batch_delete from .services.storage.uploader import put_data, put_file, put_stream +from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save from .services.compute.app import AccountClient diff --git a/qiniu/http.py b/qiniu/http.py index 0efb7ff7..5eea3e9a 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -8,7 +8,6 @@ import qiniu.auth from . import __version__ - _sys_info = '{0}; {1}'.format(platform.system(), platform.machine()) _python_ver = platform.python_version() @@ -35,12 +34,17 @@ def _init(): _session = session -def _post(url, data, files, auth): +def _post(url, data, files, auth, headers=None): if _session is None: _init() try: + post_headers = _headers + if headers is not None: + for k, v in headers.items(): + post_headers.update({k: v}) r = _session.post( - url, data=data, files=files, auth=auth, headers=_headers, timeout=config.get_default('connection_timeout')) + url, data=data, files=files, auth=auth, headers=post_headers, + timeout=config.get_default('connection_timeout')) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) @@ -77,8 +81,11 @@ def _post_with_auth(url, data, auth): return _post(url, data, None, qiniu.auth.RequestsAuth(auth)) -def _post_with_qiniu_mac(url, data, auth): +def _post_with_auth_and_headers(url, data, auth, headers): + return _post(url, data, None, qiniu.auth.RequestsAuth(auth), headers) + +def _post_with_qiniu_mac(url, data, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None timeout = config.get_default('connection_timeout') diff --git a/qiniu/services/cdn/__init__.py b/qiniu/services/cdn/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py new file mode 100644 index 00000000..4370134c --- /dev/null +++ b/qiniu/services/cdn/manager.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- + +from qiniu import http +import json + +from qiniu.compat import is_py2 +from qiniu.compat import is_py3 + +import hashlib + + +def urlencode(str): + if is_py2: + import urllib2 + return urllib2.quote(str) + elif is_py3: + import urllib.parse + return urllib.parse.quote(str) + + +class CdnManager(object): + def __init__(self, auth): + self.auth = auth + self.server = 'http://fusion.qiniuapi.com' + + def refresh_urls(self, urls): + """ + 刷新文件列表，文档 http://developer.qiniu.com/article/fusion/api/refresh.html + + Args: + urls: 待刷新的文件外链列表 + + Returns: + 一个dict变量和一个ResponseInfo对象 + 参考代码 examples/cdn_manager.py + """ + return self.refresh_urls_and_dirs(urls, None) + + def refresh_dirs(self, dirs): + """ + 刷新目录，文档 http://developer.qiniu.com/article/fusion/api/refresh.html + + Args: + urls: 待刷新的目录列表 + + Returns: + 一个dict变量和一个ResponseInfo对象 + 参考代码 examples/cdn_manager.py + """ + return self.refresh_urls_and_dirs(None, dirs) + + def refresh_urls_and_dirs(self, urls, dirs): + """ + 刷新文件目录，文档 http://developer.qiniu.com/article/fusion/api/refresh.html + + Args: + urls: 待刷新的目录列表 + dirs: 待刷新的文件列表 + + Returns: + 一个dict变量和一个ResponseInfo对象 + 参考代码 examples/cdn_manager.py + """ + req = {} + if urls is not None and len(urls) > 0: + req.update({"urls": urls}) + if dirs is not None and len(dirs) > 0: + req.update({"dirs": dirs}) + + body = json.dumps(req) + url = '{0}/v2/tune/refresh'.format(self.server) + return self.__post(url, body) + + def prefetch_urls(self, urls): + """ + 预取文件列表，文档 http://developer.qiniu.com/article/fusion/api/prefetch.html + + Args: + urls: 待预取的文件外链列表 + + Returns: + 一个dict变量和一个ResponseInfo对象 + 参考代码 examples/cdn_manager.py + """ + req = {} + req.update({"urls": urls}) + + body = json.dumps(req) + url = '{0}/v2/tune/prefetch'.format(self.server) + return self.__post(url, body) + + def get_bandwidth_data(self, domains, start_date, end_date, granularity): + """ + 预取带宽数据，文档 http://developer.qiniu.com/article/fusion/api/traffic-bandwidth.html + + Args: + domains: 域名列表 + start_date: 起始日期 + end_date: 结束日期 + granularity: 数据间隔 + + Returns: + 一个dict变量和一个ResponseInfo对象 + 参考代码 examples/cdn_manager.py + """ + req = {} + req.update({"domains": ';'.join(domains)}) + req.update({"startDate": start_date}) + req.update({"endDate": end_date}) + req.update({"granularity": granularity}) + + body = json.dumps(req) + url = '{0}/v2/tune/bandwidth'.format(self.server) + return self.__post(url, body) + + def get_flux_data(self, domains, start_date, end_date, granularity): + """ + 预取流量数据，文档 http://developer.qiniu.com/article/fusion/api/traffic-bandwidth.html + + Args: + domains: 域名列表 + start_date: 起始日期 + end_date: 结束日期 + granularity: 数据间隔 + + Returns: + 一个dict变量和一个ResponseInfo对象 + 参考代码 examples/cdn_manager.py + """ + req = {} + req.update({"domains": ';'.join(domains)}) + req.update({"startDate": start_date}) + req.update({"endDate": end_date}) + req.update({"granularity": granularity}) + + body = json.dumps(req) + url = '{0}/v2/tune/flux'.format(self.server) + return self.__post(url, body) + + def get_log_list_data(self, domains, log_date): + """ + 获取日志下载链接，文档 http://developer.qiniu.com/article/fusion/api/log.html + + Args: + domains: 域名列表 + log_date: 日志日期 + + Returns: + 一个dict变量和一个ResponseInfo对象 + 参考代码 examples/cdn_manager.py + """ + req = {} + req.update({"domains": ';'.join(domains)}) + req.update({"startDate": log_date}) + + body = json.dumps(req) + url = '{0}/v2/tune/log/list'.format(self.server) + return self.__post(url, body) + + def __post(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._post_with_auth_and_headers(url, data, self.auth, headers) + + +def create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline): + """ + 创建时间戳防盗链 + + Args: + host: 带访问协议的域名 + file_name: 原始文件名，不需要urlencode + query_string: 查询参数，不需要urlencode + encrypt_key: 时间戳防盗链密钥 + deadline: 链接有效期时间戳（以秒为单位） + + Returns: + 带时间戳防盗链鉴权访问链接 + """ + if len(query_string) > 0: + url_to_sign = '{0}/{1}?{2}'.format(host, urlencode(file_name), urlencode(query_string)) + else: + url_to_sign = '{0}/{1}'.format(host, urlencode(file_name)) + + path = '/{0}'.format(urlencode(file_name)) + expire_hex = str(hex(deadline))[2:] + str_to_sign = '{0}{1}{2}'.format(encrypt_key, path, expire_hex).encode() + sign_str = hashlib.md5(str_to_sign).hexdigest() + + if len(query_string) > 0: + signed_url = '{0}&sign={1}&t={2}'.format(url_to_sign, sign_str, expire_hex) + else: + signed_url = '{0}?sign={1}&t={2}'.format(url_to_sign, sign_str, expire_hex) + + return signed_url diff --git a/setup.py b/setup.py index 794e59b3..8c2bcca4 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ 'qiniu.services.storage', 'qiniu.services.processing', 'qiniu.services.compute', + 'qiniu.services.cdn', ] From 7966453e02b35b58ab8e440bbd015be8fc0a9515 Mon Sep 17 00:00:00 2001 From: Jemy Date: Sun, 8 Jan 2017 18:46:27 +0800 Subject: [PATCH 209/478] fix ocular fail under env 2.6.9 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 961aaa58..7534aa72 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ before_script: script: - if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 .; fi - py.test --cov qiniu -- ocular --data-file .coverage +- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then ocular --data-file .coverage; fi deploy: provider: pypi user: qiniusdk From 94622de9540cb9a84ec7a0c00f37d8047c59e4eb Mon Sep 17 00:00:00 2001 From: Jemy Date: Sun, 8 Jan 2017 22:31:38 +0800 Subject: [PATCH 210/478] refractor the code of create anti leech url --- examples/cdn_manager.py | 16 ++++++++++------ qiniu/services/cdn/manager.py | 22 +++++++++++++--------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/examples/cdn_manager.py b/examples/cdn_manager.py index 56dfc8b6..9a217a6d 100644 --- a/examples/cdn_manager.py +++ b/examples/cdn_manager.py @@ -89,14 +89,18 @@ def print_result(result): print('构建时间戳防盗链') # 时间戳防盗链密钥，后台获取 -encrypt_key = '...' +encrypt_key = 'xxx' # 原始文件名，必须是utf8编码 test_file_name1 = '基本概括.mp4' test_file_name2 = '2017/01/07/test.png' # 查询参数列表 -query_string = 'name=七牛&year=2017' +query_string_dict = { + 'name': '七牛', + 'year': 2017, + '年龄': 28, +} # 带访问协议的域名 host = 'http://video.example.com' @@ -105,17 +109,17 @@ def print_result(result): deadline = int(time.time()) + 3600 # 带查询参数，中文文件名 -signed_url1 = create_timestamp_anti_leech_url(host, test_file_name1, query_string, encrypt_key, deadline) +signed_url1 = create_timestamp_anti_leech_url(host, test_file_name1, query_string_dict, encrypt_key, deadline) print(signed_url1) # 带查询参数，英文文件名 -signed_url2 = create_timestamp_anti_leech_url(host, test_file_name2, query_string, encrypt_key, deadline) +signed_url2 = create_timestamp_anti_leech_url(host, test_file_name2, query_string_dict, encrypt_key, deadline) print(signed_url2) # 不带查询参数，中文文件名 -signed_url3 = create_timestamp_anti_leech_url(host, test_file_name1, '', encrypt_key, deadline) +signed_url3 = create_timestamp_anti_leech_url(host, test_file_name1, None, encrypt_key, deadline) print(signed_url3) # 不带查询参数，英文文件名 -signed_url4 = create_timestamp_anti_leech_url(host, test_file_name2, '', encrypt_key, deadline) +signed_url4 = create_timestamp_anti_leech_url(host, test_file_name2, None, encrypt_key, deadline) print(signed_url4) diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 4370134c..58d8091c 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -162,22 +162,26 @@ def __post(self, url, data=None): return http._post_with_auth_and_headers(url, data, self.auth, headers) -def create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline): +def create_timestamp_anti_leech_url(host, file_name, query_string_dict, encrypt_key, deadline): """ 创建时间戳防盗链 Args: - host: 带访问协议的域名 - file_name: 原始文件名，不需要urlencode - query_string: 查询参数，不需要urlencode - encrypt_key: 时间戳防盗链密钥 - deadline: 链接有效期时间戳（以秒为单位） + host: 带访问协议的域名 + file_name: 原始文件名，不需要urlencode + query_string_dict: 查询参数，不需要urlencode + encrypt_key: 时间戳防盗链密钥 + deadline: 链接有效期时间戳（以秒为单位） Returns: 带时间戳防盗链鉴权访问链接 """ - if len(query_string) > 0: - url_to_sign = '{0}/{1}?{2}'.format(host, urlencode(file_name), urlencode(query_string)) + if query_string_dict is not None and len(query_string_dict) > 0: + query_string_items = [] + for k, v in query_string_dict.items(): + query_string_items.append('{0}={1}'.format(urlencode(str(k)), urlencode(str(v)))) + query_string = '&'.join(query_string_items) + url_to_sign = '{0}/{1}?{2}'.format(host, urlencode(file_name), query_string) else: url_to_sign = '{0}/{1}'.format(host, urlencode(file_name)) @@ -186,7 +190,7 @@ def create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, str_to_sign = '{0}{1}{2}'.format(encrypt_key, path, expire_hex).encode() sign_str = hashlib.md5(str_to_sign).hexdigest() - if len(query_string) > 0: + if query_string_dict is not None and len(query_string_dict) > 0: signed_url = '{0}&sign={1}&t={2}'.format(url_to_sign, sign_str, expire_hex) else: signed_url = '{0}?sign={1}&t={2}'.format(url_to_sign, sign_str, expire_hex) From d89940056b3ac27b30b5f8ce5742360de93b55cd Mon Sep 17 00:00:00 2001 From: pqx Date: Thu, 12 Jan 2017 17:32:12 +0800 Subject: [PATCH 211/478] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dkirk=20app=20host?= =?UTF-8?q?=E4=B8=8D=E8=83=BD=E9=85=8D=E7=BD=AEbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/compute/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/compute/app.py b/qiniu/services/compute/app.py index 61b95635..614ff668 100644 --- a/qiniu/services/compute/app.py +++ b/qiniu/services/compute/app.py @@ -36,7 +36,7 @@ def __init__(self, auth, host=None): if (auth is None): self.host = KIRK_HOST['APPPROXY'] else: - self.host = KIRK_HOST['APPGLOBAL'] + self.host = host or KIRK_HOST['APPGLOBAL'] acc, info = self.get_account_info() self.uri = acc.get('name') From 0fe927721d9a99cbec69e46ec2eb058a46c13155 Mon Sep 17 00:00:00 2001 From: Jemy Date: Fri, 3 Feb 2017 17:48:06 +0800 Subject: [PATCH 212/478] use md5 to be the resumable upload recorder key --- .../storage/upload_progress_recorder.py | 27 +++++++++---------- qiniu/services/storage/uploader.py | 10 ++++--- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index 4e777883..697b2da7 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -import base64 +import hashlib import json import os import tempfile @@ -21,16 +21,15 @@ class UploadProgressRecorder(object): Attributes: record_folder: 保存上传记录的目录 """ + def __init__(self, record_folder=tempfile.gettempdir()): self.record_folder = record_folder def get_upload_record(self, file_name, key): + record_key = '{0}/{1}'.format(key, file_name) - key = '{0}/{1}'.format(key, file_name) - - record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') - upload_record_file_path = os.path.join(self.record_folder, - record_file_name) + record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() + upload_record_file_path = os.path.join(self.record_folder, record_file_name) if not os.path.isfile(upload_record_file_path): return None with open(upload_record_file_path, 'r') as f: @@ -38,16 +37,14 @@ def get_upload_record(self, file_name, key): return json_data def set_upload_record(self, file_name, key, data): - key = '{0}/{1}'.format(key, file_name) - record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') - upload_record_file_path = os.path.join(self.record_folder, - record_file_name) + record_key = '{0}/{1}'.format(key, file_name) + record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() + upload_record_file_path = os.path.join(self.record_folder, record_file_name) with open(upload_record_file_path, 'w') as f: json.dump(data, f) def delete_upload_record(self, file_name, key): - key = '{0}/{1}'.format(key, file_name) - record_file_name = base64.b64encode(key.encode('utf-8')).decode('utf-8') - record_file_path = os.path.join(self.record_folder, - record_file_name) - os.remove(record_file_path) + record_key = '{0}/{1}'.format(key, file_name) + record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() + upload_record_file_path = os.path.join(self.record_folder, record_file_name) + os.remove(upload_record_file_path) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 44cbcfeb..836a4434 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -10,7 +10,8 @@ def put_data( - up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, fname=None): + up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, + fname=None): """上传二进制流到七牛 Args: @@ -160,7 +161,7 @@ def recovery_from_record(self): try: if not record['modify_time'] or record['size'] != self.size or \ - record['modify_time'] != self.modify_time: + record['modify_time'] != self.modify_time: return 0 except KeyError: return 0 @@ -187,8 +188,8 @@ def upload(self): self.blockStatus.append(ret) offset += length self.record_upload_progress(offset) - if(callable(self.progress_handler)): - self.progress_handler(((len(self.blockStatus) - 1) * config._BLOCK_SIZE)+length, self.size) + if (callable(self.progress_handler)): + self.progress_handler(((len(self.blockStatus) - 1) * config._BLOCK_SIZE) + length, self.size) return self.make_file(host) def make_block(self, block, block_size, host): @@ -223,6 +224,7 @@ def make_file(self, host): """创建文件""" url = self.file_url(host) body = ','.join([status['ctx'] for status in self.blockStatus]) + self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) return self.post(url, body) def post(self, url, data): From f15057228de4d8f692f0f9dc8169317a30d88118 Mon Sep 17 00:00:00 2001 From: Bai Long Date: Fri, 3 Feb 2017 19:05:50 +0800 Subject: [PATCH 213/478] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 69bf6d93..84acf6ec 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.1.0' +__version__ = '7.1.1' from .auth import Auth, QiniuMacAuth From 50aea442e2cfc1ffe6d0f99e91fb81caaf8ebdcc Mon Sep 17 00:00:00 2001 From: Bai Long Date: Fri, 3 Feb 2017 19:10:12 +0800 Subject: [PATCH 214/478] update [ci skip] --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb70367b..36d25aaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ #Changelog +## 7.1.1 (2017-02-03) +### 增加 +* 增加cdn刷新，预取，日志获取，时间戳防盗链生成功能 + +### 修正 +* 修复分片上传的upload record path遇到中文时的问题，现在使用md5来计算文件名 + ## 7.1.0 (2016-12-08) ### 增加 * 通用计算支持 From 49c05c3972ffe8ea6ab643748532d2f759e5a206 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Tue, 14 Mar 2017 17:45:24 +0800 Subject: [PATCH 215/478] fix move error --- examples/move_to.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/move_to.py b/examples/move_to.py index df4d4f46..71c9c2c9 100644 --- a/examples/move_to.py +++ b/examples/move_to.py @@ -19,6 +19,6 @@ #将文件从文件key 移动到文件key2，可以实现文件的重命名 可以在不同bucket移动 key2 = 'python-logo2.png' -ret, info = bucket.move(bucket_name, key, bucket, key2) +ret, info = bucket.move(bucket_name, key, bucket_name, key2) print(info) assert ret == {} From a2e42ab92dff0aca2c86316e45c8753fb8c027a3 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Fri, 17 Mar 2017 15:35:52 +0800 Subject: [PATCH 216/478] add delete_after_days API --- examples/delete_afte_days.py | 26 ++++++++++++++++++++++++++ qiniu/services/storage/bucket.py | 22 ++++++++++++++++++++++ test_qiniu.py | 7 +++++++ 3 files changed, 55 insertions(+) create mode 100644 examples/delete_afte_days.py diff --git a/examples/delete_afte_days.py b/examples/delete_afte_days.py new file mode 100644 index 00000000..cd54b49e --- /dev/null +++ b/examples/delete_afte_days.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +access_key = '...' +secret_key = '...' + +#初始化Auth状态 +q = Auth(access_key, secret_key) + +#初始化BucketManager +bucket = BucketManager(q) + +#你要测试的空间， 并且这个key在你空间中存在 +bucket_name = 'Bucket_Name' +key = 'python-test.png' + +#您要更新的生命周期 +days = '5' + +ret, info = bucket.delete_after_days(bucket_name, key, days) +print(info) + + diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 43f03d39..906a1707 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -246,6 +246,28 @@ def buckets(self): """ return self.__rs_do('buckets') + def delete_after_days(self, bucket, key, days): + """更新文件生命周期 + + Returns: + 一个dict变量，返回结果类似： + [ + { "code": , "data": }, + { "code": }, + { "code": }, + { "code": }, + { "code": , "data": { "error": "" } }, + ... + ] + 一个ResponseInfo对象 + Args: + bucket: 目标资源空间 + key: 目标资源文件名 + days: 指定天数 + """ + resource = entry(bucket, key) + return self.__rs_do('deleteAfterDays', resource, days) + def __rs_do(self, operation, *args): return self.__server_do(config.get_default('default_rs_host'), operation, *args) diff --git a/test_qiniu.py b/test_qiniu.py index a1bd5773..d040162f 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -264,6 +264,13 @@ def test_batch_stat(self): print(info) assert ret[0]['code'] == 200 + def test_delete_after_days(self): + days = '5' + ret, info = self.bucket.delete_after_days(bucket_name,'invaild.html', days) + assert info.status_code == 612 + ret, info = self.bucket.delete_after_days(bucket_name,'fetch.html', days ) + assert info.status_code == 200 + class UploaderTestCase(unittest.TestCase): From e20f5f5a2301d3202fb7b8b8cf7eba403d3ed414 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Fri, 17 Mar 2017 16:06:33 +0800 Subject: [PATCH 217/478] fix tes_delete_after_days bug --- examples/deleteafterdays.py | 21 +++++++++++++++++++++ test_qiniu.py | 4 +++- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 examples/deleteafterdays.py diff --git a/examples/deleteafterdays.py b/examples/deleteafterdays.py new file mode 100644 index 00000000..8c98f016 --- /dev/null +++ b/examples/deleteafterdays.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +access_key = 'l6LRg3So-l_th3Ti5744bRyhp6O6kUACAep-KjEm' +secret_key = 'piOuakNoSNX0fdbggkQ4Rq2-bldLq3R90AB2LECc' + +bucket_name = 'bernie' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +key = '2.mp4' + +ret, info = bucket.delete_after_days(bucket_name, key, '5') +print(info) + + diff --git a/test_qiniu.py b/test_qiniu.py index d040162f..5702dbfc 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -268,7 +268,9 @@ def test_delete_after_days(self): days = '5' ret, info = self.bucket.delete_after_days(bucket_name,'invaild.html', days) assert info.status_code == 612 - ret, info = self.bucket.delete_after_days(bucket_name,'fetch.html', days ) + key = 'copyto'+rand_string(8) + ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + ret, info = self.bucket.delete_after_days(bucket_name, key, days) assert info.status_code == 200 From 20573b488df1a987c55d68817a8a1fe527ff3c2b Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Fri, 17 Mar 2017 16:08:22 +0800 Subject: [PATCH 218/478] fix tes_delete_after_days bug --- examples/deleteafterdays.py | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 examples/deleteafterdays.py diff --git a/examples/deleteafterdays.py b/examples/deleteafterdays.py deleted file mode 100644 index 8c98f016..00000000 --- a/examples/deleteafterdays.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa - -from qiniu import Auth -from qiniu import BucketManager - -access_key = 'l6LRg3So-l_th3Ti5744bRyhp6O6kUACAep-KjEm' -secret_key = 'piOuakNoSNX0fdbggkQ4Rq2-bldLq3R90AB2LECc' - -bucket_name = 'bernie' - -q = Auth(access_key, secret_key) - -bucket = BucketManager(q) - -key = '2.mp4' - -ret, info = bucket.delete_after_days(bucket_name, key, '5') -print(info) - - From f3822450133e95518a794891983fa2c02c5c3a84 Mon Sep 17 00:00:00 2001 From: rwifeng Date: Fri, 24 Mar 2017 14:03:04 +0800 Subject: [PATCH 219/478] add comment of delete_after_days --- CHANGELOG.md | 8 +++++++- qiniu/__init__.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36d25aaf..c7414ee9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ -#Changelog + #Changelog + +## 7.1.2 (2017-03-24) +### 增加 +* 增加设置文件生命周期的接口 ## 7.1.1 (2017-02-03) ### 增加 @@ -89,3 +93,5 @@ * 代码覆盖度报告 * policy改为dict, 便于灵活增加，并加入过期字段检查 * 文件列表支持目录形式 + + diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 84acf6ec..870e3857 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.1.1' +__version__ = '7.1.2' from .auth import Auth, QiniuMacAuth From 0818b14a5aa79d15a810b4a8b55d071c34ebefd0 Mon Sep 17 00:00:00 2001 From: rwifeng Date: Fri, 24 Mar 2017 14:05:06 +0800 Subject: [PATCH 220/478] format changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7414ee9..4f7721c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ - #Changelog +#Changelog ## 7.1.2 (2017-03-24) ### 增加 From 6cd687ae876729146f1ebff6df7d0a838681b666 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Tue, 18 Apr 2017 09:58:05 +0800 Subject: [PATCH 221/478] Update http.py --- qiniu/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/http.py b/qiniu/http.py index 5eea3e9a..5ec0d3b4 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -38,7 +38,7 @@ def _post(url, data, files, auth, headers=None): if _session is None: _init() try: - post_headers = _headers + post_headers = _headers.copy() if headers is not None: for k, v in headers.items(): post_headers.update({k: v}) From 3655d944f5237ce9cc29b7f30ec826a92a374eac Mon Sep 17 00:00:00 2001 From: jemygraw Date: Thu, 27 Apr 2017 14:36:06 +0800 Subject: [PATCH 222/478] [ISSUE-246]fix the cdn function of get log list --- qiniu/services/cdn/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 58d8091c..65d55c90 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -151,7 +151,7 @@ def get_log_list_data(self, domains, log_date): """ req = {} req.update({"domains": ';'.join(domains)}) - req.update({"startDate": log_date}) + req.update({"day": log_date}) body = json.dumps(req) url = '{0}/v2/tune/log/list'.format(self.server) @@ -159,7 +159,7 @@ def get_log_list_data(self, domains, log_date): def __post(self, url, data=None): headers = {'Content-Type': 'application/json'} - return http._post_with_auth_and_headers(url, data, self.auth, headers) + return http._post_with_auth_and_head2ers(url, data, self.auth, headers) def create_timestamp_anti_leech_url(host, file_name, query_string_dict, encrypt_key, deadline): From 761800fd8a1f361f98affdefb1c93c1ac4f7255d Mon Sep 17 00:00:00 2001 From: jemygraw Date: Thu, 27 Apr 2017 14:36:35 +0800 Subject: [PATCH 223/478] fix typo problem --- qiniu/services/cdn/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 65d55c90..a86cced8 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -159,7 +159,7 @@ def get_log_list_data(self, domains, log_date): def __post(self, url, data=None): headers = {'Content-Type': 'application/json'} - return http._post_with_auth_and_head2ers(url, data, self.auth, headers) + return http._post_with_auth_and_headers(url, data, self.auth, headers) def create_timestamp_anti_leech_url(host, file_name, query_string_dict, encrypt_key, deadline): From b97b715a44ca6507782420ee53146b552f264813 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Thu, 27 Apr 2017 16:00:22 +0800 Subject: [PATCH 224/478] improve the efficiency mentioned in issue 203 --- qiniu/http.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/http.py b/qiniu/http.py index 5ec0d3b4..0ee27cea 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -20,6 +20,7 @@ def __return_wrapper(resp): if resp.status_code != 200 or resp.headers.get('X-Reqid') is None: return None, ResponseInfo(resp) + resp.encoding = 'utf-8' ret = resp.json() if resp.text != '' else {} return ret, ResponseInfo(resp) From dce0bd785d66fbc164b9afbb381d04739e415547 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Thu, 27 Apr 2017 16:09:29 +0800 Subject: [PATCH 225/478] rename test files --- examples/pfop_vframe.py | 29 +++++++++++++++++++++++++++++ examples/pfop_watermark.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 examples/pfop_vframe.py create mode 100644 examples/pfop_watermark.py diff --git a/examples/pfop_vframe.py b/examples/pfop_vframe.py new file mode 100644 index 00000000..6c1c2ba5 --- /dev/null +++ b/examples/pfop_vframe.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth, PersistentFop, build_op, op_save, urlsafe_base64_encode + +#对已经上传到七牛的视频发起异步转码操作 +access_key = 'Access_Key' +secret_key = 'Secret_Key' +q = Auth(access_key, secret_key) + +#要转码的文件所在的空间和文件名。 +bucket = 'Bucket_Name' +key = '1.mp4' + +#转码是使用的队列名称。 +pipeline = 'mpsdemo' + +#要进行视频截图操作。 +fops = 'vframe/jpg/offset/1/w/480/h/360/rotate/90' + +#可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 +saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') +fops = fops+'|saveas/'+saveas_key + +pfop = PersistentFop(q, bucket, pipeline) +ops = [] +ops.append(fops) +ret, info = pfop.execute(key, ops, 1) +print(info) +assert ret['persistentId'] is not None \ No newline at end of file diff --git a/examples/pfop_watermark.py b/examples/pfop_watermark.py new file mode 100644 index 00000000..dd1a97d4 --- /dev/null +++ b/examples/pfop_watermark.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth, PersistentFop, build_op, op_save, urlsafe_base64_encode + +#对已经上传到七牛的视频发起异步转码操作 +access_key = 'Access_Key' +secret_key = 'Secret_Key' +q = Auth(access_key, secret_key) + +#要转码的文件所在的空间和文件名。 +bucket = 'Bucket_Name' +key = '1.mp4' + +#转码是使用的队列名称。 +pipeline = 'mpsdemo' + +#需要添加水印的图片UrlSafeBase64,可以参考http://developer.qiniu.com/code/v6/api/dora-api/av/video-watermark.html +base64URL = urlsafe_base64_encode('http://developer.qiniu.com/resource/logo-2.jpg'); + +#视频水印参数 +fops = 'avthumb/mp4/'+base64URL + +#可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 +saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') +fops = fops+'|saveas/'+saveas_key + +pfop = PersistentFop(q, bucket, pipeline) +ops = [] +ops.append(fops) +ret, info = pfop.execute(key, ops, 1) +print(info) +assert ret['persistentId'] \ No newline at end of file From 4bae4f711b811f7dcbb40616608652df2bd0afc1 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Thu, 27 Apr 2017 16:33:45 +0800 Subject: [PATCH 226/478] remove old files --- examples/pfop_ vframe.py | 29 ----------------------------- examples/pfop_ watermark.py | 32 -------------------------------- 2 files changed, 61 deletions(-) delete mode 100644 examples/pfop_ vframe.py delete mode 100644 examples/pfop_ watermark.py diff --git a/examples/pfop_ vframe.py b/examples/pfop_ vframe.py deleted file mode 100644 index 6c1c2ba5..00000000 --- a/examples/pfop_ vframe.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa -from qiniu import Auth, PersistentFop, build_op, op_save, urlsafe_base64_encode - -#对已经上传到七牛的视频发起异步转码操作 -access_key = 'Access_Key' -secret_key = 'Secret_Key' -q = Auth(access_key, secret_key) - -#要转码的文件所在的空间和文件名。 -bucket = 'Bucket_Name' -key = '1.mp4' - -#转码是使用的队列名称。 -pipeline = 'mpsdemo' - -#要进行视频截图操作。 -fops = 'vframe/jpg/offset/1/w/480/h/360/rotate/90' - -#可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 -saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') -fops = fops+'|saveas/'+saveas_key - -pfop = PersistentFop(q, bucket, pipeline) -ops = [] -ops.append(fops) -ret, info = pfop.execute(key, ops, 1) -print(info) -assert ret['persistentId'] is not None \ No newline at end of file diff --git a/examples/pfop_ watermark.py b/examples/pfop_ watermark.py deleted file mode 100644 index dd1a97d4..00000000 --- a/examples/pfop_ watermark.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa -from qiniu import Auth, PersistentFop, build_op, op_save, urlsafe_base64_encode - -#对已经上传到七牛的视频发起异步转码操作 -access_key = 'Access_Key' -secret_key = 'Secret_Key' -q = Auth(access_key, secret_key) - -#要转码的文件所在的空间和文件名。 -bucket = 'Bucket_Name' -key = '1.mp4' - -#转码是使用的队列名称。 -pipeline = 'mpsdemo' - -#需要添加水印的图片UrlSafeBase64,可以参考http://developer.qiniu.com/code/v6/api/dora-api/av/video-watermark.html -base64URL = urlsafe_base64_encode('http://developer.qiniu.com/resource/logo-2.jpg'); - -#视频水印参数 -fops = 'avthumb/mp4/'+base64URL - -#可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 -saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') -fops = fops+'|saveas/'+saveas_key - -pfop = PersistentFop(q, bucket, pipeline) -ops = [] -ops.append(fops) -ret, info = pfop.execute(key, ops, 1) -print(info) -assert ret['persistentId'] \ No newline at end of file From 1bb4cd74a403b681ce6fdaa9ec6df3b78765c831 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Mon, 5 Jun 2017 10:07:03 +0800 Subject: [PATCH 227/478] publish 7.1.3 --- CHANGELOG.md | 6 +++++- qiniu/__init__.py | 2 +- qiniu/auth.py | 14 +++----------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f7721c8..a7884879 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ -#Changelog +# Changelog + +## 7.1.3 (2017-06-05) +### 修正 +* cdn功能中获取域名日志列表的参数错误 ## 7.1.2 (2017-03-24) ### 增加 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 870e3857..fac846cb 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.1.2' +__version__ = '7.1.3' from .auth import Auth, QiniuMacAuth diff --git a/qiniu/auth.py b/qiniu/auth.py index feef3bcc..a5d8c86b 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -11,7 +11,7 @@ # 上传策略，参数规格详见 -# http://developer.qiniu.com/docs/v6/api/reference/security/put-policy.html +# https://developer.qiniu.com/kodo/manual/1206/put-policy _policy_fields = set([ 'callbackUrl', # 回调URL 'callbackBody', # 回调Body @@ -37,19 +37,14 @@ 'deleteAfterDays', # 文件多少天后自动删除 ]) -_deprecated_policy_fields = set([ - 'asyncOps' -]) - - class Auth(object): """七牛安全机制类 该类主要内容是七牛上传凭证、下载凭证、管理凭证三种凭证的签名接口的实现，以及回调验证。 Attributes: - __access_key: 账号密钥对中的accessKey，详见 https://portal.qiniu.com/setting/key - __secret_key: 账号密钥对重的secretKey，详见 https://portal.qiniu.com/setting/key + __access_key: 账号密钥对中的accessKey，详见 https://portal.qiniu.com/user/key + __secret_key: 账号密钥对重的secretKey，详见 https://portal.qiniu.com/user/key """ def __init__(self, access_key, secret_key): @@ -178,8 +173,6 @@ def verify_callback(self, origin_authorization, url, body, content_type='applica @staticmethod def __copy_policy(policy, to, strict_policy): for k, v in policy.items(): - if k in _deprecated_policy_fields: - raise ValueError(k + ' has deprecated') if (not strict_policy) or k in _policy_fields: to[k] = v @@ -189,7 +182,6 @@ def __init__(self, auth): self.auth = auth def __call__(self, r): - token = None if r.body is not None and r.headers['Content-Type'] == 'application/x-www-form-urlencoded': token = self.auth.token_of_request(r.url, r.body, 'application/x-www-form-urlencoded') else: From 5353bdd48059b2f0ece0a70aa10889201399fbe2 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Mon, 5 Jun 2017 10:57:36 +0800 Subject: [PATCH 228/478] fix build failed issues --- test_qiniu.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index 5702dbfc..0c9caf3f 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -98,10 +98,6 @@ def test_token_of_request(self): token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', 'application/x-www-form-urlencoded') assert token == 'abcdefghklmnopq:svWRNcacOE-YMsc70nuIYdaa1e4=' - def test_deprecatedPolicy(self): - with pytest.raises(ValueError): - dummy_auth.upload_token('1', None, policy={'asyncOps': 1}) - def test_verify_callback(self): body = 'name=sunflower.jpg&hash=Fn6qeQi4VDLQ347NiRm-RlQx_4O2&location=Shanghai&price=1500.00&uid=123' url = 'test.qiniu.com/callback' @@ -118,7 +114,7 @@ def test_list(self): print(info) assert eof is False assert len(ret.get('items')) == 4 - ret, eof, info = self.bucket.list(bucket_name, limit=100) + ret, eof, info = self.bucket.list(bucket_name, limit=1000) print(info) assert eof is True From 1392a6fed89a043ec2c5ad9c44bd5703da21c07c Mon Sep 17 00:00:00 2001 From: jemygraw Date: Mon, 5 Jun 2017 11:02:42 +0800 Subject: [PATCH 229/478] fix code style --- qiniu/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/auth.py b/qiniu/auth.py index a5d8c86b..7f9f87e5 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -37,6 +37,7 @@ 'deleteAfterDays', # 文件多少天后自动删除 ]) + class Auth(object): """七牛安全机制类 From 5eae0302e59c1873173e4f1f0211331b1fc00f42 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Mon, 5 Jun 2017 12:02:13 +0800 Subject: [PATCH 230/478] publish v7.1.4 --- CHANGELOG.md | 2 +- qiniu/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7884879..2ab1c3bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 7.1.3 (2017-06-05) +## 7.1.4 (2017-06-05) ### 修正 * cdn功能中获取域名日志列表的参数错误 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index fac846cb..a229565a 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.1.3' +__version__ = '7.1.4' from .auth import Auth, QiniuMacAuth From 830b693c450123aae4a86aa8341a6ed6485951cb Mon Sep 17 00:00:00 2001 From: jemygraw Date: Mon, 5 Jun 2017 21:47:16 +0800 Subject: [PATCH 231/478] [ISSUE-268] add support for infrequent storage, change type,stat,putpolicy --- qiniu/auth.py | 1 + qiniu/services/storage/bucket.py | 22 ++++++++++-- test_qiniu.py | 62 ++++++++++++++++---------------- 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index 7f9f87e5..8da00b8e 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -35,6 +35,7 @@ 'persistentNotifyUrl', # 持久化处理结果通知URL 'persistentPipeline', # 持久化处理独享队列 'deleteAfterDays', # 文件多少天后自动删除 + 'fileType', # 文件的存储类型，0为普通存储，1为低频存储 ]) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 906a1707..c93d2506 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -17,7 +17,7 @@ class BucketManager(object): def __init__(self, auth, zone=None): self.auth = auth - if(zone is None): + if (zone is None): self.zone = config.get_default('default_zone') else: self.zone = zone @@ -68,7 +68,7 @@ def stat(self, bucket, key): """获取文件信息: 获取资源的元信息，但不返回文件内容，具体规格参考： - http://developer.qiniu.com/docs/v6/api/reference/rs/stat.html + https://developer.qiniu.com/kodo/api/1308/stat Args: bucket: 待获取信息资源所在的空间 @@ -81,6 +81,7 @@ def stat(self, bucket, key): "hash": "ljfockr0lOil_bZfyaI2ZY78HWoH", "mimeType": "application/octet-stream", "putTime": 13603956734587420 + "type": 0 } 一个ResponseInfo对象 """ @@ -210,6 +211,20 @@ def change_mime(self, bucket, key, mime): encode_mime = urlsafe_base64_encode(mime) return self.__rs_do('chgm', resource, 'mime/{0}'.format(encode_mime)) + def change_type(self, bucket, key, storage_type): + """修改文件的存储类型 + + 修改文件的存储类型为普通存储或者是低频存储，参考温度： + https://developer.qiniu.com/kodo/api/3710/modify-the-file-type + + Args: + bucket: 待操作资源所在空间 + key: 待操作资源文件名 + storage_type: 待操作资源存储类型，0为普通存储，1为低频存储 + """ + resource = entry(bucket, key) + return self.__rs_do('chtype', resource, 'type/{0}'.format(storage_type)) + def batch(self, operations): """批量操作: @@ -319,4 +334,5 @@ def _one_key_batch(operation, bucket, keys): def _two_key_batch(operation, source_bucket, key_pairs, target_bucket, force='false'): if target_bucket is None: target_bucket = source_bucket - return [_build_op(operation, entry(source_bucket, k), entry(target_bucket, v), 'force/{0}'.format(force)) for k, v in key_pairs.items()] + return [_build_op(operation, entry(source_bucket, k), entry(target_bucket, v), 'force/{0}'.format(force)) for k, v + in key_pairs.items()] diff --git a/test_qiniu.py b/test_qiniu.py index 0c9caf3f..3d77aa55 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -11,7 +11,8 @@ from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone from qiniu import put_data, put_file, put_stream -from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, build_batch_delete +from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, \ + build_batch_delete from qiniu import urlsafe_base64_encode, urlsafe_base64_decode from qiniu.compat import is_py2, is_py3, b @@ -24,6 +25,7 @@ import sys import StringIO import urllib + reload(sys) sys.setdefaultencoding('utf-8') StringIO = StringIO.StringIO @@ -31,6 +33,7 @@ elif is_py3: import io import urllib + StringIO = io.StringIO urlopen = urllib.request.urlopen @@ -51,7 +54,7 @@ def rand_string(length): def create_temp_file(size): t = tempfile.mktemp() f = open(t, 'wb') - f.seek(size-1) + f.seek(size - 1) f.write(b('0')) f.close() return t @@ -69,7 +72,6 @@ def is_travis(): class UtilsTest(unittest.TestCase): - def test_urlsafe(self): a = 'hello\x96' u = urlsafe_base64_encode(a) @@ -77,7 +79,6 @@ def test_urlsafe(self): class AuthTestCase(unittest.TestCase): - def test_token(self): token = dummy_auth.token('test') assert token == 'abcdefghklmnopq:mSNBTR7uS2crJsyFr2Amwv1LaYg=' @@ -129,7 +130,8 @@ def test_prefetch(self): assert ret['key'] == 'python-sdk.html' def test_fetch(self): - ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, 'fetch.html') + ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, + 'fetch.html') print(info) assert ret['key'] == 'fetch.html' assert 'hash' in ret @@ -152,7 +154,7 @@ def test_delete(self): assert info.status_code == 612 def test_rename(self): - key = 'renameto'+rand_string(8) + key = 'renameto' + rand_string(8) self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) key2 = key + 'move' ret, info = self.bucket.rename(bucket_name, key, key2) @@ -163,7 +165,7 @@ def test_rename(self): assert ret == {} def test_copy(self): - key = 'copyto'+rand_string(8) + key = 'copyto' + rand_string(8) ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) print(info) assert ret == {} @@ -176,27 +178,24 @@ def test_change_mime(self): print(info) assert ret == {} - def test_copy(self): - key = 'copyto'+rand_string(8) - ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + def test_change_type(self): + target_key = 'copyto' + rand_string(8) + self.bucket.copy(bucket_name, 'copyfrom', bucket_name, target_key) + ret, info = self.bucket.change_type(bucket_name, target_key, 1) print(info) assert ret == {} - ret, info = self.bucket.delete(bucket_name, key) + ret, info = self.bucket.stat(bucket_name, target_key) print(info) - assert ret == {} + assert 'type' in ret + self.bucket.delete(bucket_name, target_key) def test_copy_force(self): ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, 'copyfrom', force='true') print(info) assert info.status_code == 200 - def test_change_mime(self): - ret, info = self.bucket.change_mime(bucket_name, 'python-sdk.html', 'text/html') - print(info) - assert ret == {} - def test_batch_copy(self): - key = 'copyto'+rand_string(8) + key = 'copyto' + rand_string(8) ops = build_batch_copy(bucket_name, {'copyfrom': key}, bucket_name) ret, info = self.bucket.batch(ops) print(info) @@ -213,7 +212,7 @@ def test_batch_copy_force(self): assert ret[0]['code'] == 200 def test_batch_move(self): - key = 'moveto'+rand_string(8) + key = 'moveto' + rand_string(8) self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) key2 = key + 'move' ops = build_batch_move(bucket_name, {key: key2}, bucket_name) @@ -225,16 +224,16 @@ def test_batch_move(self): assert ret == {} def test_batch_move_force(self): - ret,info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, 'copyfrom', force='true') + ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, 'copyfrom', force='true') print(info) assert info.status_code == 200 - ops = build_batch_move(bucket_name, {'copyfrom':'copyfrom'}, bucket_name,force='true') + ops = build_batch_move(bucket_name, {'copyfrom': 'copyfrom'}, bucket_name, force='true') ret, info = self.bucket.batch(ops) print(info) assert ret[0]['code'] == 200 def test_batch_rename(self): - key = 'rename'+rand_string(8) + key = 'rename' + rand_string(8) self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) key2 = key + 'rename' ops = build_batch_move(bucket_name, {key: key2}, bucket_name) @@ -246,10 +245,10 @@ def test_batch_rename(self): assert ret == {} def test_batch_rename_force(self): - ret,info = self.bucket.rename(bucket_name, 'copyfrom', 'copyfrom', force='true') + ret, info = self.bucket.rename(bucket_name, 'copyfrom', 'copyfrom', force='true') print(info) assert info.status_code == 200 - ops = build_batch_rename(bucket_name, {'copyfrom':'copyfrom'}, force='true') + ops = build_batch_rename(bucket_name, {'copyfrom': 'copyfrom'}, force='true') ret, info = self.bucket.batch(ops) print(info) assert ret[0]['code'] == 200 @@ -262,16 +261,15 @@ def test_batch_stat(self): def test_delete_after_days(self): days = '5' - ret, info = self.bucket.delete_after_days(bucket_name,'invaild.html', days) + ret, info = self.bucket.delete_after_days(bucket_name, 'invaild.html', days) assert info.status_code == 612 - key = 'copyto'+rand_string(8) + key = 'copyto' + rand_string(8) ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) ret, info = self.bucket.delete_after_days(bucket_name, key, days) assert info.status_code == 200 class UploaderTestCase(unittest.TestCase): - mime_type = "text/plain" params = {'x:a': 'a'} q = Auth(access_key, secret_key) @@ -400,7 +398,6 @@ def test_putData_without_fname2(self): class ResumableUploaderTestCase(unittest.TestCase): - mime_type = "text/plain" params = {'x:a': 'a'} q = Auth(access_key, secret_key) @@ -411,7 +408,8 @@ def test_put_stream(self): size = os.stat(localfile).st_size with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, self.params, self.mime_type) + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, self.params, + self.mime_type) print(info) assert ret['key'] == key @@ -438,13 +436,12 @@ def test_retry(self): class DownloadTestCase(unittest.TestCase): - q = Auth(access_key, secret_key) def test_private_url(self): private_bucket = 'private-res' private_key = 'gogopher.jpg' - base_url = 'http://%s/%s' % (private_bucket+'.qiniudn.com', private_key) + base_url = 'http://%s/%s' % (private_bucket + '.qiniudn.com', private_key) private_url = self.q.private_download_url(base_url, expires=3600) print(private_url) r = requests.get(private_url) @@ -469,11 +466,13 @@ def test_zero_size(self): hash = etag("x") assert hash == 'Fto5o-5ea0sNMlW_75VgGJCv2AcJ' remove_temp_file("x") + def test_small_size(self): localfile = create_temp_file(1024 * 1024) hash = etag(localfile) assert hash == 'FnlAdmDasGTQOIgrU1QIZaGDv_1D' remove_temp_file(localfile) + def test_large_size(self): localfile = create_temp_file(4 * 1024 * 1024 + 1) hash = etag(localfile) @@ -489,5 +488,6 @@ def __init__(self, str): def read(self): print(self.str) + if __name__ == '__main__': unittest.main() From 2d6429cc182f1fcaf373b45cebf494fc6675dd8e Mon Sep 17 00:00:00 2001 From: jemygraw Date: Tue, 6 Jun 2017 09:34:41 +0800 Subject: [PATCH 232/478] fix code style --- qiniu/services/storage/bucket.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index c93d2506..46c79f02 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -213,10 +213,10 @@ def change_mime(self, bucket, key, mime): def change_type(self, bucket, key, storage_type): """修改文件的存储类型 - - 修改文件的存储类型为普通存储或者是低频存储，参考温度： + + 修改文件的存储类型为普通存储或者是低频存储，参考文档： https://developer.qiniu.com/kodo/api/3710/modify-the-file-type - + Args: bucket: 待操作资源所在空间 key: 待操作资源文件名 From 8659dba82b1d5e6eedc70ed77592b3608c78afc4 Mon Sep 17 00:00:00 2001 From: raxxar1024 Date: Sat, 24 Jun 2017 09:40:18 +0800 Subject: [PATCH 233/478] raise exception when uploading file has Chinese characters. --- qiniu/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index 8da00b8e..de3b015c 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -140,7 +140,7 @@ def upload_token(self, bucket, key=None, expires=3600, policy=None, strict_polic scope = bucket if key is not None: - scope = '{0}:{1}'.format(bucket, key) + scope = u'{0}:{1}'.format(bucket, key) args = dict( scope=scope, From 6b49a8cb6a0f7a19e208b619243f586fbde8fc82 Mon Sep 17 00:00:00 2001 From: LI Daobing Date: Thu, 29 Jun 2017 00:23:49 +0800 Subject: [PATCH 234/478] fix document url --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index a229565a..facb4a2f 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -4,7 +4,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ For detailed document, please see: - + ''' # flake8: noqa From 4bf904841630a3157edf0fe8cbf471744c70c53d Mon Sep 17 00:00:00 2001 From: LI Daobing Date: Thu, 29 Jun 2017 00:30:58 +0800 Subject: [PATCH 235/478] fix doc ref url --- qiniu/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qiniu/utils.py b/qiniu/utils.py index e4316e09..29c61af6 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -19,7 +19,7 @@ def urlsafe_base64_encode(data): """urlsafe的base64编码: 对提供的数据进行urlsafe的base64编码。规格参考： - http://developer.qiniu.com/docs/v6/api/overview/appendix.html#urlsafe-base64 + https://developer.qiniu.com/kodo/manual/1231/appendix#1 Args: data: 待编码的数据，一般为字符串 @@ -108,7 +108,7 @@ def _sha1(data): def etag_stream(input_stream): """计算输入流的etag: - etag规格参考 http://developer.qiniu.com/docs/v6/api/overview/appendix.html#qiniu-etag + etag规格参考 https://developer.qiniu.com/kodo/manual/1231/appendix#3 Args: input_stream: 待计算etag的二进制流 @@ -145,7 +145,7 @@ def etag(filePath): def entry(bucket, key): """计算七牛API中的数据格式: - entry规格参考 http://developer.qiniu.com/docs/v6/api/reference/data-formats.html + entry规格参考 https://developer.qiniu.com/kodo/api/1276/data-format Args: bucket: 待操作的空间名 From cd8c91a06740e9e9c7411868f0320c3f822611a3 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Tue, 15 Aug 2017 10:14:27 +0800 Subject: [PATCH 236/478] add examples --- examples/batch.py | 0 examples/cdn_manager.py | 0 examples/change_mime_eg.py | 20 ++++++++++++++++++++ examples/change_type_eg.py | 21 +++++++++++++++++++++ examples/copy_to.py | 0 examples/delete.py | 0 examples/delete_afte_days.py | 0 examples/download.py | 0 examples/fetch.py | 0 examples/fops.py | 0 examples/kirk/README.md | 0 examples/kirk/list_apps.py | 0 examples/kirk/list_services.py | 0 examples/kirk/list_stacks.py | 0 examples/list.py | 0 examples/move_to.py | 0 examples/pfop_vframe.py | 0 examples/pfop_watermark.py | 0 examples/stat.py | 0 examples/upload.py | 0 examples/upload_callback.py | 0 examples/upload_pfops.py | 0 22 files changed, 41 insertions(+) mode change 100644 => 100755 examples/batch.py mode change 100644 => 100755 examples/cdn_manager.py create mode 100644 examples/change_mime_eg.py create mode 100644 examples/change_type_eg.py mode change 100644 => 100755 examples/copy_to.py mode change 100644 => 100755 examples/delete.py mode change 100644 => 100755 examples/delete_afte_days.py mode change 100644 => 100755 examples/download.py mode change 100644 => 100755 examples/fetch.py mode change 100644 => 100755 examples/fops.py mode change 100644 => 100755 examples/kirk/README.md mode change 100644 => 100755 examples/kirk/list_apps.py mode change 100644 => 100755 examples/kirk/list_services.py mode change 100644 => 100755 examples/kirk/list_stacks.py mode change 100644 => 100755 examples/list.py mode change 100644 => 100755 examples/move_to.py mode change 100644 => 100755 examples/pfop_vframe.py mode change 100644 => 100755 examples/pfop_watermark.py mode change 100644 => 100755 examples/stat.py mode change 100644 => 100755 examples/upload.py mode change 100644 => 100755 examples/upload_callback.py mode change 100644 => 100755 examples/upload_pfops.py diff --git a/examples/batch.py b/examples/batch.py old mode 100644 new mode 100755 diff --git a/examples/cdn_manager.py b/examples/cdn_manager.py old mode 100644 new mode 100755 diff --git a/examples/change_mime_eg.py b/examples/change_mime_eg.py new file mode 100644 index 00000000..5db61f8a --- /dev/null +++ b/examples/change_mime_eg.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +access_key = '...' +secret_key = '...' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +bucket_name = 'Bucket_Name' + +key = '...' + +ret, info = bucket.change_mime(bucket_name, key, 'image/jpg') +print(info) +assert info.status_code == 200 diff --git a/examples/change_type_eg.py b/examples/change_type_eg.py new file mode 100644 index 00000000..ff015562 --- /dev/null +++ b/examples/change_type_eg.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +access_key = '...' +secret_key = '...' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +bucket_name = 'Bucket_Name' + +key = '...' + +ret, info = bucket.change_type(bucket_name, key ,1)#1表示低频存储，0是标准存储 + +print(info) +# assert info.status_code == 200 diff --git a/examples/copy_to.py b/examples/copy_to.py old mode 100644 new mode 100755 diff --git a/examples/delete.py b/examples/delete.py old mode 100644 new mode 100755 diff --git a/examples/delete_afte_days.py b/examples/delete_afte_days.py old mode 100644 new mode 100755 diff --git a/examples/download.py b/examples/download.py old mode 100644 new mode 100755 diff --git a/examples/fetch.py b/examples/fetch.py old mode 100644 new mode 100755 diff --git a/examples/fops.py b/examples/fops.py old mode 100644 new mode 100755 diff --git a/examples/kirk/README.md b/examples/kirk/README.md old mode 100644 new mode 100755 diff --git a/examples/kirk/list_apps.py b/examples/kirk/list_apps.py old mode 100644 new mode 100755 diff --git a/examples/kirk/list_services.py b/examples/kirk/list_services.py old mode 100644 new mode 100755 diff --git a/examples/kirk/list_stacks.py b/examples/kirk/list_stacks.py old mode 100644 new mode 100755 diff --git a/examples/list.py b/examples/list.py old mode 100644 new mode 100755 diff --git a/examples/move_to.py b/examples/move_to.py old mode 100644 new mode 100755 diff --git a/examples/pfop_vframe.py b/examples/pfop_vframe.py old mode 100644 new mode 100755 diff --git a/examples/pfop_watermark.py b/examples/pfop_watermark.py old mode 100644 new mode 100755 diff --git a/examples/stat.py b/examples/stat.py old mode 100644 new mode 100755 diff --git a/examples/upload.py b/examples/upload.py old mode 100644 new mode 100755 diff --git a/examples/upload_callback.py b/examples/upload_callback.py old mode 100644 new mode 100755 diff --git a/examples/upload_pfops.py b/examples/upload_pfops.py old mode 100644 new mode 100755 From cfe91e5b6cc9961dfc930ba64ac0f375a032cf03 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Mon, 21 Aug 2017 16:18:27 +0800 Subject: [PATCH 237/478] image and bandwidth --- examples/bucket_image_unimage.py | 26 ++++++++++++++++++++++++++ examples/cdn_bandwidth.py | 27 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 examples/bucket_image_unimage.py create mode 100644 examples/cdn_bandwidth.py diff --git a/examples/bucket_image_unimage.py b/examples/bucket_image_unimage.py new file mode 100644 index 00000000..f3861a17 --- /dev/null +++ b/examples/bucket_image_unimage.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +access_key = '...' +secret_key = '...' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +bucket_name = 'Bucket_Name' + +key = '...' + +image_url = '' + +req_host = '' + +ret, info = bucket.image(bucket_name, image_url, req_host) +print(info) + +ret, info = bucket.unimage(bucket_name, image_url, req_host) +print(info) diff --git a/examples/cdn_bandwidth.py b/examples/cdn_bandwidth.py new file mode 100644 index 00000000..384998fa --- /dev/null +++ b/examples/cdn_bandwidth.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +import qiniu +from qiniu import CdnManager + + +# 账户ak，sk +access_key = '' +secret_key = '' + +auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) +cdn_manager = CdnManager(auth) + +startDate = '2017-07-20' + +endDate = '2017-08-20' + +granularity = 'day' + +urls = [ + 'a.example.com', + 'b.example.com' +] + +ret, info = cdn_manager.get_bandwidth_data(urls, startDate, endDate, granularity) + +print(ret) +print(info) From 9c6cc87ff9b081b1695697f55d208a16b1a408f2 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Mon, 21 Aug 2017 17:59:12 +0800 Subject: [PATCH 238/478] flux, log, change_mime, timestap_url --- examples/cdn_flux.py | 28 ++++++++++++++++++++++++++++ examples/cdn_log.py | 24 ++++++++++++++++++++++++ examples/change_mime.py | 21 +++++++++++++++++++++ examples/timestamp_url.py | 18 ++++++++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 examples/cdn_flux.py create mode 100644 examples/cdn_log.py create mode 100644 examples/change_mime.py create mode 100644 examples/timestamp_url.py diff --git a/examples/cdn_flux.py b/examples/cdn_flux.py new file mode 100644 index 00000000..d7629cff --- /dev/null +++ b/examples/cdn_flux.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +import qiniu +from qiniu import CdnManager + + +# 账户ak，sk +access_key = '' +secret_key = '' + +auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) +cdn_manager = CdnManager(auth) + +startDate = '2017-07-20' + +endDate = '2017-08-20' + +granularity = 'day' + +urls = [ + 'a.example.com', + 'b.example.com' +] + +# 获得指定域名流量 +ret, info = cdn_manager.get_flux_data(urls, startDate, endDate, granularity) + +print(ret) +print(info) diff --git a/examples/cdn_log.py b/examples/cdn_log.py new file mode 100644 index 00000000..77c1b49a --- /dev/null +++ b/examples/cdn_log.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +import qiniu +from qiniu import CdnManager + + +# 账户ak，sk +access_key = '' +secret_key = '' + +auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) +cdn_manager = CdnManager(auth) + +log_date = '2017-07-20' + +urls = [ + 'a.example.com', + 'b.example.com' +] + +# 获得指定域名流量 +ret, info = cdn_manager.get_log_list_data(urls, log_date) + +print(ret) +print(info) diff --git a/examples/change_mime.py b/examples/change_mime.py new file mode 100644 index 00000000..69714ea7 --- /dev/null +++ b/examples/change_mime.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +access_key = '' +secret_key = '' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +bucket_name = '' + +key = 'example.png' + +mime_type = 'image/jpeg' + +ret, info = bucket.change_mime(bucket_name, key, mime_type) +print(info) diff --git a/examples/timestamp_url.py b/examples/timestamp_url.py new file mode 100644 index 00000000..90646451 --- /dev/null +++ b/examples/timestamp_url.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +import qiniu +from qiniu import CdnManager +from qiniu.services.cdn.manager import create_timestamp_anti_leech_url + +host = 'http://ymhb.qiniuts.com' + +encrypt_key = '5e99688aeab9329af09b2ba8388b87882ba811ba' + +file_name = 'yum.png' + +query_string_dict = {'imageInfo': ''} + +deadline = 1503414248 + +p_url = create_timestamp_anti_leech_url(host, file_name, query_string_dict, encrypt_key, deadline) + +print(p_url) From 242eeb055d08fdb4e95d6442691a4498a3e78663 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Mon, 21 Aug 2017 18:11:27 +0800 Subject: [PATCH 239/478] tmp --- examples/change_type.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 examples/change_type.py diff --git a/examples/change_type.py b/examples/change_type.py new file mode 100644 index 00000000..e69de29b From 6f91efd88b55af8f40c80d2e32f161b058a2259a Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Mon, 21 Aug 2017 18:14:57 +0800 Subject: [PATCH 240/478] fix conflict --- examples/change_mime.py | 13 ++++++------- examples/change_mime_eg.py | 20 -------------------- examples/change_type.py | 21 +++++++++++++++++++++ examples/change_type_eg.py | 21 --------------------- 4 files changed, 27 insertions(+), 48 deletions(-) delete mode 100644 examples/change_mime_eg.py delete mode 100644 examples/change_type_eg.py diff --git a/examples/change_mime.py b/examples/change_mime.py index 69714ea7..5db61f8a 100644 --- a/examples/change_mime.py +++ b/examples/change_mime.py @@ -4,18 +4,17 @@ from qiniu import Auth from qiniu import BucketManager -access_key = '' -secret_key = '' +access_key = '...' +secret_key = '...' q = Auth(access_key, secret_key) bucket = BucketManager(q) -bucket_name = '' +bucket_name = 'Bucket_Name' -key = 'example.png' +key = '...' -mime_type = 'image/jpeg' - -ret, info = bucket.change_mime(bucket_name, key, mime_type) +ret, info = bucket.change_mime(bucket_name, key, 'image/jpg') print(info) +assert info.status_code == 200 diff --git a/examples/change_mime_eg.py b/examples/change_mime_eg.py deleted file mode 100644 index 5db61f8a..00000000 --- a/examples/change_mime_eg.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa - -from qiniu import Auth -from qiniu import BucketManager - -access_key = '...' -secret_key = '...' - -q = Auth(access_key, secret_key) - -bucket = BucketManager(q) - -bucket_name = 'Bucket_Name' - -key = '...' - -ret, info = bucket.change_mime(bucket_name, key, 'image/jpg') -print(info) -assert info.status_code == 200 diff --git a/examples/change_type.py b/examples/change_type.py index e69de29b..ff015562 100644 --- a/examples/change_type.py +++ b/examples/change_type.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +access_key = '...' +secret_key = '...' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +bucket_name = 'Bucket_Name' + +key = '...' + +ret, info = bucket.change_type(bucket_name, key ,1)#1表示低频存储，0是标准存储 + +print(info) +# assert info.status_code == 200 diff --git a/examples/change_type_eg.py b/examples/change_type_eg.py deleted file mode 100644 index ff015562..00000000 --- a/examples/change_type_eg.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa - -from qiniu import Auth -from qiniu import BucketManager - -access_key = '...' -secret_key = '...' - -q = Auth(access_key, secret_key) - -bucket = BucketManager(q) - -bucket_name = 'Bucket_Name' - -key = '...' - -ret, info = bucket.change_type(bucket_name, key ,1)#1表示低频存储，0是标准存储 - -print(info) -# assert info.status_code == 200 From b7f306fe30f337e072875038579341b704b23977 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Wed, 23 Aug 2017 11:36:01 +0800 Subject: [PATCH 241/478] add batch_copy, batch_delete, batch_move, batch_stat,upload_token --- examples/batch_copy.py | 26 ++++++++++++++++++++++++++ examples/batch_delete.py | 25 +++++++++++++++++++++++++ examples/batch_move.py | 27 +++++++++++++++++++++++++++ examples/batch_stat.py | 26 ++++++++++++++++++++++++++ examples/upload_token.py | 30 ++++++++++++++++++++++++++++++ 5 files changed, 134 insertions(+) create mode 100644 examples/batch_copy.py create mode 100644 examples/batch_delete.py create mode 100644 examples/batch_move.py create mode 100644 examples/batch_stat.py create mode 100644 examples/upload_token.py diff --git a/examples/batch_copy.py b/examples/batch_copy.py new file mode 100644 index 00000000..da1e3729 --- /dev/null +++ b/examples/batch_copy.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +""" +批量拷贝文件 + +https://developer.qiniu.com/kodo/api/1250/batch +""" + + +from qiniu import build_batch_copy, Auth, BucketManager + +access_key = '' + +secret_key = '' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +src_bucket_name = 'lower' + +target_bucket_name = 'simi' + +# force为true时强制同名覆盖, 字典的键为原文件，值为目标文件 +ops = build_batch_copy(src_bucket_name, {'src_key1': 'target_key1', 'src_key2': 'target_key2'}, target_bucket_name, force='true') +ret, info = bucket.batch(ops) +print(info) diff --git a/examples/batch_delete.py b/examples/batch_delete.py new file mode 100644 index 00000000..89ea1180 --- /dev/null +++ b/examples/batch_delete.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +""" +批量删除文件 + +https://developer.qiniu.com/kodo/api/1250/batch +""" + + +from qiniu import build_batch_delete, Auth, BucketManager + +access_key = '' + +secret_key = '' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +bucket_name = '' + +keys = ['1.gif', '2.txt', '3.png', '4.html'] + +ops = build_batch_delete(bucket_name, keys) +ret, info = bucket.batch(ops) +print(info) diff --git a/examples/batch_move.py b/examples/batch_move.py new file mode 100644 index 00000000..b333e295 --- /dev/null +++ b/examples/batch_move.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +""" +批量移动文件 + +https://developer.qiniu.com/kodo/api/1250/batch +""" + + +from qiniu import build_batch_move, Auth, BucketManager + +access_key = '' + +secret_key = '' + + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +src_bucket_name = '' + +target_bucket_name = '' + +# force为true时强制同名覆盖, 字典的键为原文件，值为目标文件 +ops = build_batch_move(src_bucket_name, {'src_key1': 'target_key1', 'src_key2': 'target_key2'}, target_bucket_name, force='true') +ret, info = bucket.batch(ops) +print(info) diff --git a/examples/batch_stat.py b/examples/batch_stat.py new file mode 100644 index 00000000..d8bf8391 --- /dev/null +++ b/examples/batch_stat.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +""" +批量删除文件 + +https://developer.qiniu.com/kodo/api/1250/batch +""" + + +from qiniu import build_batch_stat, Auth, BucketManager + +access_key = '' + +secret_key = '' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +bucket_name = '' + +# 需要查询的文件名 +keys = ['1.gif', '2.txt', '3.png', '4.html'] + +ops = build_batch_stat(bucket_name, keys) +ret, info = bucket.batch(ops) +print(info) diff --git a/examples/upload_token.py b/examples/upload_token.py new file mode 100644 index 00000000..8d92434c --- /dev/null +++ b/examples/upload_token.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth + +#需要填写你的 Access Key 和 Secret Key +access_key = '' +secret_key = '' + +#构建鉴权对象 +q = Auth(access_key, secret_key) + +#要上传的空间 +bucket_name = '' + +#上传到七牛后保存的文件名 +key = '' + +#生成上传 Token，可以指定过期时间等 + +# 上传策略 +policy = { + # 'callbackUrl':'https://requestb.in/1c7q2d31', + # 'callbackBody':'filename=$(fname)&filesize=$(fsize)' + } + +token = q.upload_token(bucket_name, key, 3600, policy) + +print(token) + From e6187cd426e653ed46adadcbd1bc1a7e800433df Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Wed, 23 Aug 2017 13:14:02 +0800 Subject: [PATCH 242/478] add annotation --- examples/batch_rename.py | 26 ++++++++++++++++++++++++++ examples/batch_stat.py | 2 +- examples/cdn_bandwidth.py | 3 +++ examples/cdn_flux.py | 3 +++ examples/cdn_log.py | 5 ++++- examples/timestamp_url.py | 30 +++++++++++++++++++++--------- examples/upload_token.py | 4 +++- qiniu/services/cdn/manager.py | 16 ++++++---------- 8 files changed, 67 insertions(+), 22 deletions(-) create mode 100644 examples/batch_rename.py diff --git a/examples/batch_rename.py b/examples/batch_rename.py new file mode 100644 index 00000000..dc7c35e5 --- /dev/null +++ b/examples/batch_rename.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +""" +批量重命名文件 + +https://developer.qiniu.com/kodo/api/1250/batch +""" + + +from qiniu import build_batch_rename, Auth, BucketManager + +access_key = '' + +secret_key = '' + + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +bucket_name = '' + + +# force为true时强制同名覆盖, 字典的键为原文件，值为目标文件 +ops = build_batch_rename(bucket_name, {'src_key1': 'target_key1', 'src_key2': 'target_key2'}, force='true') +ret, info = bucket.batch(ops) +print(info) diff --git a/examples/batch_stat.py b/examples/batch_stat.py index d8bf8391..4ab7592c 100644 --- a/examples/batch_stat.py +++ b/examples/batch_stat.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -批量删除文件 +批量查询文件信息 https://developer.qiniu.com/kodo/api/1250/batch """ diff --git a/examples/cdn_bandwidth.py b/examples/cdn_bandwidth.py index 384998fa..a98c0423 100644 --- a/examples/cdn_bandwidth.py +++ b/examples/cdn_bandwidth.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +""" +查询指定域名指定时间段内的带宽 +""" import qiniu from qiniu import CdnManager diff --git a/examples/cdn_flux.py b/examples/cdn_flux.py index d7629cff..c88517e3 100644 --- a/examples/cdn_flux.py +++ b/examples/cdn_flux.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +""" +查询指定域名指定时间段内的流量 +""" import qiniu from qiniu import CdnManager diff --git a/examples/cdn_log.py b/examples/cdn_log.py index 77c1b49a..2ebb7271 100644 --- a/examples/cdn_log.py +++ b/examples/cdn_log.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +""" +获取指定域名指定时间内的日志链接 +""" import qiniu from qiniu import CdnManager @@ -17,7 +20,7 @@ 'b.example.com' ] -# 获得指定域名流量 + ret, info = cdn_manager.get_log_list_data(urls, log_date) print(ret) diff --git a/examples/timestamp_url.py b/examples/timestamp_url.py index 90646451..3bff2ae5 100644 --- a/examples/timestamp_url.py +++ b/examples/timestamp_url.py @@ -1,18 +1,30 @@ # -*- coding: utf-8 -*- -import qiniu -from qiniu import CdnManager + +""" +获取一个配置时间戳防盗链的url +""" + from qiniu.services.cdn.manager import create_timestamp_anti_leech_url +import time + +host = 'http://a.example.com' + +# 配置时间戳时指定的key +encrypt_key = '' + +# 资源路径 +file_name = 'a/b/c/example.jpeg' + +# 查询字符串,不需加? +query_string = '' -host = 'http://ymhb.qiniuts.com' +# 截止日期的时间戳,秒为单位，3600为当前时间一小时之后过期 +deadline = int(time.time())+3600 -encrypt_key = '5e99688aeab9329af09b2ba8388b87882ba811ba' -file_name = 'yum.png' +timestamp_url = create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline) -query_string_dict = {'imageInfo': ''} +print(timestamp_url) -deadline = 1503414248 -p_url = create_timestamp_anti_leech_url(host, file_name, query_string_dict, encrypt_key, deadline) -print(p_url) diff --git a/examples/upload_token.py b/examples/upload_token.py index 8d92434c..911319c4 100644 --- a/examples/upload_token.py +++ b/examples/upload_token.py @@ -18,10 +18,12 @@ #生成上传 Token，可以指定过期时间等 -# 上传策略 +# 上传策略示例 +# https://developer.qiniu.com/kodo/manual/1206/put-policy policy = { # 'callbackUrl':'https://requestb.in/1c7q2d31', # 'callbackBody':'filename=$(fname)&filesize=$(fsize)' + # 'persistentOps':'imageView2/1/w/200/h/200' } token = q.upload_token(bucket_name, key, 3600, policy) diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index a86cced8..f073a77b 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -91,7 +91,7 @@ def prefetch_urls(self, urls): def get_bandwidth_data(self, domains, start_date, end_date, granularity): """ - 预取带宽数据，文档 http://developer.qiniu.com/article/fusion/api/traffic-bandwidth.html + 查询带宽数据，文档 http://developer.qiniu.com/article/fusion/api/traffic-bandwidth.html Args: domains: 域名列表 @@ -115,7 +115,7 @@ def get_bandwidth_data(self, domains, start_date, end_date, granularity): def get_flux_data(self, domains, start_date, end_date, granularity): """ - 预取流量数据，文档 http://developer.qiniu.com/article/fusion/api/traffic-bandwidth.html + 查询流量数据，文档 http://developer.qiniu.com/article/fusion/api/traffic-bandwidth.html Args: domains: 域名列表 @@ -162,25 +162,21 @@ def __post(self, url, data=None): return http._post_with_auth_and_headers(url, data, self.auth, headers) -def create_timestamp_anti_leech_url(host, file_name, query_string_dict, encrypt_key, deadline): +def create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline): """ 创建时间戳防盗链 Args: host: 带访问协议的域名 file_name: 原始文件名，不需要urlencode - query_string_dict: 查询参数，不需要urlencode + query_string: 查询参数，不需要urlencode encrypt_key: 时间戳防盗链密钥 deadline: 链接有效期时间戳（以秒为单位） Returns: 带时间戳防盗链鉴权访问链接 """ - if query_string_dict is not None and len(query_string_dict) > 0: - query_string_items = [] - for k, v in query_string_dict.items(): - query_string_items.append('{0}={1}'.format(urlencode(str(k)), urlencode(str(v)))) - query_string = '&'.join(query_string_items) + if query_string: url_to_sign = '{0}/{1}?{2}'.format(host, urlencode(file_name), query_string) else: url_to_sign = '{0}/{1}'.format(host, urlencode(file_name)) @@ -190,7 +186,7 @@ def create_timestamp_anti_leech_url(host, file_name, query_string_dict, encrypt_ str_to_sign = '{0}{1}{2}'.format(encrypt_key, path, expire_hex).encode() sign_str = hashlib.md5(str_to_sign).hexdigest() - if query_string_dict is not None and len(query_string_dict) > 0: + if query_string: signed_url = '{0}&sign={1}&t={2}'.format(url_to_sign, sign_str, expire_hex) else: signed_url = '{0}?sign={1}&t={2}'.format(url_to_sign, sign_str, expire_hex) From 6eb625204bcf24df58bcdcf04a78c015c642fe4f Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Wed, 23 Aug 2017 13:19:10 +0800 Subject: [PATCH 243/478] fix real information --- examples/batch_copy.py | 4 ++-- examples/bucket_image_unimage.py | 26 -------------------------- 2 files changed, 2 insertions(+), 28 deletions(-) delete mode 100644 examples/bucket_image_unimage.py diff --git a/examples/batch_copy.py b/examples/batch_copy.py index da1e3729..ccb8781f 100644 --- a/examples/batch_copy.py +++ b/examples/batch_copy.py @@ -16,9 +16,9 @@ bucket = BucketManager(q) -src_bucket_name = 'lower' +src_bucket_name = '' -target_bucket_name = 'simi' +target_bucket_name = '' # force为true时强制同名覆盖, 字典的键为原文件，值为目标文件 ops = build_batch_copy(src_bucket_name, {'src_key1': 'target_key1', 'src_key2': 'target_key2'}, target_bucket_name, force='true') diff --git a/examples/bucket_image_unimage.py b/examples/bucket_image_unimage.py deleted file mode 100644 index f3861a17..00000000 --- a/examples/bucket_image_unimage.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa - -from qiniu import Auth -from qiniu import BucketManager - -access_key = '...' -secret_key = '...' - -q = Auth(access_key, secret_key) - -bucket = BucketManager(q) - -bucket_name = 'Bucket_Name' - -key = '...' - -image_url = '' - -req_host = '' - -ret, info = bucket.image(bucket_name, image_url, req_host) -print(info) - -ret, info = bucket.unimage(bucket_name, image_url, req_host) -print(info) From af4bd19d1c6b2b4222cc1d924b39a9701f139a7a Mon Sep 17 00:00:00 2001 From: jemygraw Date: Thu, 24 Aug 2017 14:10:37 +0800 Subject: [PATCH 244/478] remove kirk api --- examples/kirk/README.md | 5 - examples/kirk/list_apps.py | 18 - examples/kirk/list_services.py | 26 -- examples/kirk/list_stacks.py | 23 - qiniu/services/compute/__init__.py | 0 qiniu/services/compute/app.py | 224 ---------- qiniu/services/compute/config.py | 21 - qiniu/services/compute/qcos_api.py | 694 ----------------------------- 8 files changed, 1011 deletions(-) delete mode 100755 examples/kirk/README.md delete mode 100755 examples/kirk/list_apps.py delete mode 100755 examples/kirk/list_services.py delete mode 100755 examples/kirk/list_stacks.py delete mode 100644 qiniu/services/compute/__init__.py delete mode 100644 qiniu/services/compute/app.py delete mode 100644 qiniu/services/compute/config.py delete mode 100644 qiniu/services/compute/qcos_api.py diff --git a/examples/kirk/README.md b/examples/kirk/README.md deleted file mode 100755 index 23e5a852..00000000 --- a/examples/kirk/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Examples - -``` -$ python list_apps.py -``` diff --git a/examples/kirk/list_apps.py b/examples/kirk/list_apps.py deleted file mode 100755 index 2062b8b1..00000000 --- a/examples/kirk/list_apps.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa - -import sys -from qiniu import QiniuMacAuth -from qiniu import AccountClient - -access_key = sys.argv[1] -secret_key = sys.argv[2] - -acc_client = AccountClient(QiniuMacAuth(access_key, secret_key)) - -ret, info = acc_client.list_apps() - -print(ret) -print(info) - -assert len(ret) is not None diff --git a/examples/kirk/list_services.py b/examples/kirk/list_services.py deleted file mode 100755 index 9ec683fb..00000000 --- a/examples/kirk/list_services.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa - -import sys -from qiniu import QiniuMacAuth -from qiniu import AccountClient - -access_key = sys.argv[1] -secret_key = sys.argv[2] - -acc_client = AccountClient(QiniuMacAuth(access_key, secret_key)) -apps, info = acc_client.list_apps() - -for app in apps: - if app.get('runMode') == 'Private': - uri = app.get('uri') - qcos = acc_client.get_qcos_client(uri) - if qcos != None: - stacks, info = qcos.list_stacks() - for stack in stacks: - stack_name = stack.get('name') - services, info = qcos.list_services(stack_name) - print("list_services of '%s : %s':"%(uri, stack_name)) - print(services) - print(info) - assert len(services) is not None diff --git a/examples/kirk/list_stacks.py b/examples/kirk/list_stacks.py deleted file mode 100755 index 67815603..00000000 --- a/examples/kirk/list_stacks.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa - -import sys -from qiniu import QiniuMacAuth -from qiniu import AccountClient - -access_key = sys.argv[1] -secret_key = sys.argv[2] - -acc_client = AccountClient(QiniuMacAuth(access_key, secret_key)) -apps, info = acc_client.list_apps() - -for app in apps: - if app.get('runMode') == 'Private': - uri = app.get('uri') - qcos = acc_client.get_qcos_client(uri) - if qcos != None: - stacks, info = qcos.list_stacks() - print("list_stacks of '%s':"%uri) - print(stacks) - print(info) - assert len(stacks) is not None diff --git a/qiniu/services/compute/__init__.py b/qiniu/services/compute/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/qiniu/services/compute/app.py b/qiniu/services/compute/app.py deleted file mode 100644 index 614ff668..00000000 --- a/qiniu/services/compute/app.py +++ /dev/null @@ -1,224 +0,0 @@ -# -*- coding: utf-8 -*- -from qiniu import http, QiniuMacAuth -from .config import KIRK_HOST -from .qcos_api import QcosClient - - -class AccountClient(object): - """客户端入口 - - 使用账号密钥生成账号客户端，可以进一步： - 1、获取和操作账号数据 - 2、获得部署的应用的客户端 - - 属性： - auth: 账号管理密钥对，QiniuMacAuth对象 - host: API host，在『内网模式』下使用时，auth=None，会自动使用 apiproxy 服务 - - 接口： - get_qcos_client(app_uri) - create_qcos_client(app_uri) - get_app_keys(app_uri) - get_valid_app_auth(app_uri) - get_account_info() - get_app_region_products(app_uri) - get_region_products(region) - list_regions() - list_apps() - create_app(args) - delete_app(app_uri) - - """ - - def __init__(self, auth, host=None): - self.auth = auth - self.qcos_clients = {} - if (auth is None): - self.host = KIRK_HOST['APPPROXY'] - else: - self.host = host or KIRK_HOST['APPGLOBAL'] - acc, info = self.get_account_info() - self.uri = acc.get('name') - - def get_qcos_client(self, app_uri): - """获得资源管理客户端 - 缓存，但不是线程安全的 - """ - - client = self.qcos_clients.get(app_uri) - if (client is None): - client = self.create_qcos_client(app_uri) - self.qcos_clients[app_uri] = client - - return client - - def create_qcos_client(self, app_uri): - """创建资源管理客户端 - - """ - - if (self.auth is None): - return QcosClient(None) - - products = self.get_app_region_products(app_uri) - auth = self.get_valid_app_auth(app_uri) - - if products is None or auth is None: - return None - - return QcosClient(auth, products.get('api')) - - def get_app_keys(self, app_uri): - """获得账号下应用的密钥 - - 列出指定应用的密钥，仅当访问者对指定应用有管理权限时有效： - 用户对创建的应用有管理权限。 - 用户对使用的第三方应用没有管理权限，第三方应用的运维方有管理权限。 - - Args: - - app_uri: 应用的完整标识 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回秘钥列表，失败返回None - - ResponseInfo 请求的Response信息 - """ - - url = '{0}/v3/apps/{1}/keys'.format(self.host, app_uri) - return http._get_with_qiniu_mac(url, None, self.auth) - - def get_valid_app_auth(self, app_uri): - """获得账号下可用的应用的密钥 - - 列出指定应用的可用密钥 - - Args: - - app_uri: 应用的完整标识 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回可用秘钥列表，失败返回None - - ResponseInfo 请求的Response信息 - """ - - ret, retInfo = self.get_app_keys(app_uri) - - if ret is None: - return None - - for k in ret: - if (k.get('state') == 'enabled'): - return QiniuMacAuth(k.get('ak'), k.get('sk')) - - return None - - def get_account_info(self): - """获得当前账号的信息 - - 查看当前请求方（请求鉴权使用的 AccessKey 的属主）的账号信息。 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回用户信息，失败返回None - - ResponseInfo 请求的Response信息 - """ - - url = '{0}/v3/info'.format(self.host) - return http._get_with_qiniu_mac(url, None, self.auth) - - def get_app_region_products(self, app_uri): - """获得指定应用所在区域的产品信息 - - Args: - - app_uri: 应用的完整标识 - - Returns: - 返回产品信息列表，若失败则返回None - """ - apps, retInfo = self.list_apps() - if apps is None: - return None - - for app in apps: - if (app.get('uri') == app_uri): - return self.get_region_products(app.get('region')) - - return - - def get_region_products(self, region): - """获得指定区域的产品信息 - - Args: - - region: 区域，如："nq" - - Returns: - 返回该区域的产品信息，若失败则返回None - """ - - regions, retInfo = self.list_regions() - if regions is None: - return None - - for r in regions: - if r.get('name') == region: - return r.get('products') - - def list_regions(self): - """获得账号可见的区域的信息 - - 列出当前用户所有可使用的区域。 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回区域列表，失败返回None - - ResponseInfo 请求的Response信息 - """ - - url = '{0}/v3/regions'.format(self.host) - return http._get_with_qiniu_mac(url, None, self.auth) - - def list_apps(self): - """获得当前账号的应用列表 - - 列出所属应用为当前请求方的应用列表。 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回应用列表，失败返回None - - ResponseInfo 请求的Response信息 - """ - - url = '{0}/v3/apps'.format(self.host) - return http._get_with_qiniu_mac(url, None, self.auth) - - def create_app(self, args): - """创建应用 - - 在指定区域创建一个新应用，所属应用为当前请求方。 - - Args: - - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ - - Returns: - - result 成功返回所创建的应用信息，若失败则返回None - - ResponseInfo 请求的Response信息 - """ - - url = '{0}/v3/apps'.format(self.host) - return http._post_with_qiniu_mac(url, args, self.auth) - - def delete_app(self, app_uri): - """删除应用 - - 删除指定标识的应用，当前请求方对该应用应有删除权限。 - - Args: - - app_uri: 应用的完整标识 - - Returns: - - result 成功返回空dict{}，若失败则返回None - - ResponseInfo 请求的Response信息 - """ - - url = '{0}/v3/apps/{1}'.format(self.host, app_uri) - return http._delete_with_qiniu_mac(url, None, self.auth) diff --git a/qiniu/services/compute/config.py b/qiniu/services/compute/config.py deleted file mode 100644 index 045ed784..00000000 --- a/qiniu/services/compute/config.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- - -KIRK_HOST = { - 'APPGLOBAL': "https://app-api.qiniu.com", # 公有云 APP API - 'APPPROXY': "http://app.qcos.qiniu", # 内网 APP API - 'APIPROXY': "http://api.qcos.qiniu", # 内网 API -} - -CONTAINER_UINT_TYPE = { - '1U1G': '单核(CPU)，1GB(内存)', - '1U2G': '单核(CPU)，2GB(内存)', - '1U4G': '单核(CPU)，4GB(内存)', - '1U8G': '单核(CPU)，8GB(内存)', - '2U2G': '双核(CPU)，2GB(内存)', - '2U4G': '双核(CPU)，4GB(内存)', - '2U8G': '双核(CPU)，8GB(内存)', - '2U16G': '双核(CPU)，16GB(内存)', - '4U8G': '四核(CPU)，8GB(内存)', - '4U16G': '四核(CPU)，16GB(内存)', - '8U16G': '八核(CPU)，16GB(内存)', -} diff --git a/qiniu/services/compute/qcos_api.py b/qiniu/services/compute/qcos_api.py deleted file mode 100644 index 250a4bf6..00000000 --- a/qiniu/services/compute/qcos_api.py +++ /dev/null @@ -1,694 +0,0 @@ -# -*- coding: utf-8 -*- -from qiniu import http -from .config import KIRK_HOST - - -class QcosClient(object): - """资源管理客户端 - - 使用应用密钥生成资源管理客户端，可以进一步： - 1、部署服务和容器，获得信息 - 2、创建网络资源，获得信息 - - 属性： - auth: 应用密钥对，QiniuMacAuth对象 - host: API host，在『内网模式』下使用时，auth=None，会自动使用 apiproxy 服务，只能管理当前容器所在的应用资源。 - - 接口： - list_stacks() - create_stack(args) - delete_stack(stack) - get_stack(stack) - start_stack(stack) - stop_stack(stack) - - list_services(stack) - create_service(stack, args) - get_service_inspect(stack, service) - start_service(stack, service) - stop_service(stack, service) - update_service(stack, service, args) - scale_service(stack, service, args) - delete_service(stack, service) - create_service_volume(stack, service, volume, args) - extend_service_volume(stack, service, volume, args) - delete_service_volume(stack, service, volume) - - list_containers(args) - get_container_inspect(ip) - start_container(ip) - stop_container(ip) - restart_container(ip) - - list_aps() - create_ap(args) - search_ap(mode, query) - get_ap(apid) - update_ap(apid, args) - set_ap_port(apid, port, args) - delete_ap(apid) - publish_ap(apid, args) - unpublish_ap(apid) - get_ap_port_healthcheck(apid, port) - set_ap_port_container(apid, port, args) - disable_ap_port(apid, port) - enable_ap_port(apid, port) - get_ap_providers() - get_web_proxy(backend) - """ - - def __init__(self, auth, host=None): - self.auth = auth - if auth is None: - self.host = KIRK_HOST['APIPROXY'] - else: - self.host = host - - def list_stacks(self): - """获得服务组列表 - - 列出当前应用的所有服务组信息。 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回服务组列表[, , ...]，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks'.format(self.host) - return self.__get(url) - - def create_stack(self, args): - """创建服务组 - - 创建新一个指定名称的服务组，并创建其下的服务。 - - Args: - - args: 服务组描述，参考 http://kirk-docs.qiniu.com/apidocs/ - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks'.format(self.host) - return self.__post(url, args) - - def delete_stack(self, stack): - """删除服务组 - - 删除服务组内所有服务并销毁服务组。 - - Args: - - stack: 服务所属的服务组名称 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}'.format(self.host, stack) - return self.__delete(url) - - def get_stack(self, stack): - """获取服务组 - - 查看服务组的属性信息。 - - Args: - - stack: 服务所属的服务组名称 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回stack信息，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}'.format(self.host, stack) - return self.__get(url) - - def start_stack(self, stack): - """启动服务组 - - 启动服务组中的所有停止状态的服务。 - - Args: - - stack: 服务所属的服务组名称 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/start'.format(self.host, stack) - return self.__post(url) - - def stop_stack(self, stack): - """停止服务组 - - 停止服务组中所有运行状态的服务。 - - Args: - - stack: 服务所属的服务组名称 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/stop'.format(self.host, stack) - return self.__post(url) - - def list_services(self, stack): - """获得服务列表 - - 列出指定名称的服务组内所有的服务, 返回一组详细的服务信息。 - - Args: - - stack: 服务所属的服务组名称 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回服务信息列表[, , ...]，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/services'.format(self.host, stack) - return self.__get(url) - - def create_service(self, stack, args): - """创建服务 - - 创建一个服务，平台会异步地按模板分配资源并部署所有容器。 - - Args: - - stack: 服务所属的服务组名称 - - args: 服务具体描述请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/services'.format(self.host, stack) - return self.__post(url, args) - - def delete_service(self, stack, service): - """删除服务 - - 删除指定名称服务，并自动销毁服务已部署的所有容器和存储卷。 - - Args: - - stack: 服务所属的服务组名称 - - service: 服务名 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/services/{2}'.format(self.host, stack, service) - return self.__delete(url) - - def get_service_inspect(self, stack, service): - """查看服务 - - 查看指定名称服务的属性。 - - Args: - - stack: 服务所属的服务组名称 - - service: 服务名 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回服务信息，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/services/{2}/inspect'.format(self.host, stack, service) - return self.__get(url) - - def start_service(self, stack, service): - """启动服务 - - 启动指定名称服务的所有容器。 - - Args: - - stack: 服务所属的服务组名称 - - service: 服务名 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/services/{2}/start'.format(self.host, stack, service) - return self.__post(url) - - def stop_service(self, stack, service): - """停止服务 - - 停止指定名称服务的所有容器。 - - Args: - - stack: 服务所属的服务组名称 - - service: 服务名 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/services/{2}/stop'.format(self.host, stack, service) - return self.__post(url) - - def update_service(self, stack, service, args): - """更新服务 - - 更新指定名称服务的配置如容器镜像等参数，容器被重新部署后生效。 - 如果指定manualUpdate参数，则需要额外调用 部署服务 接口并指定参数进行部署；处于人工升级模式的服务禁止执行其他修改操作。 - 如果不指定manualUpdate参数，平台会自动完成部署。 - - Args: - - stack: 服务所属的服务组名称 - - service: 服务名 - - args: 服务具体描述请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/services/{2}'.format(self.host, stack, service) - return self.__post(url, args) - - def scale_service(self, stack, service, args): - """扩容/缩容服务 - - 更新指定名称服务的配置如容器镜像等参数，容器被重新部署后生效。 - 如果指定manualUpdate参数，则需要额外调用 部署服务 接口并指定参数进行部署；处于人工升级模式的服务禁止执行其他修改操作。 - 如果不指定manualUpdate参数，平台会自动完成部署。 - - Args: - - stack: 服务所属的服务组名称 - - service: 服务名 - - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/services/{2}/scale'.format(self.host, stack, service) - return self.__post(url, args) - - def create_service_volume(self, stack, service, args): - """创建存储卷 - - 为指定名称的服务增加存储卷资源，并挂载到部署的容器中。 - - Args: - - stack: 服务所属的服务组名称 - - service: 服务名 - - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/services/{2}/volumes'.format(self.host, stack, service) - return self.__post(url, args) - - def extend_service_volume(self, stack, service, volume, args): - """扩容存储卷 - - 为指定名称的服务增加存储卷资源，并挂载到部署的容器中。 - - Args: - - stack: 服务所属的服务组名称 - - service: 服务名 - - volume: 存储卷名 - - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/services/{2}/volumes/{3}/extend'.format(self.host, stack, service, volume) - return self.__post(url, args) - - def delete_service_volume(self, stack, service, volume): - """删除存储卷 - - 从部署的容器中移除挂载，并销毁指定服务下指定名称的存储卷, 并重新启动该容器。 - - Args: - - stack: 服务所属的服务组名称 - - service: 服务名 - - volume: 存储卷名 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/stacks/{1}/services/{2}/volumes/{3}'.format(self.host, stack, service, volume) - return self.__delete(url) - - def list_containers(self, stack=None, service=None): - """列出容器列表 - - 列出应用内所有部署的容器, 返回一组容器IP。 - - Args: - - stack: 要列出容器的服务组名(可不填，表示默认列出所有) - - service: 要列出容器服务的服务名(可不填，表示默认列出所有) - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回容器的ip数组，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/containers'.format(self.host) - params = {} - if stack is not None: - params['stack'] = stack - if service is not None: - params['service'] = service - return self.__get(url, params or None) - - def get_container_inspect(self, ip): - """查看容器 - - 查看指定IP的容器，返回容器属性。 - - Args: - - ip: 容器ip - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回容器的信息，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/containers/{1}/inspect'.format(self.host, ip) - return self.__get(url) - - def start_container(self, ip): - """启动容器 - - 启动指定IP的容器。 - - Args: - - ip: 容器ip - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/containers/{1}/start'.format(self.host, ip) - return self.__post(url) - - def stop_container(self, ip): - """停止容器 - - 停止指定IP的容器。 - - Args: - - ip: 容器ip - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/containers/{1}/stop'.format(self.host, ip) - return self.__post(url) - - def restart_container(self, ip): - """重启容器 - - 重启指定IP的容器。 - - Args: - - ip: 容器ip - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/containers/{1}/restart'.format(self.host, ip) - return self.__post(url) - - def list_aps(self): - """列出接入点 - - 列出当前应用的所有接入点。 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回接入点列表，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps'.format(self.host) - return self.__get(url) - - def create_ap(self, args): - """申请接入点 - - 申请指定配置的接入点资源。 - - Args: - - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回申请到的接入点信息，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps'.format(self.host) - return self.__post(url, args) - - def search_ap(self, mode, query): - """搜索接入点 - - 查看指定接入点的所有配置信息，包括所有监听端口的配置。 - - Args: - - mode: 搜索模式，可以是domain、ip、host - - query: 搜索文本 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回搜索结果，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps/search?{1}={2}'.format(self.host, mode, query) - return self.__get(url) - - def get_ap(self, apid): - """查看接入点 - - 给出接入点的域名或IP，查看配置信息，包括所有监听端口的配置。 - - Args: - - apid: 接入点ID - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回接入点信息，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps/{1}'.format(self.host, apid) - return self.__get(url) - - def update_ap(self, apid, args): - """更新接入点 - - 更新指定接入点的配置，如带宽。 - - Args: - - apid: 接入点ID - - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps/{1}'.format(self.host, apid) - return self.__post(url, args) - - def set_ap_port(self, apid, port, args): - """更新接入点端口配置 - - 更新接入点指定端口的配置。 - - Args: - - apid: 接入点ID - - port: 要设置的端口号 - - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps/{1}/{2}'.format(self.host, apid, port) - return self.__post(url, args) - - def delete_ap(self, apid): - """释放接入点 - - 销毁指定接入点资源。 - - Args: - - apid: 接入点ID - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps/{1}'.format(self.host, apid) - return self.__delete(url) - - def publish_ap(self, apid, args): - """绑定自定义域名 - - 绑定用户自定义的域名，仅对公网域名模式接入点生效。 - - Args: - - apid: 接入点ID - - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps/{1}/publish'.format(self.host, apid) - return self.__post(url, args) - - def unpublish_ap(self, apid, args): - """解绑自定义域名 - - 解绑用户自定义的域名，仅对公网域名模式接入点生效。 - - Args: - - apid: 接入点ID - - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps/{1}/unpublish'.format(self.host, apid) - return self.__post(url, args) - - def get_ap_port_healthcheck(self, apid, port): - """查看健康检查结果 - - 检查接入点的指定端口的后端健康状况。 - - Args: - - apid: 接入点ID - - port: 要设置的端口号 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回健康状况，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps/{1}/{2}/healthcheck'.format(self.host, apid, port) - return self.__get(url) - - def set_ap_port_container(self, apid, port, args): - """调整后端实例配置 - - 调整接入点指定后端实例（容器）的配置，例如临时禁用流量等。 - - Args: - - apid: 接入点ID - - port: 要设置的端口号 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps/{1}/{2}/setcontainer'.format(self.host, apid, port) - return self.__post(url, args) - - def disable_ap_port(self, apid, port): - """临时关闭接入点端口 - - 临时关闭接入点端口，仅对公网域名，公网ip有效。 - - Args: - - apid: 接入点ID - - port: 要设置的端口号 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps/{1}/{2}/disable'.format(self.host, apid, port) - return self.__post(url) - - def enable_ap_port(self, apid, port): - """开启接入点端口 - - 开启临时关闭的接入点端口，仅对公网域名，公网ip有效。 - - Args: - - apid: 接入点ID - - port: 要设置的端口号 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回空dict{}，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps/{1}/{2}/enable'.format(self.host, apid, port) - return self.__post(url) - - def get_ap_providers(self): - """列出入口提供商 - - 列出当前支持的入口提供商，仅对申请公网IP模式接入点有效。 - 注：公网IP供应商telecom=电信，unicom=联通，mobile=移动。 - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回接入商列表，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/aps/providers'.format(self.host) - return self.__get(url) - - def get_web_proxy(self, backend): - """获取一次性代理地址 - - 对内网地址获取一个一次性的外部可访问的代理地址 - - Args: - - backend: 后端地址，如："10.128.0.1:8080" - - Returns: - 返回一个tuple对象，其格式为(, ) - - result 成功返回代理地址信息，失败返回{"error": ""} - - ResponseInfo 请求的Response信息 - """ - url = '{0}/v3/webproxy'.format(self.host) - return self.__post(url, {'backend': backend}) - - def __post(self, url, data=None): - return http._post_with_qiniu_mac(url, data, self.auth) - - def __get(self, url, params=None): - return http._get_with_qiniu_mac(url, params, self.auth) - - def __delete(self, url): - return http._delete_with_qiniu_mac(url, None, self.auth) From 3f38843ddbccb21fa47eb570515c2b7c326dcf50 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Thu, 24 Aug 2017 14:11:11 +0800 Subject: [PATCH 245/478] set crc32 check as the default action in form upload --- qiniu/services/storage/uploader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 836a4434..b95f8a44 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -27,7 +27,7 @@ def put_data( 一个dict变量，类似 {"hash": "", "key": ""} 一个ResponseInfo对象 """ - crc = crc32(data) if check_crc else None + crc = crc32(data) return _form_put(up_token, key, data, params, mime_type, crc, progress_handler, fname) @@ -61,7 +61,7 @@ def put_file(up_token, key, file_path, params=None, upload_progress_recorder=upload_progress_recorder, modify_time=(int)(os.path.getmtime(file_path))) else: - crc = file_crc32(file_path) if check_crc else None + crc = file_crc32(file_path) ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler, file_name) # ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler) return ret, info From c4e0dace35f6922b45ca2a44232445240e4699bf Mon Sep 17 00:00:00 2001 From: jemygraw Date: Thu, 24 Aug 2017 14:52:31 +0800 Subject: [PATCH 246/478] clear useless test cases --- manual_test_kirk.py | 334 -------------------------------------------- qiniu/__init__.py | 2 - setup.py | 1 - test_qiniu.py | 28 ---- 4 files changed, 365 deletions(-) delete mode 100644 manual_test_kirk.py diff --git a/manual_test_kirk.py b/manual_test_kirk.py deleted file mode 100644 index e6791f30..00000000 --- a/manual_test_kirk.py +++ /dev/null @@ -1,334 +0,0 @@ -# -*- coding: utf-8 -*- -""" -======================= - 注意：必须手动运行 -======================= -""" -import os -import sys -import time -import logging -import pytest -from qiniu import auth -from qiniu.services import compute - - -access_key = os.getenv('QINIU_ACCESS_KEY') -secret_key = os.getenv('QINIU_SECRET_KEY') -qn_auth = auth.QiniuMacAuth(access_key, secret_key) -acc_client = compute.app.AccountClient(qn_auth) -qcos_client = None -user_name = '' -app_uri = '' -app_name = 'appjust4test' -app_region = 'nq' - - -def setup_module(module): - acc_client = compute.app.AccountClient(qn_auth) - user_info = acc_client.get_account_info()[0] - acc_client.create_app({'name': app_name, 'title': 'whatever', 'region': app_region}) - - module.user_name = user_info['name'] - module.app_uri = '{0}.{1}'.format(module.user_name, app_name) - module.qcos_client = acc_client.create_qcos_client(module.app_uri) - - -def teardown_module(module): - module.app_uri - acc_client.delete_app(module.app_uri) - - -class TestApp: - """应用测试用例""" - - def test_create_and_delete_app(self): - _name_create = 'appjust4testcreate' - _uri_create = '' - _args = {'name': _name_create, 'title': 'whatever', 'region': app_region} - - with Call(acc_client, 'create_app', _args) as r: - assert r[0] is not None - _uri_create = r[0]['uri'] - - with Call(acc_client, 'delete_app', _uri_create) as r: - assert r[0] == {} - - def test_get_app_keys(self): - with Call(acc_client, 'get_app_keys', app_uri) as r: - assert len(r[0]) > 0 - - def test_get_account_info(self): - with Call(acc_client, 'get_account_info') as r: - assert r[0] is not None - - -class TestStack: - """服务组测试用例""" - - _name = 'just4test' - _name_del = 'just4del' - _name_create = 'just4create' - - @classmethod - def setup_class(cls): - qcos_client.create_stack({'name': cls._name}) - qcos_client.create_stack({'name': cls._name_del}) - - @classmethod - def teardown_class(cls): - qcos_client.delete_stack(cls._name) - qcos_client.delete_stack(cls._name_create) - qcos_client.delete_stack(cls._name_del) - - def test_create_stack(self): - with Call(qcos_client, 'create_stack', {'name': self._name_create}) as r: - assert r[0] == {} - - def test_delete_stack(self): - with Call(qcos_client, 'delete_stack', self._name_del) as r: - assert r[0] == {} - - def test_list_stacks(self): - with Call(qcos_client, 'list_stacks') as r: - assert len(r) > 0 - assert self._name in [stack['name'] for stack in r[0]] - - def test_get_stack(self): - with Call(qcos_client, 'get_stack', self._name) as r: - assert r[0]['name'] == self._name - - def test_start_stack(self): - with Call(qcos_client, 'start_stack', self._name) as r: - assert r[0] == {} - - def test_stop_stack(self): - with Call(qcos_client, 'stop_stack', self._name) as r: - assert r[0] == {} - - -class TestService: - """服务测试用例""" - - _stack = 'just4test2' - _name = 'spaceship' - _name_del = 'spaceship4del' - _name_create = 'spaceship4create' - _image = 'library/nginx:stable' - _unit = '1U1G' - _spec = {'image': _image, 'unitType': _unit} - - @classmethod - def setup_class(cls): - qcos_client.delete_stack(cls._stack) - qcos_client.create_stack({'name': cls._stack}) - qcos_client.create_service(cls._stack, {'name': cls._name, 'spec': cls._spec}) - qcos_client.create_service(cls._stack, {'name': cls._name_del, 'spec': cls._spec}) - - _debug_info('waiting for services to setup ...') - time.sleep(10) - - @classmethod - def teardown_class(cls): - # 删除stack会清理所有相关服务 - qcos_client.delete_stack(cls._stack) - - def test_create_service(self): - service = {'name': self._name_create, 'spec': self._spec} - with Call(qcos_client, 'create_service', self._stack, service) as r: - assert r[0] == {} - - def test_delete_service(self): - with Call(qcos_client, 'delete_service', self._stack, self._name_del) as r: - assert r[0] == {} - - def test_list_services(self): - with Call(qcos_client, 'list_services', self._stack) as r: - assert len(r) > 0 - assert self._name in [service['name'] for service in r[0]] - - def test_get_service_inspect(self): - with Call(qcos_client, 'get_service_inspect', self._stack, self._name) as r: - assert r[0]['name'] == self._name - assert r[0]['spec']['unitType'] == self._unit - - def test_update_service(self): - data = {'spec': {'autoRestart': 'ON_FAILURE'}} - with Call(qcos_client, 'update_service', self._stack, self._name, data) as r: - assert r[0] == {} - - _debug_info('waiting for update services to ready ...') - time.sleep(10) - - def test_scale_service(self): - data = {'instanceNum': 2} - with Call(qcos_client, 'scale_service', self._stack, self._name, data) as r: - assert r[0] == {} - - _debug_info('waiting for scale services to ready ...') - time.sleep(10) - - -class TestContainer: - """容器测试用例""" - - _stack = 'just4test3' - _service = 'spaceship' - _spec = {'image': 'library/nginx:stable', 'unitType': '1U1G'} - # 为了方便测试，容器数量最少为2 - _instanceNum = 2 - - @classmethod - def setup_class(cls): - qcos_client.delete_stack(cls._stack) - qcos_client.create_stack({'name': cls._stack}) - qcos_client.create_service(cls._stack, {'name': cls._service, 'spec': cls._spec, 'instanceNum': cls._instanceNum}) - - _debug_info('waiting for containers to setup ...') - time.sleep(10) - - @classmethod - def teardown_class(cls): - qcos_client.delete_stack(cls._stack) - - def test_list_containers(self): - with Call(qcos_client, 'list_containers', self._stack, self._service) as r: - assert len(r[0]) > 0 - assert len(r[0]) <= self._instanceNum - - def test_get_container_inspect(self): - ips = qcos_client.list_containers(self._stack, self._service)[0] - # 查看第1个容器 - with Call(qcos_client, 'get_container_inspect', ips[0]) as r: - assert r[0]['ip'] == ips[0] - - def test_stop_and_strat_container(self): - ips = qcos_client.list_containers(self._stack, self._service)[0] - # 停止第2个容器 - with Call(qcos_client, 'stop_container', ips[1]) as r: - assert r[0] == {} - - _debug_info('waiting for containers to stop ...') - time.sleep(3) - - # 启动第2个容器 - with Call(qcos_client, 'start_container', ips[1]) as r: - assert r[0] == {} - - def test_restart_container(self): - ips = qcos_client.list_containers(self._stack, self._service)[0] - # 重启第1个容器 - with Call(qcos_client, 'restart_container', ips[0]) as r: - assert r[0] == {} - - -class TestAp: - """接入点测试用例""" - - _stack = 'just4test4' - _service = 'spaceship' - _spec = {'image': 'library/nginx:stable', 'unitType': '1U1G'} - # 为了方便测试，容器数量最少为2 - _instanceNum = 2 - _apid_domain = {} - _apid_ip = {} - _apid_ip_port = 8080 - _user_domain = 'just4test001.example.com' - - @classmethod - def setup_class(cls): - qcos_client.delete_stack(cls._stack) - qcos_client.create_stack({'name': cls._stack}) - qcos_client.create_service(cls._stack, {'name': cls._service, 'spec': cls._spec, 'instanceNum': cls._instanceNum}) - cls._ap_domain = qcos_client.create_ap({'type': 'DOMAIN', 'provider': 'Telecom', 'unitType': 'BW_10M', 'title': 'public1'})[0] - cls._ap_ip = qcos_client.create_ap({'type': 'PUBLIC_IP', 'provider': 'Telecom', 'unitType': 'BW_10M', 'title': 'public2'})[0] - qcos_client.set_ap_port(cls._ap_ip['apid'], cls._apid_ip_port, {'proto': 'http'}) - - @classmethod - def teardown_class(cls): - qcos_client.delete_stack(cls._stack) - qcos_client.delete_ap(cls._ap_domain['apid']) - qcos_client.delete_ap(cls._ap_ip['apid']) - - def test_list_aps(self): - with Call(qcos_client, 'list_aps') as r: - assert len(r[0]) > 0 - assert self._ap_domain['apid'] in [ap['apid'] for ap in r[0]] - assert self._ap_domain['apid'] in [ap['apid'] for ap in r[0]] - - def test_create_and_delete_ap(self): - apid = 0 - ap = {'type': 'DOMAIN', 'provider': 'Telecom', 'unitType': 'BW_10M', 'title': 'public1'} - - with Call(qcos_client, 'create_ap', ap) as r: - assert r[0] is not None and r[0]['apid'] > 0 - apid = r[0]['apid'] - - with Call(qcos_client, 'delete_ap', apid) as r: - assert r[0] == {} - - def test_search_ap(self): - with Call(qcos_client, 'search_ap', 'ip', self._ap_ip['ip']) as r: - assert str(r[0]['apid']) == self._ap_ip['apid'] - - def test_get_ap(self): - with Call(qcos_client, 'get_ap', self._ap_ip['apid']) as r: - assert str(r[0]['apid']) == self._ap_ip['apid'] - - def test_update_ap(self): - with Call(qcos_client, 'update_ap', self._ap_ip['apid'], {}) as r: - assert r[0] == {} - - def test_set_ap_port(self): - with Call(qcos_client, 'set_ap_port', self._ap_ip['apid'], 80, {'proto': 'http'}) as r: - assert r[0] == {} - - def test_publish_ap(self): - domain = {'userDomain': self._user_domain} - with Call(qcos_client, 'publish_ap', self._ap_domain['apid'], domain) as r: - assert r[0] == {} - - def test_unpublish_ap(self): - domain = {'userDomain': self._user_domain} - with Call(qcos_client, 'unpublish_ap', self._ap_domain['apid'], domain) as r: - assert r[0] == {} - - def test_get_ap_port_healthcheck(self): - with Call(qcos_client, 'get_ap_port_healthcheck', self._ap_ip['apid'], self._apid_ip_port) as r: - assert r[0] is not None - - def test_disable_ap_port(self): - with Call(qcos_client, 'disable_ap_port', self._ap_ip['apid'], self._apid_ip_port) as r: - assert r[0] == {} - - def test_enable_ap_port(self): - with Call(qcos_client, 'enable_ap_port', self._ap_ip['apid'], self._apid_ip_port) as r: - assert r[0] == {} - - def test_get_ap_providers(self): - with Call(qcos_client, 'get_ap_providers') as r: - assert len(r[0]) > 0 - - -class Call(object): - def __init__(self, obj, method, *args): - self.context = (obj, method, args) - self.result = None - - def __enter__(self): - self.result = getattr(self.context[0], self.context[1])(*self.context[2]) - assert self.result is not None - return self.result - - def __exit__(self, type, value, traceback): - _debug_info('\033[94m%s.%s\x1b[0m: %s', self.context[0].__class__, self.context[1], self.result) - - -def _debug_info(*args): - logger = logging.getLogger(__name__) - logger.debug(*args) - - -if __name__ == '__main__': - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - pytest.main() diff --git a/qiniu/__init__.py b/qiniu/__init__.py index facb4a2f..63fb338b 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -22,7 +22,5 @@ from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save -from .services.compute.app import AccountClient -from .services.compute.qcos_api import QcosClient from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry diff --git a/setup.py b/setup.py index 8c2bcca4..31413955 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,6 @@ 'qiniu.services', 'qiniu.services.storage', 'qiniu.services.processing', - 'qiniu.services.compute', 'qiniu.services.cdn', ] diff --git a/test_qiniu.py b/test_qiniu.py index 3d77aa55..1a42e6fb 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -335,34 +335,6 @@ def test_withoutRead_withoutSeek_retry(self): assert ret['key'] == key assert ret['hash'] == 'FlYu0iBR1WpvYi4whKXiBuQpyLLk' - def test_hasRead_hasSeek_retry(self): - key = 'withReadAndSeek_retry' - data = StringIO('hello retry again!') - set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) - token = self.q.upload_token(bucket_name) - ret, info = put_data(token, key, data) - print(info) - assert ret['key'] == key - assert ret['hash'] == 'FuEbdt6JP2BqwQJi7PezYhmuVYOo' - - def test_hasRead_withoutSeek_retry(self): - key = 'withReadAndWithoutSeek_retry' - data = ReadWithoutSeek('I only have read attribute!') - set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) - token = self.q.upload_token(bucket_name) - ret, info = put_data(token, key, data) - print(info) - assert ret is None - - def test_hasRead_WithoutSeek_retry2(self): - key = 'withReadAndWithoutSeek_retry2' - data = urlopen("http://www.qiniu.com") - set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) - token = self.q.upload_token(bucket_name, key) - ret, info = put_data(token, key, data) - print(info) - assert ret is not None - def test_putData_without_fname(self): if is_travis(): return From af8f9ae7bb813e9fe60c416f987099d325e46a2d Mon Sep 17 00:00:00 2001 From: jemygraw Date: Thu, 24 Aug 2017 17:20:43 +0800 Subject: [PATCH 247/478] [ISSUE-271] fix the manually set hosts not work bug --- qiniu/zone.py | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/qiniu/zone.py b/qiniu/zone.py index 376cd73b..21ebc047 100644 --- a/qiniu/zone.py +++ b/qiniu/zone.py @@ -7,7 +7,7 @@ from qiniu import compat from qiniu import utils -UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host +UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host class Zone(object): @@ -19,7 +19,9 @@ class Zone(object): up_host: 首选上传地址 up_host_backup: 备用上传地址 """ - def __init__(self, up_host=None, up_host_backup=None, io_host=None, host_cache={}, scheme="http", home_dir=tempfile.gettempdir()): + + def __init__(self, up_host=None, up_host_backup=None, io_host=None, host_cache={}, scheme="http", + home_dir=os.getcwd()): """初始化Zone类""" self.up_host, self.up_host_backup, self.io_host = up_host, up_host_backup, io_host self.host_cache = host_cache @@ -53,7 +55,7 @@ def get_up_host(self, ak, bucket): def unmarshal_up_token(self, up_token): token = up_token.split(':') - if(len(token) != 3): + if (len(token) != 3): raise ValueError('invalid up_token') ak = token[0] @@ -61,7 +63,7 @@ def unmarshal_up_token(self, up_token): scope = policy["scope"] bucket = scope - if(':' in scope): + if (':' in scope): bucket = scope.split(':')[0] return ak, bucket @@ -70,10 +72,30 @@ def get_bucket_hosts(self, ak, bucket): key = self.scheme + ":" + ak + ":" + bucket bucket_hosts = self.get_bucket_hosts_to_cache(key) - if(len(bucket_hosts) > 0): + if (len(bucket_hosts) > 0): return bucket_hosts - hosts = compat.json.loads(self.bucket_hosts(ak, bucket)) + hosts = {} + hosts.update({self.scheme: {}}) + + hosts[self.scheme].update({'up': []}) + hosts[self.scheme].update({'io': []}) + + if self.up_host != None: + hosts[self.scheme]['up'].append(self.scheme + "://" + self.up_host) + + if self.up_host_backup != None: + hosts[self.scheme]['up'].append(self.scheme + "://" + self.up_host_backup) + + if self.io_host != None: + hosts[self.scheme]['io'].append(self.scheme + "://" + self.io_host) + + if len(hosts[self.scheme]) == 0 or self.io_host == None: + # print(hosts) + hosts = compat.json.loads(self.bucket_hosts(ak, bucket)) + else: + # 1 year + hosts['ttl'] = int(time.time()) + 31536000 scheme_hosts = hosts[self.scheme] bucket_hosts = { @@ -88,13 +110,13 @@ def get_bucket_hosts(self, ak, bucket): def get_bucket_hosts_to_cache(self, key): ret = [] - if(len(self.host_cache) == 0): + if (len(self.host_cache) == 0): self.host_cache_from_file() - if(not (key in self.host_cache)): + if (not (key in self.host_cache)): return ret - if(self.host_cache[key]['deadline'] > time.time()): + if (self.host_cache[key]['deadline'] > time.time()): ret = self.host_cache[key] return ret @@ -126,6 +148,5 @@ def host_cache_to_file(self): def bucket_hosts(self, ak, bucket): url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) ret = requests.get(url) - # ret, info = http._get(url, None, None) data = compat.json.dumps(ret.json(), separators=(',', ':')) return data From ae3e2f3b523d32b86f8a2e507a03997e66451956 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Thu, 24 Aug 2017 17:21:15 +0800 Subject: [PATCH 248/478] remove useless imports --- qiniu/zone.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qiniu/zone.py b/qiniu/zone.py index 21ebc047..4716159e 100644 --- a/qiniu/zone.py +++ b/qiniu/zone.py @@ -3,7 +3,6 @@ import os import time import requests -import tempfile from qiniu import compat from qiniu import utils From afb962f3083e31f2437dbaa5b5a0b1b60dbf5b37 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Sat, 26 Aug 2017 10:53:56 +0800 Subject: [PATCH 249/478] publish v7.1.5 --- examples/upload_with_zone.py | 34 ++++++++++++++++++++++++++++++++++ qiniu/__init__.py | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 examples/upload_with_zone.py diff --git a/examples/upload_with_zone.py b/examples/upload_with_zone.py new file mode 100644 index 00000000..5151a970 --- /dev/null +++ b/examples/upload_with_zone.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth, put_file, etag, urlsafe_base64_encode +import qiniu.config +from qiniu import Zone, set_default + +# 需要填写你的 Access Key 和 Secret Key +access_key = '...' +secret_key = '...' + +# 构建鉴权对象 +q = Auth(access_key, secret_key) + +# 要上传的空间 +bucket_name = 'Bucket_Name' + +# 上传到七牛后保存的文件名 +key = 'my-python-logo.png'; + +# 生成上传 Token，可以指定过期时间等 +token = q.upload_token(bucket_name, key, 3600) + +# 要上传文件的本地路径 +localfile = 'stat.py' + +# 指定固定的zone +zone = Zone(up_host='uptest.qiniu.com', up_host_backup='uptest.qiniu.com', io_host='iovip.qbox.me', scheme='http') +set_default(default_zone=zone) + +ret, info = put_file(token, key, localfile) +print(info) +assert ret['key'] == key +assert ret['hash'] == etag(localfile) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 63fb338b..957eae4e 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.1.4' +__version__ = '7.1.5' from .auth import Auth, QiniuMacAuth From 39bc7cf2cb3890e9aba82bce97d1d84183011fae Mon Sep 17 00:00:00 2001 From: jemygraw Date: Sat, 26 Aug 2017 10:58:33 +0800 Subject: [PATCH 250/478] fix is none check logic --- examples/timestamp_url.py | 3 --- qiniu/zone.py | 8 ++++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/examples/timestamp_url.py b/examples/timestamp_url.py index 3bff2ae5..d1e548b3 100644 --- a/examples/timestamp_url.py +++ b/examples/timestamp_url.py @@ -25,6 +25,3 @@ timestamp_url = create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline) print(timestamp_url) - - - diff --git a/qiniu/zone.py b/qiniu/zone.py index 4716159e..68f7c53d 100644 --- a/qiniu/zone.py +++ b/qiniu/zone.py @@ -80,16 +80,16 @@ def get_bucket_hosts(self, ak, bucket): hosts[self.scheme].update({'up': []}) hosts[self.scheme].update({'io': []}) - if self.up_host != None: + if self.up_host is not None: hosts[self.scheme]['up'].append(self.scheme + "://" + self.up_host) - if self.up_host_backup != None: + if self.up_host_backup is not None: hosts[self.scheme]['up'].append(self.scheme + "://" + self.up_host_backup) - if self.io_host != None: + if self.io_host is not None: hosts[self.scheme]['io'].append(self.scheme + "://" + self.io_host) - if len(hosts[self.scheme]) == 0 or self.io_host == None: + if len(hosts[self.scheme]) == 0 or self.io_host is None: # print(hosts) hosts = compat.json.loads(self.bucket_hosts(ak, bucket)) else: From d5fa0416128430b990a1b87719277dae6c42c223 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Sat, 26 Aug 2017 11:04:20 +0800 Subject: [PATCH 251/478] add isPrefixalScope parameter --- qiniu/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/auth.py b/qiniu/auth.py index de3b015c..02525118 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -36,6 +36,7 @@ 'persistentPipeline', # 持久化处理独享队列 'deleteAfterDays', # 文件多少天后自动删除 'fileType', # 文件的存储类型，0为普通存储，1为低频存储 + 'isPrefixalScope' # 指定上传文件必须使用的前缀 ]) From e14f28f13a7499a9d957ab83f9db422bbaa15af5 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Thu, 31 Aug 2017 12:49:11 +0800 Subject: [PATCH 252/478] add prefetch, refresh --- examples/batch.py | 10 +-- examples/cdn_manager.py | 125 --------------------------------- examples/change_type.py | 4 +- examples/copy_to.py | 8 +-- examples/delete.py | 8 +-- examples/delete_afte_days.py | 8 +-- examples/download.py | 8 +-- examples/fetch.py | 4 +- examples/fops.py | 16 ++--- examples/move_to.py | 8 +-- examples/pfop_vframe.py | 14 ++-- examples/pfop_watermark.py | 21 +++--- examples/prefetch_to_bucket.py | 28 ++++++++ examples/prefetch_to_cdn.py | 29 ++++++++ examples/refresh_dirs.py | 21 ++++++ examples/refresh_urls.py | 20 ++++++ examples/stat.py | 8 +-- examples/timestamp_url.py | 3 - examples/upload.py | 3 +- examples/upload_callback.py | 7 +- examples/upload_pfops.py | 9 ++- 21 files changed, 164 insertions(+), 198 deletions(-) delete mode 100755 examples/cdn_manager.py create mode 100644 examples/prefetch_to_bucket.py create mode 100644 examples/prefetch_to_cdn.py create mode 100644 examples/refresh_dirs.py create mode 100644 examples/refresh_urls.py diff --git a/examples/batch.py b/examples/batch.py index 4f653661..dd05bde9 100755 --- a/examples/batch.py +++ b/examples/batch.py @@ -2,9 +2,9 @@ # flake8: noqa from qiniu import Auth -from qiniu import BucketManager,build_batch_rename -# from qiniu import build_batch_copy, -# from qiniu import build_batch_move,build_batch_rename +from qiniu import BucketManager +from qiniu import build_batch_copy +from qiniu import build_batch_move,build_batch_rename access_key = '...' secret_key = '...' @@ -14,11 +14,11 @@ # 初始化BucketManager bucket = BucketManager(q) -keys = {'123.jpg':'123.jpg'} +keys = {'123.jpg': '123.jpg'} # ops = build_batch_copy( 'teest', keys, 'teest',force='true') # ops = build_batch_move('teest', keys, 'teest', force='true') -ops = build_batch_rename('teest', keys,force='true') +ops = build_batch_rename('teest', keys, force='true') ret, info = bucket.batch(ops) print(ret) diff --git a/examples/cdn_manager.py b/examples/cdn_manager.py deleted file mode 100755 index 9a217a6d..00000000 --- a/examples/cdn_manager.py +++ /dev/null @@ -1,125 +0,0 @@ -# -*- coding: utf-8 -*- -import qiniu -from qiniu import CdnManager -from qiniu import create_timestamp_anti_leech_url -import time - - -# 演示函数调用结果 -def print_result(result): - if result[0] is not None: - print(type(result[0])) - print(result[0]) - - print(type(result[1])) - print(result[1]) - - -# 账户ak，sk -access_key = '...' -secret_key = '...' - -auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) -cdn_manager = CdnManager(auth) - -urls = [ - 'http://if-pbl.qiniudn.com/qiniu.jpg', - 'http://if-pbl.qiniudn.com/qiniu2.jpg' -] - -# 注意链接最后的斜杠表示目录 -dirs = [ - 'http://if-pbl.qiniudn.com/test1/', - 'http://if-pbl.qiniudn.com/test2/' -] -"""刷新文件，目录""" - -# 刷新链接 -print('刷新文件') -refresh_url_result = cdn_manager.refresh_urls(urls) -print_result(refresh_url_result) - -# 刷新目录需要联系七牛技术支持开通权限 -print('刷新目录') -refresh_dir_result = cdn_manager.refresh_dirs(dirs) -print_result(refresh_dir_result) - -# 同时刷新文件和目录 -print('刷新文件和目录') -refresh_all_result = cdn_manager.refresh_urls_and_dirs(urls, dirs) -print_result(refresh_all_result) - -"""预取文件""" - -# 预取文件链接 -print('预取文件链接') -prefetch_url_result = cdn_manager.prefetch_urls(urls) -print_result(prefetch_url_result) - -"""获取带宽和流量数据""" - -domains = ['if-pbl.qiniudn.com', 'qdisk.qiniudn.com'] - -start_date = '2017-01-01' -end_date = '2017-01-02' - -# 5min or hour or day -granularity = 'day' - -# 获取带宽数据 -print('获取带宽数据') -bandwidth_data = cdn_manager.get_bandwidth_data(domains, start_date, end_date, granularity) -print_result(bandwidth_data) - -# 获取流量数据 -print('获取流量数据') -flux_data = cdn_manager.get_flux_data(domains, start_date, end_date, granularity) -print_result(flux_data) - -"""获取日志文件下载地址列表""" -# 获取日志列表 -print('获取日志列表') -log_date = '2017-01-01' -log_data = cdn_manager.get_log_list_data(domains, log_date) -print_result(log_data) - -"""构建时间戳防盗链""" - -# 构建时间戳防盗链 -print('构建时间戳防盗链') - -# 时间戳防盗链密钥，后台获取 -encrypt_key = 'xxx' - -# 原始文件名，必须是utf8编码 -test_file_name1 = '基本概括.mp4' -test_file_name2 = '2017/01/07/test.png' - -# 查询参数列表 -query_string_dict = { - 'name': '七牛', - 'year': 2017, - '年龄': 28, -} - -# 带访问协议的域名 -host = 'http://video.example.com' - -# unix时间戳 -deadline = int(time.time()) + 3600 - -# 带查询参数，中文文件名 -signed_url1 = create_timestamp_anti_leech_url(host, test_file_name1, query_string_dict, encrypt_key, deadline) -print(signed_url1) - -# 带查询参数，英文文件名 -signed_url2 = create_timestamp_anti_leech_url(host, test_file_name2, query_string_dict, encrypt_key, deadline) -print(signed_url2) - -# 不带查询参数，中文文件名 -signed_url3 = create_timestamp_anti_leech_url(host, test_file_name1, None, encrypt_key, deadline) -print(signed_url3) - -# 不带查询参数，英文文件名 -signed_url4 = create_timestamp_anti_leech_url(host, test_file_name2, None, encrypt_key, deadline) -print(signed_url4) diff --git a/examples/change_type.py b/examples/change_type.py index ff015562..549f87ea 100644 --- a/examples/change_type.py +++ b/examples/change_type.py @@ -15,7 +15,7 @@ key = '...' -ret, info = bucket.change_type(bucket_name, key ,1)#1表示低频存储，0是标准存储 +# 1表示低频存储，0是标准存储 +ret, info = bucket.change_type(bucket_name, key, 1) print(info) -# assert info.status_code == 200 diff --git a/examples/copy_to.py b/examples/copy_to.py index bab02f7a..a77e9397 100755 --- a/examples/copy_to.py +++ b/examples/copy_to.py @@ -7,17 +7,17 @@ access_key = '...' secret_key = '...' -#初始化Auth状态 +# 初始化Auth状态 q = Auth(access_key, secret_key) -#初始化BucketManager +# 初始化BucketManager bucket = BucketManager(q) -#你要测试的空间， 并且这个key在你空间中存在 +# 你要测试的空间， 并且这个key在你空间中存在 bucket_name = 'Bucket_Name' key = 'python-logo.png' -#将文件从文件key 复制到文件key2。 可以在不同bucket复制 +# 将文件从文件key 复制到文件key2。 可以在不同bucket复制 key2 = 'python-logo2.png' ret, info = bucket.copy(bucket_name, key, bucket_name, key2) diff --git a/examples/delete.py b/examples/delete.py index ab5570a2..5466ba3a 100755 --- a/examples/delete.py +++ b/examples/delete.py @@ -7,17 +7,17 @@ access_key = '...' secret_key = '...' -#初始化Auth状态 +# 初始化Auth状态 q = Auth(access_key, secret_key) -#初始化BucketManager +# 初始化BucketManager bucket = BucketManager(q) -#你要测试的空间， 并且这个key在你空间中存在 +# 你要测试的空间， 并且这个key在你空间中存在 bucket_name = 'Bucket_Name' key = 'python-logo.png' -#删除bucket_name 中的文件 key +# 删除bucket_name 中的文件 key ret, info = bucket.delete(bucket_name, key) print(info) assert ret == {} diff --git a/examples/delete_afte_days.py b/examples/delete_afte_days.py index cd54b49e..bfffb378 100755 --- a/examples/delete_afte_days.py +++ b/examples/delete_afte_days.py @@ -7,17 +7,17 @@ access_key = '...' secret_key = '...' -#初始化Auth状态 +# 初始化Auth状态 q = Auth(access_key, secret_key) -#初始化BucketManager +# 初始化BucketManager bucket = BucketManager(q) -#你要测试的空间， 并且这个key在你空间中存在 +# 你要测试的空间， 并且这个key在你空间中存在 bucket_name = 'Bucket_Name' key = 'python-test.png' -#您要更新的生命周期 +# 您要更新的生命周期，单位为天 days = '5' ret, info = bucket.delete_after_days(bucket_name, key, days) diff --git a/examples/download.py b/examples/download.py index 3ca0fa08..f8437959 100755 --- a/examples/download.py +++ b/examples/download.py @@ -11,13 +11,13 @@ bucket_domain = "..." key = "..." -#有两种方式构造base_url的形式 +# 有两种方式构造base_url的形式 base_url = 'http://%s/%s' % (bucket_domain, key) -#或者直接输入url的方式下载 -base_url = 'http://domain/key' +# 或者直接输入url的方式下载 +# base_url = 'http://domain/key' -#可以设置token过期时间 +# 可以设置token过期时间 private_url = q.private_download_url(base_url, expires=3600) print(private_url) diff --git a/examples/fetch.py b/examples/fetch.py index bc19d7fb..c66c406e 100755 --- a/examples/fetch.py +++ b/examples/fetch.py @@ -13,10 +13,10 @@ bucket = BucketManager(q) -url = 'http://7xr875.com1.z0.glb.clouddn.com/test.jpg' +url = 'http://aaa.example.com/test.jpg' key = 'test.jpg' -ret, info = bucket.fetch( url, bucket_name, key) +ret, info = bucket.fetch(url, bucket_name, key) print(info) assert ret['key'] == key diff --git a/examples/fops.py b/examples/fops.py index 74cfe5df..90983a23 100755 --- a/examples/fops.py +++ b/examples/fops.py @@ -1,28 +1,28 @@ # -*- coding: utf-8 -*- # flake8: noqa -from qiniu import Auth, PersistentFop, build_op, op_save, urlsafe_base64_encode +from qiniu import Auth, PersistentFop, urlsafe_base64_encode -#对已经上传到七牛的视频发起异步转码操作 +# 对已经上传到七牛的视频发起异步转码操作 access_key = '...' secret_key = '...' q = Auth(access_key, secret_key) -#要转码的文件所在的空间和文件名。 +# 要转码的文件所在的空间和文件名。 bucket_name = 'Bucket_Name' key = '1.mp4' -#转码是使用的队列名称。 +# 转码是使用的队列名称。 pipeline = 'your_pipeline' -#要进行转码的转码操作。 +# 要进行转码的转码操作，下面是一个例子。 fops = 'avthumb/mp4/s/640x360/vb/1.25m' -#可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 +# 可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') fops = fops+'|saveas/'+saveas_key - -pfop = PersistentFop(q, bucket_name, pipeline) ops = [] +pfop = PersistentFop(q, bucket_name, pipeline) + ops.append(fops) ret, info = pfop.execute(key, ops, 1) print(info) diff --git a/examples/move_to.py b/examples/move_to.py index 71c9c2c9..edf9eeeb 100755 --- a/examples/move_to.py +++ b/examples/move_to.py @@ -6,17 +6,17 @@ access_key = '...' secret_key = '...' -#初始化Auth状态 +# 初始化Auth状态 q = Auth(access_key, secret_key) -#初始化BucketManager +# 初始化BucketManager bucket = BucketManager(q) -#你要测试的空间， 并且这个key在你空间中存在 +# 你要测试的空间， 并且这个key在你空间中存在 bucket_name = 'Bucket_Name' key = 'python-logo.png' -#将文件从文件key 移动到文件key2，可以实现文件的重命名 可以在不同bucket移动 +# 将文件从文件key 移动到文件key2，可以实现文件的重命名 可以在不同bucket移动 key2 = 'python-logo2.png' ret, info = bucket.move(bucket_name, key, bucket_name, key2) diff --git a/examples/pfop_vframe.py b/examples/pfop_vframe.py index 6c1c2ba5..e18564ec 100755 --- a/examples/pfop_vframe.py +++ b/examples/pfop_vframe.py @@ -1,23 +1,23 @@ # -*- coding: utf-8 -*- # flake8: noqa -from qiniu import Auth, PersistentFop, build_op, op_save, urlsafe_base64_encode +from qiniu import Auth, PersistentFop, urlsafe_base64_encode -#对已经上传到七牛的视频发起异步转码操作 +# 对已经上传到七牛的视频发起异步转码操作 access_key = 'Access_Key' secret_key = 'Secret_Key' q = Auth(access_key, secret_key) -#要转码的文件所在的空间和文件名。 +# 要转码的文件所在的空间和文件名。 bucket = 'Bucket_Name' key = '1.mp4' -#转码是使用的队列名称。 -pipeline = 'mpsdemo' +# 转码是使用的队列名称。 +pipeline = 'pipeline_name' -#要进行视频截图操作。 +# 要进行视频截图操作。 fops = 'vframe/jpg/offset/1/w/480/h/360/rotate/90' -#可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 +# 可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') fops = fops+'|saveas/'+saveas_key diff --git a/examples/pfop_watermark.py b/examples/pfop_watermark.py index dd1a97d4..da5cd43a 100755 --- a/examples/pfop_watermark.py +++ b/examples/pfop_watermark.py @@ -1,31 +1,30 @@ # -*- coding: utf-8 -*- # flake8: noqa -from qiniu import Auth, PersistentFop, build_op, op_save, urlsafe_base64_encode +from qiniu import Auth, PersistentFop, urlsafe_base64_encode -#对已经上传到七牛的视频发起异步转码操作 +# 对已经上传到七牛的视频发起异步转码操作 access_key = 'Access_Key' secret_key = 'Secret_Key' q = Auth(access_key, secret_key) -#要转码的文件所在的空间和文件名。 +# 要转码的文件所在的空间和文件名。 bucket = 'Bucket_Name' key = '1.mp4' -#转码是使用的队列名称。 -pipeline = 'mpsdemo' +# 转码是使用的队列名称。 +pipeline = 'pipeline_name' -#需要添加水印的图片UrlSafeBase64,可以参考http://developer.qiniu.com/code/v6/api/dora-api/av/video-watermark.html -base64URL = urlsafe_base64_encode('http://developer.qiniu.com/resource/logo-2.jpg'); +# 需要添加水印的图片UrlSafeBase64,可以参考http://developer.qiniu.com/code/v6/api/dora-api/av/video-watermark.html +base64URL = urlsafe_base64_encode('http://developer.qiniu.com/resource/logo-2.jpg') -#视频水印参数 +# 视频水印参数 fops = 'avthumb/mp4/'+base64URL -#可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 +# 可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') fops = fops+'|saveas/'+saveas_key - -pfop = PersistentFop(q, bucket, pipeline) ops = [] +pfop = PersistentFop(q, bucket, pipeline) ops.append(fops) ret, info = pfop.execute(key, ops, 1) print(info) diff --git a/examples/prefetch_to_bucket.py b/examples/prefetch_to_bucket.py new file mode 100644 index 00000000..2d447ff4 --- /dev/null +++ b/examples/prefetch_to_bucket.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +""" +拉取镜像源资源到空间 + +https://developer.qiniu.com/kodo/api/1293/prefetch +""" + +from qiniu import Auth +from qiniu import BucketManager + +access_key = '...' +secret_key = '...' + + +bucket_name = 'Bucket_Name' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +# 要拉取的文件名 +key = 'test.jpg' + +ret, info = bucket.prefetch(bucket_name, key) +print(info) +assert ret['key'] == key diff --git a/examples/prefetch_to_cdn.py b/examples/prefetch_to_cdn.py new file mode 100644 index 00000000..8bdaa0a7 --- /dev/null +++ b/examples/prefetch_to_cdn.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +""" +预取资源到cdn节点 + +https://developer.qiniu.com/fusion/api/1227/file-prefetching +""" + + +import qiniu +from qiniu import CdnManager + + +# 账户ak，sk +access_key = '...' +secret_key = '...' + +auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) +cdn_manager = CdnManager(auth) + +# 需要刷新的文件链接 +urls = [ + 'http://aaa.example.com/doc/img/', + 'http://bbb.example.com/doc/video/' +] + + +# 刷新链接 +refresh_dir_result = cdn_manager.prefetch_urls(urls) diff --git a/examples/refresh_dirs.py b/examples/refresh_dirs.py new file mode 100644 index 00000000..1db73015 --- /dev/null +++ b/examples/refresh_dirs.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +import qiniu +from qiniu import CdnManager + + +# 账户ak，sk +access_key = '...' +secret_key = '...' + +auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) +cdn_manager = CdnManager(auth) + +# 需要刷新的目录链接 +dirs = [ + 'http://aaa.example.com/doc/img/', + 'http://bbb.example.com/doc/video/' +] + + +# 刷新链接 +refresh_dir_result = cdn_manager.refresh_dirs(dirs) diff --git a/examples/refresh_urls.py b/examples/refresh_urls.py new file mode 100644 index 00000000..43b62baa --- /dev/null +++ b/examples/refresh_urls.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +import qiniu +from qiniu import CdnManager + +# 账户ak，sk +access_key = '...' +secret_key = '...' + +auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) +cdn_manager = CdnManager(auth) + +# 需要刷新的文件链接 +urls = [ + 'http://aaa.example.com/a.gif', + 'http://bbb.example.com/b.jpg' +] + +# 刷新链接 +refresh_url_result = cdn_manager.refresh_urls(urls) +print(refresh_url_result) diff --git a/examples/stat.py b/examples/stat.py index ca07e125..6ee62209 100755 --- a/examples/stat.py +++ b/examples/stat.py @@ -6,17 +6,17 @@ access_key = '...' secret_key = '...' -#初始化Auth状态 +# 初始化Auth状态 q = Auth(access_key, secret_key) -#初始化BucketManager +# 初始化BucketManager bucket = BucketManager(q) -#你要测试的空间， 并且这个key在你空间中存在 +# 你要测试的空间， 并且这个key在你空间中存在 bucket_name = 'Bucket_Name' key = 'python-logo.png' -#获取文件的状态信息 +# 获取文件的状态信息 ret, info = bucket.stat(bucket_name, key) print(info) assert 'hash' in ret diff --git a/examples/timestamp_url.py b/examples/timestamp_url.py index 3bff2ae5..d1e548b3 100644 --- a/examples/timestamp_url.py +++ b/examples/timestamp_url.py @@ -25,6 +25,3 @@ timestamp_url = create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline) print(timestamp_url) - - - diff --git a/examples/upload.py b/examples/upload.py index a8920de9..3d6220ad 100755 --- a/examples/upload.py +++ b/examples/upload.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- # flake8: noqa -from qiniu import Auth, put_file, etag, urlsafe_base64_encode -import qiniu.config +from qiniu import Auth, put_file, etag #需要填写你的 Access Key 和 Secret Key access_key = '...' diff --git a/examples/upload_callback.py b/examples/upload_callback.py index 86d146d0..1cc8cd38 100755 --- a/examples/upload_callback.py +++ b/examples/upload_callback.py @@ -2,7 +2,6 @@ # flake8: noqa from qiniu import Auth, put_file, etag -import qiniu.config access_key = '...' secret_key = '...' @@ -14,9 +13,9 @@ key = 'my-python-logo.png' #上传文件到七牛后， 七牛将文件名和文件大小回调给业务服务器。 -policy={ - 'callbackUrl':'http://your.domain.com/callback.php', - 'callbackBody':'filename=$(fname)&filesize=$(fsize)' +policy = { + 'callbackUrl': 'http://your.domain.com/callback.php', + 'callbackBody': 'filename=$(fname)&filesize=$(fsize)' } token = q.upload_token(bucket_name, key, 3600, policy) diff --git a/examples/upload_pfops.py b/examples/upload_pfops.py index b0e95493..06670a60 100755 --- a/examples/upload_pfops.py +++ b/examples/upload_pfops.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # flake8: noqa from qiniu import Auth, put_file, etag, urlsafe_base64_encode -import qiniu.config access_key = '...' secret_key = '...' @@ -20,14 +19,14 @@ fops = 'avthumb/mp4/vcodec/libx264' # 通过添加'|saveas'参数，指定处理后的文件保存的bucket和key，不指定默认保存在当前空间，bucket_saved为目标bucket，bucket_saved为目标key -saveas_key = urlsafe_base64_encode('bucket_saved:bucket_saved')# +saveas_key = urlsafe_base64_encode('bucket_saved:bucket_saved') fops = fops+'|saveas/'+saveas_key # 在上传策略中指定fobs和pipeline -policy={ - 'persistentOps':fops, - 'persistentPipeline':pipeline +policy = { + 'persistentOps': fops, + 'persistentPipeline': pipeline } token = q.upload_token(bucket_name, key, 3600, policy) From b3252b67776cebea95a70b62a3a2d3753d31fb88 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Tue, 26 Sep 2017 17:06:04 +0800 Subject: [PATCH 253/478] [PDEX-1086] support the keep last modified not changed for files uploaded --- qiniu/services/storage/uploader.py | 31 +++++++++++++++++++++--------- qiniu/utils.py | 14 +++++++++++++- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index b95f8a44..1340d06f 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -4,7 +4,7 @@ import time from qiniu import config -from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter +from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter, rfc_from_timestamp from qiniu import http from .upload_progress_recorder import UploadProgressRecorder @@ -33,7 +33,7 @@ def put_data( def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, - progress_handler=None, upload_progress_recorder=None): + progress_handler=None, upload_progress_recorder=None, keep_last_modified=False): """上传文件到七牛 Args: @@ -55,19 +55,22 @@ def put_file(up_token, key, file_path, params=None, # fname = os.path.basename(file_path) with open(file_path, 'rb') as input_stream: file_name = os.path.basename(file_path) + modify_time = int(os.path.getmtime(file_path)) if size > config._BLOCK_SIZE * 2: ret, info = put_stream(up_token, key, input_stream, file_name, size, params, mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, - modify_time=(int)(os.path.getmtime(file_path))) + modify_time=modify_time, keep_last_modified=keep_last_modified) else: crc = file_crc32(file_path) - ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler, file_name) - # ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, progress_handler) + ret, info = _form_put(up_token, key, input_stream, params, mime_type, + crc, progress_handler, file_name, + modify_time=modify_time, keep_last_modified=keep_last_modified) return ret, info -def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None, file_name=None): +def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None, file_name=None, modify_time=None, + keep_last_modified=False): fields = {} if params: for k, v in params.items(): @@ -85,6 +88,10 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None if not fname or not fname.strip(): fname = 'file_name' + # last modify time + if modify_time and keep_last_modified: + fields['x-qn-meta-!Last-Modified'] = rfc_from_timestamp(modify_time) + r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) if r is None and info.need_retry(): if info.connect_failed: @@ -102,9 +109,9 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None def put_stream(up_token, key, input_stream, file_name, data_size, params=None, mime_type=None, progress_handler=None, - upload_progress_recorder=None, modify_time=None): + upload_progress_recorder=None, modify_time=None, keep_last_modified=False): task = _Resume(up_token, key, input_stream, data_size, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, file_name) + progress_handler, upload_progress_recorder, modify_time, file_name, keep_last_modified) return task.upload() @@ -128,7 +135,7 @@ class _Resume(object): """ def __init__(self, up_token, key, input_stream, data_size, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, file_name): + progress_handler, upload_progress_recorder, modify_time, file_name, keep_last_modified): """初始化断点续上传""" self.up_token = up_token self.key = key @@ -140,6 +147,7 @@ def __init__(self, up_token, key, input_stream, data_size, params, mime_type, self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() self.modify_time = modify_time or time.time() self.file_name = file_name + self.keep_last_modified = keep_last_modified # print(self.modify_time) # print(modify_time) @@ -216,6 +224,11 @@ def file_url(self, host): for k, v in self.params.items(): url.append('{0}/{1}'.format(k, urlsafe_base64_encode(v))) pass + + if self.modify_time and self.keep_last_modified: + url.append( + "x-qn-meta-!Last-Modified/{0}".format(urlsafe_base64_encode(rfc_from_timestamp(self.modify_time)))) + url = '/'.join(url) # print url return url diff --git a/qiniu/utils.py b/qiniu/utils.py index 29c61af6..d0670612 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -2,11 +2,12 @@ from hashlib import sha1 from base64 import urlsafe_b64encode, urlsafe_b64decode - +from datetime import datetime from .compat import b, s try: import zlib + binascii = zlib except ImportError: zlib = None @@ -158,3 +159,14 @@ def entry(bucket, key): return urlsafe_base64_encode('{0}'.format(bucket)) else: return urlsafe_base64_encode('{0}:{1}'.format(bucket, key)) + + +def rfc_from_timestamp(timestamp): + """将时间戳转换为HTTP RFC格式 + + Args: + timestamp: 整型Unix时间戳（单位秒） + """ + last_modified_date = datetime.fromtimestamp(timestamp) + last_modified_str = last_modified_date.strftime('%a, %d %b %Y %H:%M:%S GMT') + return last_modified_str From f7ec17f1b00670e25abc79055bb0f5805428d2d1 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Tue, 26 Sep 2017 17:09:18 +0800 Subject: [PATCH 254/478] publish v7.1.6 --- CHANGELOG.md | 10 ++++++++++ qiniu/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ab1c3bb..df52d50e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 7.1.6 (2017-09-26) + +* 给 `put_file` 功能增加保持本地文件Last Modified功能，以支持切换源站的客户CDN不回源 + +## 7.1.5 (2017-08-26) + +* 设置表单上传默认校验crc32 +* 增加PutPolicy新参数isPrefixalScope +* 修复手动指定的zone无效的问题 + ## 7.1.4 (2017-06-05) ### 修正 * cdn功能中获取域名日志列表的参数错误 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 957eae4e..0615ee51 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.1.5' +__version__ = '7.1.6' from .auth import Auth, QiniuMacAuth From f1e67d442d2c51d0318684eaa586889b0f489d27 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Wed, 27 Sep 2017 20:02:10 +0800 Subject: [PATCH 255/478] fix the http rft time format from timestamp using utc timezone --- qiniu/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/utils.py b/qiniu/utils.py index d0670612..716f4c6d 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -167,6 +167,6 @@ def rfc_from_timestamp(timestamp): Args: timestamp: 整型Unix时间戳（单位秒） """ - last_modified_date = datetime.fromtimestamp(timestamp) + last_modified_date = datetime.utcfromtimestamp(timestamp) last_modified_str = last_modified_date.strftime('%a, %d %b %Y %H:%M:%S GMT') return last_modified_str From dc853d6e5fa05ff5104bb4c06dbc8627e7df8b08 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Wed, 27 Sep 2017 20:02:35 +0800 Subject: [PATCH 256/478] publish 7.1.7 --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 0615ee51..5321601c 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.1.6' +__version__ = '7.1.7' from .auth import Auth, QiniuMacAuth From 363c030b130fbc05c3bc10ecc723249a48cd5ed2 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Wed, 27 Sep 2017 20:04:08 +0800 Subject: [PATCH 257/478] update readme --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index df52d50e..7a01d7b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 7.1.7 (2017-09-27) + +* 修复从时间戳获取rfc http格式的时间字符串问题 + ## 7.1.6 (2017-09-26) * 给 `put_file` 功能增加保持本地文件Last Modified功能，以支持切换源站的客户CDN不回源 From c9fa719d164789aded7c446eb2bdf5c3f70cfbad Mon Sep 17 00:00:00 2001 From: jemygraw Date: Wed, 18 Oct 2017 11:43:29 +0800 Subject: [PATCH 258/478] remove ak sk from travis config --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7534aa72..45775088 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,8 +12,6 @@ install: - pip install requests - pip install scrutinizer-ocular before_script: -- export QINIU_ACCESS_KEY="QWYn5TFQsLLU1pL5MFEmX3s5DmHdUThav9WyOWOm" -- export QINIU_SECRET_KEY="Bxckh6FA-Fbs9Yt3i3cbKVK22UPBmAOHJcL95pGz" - export QINIU_TEST_BUCKET="pythonsdk" - export QINIU_TEST_DOMAIN="pythonsdk.qiniudn.com" - export QINIU_TEST_ENV="travis" From cf61bedbdf299923d930af1797bbea8cc8aa6b99 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Wed, 18 Oct 2017 17:28:48 +0800 Subject: [PATCH 259/478] add kirk api code files --- examples/kirk/README.md | 0 examples/kirk/compute/__init__.py | 0 examples/kirk/compute/app.py | 0 examples/kirk/compute/config.py | 0 examples/kirk/compute/qcos_api.py | 0 examples/kirk/list_apps.py | 0 examples/kirk/list_services.py | 0 examples/kirk/list_stacks.py | 0 8 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 examples/kirk/README.md create mode 100644 examples/kirk/compute/__init__.py create mode 100644 examples/kirk/compute/app.py create mode 100644 examples/kirk/compute/config.py create mode 100644 examples/kirk/compute/qcos_api.py create mode 100644 examples/kirk/list_apps.py create mode 100644 examples/kirk/list_services.py create mode 100644 examples/kirk/list_stacks.py diff --git a/examples/kirk/README.md b/examples/kirk/README.md new file mode 100644 index 00000000..e69de29b diff --git a/examples/kirk/compute/__init__.py b/examples/kirk/compute/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/kirk/compute/app.py b/examples/kirk/compute/app.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/kirk/compute/config.py b/examples/kirk/compute/config.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/kirk/compute/qcos_api.py b/examples/kirk/compute/qcos_api.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/kirk/list_apps.py b/examples/kirk/list_apps.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/kirk/list_services.py b/examples/kirk/list_services.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/kirk/list_stacks.py b/examples/kirk/list_stacks.py new file mode 100644 index 00000000..e69de29b From a26adb857c56954be711dd3af5c27fccb87ee683 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Wed, 18 Oct 2017 17:32:43 +0800 Subject: [PATCH 260/478] add qiniu kirk api code --- {examples/kirk => qiniu/services}/compute/__init__.py | 0 {examples/kirk => qiniu/services}/compute/app.py | 0 {examples/kirk => qiniu/services}/compute/config.py | 0 {examples/kirk => qiniu/services}/compute/qcos_api.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {examples/kirk => qiniu/services}/compute/__init__.py (100%) rename {examples/kirk => qiniu/services}/compute/app.py (100%) rename {examples/kirk => qiniu/services}/compute/config.py (100%) rename {examples/kirk => qiniu/services}/compute/qcos_api.py (100%) diff --git a/examples/kirk/compute/__init__.py b/qiniu/services/compute/__init__.py similarity index 100% rename from examples/kirk/compute/__init__.py rename to qiniu/services/compute/__init__.py diff --git a/examples/kirk/compute/app.py b/qiniu/services/compute/app.py similarity index 100% rename from examples/kirk/compute/app.py rename to qiniu/services/compute/app.py diff --git a/examples/kirk/compute/config.py b/qiniu/services/compute/config.py similarity index 100% rename from examples/kirk/compute/config.py rename to qiniu/services/compute/config.py diff --git a/examples/kirk/compute/qcos_api.py b/qiniu/services/compute/qcos_api.py similarity index 100% rename from examples/kirk/compute/qcos_api.py rename to qiniu/services/compute/qcos_api.py From d321ca114d860661066e2ce11a597f3ad271019d Mon Sep 17 00:00:00 2001 From: jemygraw Date: Wed, 18 Oct 2017 17:36:19 +0800 Subject: [PATCH 261/478] reset the kirk api to the previous version --- examples/kirk/README.md | 5 + examples/kirk/list_apps.py | 18 + examples/kirk/list_services.py | 26 ++ examples/kirk/list_stacks.py | 23 + qiniu/services/compute/app.py | 224 ++++++++++ qiniu/services/compute/config.py | 21 + qiniu/services/compute/qcos_api.py | 694 +++++++++++++++++++++++++++++ 7 files changed, 1011 insertions(+) diff --git a/examples/kirk/README.md b/examples/kirk/README.md index e69de29b..23e5a852 100644 --- a/examples/kirk/README.md +++ b/examples/kirk/README.md @@ -0,0 +1,5 @@ +# Examples + +``` +$ python list_apps.py +``` diff --git a/examples/kirk/list_apps.py b/examples/kirk/list_apps.py index e69de29b..2062b8b1 100644 --- a/examples/kirk/list_apps.py +++ b/examples/kirk/list_apps.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +import sys +from qiniu import QiniuMacAuth +from qiniu import AccountClient + +access_key = sys.argv[1] +secret_key = sys.argv[2] + +acc_client = AccountClient(QiniuMacAuth(access_key, secret_key)) + +ret, info = acc_client.list_apps() + +print(ret) +print(info) + +assert len(ret) is not None diff --git a/examples/kirk/list_services.py b/examples/kirk/list_services.py index e69de29b..9ec683fb 100644 --- a/examples/kirk/list_services.py +++ b/examples/kirk/list_services.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +import sys +from qiniu import QiniuMacAuth +from qiniu import AccountClient + +access_key = sys.argv[1] +secret_key = sys.argv[2] + +acc_client = AccountClient(QiniuMacAuth(access_key, secret_key)) +apps, info = acc_client.list_apps() + +for app in apps: + if app.get('runMode') == 'Private': + uri = app.get('uri') + qcos = acc_client.get_qcos_client(uri) + if qcos != None: + stacks, info = qcos.list_stacks() + for stack in stacks: + stack_name = stack.get('name') + services, info = qcos.list_services(stack_name) + print("list_services of '%s : %s':"%(uri, stack_name)) + print(services) + print(info) + assert len(services) is not None diff --git a/examples/kirk/list_stacks.py b/examples/kirk/list_stacks.py index e69de29b..67815603 100644 --- a/examples/kirk/list_stacks.py +++ b/examples/kirk/list_stacks.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +import sys +from qiniu import QiniuMacAuth +from qiniu import AccountClient + +access_key = sys.argv[1] +secret_key = sys.argv[2] + +acc_client = AccountClient(QiniuMacAuth(access_key, secret_key)) +apps, info = acc_client.list_apps() + +for app in apps: + if app.get('runMode') == 'Private': + uri = app.get('uri') + qcos = acc_client.get_qcos_client(uri) + if qcos != None: + stacks, info = qcos.list_stacks() + print("list_stacks of '%s':"%uri) + print(stacks) + print(info) + assert len(stacks) is not None diff --git a/qiniu/services/compute/app.py b/qiniu/services/compute/app.py index e69de29b..614ff668 100644 --- a/qiniu/services/compute/app.py +++ b/qiniu/services/compute/app.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- +from qiniu import http, QiniuMacAuth +from .config import KIRK_HOST +from .qcos_api import QcosClient + + +class AccountClient(object): + """客户端入口 + + 使用账号密钥生成账号客户端，可以进一步： + 1、获取和操作账号数据 + 2、获得部署的应用的客户端 + + 属性： + auth: 账号管理密钥对，QiniuMacAuth对象 + host: API host，在『内网模式』下使用时，auth=None，会自动使用 apiproxy 服务 + + 接口： + get_qcos_client(app_uri) + create_qcos_client(app_uri) + get_app_keys(app_uri) + get_valid_app_auth(app_uri) + get_account_info() + get_app_region_products(app_uri) + get_region_products(region) + list_regions() + list_apps() + create_app(args) + delete_app(app_uri) + + """ + + def __init__(self, auth, host=None): + self.auth = auth + self.qcos_clients = {} + if (auth is None): + self.host = KIRK_HOST['APPPROXY'] + else: + self.host = host or KIRK_HOST['APPGLOBAL'] + acc, info = self.get_account_info() + self.uri = acc.get('name') + + def get_qcos_client(self, app_uri): + """获得资源管理客户端 + 缓存，但不是线程安全的 + """ + + client = self.qcos_clients.get(app_uri) + if (client is None): + client = self.create_qcos_client(app_uri) + self.qcos_clients[app_uri] = client + + return client + + def create_qcos_client(self, app_uri): + """创建资源管理客户端 + + """ + + if (self.auth is None): + return QcosClient(None) + + products = self.get_app_region_products(app_uri) + auth = self.get_valid_app_auth(app_uri) + + if products is None or auth is None: + return None + + return QcosClient(auth, products.get('api')) + + def get_app_keys(self, app_uri): + """获得账号下应用的密钥 + + 列出指定应用的密钥，仅当访问者对指定应用有管理权限时有效： + 用户对创建的应用有管理权限。 + 用户对使用的第三方应用没有管理权限，第三方应用的运维方有管理权限。 + + Args: + - app_uri: 应用的完整标识 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回秘钥列表，失败返回None + - ResponseInfo 请求的Response信息 + """ + + url = '{0}/v3/apps/{1}/keys'.format(self.host, app_uri) + return http._get_with_qiniu_mac(url, None, self.auth) + + def get_valid_app_auth(self, app_uri): + """获得账号下可用的应用的密钥 + + 列出指定应用的可用密钥 + + Args: + - app_uri: 应用的完整标识 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回可用秘钥列表，失败返回None + - ResponseInfo 请求的Response信息 + """ + + ret, retInfo = self.get_app_keys(app_uri) + + if ret is None: + return None + + for k in ret: + if (k.get('state') == 'enabled'): + return QiniuMacAuth(k.get('ak'), k.get('sk')) + + return None + + def get_account_info(self): + """获得当前账号的信息 + + 查看当前请求方（请求鉴权使用的 AccessKey 的属主）的账号信息。 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回用户信息，失败返回None + - ResponseInfo 请求的Response信息 + """ + + url = '{0}/v3/info'.format(self.host) + return http._get_with_qiniu_mac(url, None, self.auth) + + def get_app_region_products(self, app_uri): + """获得指定应用所在区域的产品信息 + + Args: + - app_uri: 应用的完整标识 + + Returns: + 返回产品信息列表，若失败则返回None + """ + apps, retInfo = self.list_apps() + if apps is None: + return None + + for app in apps: + if (app.get('uri') == app_uri): + return self.get_region_products(app.get('region')) + + return + + def get_region_products(self, region): + """获得指定区域的产品信息 + + Args: + - region: 区域，如："nq" + + Returns: + 返回该区域的产品信息，若失败则返回None + """ + + regions, retInfo = self.list_regions() + if regions is None: + return None + + for r in regions: + if r.get('name') == region: + return r.get('products') + + def list_regions(self): + """获得账号可见的区域的信息 + + 列出当前用户所有可使用的区域。 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回区域列表，失败返回None + - ResponseInfo 请求的Response信息 + """ + + url = '{0}/v3/regions'.format(self.host) + return http._get_with_qiniu_mac(url, None, self.auth) + + def list_apps(self): + """获得当前账号的应用列表 + + 列出所属应用为当前请求方的应用列表。 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回应用列表，失败返回None + - ResponseInfo 请求的Response信息 + """ + + url = '{0}/v3/apps'.format(self.host) + return http._get_with_qiniu_mac(url, None, self.auth) + + def create_app(self, args): + """创建应用 + + 在指定区域创建一个新应用，所属应用为当前请求方。 + + Args: + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + - result 成功返回所创建的应用信息，若失败则返回None + - ResponseInfo 请求的Response信息 + """ + + url = '{0}/v3/apps'.format(self.host) + return http._post_with_qiniu_mac(url, args, self.auth) + + def delete_app(self, app_uri): + """删除应用 + + 删除指定标识的应用，当前请求方对该应用应有删除权限。 + + Args: + - app_uri: 应用的完整标识 + + Returns: + - result 成功返回空dict{}，若失败则返回None + - ResponseInfo 请求的Response信息 + """ + + url = '{0}/v3/apps/{1}'.format(self.host, app_uri) + return http._delete_with_qiniu_mac(url, None, self.auth) diff --git a/qiniu/services/compute/config.py b/qiniu/services/compute/config.py index e69de29b..045ed784 100644 --- a/qiniu/services/compute/config.py +++ b/qiniu/services/compute/config.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +KIRK_HOST = { + 'APPGLOBAL': "https://app-api.qiniu.com", # 公有云 APP API + 'APPPROXY': "http://app.qcos.qiniu", # 内网 APP API + 'APIPROXY': "http://api.qcos.qiniu", # 内网 API +} + +CONTAINER_UINT_TYPE = { + '1U1G': '单核(CPU)，1GB(内存)', + '1U2G': '单核(CPU)，2GB(内存)', + '1U4G': '单核(CPU)，4GB(内存)', + '1U8G': '单核(CPU)，8GB(内存)', + '2U2G': '双核(CPU)，2GB(内存)', + '2U4G': '双核(CPU)，4GB(内存)', + '2U8G': '双核(CPU)，8GB(内存)', + '2U16G': '双核(CPU)，16GB(内存)', + '4U8G': '四核(CPU)，8GB(内存)', + '4U16G': '四核(CPU)，16GB(内存)', + '8U16G': '八核(CPU)，16GB(内存)', +} diff --git a/qiniu/services/compute/qcos_api.py b/qiniu/services/compute/qcos_api.py index e69de29b..250a4bf6 100644 --- a/qiniu/services/compute/qcos_api.py +++ b/qiniu/services/compute/qcos_api.py @@ -0,0 +1,694 @@ +# -*- coding: utf-8 -*- +from qiniu import http +from .config import KIRK_HOST + + +class QcosClient(object): + """资源管理客户端 + + 使用应用密钥生成资源管理客户端，可以进一步： + 1、部署服务和容器，获得信息 + 2、创建网络资源，获得信息 + + 属性： + auth: 应用密钥对，QiniuMacAuth对象 + host: API host，在『内网模式』下使用时，auth=None，会自动使用 apiproxy 服务，只能管理当前容器所在的应用资源。 + + 接口： + list_stacks() + create_stack(args) + delete_stack(stack) + get_stack(stack) + start_stack(stack) + stop_stack(stack) + + list_services(stack) + create_service(stack, args) + get_service_inspect(stack, service) + start_service(stack, service) + stop_service(stack, service) + update_service(stack, service, args) + scale_service(stack, service, args) + delete_service(stack, service) + create_service_volume(stack, service, volume, args) + extend_service_volume(stack, service, volume, args) + delete_service_volume(stack, service, volume) + + list_containers(args) + get_container_inspect(ip) + start_container(ip) + stop_container(ip) + restart_container(ip) + + list_aps() + create_ap(args) + search_ap(mode, query) + get_ap(apid) + update_ap(apid, args) + set_ap_port(apid, port, args) + delete_ap(apid) + publish_ap(apid, args) + unpublish_ap(apid) + get_ap_port_healthcheck(apid, port) + set_ap_port_container(apid, port, args) + disable_ap_port(apid, port) + enable_ap_port(apid, port) + get_ap_providers() + get_web_proxy(backend) + """ + + def __init__(self, auth, host=None): + self.auth = auth + if auth is None: + self.host = KIRK_HOST['APIPROXY'] + else: + self.host = host + + def list_stacks(self): + """获得服务组列表 + + 列出当前应用的所有服务组信息。 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回服务组列表[, , ...]，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks'.format(self.host) + return self.__get(url) + + def create_stack(self, args): + """创建服务组 + + 创建新一个指定名称的服务组，并创建其下的服务。 + + Args: + - args: 服务组描述，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks'.format(self.host) + return self.__post(url, args) + + def delete_stack(self, stack): + """删除服务组 + + 删除服务组内所有服务并销毁服务组。 + + Args: + - stack: 服务所属的服务组名称 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}'.format(self.host, stack) + return self.__delete(url) + + def get_stack(self, stack): + """获取服务组 + + 查看服务组的属性信息。 + + Args: + - stack: 服务所属的服务组名称 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回stack信息，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}'.format(self.host, stack) + return self.__get(url) + + def start_stack(self, stack): + """启动服务组 + + 启动服务组中的所有停止状态的服务。 + + Args: + - stack: 服务所属的服务组名称 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/start'.format(self.host, stack) + return self.__post(url) + + def stop_stack(self, stack): + """停止服务组 + + 停止服务组中所有运行状态的服务。 + + Args: + - stack: 服务所属的服务组名称 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/stop'.format(self.host, stack) + return self.__post(url) + + def list_services(self, stack): + """获得服务列表 + + 列出指定名称的服务组内所有的服务, 返回一组详细的服务信息。 + + Args: + - stack: 服务所属的服务组名称 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回服务信息列表[, , ...]，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services'.format(self.host, stack) + return self.__get(url) + + def create_service(self, stack, args): + """创建服务 + + 创建一个服务，平台会异步地按模板分配资源并部署所有容器。 + + Args: + - stack: 服务所属的服务组名称 + - args: 服务具体描述请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services'.format(self.host, stack) + return self.__post(url, args) + + def delete_service(self, stack, service): + """删除服务 + + 删除指定名称服务，并自动销毁服务已部署的所有容器和存储卷。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}'.format(self.host, stack, service) + return self.__delete(url) + + def get_service_inspect(self, stack, service): + """查看服务 + + 查看指定名称服务的属性。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回服务信息，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/inspect'.format(self.host, stack, service) + return self.__get(url) + + def start_service(self, stack, service): + """启动服务 + + 启动指定名称服务的所有容器。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/start'.format(self.host, stack, service) + return self.__post(url) + + def stop_service(self, stack, service): + """停止服务 + + 停止指定名称服务的所有容器。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/stop'.format(self.host, stack, service) + return self.__post(url) + + def update_service(self, stack, service, args): + """更新服务 + + 更新指定名称服务的配置如容器镜像等参数，容器被重新部署后生效。 + 如果指定manualUpdate参数，则需要额外调用 部署服务 接口并指定参数进行部署；处于人工升级模式的服务禁止执行其他修改操作。 + 如果不指定manualUpdate参数，平台会自动完成部署。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + - args: 服务具体描述请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}'.format(self.host, stack, service) + return self.__post(url, args) + + def scale_service(self, stack, service, args): + """扩容/缩容服务 + + 更新指定名称服务的配置如容器镜像等参数，容器被重新部署后生效。 + 如果指定manualUpdate参数，则需要额外调用 部署服务 接口并指定参数进行部署；处于人工升级模式的服务禁止执行其他修改操作。 + 如果不指定manualUpdate参数，平台会自动完成部署。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/scale'.format(self.host, stack, service) + return self.__post(url, args) + + def create_service_volume(self, stack, service, args): + """创建存储卷 + + 为指定名称的服务增加存储卷资源，并挂载到部署的容器中。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/volumes'.format(self.host, stack, service) + return self.__post(url, args) + + def extend_service_volume(self, stack, service, volume, args): + """扩容存储卷 + + 为指定名称的服务增加存储卷资源，并挂载到部署的容器中。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + - volume: 存储卷名 + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/volumes/{3}/extend'.format(self.host, stack, service, volume) + return self.__post(url, args) + + def delete_service_volume(self, stack, service, volume): + """删除存储卷 + + 从部署的容器中移除挂载，并销毁指定服务下指定名称的存储卷, 并重新启动该容器。 + + Args: + - stack: 服务所属的服务组名称 + - service: 服务名 + - volume: 存储卷名 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/stacks/{1}/services/{2}/volumes/{3}'.format(self.host, stack, service, volume) + return self.__delete(url) + + def list_containers(self, stack=None, service=None): + """列出容器列表 + + 列出应用内所有部署的容器, 返回一组容器IP。 + + Args: + - stack: 要列出容器的服务组名(可不填，表示默认列出所有) + - service: 要列出容器服务的服务名(可不填，表示默认列出所有) + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回容器的ip数组，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/containers'.format(self.host) + params = {} + if stack is not None: + params['stack'] = stack + if service is not None: + params['service'] = service + return self.__get(url, params or None) + + def get_container_inspect(self, ip): + """查看容器 + + 查看指定IP的容器，返回容器属性。 + + Args: + - ip: 容器ip + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回容器的信息，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/containers/{1}/inspect'.format(self.host, ip) + return self.__get(url) + + def start_container(self, ip): + """启动容器 + + 启动指定IP的容器。 + + Args: + - ip: 容器ip + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/containers/{1}/start'.format(self.host, ip) + return self.__post(url) + + def stop_container(self, ip): + """停止容器 + + 停止指定IP的容器。 + + Args: + - ip: 容器ip + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/containers/{1}/stop'.format(self.host, ip) + return self.__post(url) + + def restart_container(self, ip): + """重启容器 + + 重启指定IP的容器。 + + Args: + - ip: 容器ip + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/containers/{1}/restart'.format(self.host, ip) + return self.__post(url) + + def list_aps(self): + """列出接入点 + + 列出当前应用的所有接入点。 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回接入点列表，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps'.format(self.host) + return self.__get(url) + + def create_ap(self, args): + """申请接入点 + + 申请指定配置的接入点资源。 + + Args: + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回申请到的接入点信息，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps'.format(self.host) + return self.__post(url, args) + + def search_ap(self, mode, query): + """搜索接入点 + + 查看指定接入点的所有配置信息，包括所有监听端口的配置。 + + Args: + - mode: 搜索模式，可以是domain、ip、host + - query: 搜索文本 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回搜索结果，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/search?{1}={2}'.format(self.host, mode, query) + return self.__get(url) + + def get_ap(self, apid): + """查看接入点 + + 给出接入点的域名或IP，查看配置信息，包括所有监听端口的配置。 + + Args: + - apid: 接入点ID + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回接入点信息，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}'.format(self.host, apid) + return self.__get(url) + + def update_ap(self, apid, args): + """更新接入点 + + 更新指定接入点的配置，如带宽。 + + Args: + - apid: 接入点ID + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}'.format(self.host, apid) + return self.__post(url, args) + + def set_ap_port(self, apid, port, args): + """更新接入点端口配置 + + 更新接入点指定端口的配置。 + + Args: + - apid: 接入点ID + - port: 要设置的端口号 + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/{2}'.format(self.host, apid, port) + return self.__post(url, args) + + def delete_ap(self, apid): + """释放接入点 + + 销毁指定接入点资源。 + + Args: + - apid: 接入点ID + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}'.format(self.host, apid) + return self.__delete(url) + + def publish_ap(self, apid, args): + """绑定自定义域名 + + 绑定用户自定义的域名，仅对公网域名模式接入点生效。 + + Args: + - apid: 接入点ID + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/publish'.format(self.host, apid) + return self.__post(url, args) + + def unpublish_ap(self, apid, args): + """解绑自定义域名 + + 解绑用户自定义的域名，仅对公网域名模式接入点生效。 + + Args: + - apid: 接入点ID + - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/unpublish'.format(self.host, apid) + return self.__post(url, args) + + def get_ap_port_healthcheck(self, apid, port): + """查看健康检查结果 + + 检查接入点的指定端口的后端健康状况。 + + Args: + - apid: 接入点ID + - port: 要设置的端口号 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回健康状况，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/{2}/healthcheck'.format(self.host, apid, port) + return self.__get(url) + + def set_ap_port_container(self, apid, port, args): + """调整后端实例配置 + + 调整接入点指定后端实例（容器）的配置，例如临时禁用流量等。 + + Args: + - apid: 接入点ID + - port: 要设置的端口号 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/{2}/setcontainer'.format(self.host, apid, port) + return self.__post(url, args) + + def disable_ap_port(self, apid, port): + """临时关闭接入点端口 + + 临时关闭接入点端口，仅对公网域名，公网ip有效。 + + Args: + - apid: 接入点ID + - port: 要设置的端口号 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/{2}/disable'.format(self.host, apid, port) + return self.__post(url) + + def enable_ap_port(self, apid, port): + """开启接入点端口 + + 开启临时关闭的接入点端口，仅对公网域名，公网ip有效。 + + Args: + - apid: 接入点ID + - port: 要设置的端口号 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回空dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/{1}/{2}/enable'.format(self.host, apid, port) + return self.__post(url) + + def get_ap_providers(self): + """列出入口提供商 + + 列出当前支持的入口提供商，仅对申请公网IP模式接入点有效。 + 注：公网IP供应商telecom=电信，unicom=联通，mobile=移动。 + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回接入商列表，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/aps/providers'.format(self.host) + return self.__get(url) + + def get_web_proxy(self, backend): + """获取一次性代理地址 + + 对内网地址获取一个一次性的外部可访问的代理地址 + + Args: + - backend: 后端地址，如："10.128.0.1:8080" + + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回代理地址信息，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/v3/webproxy'.format(self.host) + return self.__post(url, {'backend': backend}) + + def __post(self, url, data=None): + return http._post_with_qiniu_mac(url, data, self.auth) + + def __get(self, url, params=None): + return http._get_with_qiniu_mac(url, params, self.auth) + + def __delete(self, url): + return http._delete_with_qiniu_mac(url, None, self.auth) From 84cbfc23640cebdae385e70b0846132e8299110d Mon Sep 17 00:00:00 2001 From: jemygraw Date: Wed, 18 Oct 2017 17:41:02 +0800 Subject: [PATCH 262/478] reset kirk api to previous version --- CHANGELOG.md | 3 + manual_test_kirk.py | 334 ++++++++++++++++++++++++++++++++++++++++++++ qiniu/__init__.py | 4 +- setup.py | 3 +- 4 files changed, 342 insertions(+), 2 deletions(-) create mode 100644 manual_test_kirk.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a01d7b3..1dcd5d9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +# 7.1.8 （2017-10-18） +* 恢复kirk的API为原来的状态 + ## 7.1.7 (2017-09-27) * 修复从时间戳获取rfc http格式的时间字符串问题 diff --git a/manual_test_kirk.py b/manual_test_kirk.py new file mode 100644 index 00000000..e6791f30 --- /dev/null +++ b/manual_test_kirk.py @@ -0,0 +1,334 @@ +# -*- coding: utf-8 -*- +""" +======================= + 注意：必须手动运行 +======================= +""" +import os +import sys +import time +import logging +import pytest +from qiniu import auth +from qiniu.services import compute + + +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') +qn_auth = auth.QiniuMacAuth(access_key, secret_key) +acc_client = compute.app.AccountClient(qn_auth) +qcos_client = None +user_name = '' +app_uri = '' +app_name = 'appjust4test' +app_region = 'nq' + + +def setup_module(module): + acc_client = compute.app.AccountClient(qn_auth) + user_info = acc_client.get_account_info()[0] + acc_client.create_app({'name': app_name, 'title': 'whatever', 'region': app_region}) + + module.user_name = user_info['name'] + module.app_uri = '{0}.{1}'.format(module.user_name, app_name) + module.qcos_client = acc_client.create_qcos_client(module.app_uri) + + +def teardown_module(module): + module.app_uri + acc_client.delete_app(module.app_uri) + + +class TestApp: + """应用测试用例""" + + def test_create_and_delete_app(self): + _name_create = 'appjust4testcreate' + _uri_create = '' + _args = {'name': _name_create, 'title': 'whatever', 'region': app_region} + + with Call(acc_client, 'create_app', _args) as r: + assert r[0] is not None + _uri_create = r[0]['uri'] + + with Call(acc_client, 'delete_app', _uri_create) as r: + assert r[0] == {} + + def test_get_app_keys(self): + with Call(acc_client, 'get_app_keys', app_uri) as r: + assert len(r[0]) > 0 + + def test_get_account_info(self): + with Call(acc_client, 'get_account_info') as r: + assert r[0] is not None + + +class TestStack: + """服务组测试用例""" + + _name = 'just4test' + _name_del = 'just4del' + _name_create = 'just4create' + + @classmethod + def setup_class(cls): + qcos_client.create_stack({'name': cls._name}) + qcos_client.create_stack({'name': cls._name_del}) + + @classmethod + def teardown_class(cls): + qcos_client.delete_stack(cls._name) + qcos_client.delete_stack(cls._name_create) + qcos_client.delete_stack(cls._name_del) + + def test_create_stack(self): + with Call(qcos_client, 'create_stack', {'name': self._name_create}) as r: + assert r[0] == {} + + def test_delete_stack(self): + with Call(qcos_client, 'delete_stack', self._name_del) as r: + assert r[0] == {} + + def test_list_stacks(self): + with Call(qcos_client, 'list_stacks') as r: + assert len(r) > 0 + assert self._name in [stack['name'] for stack in r[0]] + + def test_get_stack(self): + with Call(qcos_client, 'get_stack', self._name) as r: + assert r[0]['name'] == self._name + + def test_start_stack(self): + with Call(qcos_client, 'start_stack', self._name) as r: + assert r[0] == {} + + def test_stop_stack(self): + with Call(qcos_client, 'stop_stack', self._name) as r: + assert r[0] == {} + + +class TestService: + """服务测试用例""" + + _stack = 'just4test2' + _name = 'spaceship' + _name_del = 'spaceship4del' + _name_create = 'spaceship4create' + _image = 'library/nginx:stable' + _unit = '1U1G' + _spec = {'image': _image, 'unitType': _unit} + + @classmethod + def setup_class(cls): + qcos_client.delete_stack(cls._stack) + qcos_client.create_stack({'name': cls._stack}) + qcos_client.create_service(cls._stack, {'name': cls._name, 'spec': cls._spec}) + qcos_client.create_service(cls._stack, {'name': cls._name_del, 'spec': cls._spec}) + + _debug_info('waiting for services to setup ...') + time.sleep(10) + + @classmethod + def teardown_class(cls): + # 删除stack会清理所有相关服务 + qcos_client.delete_stack(cls._stack) + + def test_create_service(self): + service = {'name': self._name_create, 'spec': self._spec} + with Call(qcos_client, 'create_service', self._stack, service) as r: + assert r[0] == {} + + def test_delete_service(self): + with Call(qcos_client, 'delete_service', self._stack, self._name_del) as r: + assert r[0] == {} + + def test_list_services(self): + with Call(qcos_client, 'list_services', self._stack) as r: + assert len(r) > 0 + assert self._name in [service['name'] for service in r[0]] + + def test_get_service_inspect(self): + with Call(qcos_client, 'get_service_inspect', self._stack, self._name) as r: + assert r[0]['name'] == self._name + assert r[0]['spec']['unitType'] == self._unit + + def test_update_service(self): + data = {'spec': {'autoRestart': 'ON_FAILURE'}} + with Call(qcos_client, 'update_service', self._stack, self._name, data) as r: + assert r[0] == {} + + _debug_info('waiting for update services to ready ...') + time.sleep(10) + + def test_scale_service(self): + data = {'instanceNum': 2} + with Call(qcos_client, 'scale_service', self._stack, self._name, data) as r: + assert r[0] == {} + + _debug_info('waiting for scale services to ready ...') + time.sleep(10) + + +class TestContainer: + """容器测试用例""" + + _stack = 'just4test3' + _service = 'spaceship' + _spec = {'image': 'library/nginx:stable', 'unitType': '1U1G'} + # 为了方便测试，容器数量最少为2 + _instanceNum = 2 + + @classmethod + def setup_class(cls): + qcos_client.delete_stack(cls._stack) + qcos_client.create_stack({'name': cls._stack}) + qcos_client.create_service(cls._stack, {'name': cls._service, 'spec': cls._spec, 'instanceNum': cls._instanceNum}) + + _debug_info('waiting for containers to setup ...') + time.sleep(10) + + @classmethod + def teardown_class(cls): + qcos_client.delete_stack(cls._stack) + + def test_list_containers(self): + with Call(qcos_client, 'list_containers', self._stack, self._service) as r: + assert len(r[0]) > 0 + assert len(r[0]) <= self._instanceNum + + def test_get_container_inspect(self): + ips = qcos_client.list_containers(self._stack, self._service)[0] + # 查看第1个容器 + with Call(qcos_client, 'get_container_inspect', ips[0]) as r: + assert r[0]['ip'] == ips[0] + + def test_stop_and_strat_container(self): + ips = qcos_client.list_containers(self._stack, self._service)[0] + # 停止第2个容器 + with Call(qcos_client, 'stop_container', ips[1]) as r: + assert r[0] == {} + + _debug_info('waiting for containers to stop ...') + time.sleep(3) + + # 启动第2个容器 + with Call(qcos_client, 'start_container', ips[1]) as r: + assert r[0] == {} + + def test_restart_container(self): + ips = qcos_client.list_containers(self._stack, self._service)[0] + # 重启第1个容器 + with Call(qcos_client, 'restart_container', ips[0]) as r: + assert r[0] == {} + + +class TestAp: + """接入点测试用例""" + + _stack = 'just4test4' + _service = 'spaceship' + _spec = {'image': 'library/nginx:stable', 'unitType': '1U1G'} + # 为了方便测试，容器数量最少为2 + _instanceNum = 2 + _apid_domain = {} + _apid_ip = {} + _apid_ip_port = 8080 + _user_domain = 'just4test001.example.com' + + @classmethod + def setup_class(cls): + qcos_client.delete_stack(cls._stack) + qcos_client.create_stack({'name': cls._stack}) + qcos_client.create_service(cls._stack, {'name': cls._service, 'spec': cls._spec, 'instanceNum': cls._instanceNum}) + cls._ap_domain = qcos_client.create_ap({'type': 'DOMAIN', 'provider': 'Telecom', 'unitType': 'BW_10M', 'title': 'public1'})[0] + cls._ap_ip = qcos_client.create_ap({'type': 'PUBLIC_IP', 'provider': 'Telecom', 'unitType': 'BW_10M', 'title': 'public2'})[0] + qcos_client.set_ap_port(cls._ap_ip['apid'], cls._apid_ip_port, {'proto': 'http'}) + + @classmethod + def teardown_class(cls): + qcos_client.delete_stack(cls._stack) + qcos_client.delete_ap(cls._ap_domain['apid']) + qcos_client.delete_ap(cls._ap_ip['apid']) + + def test_list_aps(self): + with Call(qcos_client, 'list_aps') as r: + assert len(r[0]) > 0 + assert self._ap_domain['apid'] in [ap['apid'] for ap in r[0]] + assert self._ap_domain['apid'] in [ap['apid'] for ap in r[0]] + + def test_create_and_delete_ap(self): + apid = 0 + ap = {'type': 'DOMAIN', 'provider': 'Telecom', 'unitType': 'BW_10M', 'title': 'public1'} + + with Call(qcos_client, 'create_ap', ap) as r: + assert r[0] is not None and r[0]['apid'] > 0 + apid = r[0]['apid'] + + with Call(qcos_client, 'delete_ap', apid) as r: + assert r[0] == {} + + def test_search_ap(self): + with Call(qcos_client, 'search_ap', 'ip', self._ap_ip['ip']) as r: + assert str(r[0]['apid']) == self._ap_ip['apid'] + + def test_get_ap(self): + with Call(qcos_client, 'get_ap', self._ap_ip['apid']) as r: + assert str(r[0]['apid']) == self._ap_ip['apid'] + + def test_update_ap(self): + with Call(qcos_client, 'update_ap', self._ap_ip['apid'], {}) as r: + assert r[0] == {} + + def test_set_ap_port(self): + with Call(qcos_client, 'set_ap_port', self._ap_ip['apid'], 80, {'proto': 'http'}) as r: + assert r[0] == {} + + def test_publish_ap(self): + domain = {'userDomain': self._user_domain} + with Call(qcos_client, 'publish_ap', self._ap_domain['apid'], domain) as r: + assert r[0] == {} + + def test_unpublish_ap(self): + domain = {'userDomain': self._user_domain} + with Call(qcos_client, 'unpublish_ap', self._ap_domain['apid'], domain) as r: + assert r[0] == {} + + def test_get_ap_port_healthcheck(self): + with Call(qcos_client, 'get_ap_port_healthcheck', self._ap_ip['apid'], self._apid_ip_port) as r: + assert r[0] is not None + + def test_disable_ap_port(self): + with Call(qcos_client, 'disable_ap_port', self._ap_ip['apid'], self._apid_ip_port) as r: + assert r[0] == {} + + def test_enable_ap_port(self): + with Call(qcos_client, 'enable_ap_port', self._ap_ip['apid'], self._apid_ip_port) as r: + assert r[0] == {} + + def test_get_ap_providers(self): + with Call(qcos_client, 'get_ap_providers') as r: + assert len(r[0]) > 0 + + +class Call(object): + def __init__(self, obj, method, *args): + self.context = (obj, method, args) + self.result = None + + def __enter__(self): + self.result = getattr(self.context[0], self.context[1])(*self.context[2]) + assert self.result is not None + return self.result + + def __exit__(self, type, value, traceback): + _debug_info('\033[94m%s.%s\x1b[0m: %s', self.context[0].__class__, self.context[1], self.result) + + +def _debug_info(*args): + logger = logging.getLogger(__name__) + logger.debug(*args) + + +if __name__ == '__main__': + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + pytest.main() diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 5321601c..3cf38ff0 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.1.7' +__version__ = '7.1.8' from .auth import Auth, QiniuMacAuth @@ -22,5 +22,7 @@ from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save +from .services.compute.app import AccountClient +from .services.compute.qcos_api import QcosClient from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry diff --git a/setup.py b/setup.py index 31413955..f686b5cf 100644 --- a/setup.py +++ b/setup.py @@ -7,17 +7,18 @@ try: import setuptools + setup = setuptools.setup except ImportError: setuptools = None from distutils.core import setup - packages = [ 'qiniu', 'qiniu.services', 'qiniu.services.storage', 'qiniu.services.processing', + 'qiniu.services.compute', 'qiniu.services.cdn', ] From 3cd52acadb206b0e94fc72fbfec11aec0170f02d Mon Sep 17 00:00:00 2001 From: jemygraw Date: Fri, 3 Nov 2017 16:21:32 +0800 Subject: [PATCH 263/478] fix chinese character bug in python2 and python3 --- examples/upload.py | 27 +++++++++++++++++---------- qiniu/auth.py | 2 +- qiniu/http.py | 8 ++++++-- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/examples/upload.py b/examples/upload.py index a8920de9..29a071e7 100755 --- a/examples/upload.py +++ b/examples/upload.py @@ -3,27 +3,34 @@ from qiniu import Auth, put_file, etag, urlsafe_base64_encode import qiniu.config +from qiniu.compat import is_py2, is_py3 -#需要填写你的 Access Key 和 Secret Key +# 需要填写你的 Access Key 和 Secret Key access_key = '...' secret_key = '...' -#构建鉴权对象 +# 构建鉴权对象 q = Auth(access_key, secret_key) -#要上传的空间 -bucket_name = 'Bucket_Name' +# 要上传的空间 +bucket_name = 'if-bc' -#上传到七牛后保存的文件名 -key = 'my-python-logo.png'; +# 上传到七牛后保存的文件名 +key = 'my-python-七牛.png' -#生成上传 Token，可以指定过期时间等 +# 生成上传 Token，可以指定过期时间等 token = q.upload_token(bucket_name, key, 3600) -#要上传文件的本地路径 -localfile = './sync/bbb.jpg' +# 要上传文件的本地路径 +localfile = '/Users/jemy/Documents/qiniu.png' ret, info = put_file(token, key, localfile) +print(ret) print(info) -assert ret['key'] == key + +if is_py2: + assert ret['key'].encode('utf-8') == key +elif is_py3: + assert ret['key'] == key + assert ret['hash'] == etag(localfile) diff --git a/qiniu/auth.py b/qiniu/auth.py index 02525118..bc4bfa41 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -141,7 +141,7 @@ def upload_token(self, bucket, key=None, expires=3600, policy=None, strict_polic scope = bucket if key is not None: - scope = u'{0}:{1}'.format(bucket, key) + scope = '{0}:{1}'.format(bucket, key) args = dict( scope=scope, diff --git a/qiniu/http.py b/qiniu/http.py index 0ee27cea..45178114 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -4,6 +4,7 @@ import requests from requests.auth import AuthBase +from qiniu.compat import is_py2, is_py3 from qiniu import config import qiniu.auth from . import __version__ @@ -21,7 +22,7 @@ def __return_wrapper(resp): if resp.status_code != 200 or resp.headers.get('X-Reqid') is None: return None, ResponseInfo(resp) resp.encoding = 'utf-8' - ret = resp.json() if resp.text != '' else {} + ret = resp.json(encoding='utf-8') if resp.text != '' else {} return ret, ResponseInfo(resp) @@ -169,7 +170,10 @@ def connect_failed(self): return self.__response is None or self.req_id is None def __str__(self): - return ', '.join(['%s:%s' % item for item in self.__dict__.items()]) + if is_py2: + return ', '.join(['%s:%s' % item for item in self.__dict__.items()]).encode('utf-8') + elif is_py3: + return ', '.join(['%s:%s' % item for item in self.__dict__.items()]) def __repr__(self): return self.__str__() From 4813ad737f132765b395506b710e3b03e792a423 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Fri, 3 Nov 2017 16:33:22 +0800 Subject: [PATCH 264/478] fix the resume uploader recorder key error under python2 when using chinese character --- .../storage/upload_progress_recorder.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index 697b2da7..cac64529 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -4,6 +4,7 @@ import json import os import tempfile +from qiniu.compat import is_py2, is_py3 class UploadProgressRecorder(object): @@ -27,8 +28,11 @@ def __init__(self, record_folder=tempfile.gettempdir()): def get_upload_record(self, file_name, key): record_key = '{0}/{1}'.format(key, file_name) + if is_py2: + record_file_name = hashlib.md5(record_key).hexdigest() + else: + record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() - record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() upload_record_file_path = os.path.join(self.record_folder, record_file_name) if not os.path.isfile(upload_record_file_path): return None @@ -38,13 +42,21 @@ def get_upload_record(self, file_name, key): def set_upload_record(self, file_name, key, data): record_key = '{0}/{1}'.format(key, file_name) - record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() + if is_py2: + record_file_name = hashlib.md5(record_key).hexdigest() + else: + record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() + upload_record_file_path = os.path.join(self.record_folder, record_file_name) with open(upload_record_file_path, 'w') as f: json.dump(data, f) def delete_upload_record(self, file_name, key): record_key = '{0}/{1}'.format(key, file_name) - record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() + if is_py2: + record_file_name = hashlib.md5(record_key).hexdigest() + else: + record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() + upload_record_file_path = os.path.join(self.record_folder, record_file_name) os.remove(upload_record_file_path) From 659028840edba9b17f69cf31c7c72d824092516f Mon Sep 17 00:00:00 2001 From: jemygraw Date: Fri, 3 Nov 2017 16:55:06 +0800 Subject: [PATCH 265/478] remove unused import --- qiniu/services/storage/upload_progress_recorder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index cac64529..4b4af7ac 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -4,7 +4,7 @@ import json import os import tempfile -from qiniu.compat import is_py2, is_py3 +from qiniu.compat import is_py2 class UploadProgressRecorder(object): From 26217ddce8be84d8902a8713445384bf015a8c7a Mon Sep 17 00:00:00 2001 From: jemygraw Date: Fri, 3 Nov 2017 23:20:12 +0800 Subject: [PATCH 266/478] publish v7.1.9 --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 3cf38ff0..3e54e03a 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.1.8' +__version__ = '7.1.9' from .auth import Auth, QiniuMacAuth From 99597d062e941f3dd19169bbb80d8b073d3b5ed0 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Tue, 21 Nov 2017 13:59:24 +0800 Subject: [PATCH 267/478] create_bucket --- examples/create_bucket.py | 25 +++++++++++++++++++++++++ qiniu/services/storage/bucket.py | 12 ++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 examples/create_bucket.py diff --git a/examples/create_bucket.py b/examples/create_bucket.py new file mode 100644 index 00000000..e77099d7 --- /dev/null +++ b/examples/create_bucket.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +""" +创建存储空间 +""" + +from qiniu import Auth +from qiniu import BucketManager + + +access_key = '...' +secret_key = '...' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +bucket_name = 'Bucket_Name' + +# "填写存储区域代号 z0:华东, z1:华北, z2:华南, na0:北美" +region = 'z0' + +ret, info = bucket.mkbucketv2(bucket_name, region) +print(info) +print(ret) +assert info.status_code == 200 diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 46c79f02..30e0d100 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -283,6 +283,18 @@ def delete_after_days(self, bucket, key, days): resource = entry(bucket, key) return self.__rs_do('deleteAfterDays', resource, days) + def mkbucketv2(self, bucket_name, region): + """ + 创建存储空间 + https://developer.qiniu.com/kodo/api/1382/mkbucketv2 + + Args: + bucket_name: 存储空间名 + region: 存储区域 + """ + bucket_name = urlsafe_base64_encode(bucket_name) + return self.__rs_do('mkbucketv2', bucket_name, 'region', region) + def __rs_do(self, operation, *args): return self.__server_do(config.get_default('default_rs_host'), operation, *args) From b82064e118b008be13140bd67203af8626d75043 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Tue, 21 Nov 2017 14:02:32 +0800 Subject: [PATCH 268/478] fix ref annotate pfop.py line 32 --- qiniu/services/processing/pfop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/processing/pfop.py b/qiniu/services/processing/pfop.py index c8f1830c..9829b6c9 100644 --- a/qiniu/services/processing/pfop.py +++ b/qiniu/services/processing/pfop.py @@ -29,7 +29,7 @@ def execute(self, key, fops, force=None): Args: key: 待处理的源文件 - fops: 处理详细操作，规格详见 http://developer.qiniu.com/docs/v6/api/reference/fop/ + fops: 处理详细操作，规格详见 https://developer.qiniu.com/dora/manual/1291/persistent-data-processing-pfop force: 强制执行持久化处理开关 Returns: From 9e9effa29b7e7a6613561c9933f540b1ac4ed3fd Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Thu, 23 Nov 2017 11:24:06 +0800 Subject: [PATCH 269/478] raise a execption when bucket is error --- qiniu/zone.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qiniu/zone.py b/qiniu/zone.py index 68f7c53d..349a597d 100644 --- a/qiniu/zone.py +++ b/qiniu/zone.py @@ -95,8 +95,10 @@ def get_bucket_hosts(self, ak, bucket): else: # 1 year hosts['ttl'] = int(time.time()) + 31536000 - - scheme_hosts = hosts[self.scheme] + try: + scheme_hosts = hosts[self.scheme] + except KeyError: + raise KeyError("Please check your BUCKET_NAME! The UpHosts is %s" % hosts) bucket_hosts = { 'upHosts': scheme_hosts['up'], 'ioHosts': scheme_hosts['io'], From 918989db5af6fcc31a355e4af620bf63c086d589 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Thu, 23 Nov 2017 13:30:06 +0800 Subject: [PATCH 270/478] fix the file like object upload problem mentioned in issue 281 --- qiniu/services/storage/uploader.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 1340d06f..4e1a2157 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -27,8 +27,19 @@ def put_data( 一个dict变量，类似 {"hash": "", "key": ""} 一个ResponseInfo对象 """ - crc = crc32(data) - return _form_put(up_token, key, data, params, mime_type, crc, progress_handler, fname) + final_data = '' + if hasattr(data, 'read'): + while True: + tmp_data = data.read(config._BLOCK_SIZE) + if len(tmp_data) == 0: + break + else: + final_data += tmp_data + else: + final_data = data + + crc = crc32(final_data) + return _form_put(up_token, key, final_data, params, mime_type, crc, progress_handler, fname) def put_file(up_token, key, file_path, params=None, From af969583b20417f8f324613d6fdc205c9c25f313 Mon Sep 17 00:00:00 2001 From: jemygraw Date: Thu, 23 Nov 2017 14:07:41 +0800 Subject: [PATCH 271/478] add 7.2.0 release note --- CHANGELOG.md | 9 +++++++++ qiniu/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dcd5d9a..26f2b5d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +# 7.2.0（2017-11-23） +* 修复put_data不支持file like object的问题 +* 增加空间写错时，抛出异常提示客户的功能 +* 增加创建空间的接口功能 + +# 7.1.9（2017-11-01） +* 修复python2情况下，中文文件名上传失败的问题 +* 修复python2环境下，中文文件使用分片上传时失败的问题 + # 7.1.8 （2017-10-18） * 恢复kirk的API为原来的状态 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 3e54e03a..6d45a843 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.1.9' +__version__ = '7.2.0' from .auth import Auth, QiniuMacAuth From 511d56d72034c3c4857aa5a9302bf719b359ef72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=B1=E4=BA=89=E5=85=89?= Date: Sat, 17 Mar 2018 22:07:22 +0800 Subject: [PATCH 272/478] add script for update the ca of cdn --- examples/update_cdn_sslcert.py | 29 +++++++++ qiniu/__init__.py | 2 +- qiniu/http.py | 20 ++++++ qiniu/services/cdn/manager.py | 107 +++++++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 examples/update_cdn_sslcert.py diff --git a/examples/update_cdn_sslcert.py b/examples/update_cdn_sslcert.py new file mode 100644 index 00000000..dbdffca5 --- /dev/null +++ b/examples/update_cdn_sslcert.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +""" +更新cdn证书(可配合let's encrypt 等完成自动证书更新) +""" +import qiniu +from qiniu import DomainManager + +# 账户ak，sk +access_key = '' +secret_key = '' + +auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) +domain_manager = DomainManager(auth) + +privatekey = "ssl/www.qiniu.com/privkey.pem" +ca = "ssl/www.qiniu.com/fullchain.pem" +domain_name='www.qiniu.com' + +with open(privatekey,'r') as f: + privatekey_str=f.read() + +with open(ca,'r') as f: + ca_str=f.read() + +ret, info = domain_manager.create_sslcert(domain_name, domain_name, privatekey_str, ca_str) +print(ret['certID']) + +ret, info = domain_manager.put_httpsconf(domain_name, ret['certID'],False) +print(info) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 6d45a843..6d115087 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -19,7 +19,7 @@ from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ build_batch_stat, build_batch_delete from .services.storage.uploader import put_data, put_file, put_stream -from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url +from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url,DomainManager from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save from .services.compute.app import AccountClient diff --git a/qiniu/http.py b/qiniu/http.py index 45178114..b44a207d 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -51,6 +51,20 @@ def _post(url, data, files, auth, headers=None): return None, ResponseInfo(None, e) return __return_wrapper(r) +def _put(url, data, files, auth, headers=None): + if _session is None: + _init() + try: + post_headers = _headers.copy() + if headers is not None: + for k, v in headers.items(): + post_headers.update({k: v}) + r = _session.put( + url, data=data, files=files, auth=auth, headers=post_headers, + timeout=config.get_default('connection_timeout')) + except Exception as e: + return None, ResponseInfo(None, e) + return __return_wrapper(r) def _get(url, params, auth): try: @@ -86,6 +100,12 @@ def _post_with_auth(url, data, auth): def _post_with_auth_and_headers(url, data, auth, headers): return _post(url, data, None, qiniu.auth.RequestsAuth(auth), headers) +def _put_with_auth(url, data, auth): + return _put(url, data, None, qiniu.auth.RequestsAuth(auth)) + +def _put_with_auth_and_headers(url, data, auth, headers): + return _put(url, data, None, qiniu.auth.RequestsAuth(auth), headers) + def _post_with_qiniu_mac(url, data, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index f073a77b..313736ef 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -157,10 +157,117 @@ def get_log_list_data(self, domains, log_date): url = '{0}/v2/tune/log/list'.format(self.server) return self.__post(url, body) + def put_httpsconf(self, name, certid, forceHttps=False): + """ + 修改证书，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name#11 + + Args: + domains: 域名name + CertID: 证书id，从上传或者获取证书列表里拿到证书id + ForceHttps: 是否强制https跳转 + + Returns: + {} + """ + req = {} + req.update({"certid": certid}) + req.update({"forceHttps": forceHttps}) + + body = json.dumps(req) + url = '{0}/domain/{1}/httpsconf'.format(self.server, name) + return self.__post(url, body) + + def __post(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._post_with_auth_and_headers(url, data, self.auth, headers) + + +class DomainManager(object): + def __init__(self, auth): + self.auth = auth + self.server = 'http://api.qiniu.com' + + def create_domain(self, name, body): + """ + 创建域名，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name + + Args: + name: 域名, 如果是泛域名，必须以点号 . 开头 + bosy: 创建域名参数 + Returns: + {} + """ + url = '{0}/domain/{1}'.format(self.server, name) + return self.__post(url, body) + + def get_domain(self, name): + """ + 获取域名信息，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name + + Args: + name: 域名, 如果是泛域名，必须以点号 . 开头 + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/domain/{1}'.format(self.server, name) + return self.__post(url) + + def put_httpsconf(self, name, certid, forceHttps): + """ + 修改证书，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name#11 + + Args: + domains: 域名name + CertID: 证书id，从上传或者获取证书列表里拿到证书id + ForceHttps: 是否强制https跳转 + + Returns: + {} + """ + req = {} + req.update({"certid": certid}) + req.update({"forceHttps": forceHttps}) + + body = json.dumps(req) + url = '{0}/domain/{1}/httpsconf'.format(self.server, name) + return self.__put(url, body) + + def create_sslcert(self, name, common_name, pri, ca): + """ + 修改证书，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name#11 + + Args: + name: 证书名称 + common_name: 相关域名 + pri: 证书私钥 + ca: 证书内容 + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回dict{certID: }，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + + + """ + req = {} + req.update({"name": name}) + req.update({"common_name": common_name}) + req.update({"pri": pri}) + req.update({"ca": ca}) + + body = json.dumps(req) + url = '{0}/sslcert'.format(self.server) + return self.__post(url, body) + def __post(self, url, data=None): headers = {'Content-Type': 'application/json'} return http._post_with_auth_and_headers(url, data, self.auth, headers) + def __put(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._put_with_auth_and_headers(url, data, self.auth, headers) + def create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline): """ From 1111bb33ec302c9274edfa0853968724514d77ce Mon Sep 17 00:00:00 2001 From: zhangyunchuan Date: Wed, 18 Apr 2018 15:37:08 +0800 Subject: [PATCH 273/478] edit --- examples/rtc_server.py | 126 +++++++ qiniu/__init__.py | 2 + qiniu/auth.py | 2 +- qiniu/services/pili/__init__.py | 0 qiniu/services/pili/rtc_server_manager.py | 433 ++++++++++++++++++++++ 5 files changed, 562 insertions(+), 1 deletion(-) create mode 100644 examples/rtc_server.py create mode 100644 qiniu/services/pili/__init__.py create mode 100644 qiniu/services/pili/rtc_server_manager.py diff --git a/examples/rtc_server.py b/examples/rtc_server.py new file mode 100644 index 00000000..f8789e2b --- /dev/null +++ b/examples/rtc_server.py @@ -0,0 +1,126 @@ +#-*- coding: utf-8 -*- +#flake8: noqa +# from qiniu import Auth +# from qiniu import BucketManager + +# access_key = '...' +# secret_key = '...' +# +# q = Auth(access_key, secret_key) +# bucket = BucketManager(q) +# +# bucket_name = 'Bucket_Name' +# # 前缀 +# prefix = None +# # 列举条目 +# limit = 10 +# # 列举出除'/'的所有文件以及以'/'为分隔的所有前缀 +# delimiter = None +# # 标记 +# marker = None +# +# ret, eof, info = bucket.list(bucket_name, prefix, marker, limit, delimiter) +# +# print(info) +# +# assert len(ret.get('items')) is not None + +import sys +sys.path.append('/Users/yunchuanzhang/python3/lib/python3.5/site-packages/python-sdk/qiniu') + +from qiniu import QiniuMacAuth +from qiniu import RtcServer, RtcRoomToken + + +# UID 1380668373 PILI—VDN内部测试账号 +access_key = 'DXFtikq1YuDT_WMUntOpzpWPm2UZVtEnYvN3-CUD' +secret_key = 'F397hzMohpORVZ-bBbb-IVbpdWlI4SWu8sWq78v3' + +q = QiniuMacAuth(access_key, secret_key ) + +rtc = RtcServer(q) + + +#print ( rtc.GetApp() ) +#print ('\n\n\n') +# +# +# create_data={ +# "hub": 'python_test_hub', +# "title": 'python_test_app', +# # "maxUsers": MaxUsers, +# # "noAutoCloseRoom": NoAutoCloseRoom, +# # "noAutoCreateRoom": NoAutoCreateRoom, +# # "noAutoKickUser": NoAutoKickUser +# } +# print ( rtc.CreateApp(create_data) ) +# print ('\n\n\n') +# +# +# print ( rtc.DeleteApp('desls83s2') ) +# print ('\n\n\n') +# +# +# update_data={ +# "hub": "python_new_hub", +# "title": "python_new_app", +# # "maxUsers": , +# # "noAutoCloseRoom": , +# # "noAutoCreateRoom": , +# # "noAutoKickUser": , +# # "mergePublishRtmp": { +# # "enable": , +# # "audioOnly": , +# # "height": , +# # "width": , +# # "fps": , +# # "kbps": , +# # "url": "", +# # "streamTitle": "" +# # } +# } +# print ( rtc.UpdateApp('desmfnkw5', update_data) ) +# print ('\n\n\n') +# + +# print ( rtc.ListUser( 'd7rqwfxqd','test' ) ) +# print ('\n\n\n') +# +# print ( rtc.KickUser( 'd7rqwfxqd', 'test' ,'test' ) ) +# print ('\n\n\n') +# +# print ( rtc.ListActiveRoom( 'd7rqwfxqd' ) ) +# print ('\n\n\n') + +# roomAccess = { +# "appId": "d7rqwfxqd" , +# "roomName": "RoomName" , +# "userId": "UserID" , +# "expireAt": "1524056400" , +# "permission": "user" +# } +# print (RtcRoomToken ( access_key, secret_key, roomAccess ) ) +# print ('\n\n\n') + + +# access_key='gwd_gV4gPKZZsmEOvAuNU1AcumicmuHooTfu64q5' +# secret_key='9G4isTkVuj5ITPqH1ajhljJMTc2k4m-hZh5r5ZsK' +# +# roomAccess = { +# "appId": "desobxqpx" , +# "roomName": "lfxl" , +# "userId": "1111" , +# "expireAt": "" , +# "permission": "" +# } +# print (RtcRoomToken ( access_key, secret_key, roomAccess ) ) +# + + + + + + + + + diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 6d45a843..7543850e 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -25,4 +25,6 @@ from .services.compute.app import AccountClient from .services.compute.qcos_api import QcosClient +from .services.pili.rtc_server_manager import RtcServer, RtcRoomToken + from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry diff --git a/qiniu/auth.py b/qiniu/auth.py index bc4bfa41..00c629ff 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -245,7 +245,7 @@ def token_of_request(self, method, host, url, qheaders, content_type=None, body= data += "\n" if content_type and content_type != "application/octet-stream" and body: - data += body + data += body.decode(encoding='UTF-8') return '{0}:{1}'.format(self.__access_key, self.__token(data)) diff --git a/qiniu/services/pili/__init__.py b/qiniu/services/pili/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qiniu/services/pili/rtc_server_manager.py b/qiniu/services/pili/rtc_server_manager.py new file mode 100644 index 00000000..07a64c4e --- /dev/null +++ b/qiniu/services/pili/rtc_server_manager.py @@ -0,0 +1,433 @@ +# -*- coding: utf-8 -*- +from qiniu import http +import json, hashlib, hmac, base64 + + +class RtcServer(object): + """ + + """ + def __init__(self, auth): + self.auth = auth + self.host = 'http://rtc.qiniuapi.com' + + def CreateApp(self,data): + """ + Host rtc.qiniuapi.com + POST /v3/apps + Authorization: qiniu mac + Content-Type: application/json + + { + "hub": "", + "title": "", + "maxUsers": <MaxUsers>, + "noAutoCloseRoom": <NoAutoCloseRoom>, + "noAutoCreateRoom": <NoAutoCreateRoom>, + "noAutoKickUser": <NoAutoKickUser> + } + + :param appid: + Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 + + Title: app 的名称，可选，注意，Title 不是唯一标识，重复 create 动作将生成多个 app。 + + MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 + + NoAutoCloseRoom: bool 类型，可选，禁止自动关闭房间。默认为 false ，即用户退出房间后，房间会被主动清理释放。 + + NoAutoCreateRoom: bool 类型，可选，禁止自动创建房间。默认为 false ，即不需要主动调用接口创建即可加入房间。 + + NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ，即同一个身份的 client (app/room/user) ，新的连麦请求可以成功，旧连接被关闭。 + + :return: + 200 OK + { + "appId": "<AppID>", + "hub": "<Hub>", + "title": "<Title>", + "maxUsers": <MaxUsers>, + "noAutoCloseRoom": <NoAutoCloseRoom>, + "noAutoCreateRoom": <NoAutoCreateRoom>, + "noAutoKickUser": <NoAutoKickUser>, + "createdAt": <CreatedAt>, + "updatedAt": <UpdatedAt> + } + 616 + { + "error": "hub not match" + } + """ + + + return self.__post(self.host+'/v3/apps',data,) + + + def GetApp(self, appid=None): + """ + Host rtc.qiniuapi.com + GET /v3/apps/<AppID> + Authorization: qiniu mac + + :param appid: + AppID: app 的唯一标识。 可以不填写，不填写的话，默认就是输出所有app的相关信息 + + :return: + 200 OK + { + "appId": "<AppID>", + "hub": "<Hub>", + "title": "<Title>", + "maxUsers": <MaxUsers>, + "noAutoCloseRoom": <NoAutoCloseRoom>, + "noAutoCreateRoom": <NoAutoCreateRoom>, + "noAutoKickUser": <NoAutoKickUser>, + "mergePublishRtmp": { + "audioOnly": <AudioOnly>, + "height": <OutputHeight>, + "width": <OutputHeight>, + "fps": <OutputFps>, + "kbps": <OutputKbps>, + "url": "<URL>", + "streamTitle": "<StreamTitle>" + }, + "createdAt": <CreatedAt>, + "updatedAt": <UpdatedAt> + } + + 612 + { + "error": "app not found" + } + + #### + AppID: app 的唯一标识。 + + UID: 客户的七牛帐号。 + + Hub: 绑定的直播 hub，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 + + Title: app 的名称，注意，Title不是唯一标识。 + + MaxUsers: int 类型，连麦房间支持的最大在线人数。 + + NoAutoCloseRoom: bool 类型，禁止自动关闭房间。 + + NoAutoCreateRoom: bool 类型，禁止自动创建房间。 + + NoAutoKickUser: bool 类型，禁止自动踢人。 + + MergePublishRtmp: 连麦合流转推 RTMP 的配置。 + + CreatedAt: time 类型，app 创建的时间。 + + UpdatedAt: time 类型，app 更新的时间。 + """ + if appid: + return self.__get(self.host+'/v3/apps/%s' % appid ) + else: + return self.__get(self.host+'/v3/apps' ) + + def DeleteApp(self, appid): + """ + Host rtc.qiniuapi.com + DELETE /v3/apps/<AppID> + Authorization: qiniu mac + + :return: + 200 OK + + 612 + { + "error": "app not found" + } + """ + return self.__delete(self.host+'/v3/apps/%s' % appid) + + def UpdateApp(self, appid, data): + """ + Host rtc.qiniuapi.com + Post /v3/apps/<AppID> + Authorization: qiniu mac + + :param appid: + AppID: app 的唯一标识，创建的时候由系统生成。 + + Title: app 的名称， 可选。 + + Hub: 绑定的直播 hub，可选，用于合流后 rtmp 推流。 + + MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 + + NoAutoCloseRoom: bool 指针类型，可选，true 表示禁止自动关闭房间。 + + NoAutoCreateRoom: bool 指针指型，可选，true 表示禁止自动创建房间。 + + NoAutoKickUser: bool 类型，可选，禁止自动踢人。 + + MergePublishRtmp: 连麦合流转推 RTMP 的配置，可选择。其详细配置包括如下 + + Enable: 布尔类型，用于开启和关闭所有房间的合流功能。 + AudioOnly: 布尔类型，可选，指定是否只合成音频。 + Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 + OutputFps: int64，可选，指定合流输出的帧率，默认为 25 fps 。 + OutputKbps: int64，可选，指定合流输出的码率，默认为 1000 。 + URL: 合流后转推旁路直播的地址，可选，支持魔法变量配置按照连麦房间号生成不同的推流地址。如果是转推到七牛直播云，不建议使用该配置。 + StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号生成不同的流名。例如，配置 Hub 为 qn-zhibo ，配置 StreamTitle 为 $(roomName) ，则房间 meeting-001 的合流将会被转推到 rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。详细配置细则，请咨询七牛技术支持。 + + :return: + 200 OK + { + "appId": "<AppID>", + "hub": "<Hub>", + "title": "<Title>", + "maxUsers": <MaxUsers>, + "noAutoCloseRoom": <NoAutoCloseRoom>, + "noAutoCreateRoom": <NoAutoCreateRoom>, + "noAutoKickUser": <NoAutoKickUser>, + "mergePublishRtmp": { + "enable": <Enable>, + "audioOnly": <AudioOnly>, + "height": <OutputHeight>, + "width": <OutputHeight>, + "fps": <OutputFps>, + "kbps": <OutputKbps>, + "url": "<URL>", + "streamTitle": "<StreamTitle>" + }, + "createdAt": <CreatedAt>, + "updatedAt": <UpdatedAt> + } + + 612 + { + "error": "app not found" + } + 616 + { + "error": "hub not match" + } + """ + + return self.__post(self.host+'/v3/apps/%s' % appid , data,) + + def ListUser(self, AppID, RoomName): + """ + Host rtc.qiniuapi.com + GET /v3/apps/<AppID>/rooms/<RoomName>/users + Authorization: qiniu mac + + :param: + AppID: 连麦房间所属的 app 。 + + RoomName: 操作所查询的连麦房间。 + + :return: + 200 OK + { + "users": [ + { + "userId": "<UserID>" + }, + ] + } + 612 + { + "error": "app not found" + } + """ + return self.__get( self.host+'/v3/apps/%s/rooms/%s/users' % (AppID, RoomName)) + + def KickUser(self, AppID, RoomName, UserID): + """ + Host rtc.qiniuapi.com + DELETE /v3/apps/<AppID>/rooms/<RoomName>/users/<UserID> + Authorization: qiniu mac + + :param: + AppID: 连麦房间所属的 app 。 + + RoomName: 连麦房间。 + + UserID: 操作所剔除的用户。 + + :return: + 200 OK + 612 + { + "error": "app not found" + } + 612 + { + "error": "user not found" + } + 615 + { + "error": "room not active" + } + """ + return self.__delete( self.host+'/v3/apps/%s/rooms/%s/users/%s' % (AppID, RoomName, UserID) ) + + def ListActiveRoom(self, AppID, RoomNamePrefix=None): + """ + Host rtc.qiniuapi.com + GET /v3/apps/<AppID>/rooms?prefix=<RoomNamePrefix>&offset=<Offset>&limit=<Limit> + Authorization: qiniu mac + + :param: + AppID: 连麦房间所属的 app 。 + + RoomNamePrefix: 所查询房间名的前缀索引，可以为空。 + + Offset: int 类型，分页查询的位移标记。 + + Limit: int 类型，此次查询的最大长度。 + + :return: + 200 OK + { + "end": <IsEnd>, + "offset": <Offset>, + "rooms": [ + "<RoomName>", + ... + ] + } + 612 + { + "error": "app not found" + } + ### + IsEnd: bool 类型，分页查询是否已经查完所有房间。 + + Offset: int 类型，下次分页查询使用的位移标记。 + + RoomName: 当前活跃的房间名。 + """ + if RoomNamePrefix: + return self.__get( self.host+'/v3/apps/%s/rooms?prefix=%s' % (AppID, RoomNamePrefix) ) + else: + return self.__get( self.host+'/v3/apps/%s/rooms' % AppID ) + + def __post(self, url, data=None): + return http._post_with_qiniu_mac(url, data, self.auth) + + def __get(self, url, params=None): + return http._get_with_qiniu_mac(url,params,self.auth) + + def __delete(self, url, params=None): + return http._delete_with_qiniu_mac(url,params,self.auth) + + +def RtcRoomToken(access_key, secret_key, roomAccess ): + """ + :arg: + AppID: 房间所属帐号的 app 。 + + RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ + + UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ + + ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间，token 将在该时间后失效。 + + Permission: 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时，拥有将其他用户移除出房间等特权. + + :method: + # 1. 定义房间管理凭证，并对凭证字符做URL安全的Base64编码 + roomAccess = { + "appId": "<AppID>" + "roomName": "<RoomName>", + "userId": "<UserID>", + "expireAt": <ExpireAt>, + "permission": "<Permission>" + } + roomAccessString = json_to_string(roomAccess) + encodedRoomAccess = urlsafe_base64_encode(roomAccessString) + + # 2. 计算HMAC-SHA1签名，并对签名结果做URL安全的Base64编码 + sign = hmac_sha1(encodedRoomAccess, <SecretKey>) + encodedSign = urlsafe_base64_encode(sign) + + # 3. 将AccessKey与以上两者拼接得到房间鉴权 + roomToken = "<AccessKey>" + ":" + encodedSign + ":" + encodedRoomAccess + """ + def b(data): + return bytes(data) + + def urlsafe_base64_encode(data): + ret = base64.urlsafe_b64encode(b(data)) + return b(ret) + + def pre_token(data, SecretKey): + data = b(data) + SecretKey = b(SecretKey) + hashed = hmac.new(SecretKey, data, hashlib.sha1) + return urlsafe_base64_encode(hashed.digest()) + + + roomAccessString = json.dumps(roomAccess) + byte_result = bytes(roomAccessString, 'utf-8' ) + encodedRoomAccess = base64.urlsafe_b64encode( byte_result ) + + encodedSign = pre_token( encodedRoomAccess, secret_key ) + + #roomToken = bytes(access_key, 'utf-8') + bytes(':','utf-8') + encodedSign + bytes(':','utf-8') + encodedRoomAccess + roomToken = access_key + ':' + str(encodedSign, encoding = "utf-8") + ':' + str(encodedRoomAccess, encoding = "utf-8") + + print ( access_key ) + print ( encodedSign ) + print ( encodedRoomAccess ) + return roomToken + + + + + + # + # roomAccessString = json.dumps(roomAccess) + # byte_result = bytes(roomAccessString, 'utf-8' ) + # encodedRoomAccess = base64.urlsafe_b64encode( byte_result ) + # + # + # sign = hmac.new( bytes(secret_key, 'utf-8') , encodedRoomAccess, hashlib.sha1).digest() + # encodedSign = base64.urlsafe_b64encode( sign ) + # #roomToken = bytes(access_key, 'utf-8') + bytes(':','utf-8') + encodedSign + bytes(':','utf-8') + encodedRoomAccess + # roomToken = access_key + ':' + str(encodedSign, encoding = "utf-8") + ':' + str(encodedRoomAccess, encoding = "utf-8") + # + # print ( access_key ) + # print ( encodedSign ) + # print ( encodedRoomAccess ) + # return roomToken + + + + + + + + + # roomAccessString = json.dumps(roomAccess) + # byte_result = bytes(roomAccessString, 'utf-8' ) + # encodedRoomAccess = base64.urlsafe_b64encode( byte_result ) + # print ( type(encodedRoomAccess ) ) + # #print (encodedRoomAccess) + # + # sign = hmac.new( bytes( secret_key, 'utf-8' ) , encodedRoomAccess, hashlib.sha1).digest() + # encodedSign = base64.urlsafe_b64encode( sign ) + # #roomToken = bytes(access_key, 'utf-8') + bytes(':','utf-8') + encodedSign + bytes(':','utf-8') + encodedRoomAccess + # roomToken = access_key + ':' + str(encodedSign, encoding = "utf-8") + ':' + str(encodedRoomAccess, encoding = "utf-8") + # + # print ( access_key ) + # print ( encodedSign ) + # print ( encodedRoomAccess ) + # return roomToken + + + + + + + + + + From 65ecf735812ab633ea76790b2306cc4b8ccefbe0 Mon Sep 17 00:00:00 2001 From: zhangyunchuan <zhangyunchuan@qiniu.com> Date: Thu, 19 Apr 2018 11:05:27 +0800 Subject: [PATCH 274/478] 1st edition --- examples/rtc_server.py | 199 ++++++++++++++++++----------------------- 1 file changed, 89 insertions(+), 110 deletions(-) diff --git a/examples/rtc_server.py b/examples/rtc_server.py index f8789e2b..92476559 100644 --- a/examples/rtc_server.py +++ b/examples/rtc_server.py @@ -1,125 +1,104 @@ #-*- coding: utf-8 -*- -#flake8: noqa -# from qiniu import Auth -# from qiniu import BucketManager - -# access_key = '...' -# secret_key = '...' -# -# q = Auth(access_key, secret_key) -# bucket = BucketManager(q) -# -# bucket_name = 'Bucket_Name' -# # 前缀 -# prefix = None -# # 列举条目 -# limit = 10 -# # 列举出除'/'的所有文件以及以'/'为分隔的所有前缀 -# delimiter = None -# # 标记 -# marker = None -# -# ret, eof, info = bucket.list(bucket_name, prefix, marker, limit, delimiter) -# -# print(info) -# -# assert len(ret.get('items')) is not None - -import sys -sys.path.append('/Users/yunchuanzhang/python3/lib/python3.5/site-packages/python-sdk/qiniu') from qiniu import QiniuMacAuth from qiniu import RtcServer, RtcRoomToken - +import time # UID 1380668373 PILI—VDN内部测试账号 -access_key = 'DXFtikq1YuDT_WMUntOpzpWPm2UZVtEnYvN3-CUD' -secret_key = 'F397hzMohpORVZ-bBbb-IVbpdWlI4SWu8sWq78v3' - -q = QiniuMacAuth(access_key, secret_key ) - -rtc = RtcServer(q) - -#print ( rtc.GetApp() ) -#print ('\n\n\n') -# -# -# create_data={ -# "hub": 'python_test_hub', -# "title": 'python_test_app', -# # "maxUsers": MaxUsers, -# # "noAutoCloseRoom": NoAutoCloseRoom, -# # "noAutoCreateRoom": NoAutoCreateRoom, -# # "noAutoKickUser": NoAutoKickUser -# } -# print ( rtc.CreateApp(create_data) ) -# print ('\n\n\n') -# -# -# print ( rtc.DeleteApp('desls83s2') ) -# print ('\n\n\n') -# -# -# update_data={ -# "hub": "python_new_hub", -# "title": "python_new_app", -# # "maxUsers": <MaxUsers>, -# # "noAutoCloseRoom": <NoAutoCloseRoom>, -# # "noAutoCreateRoom": <NoAutoCreateRoom>, -# # "noAutoKickUser": <NoAutoKickUser>, -# # "mergePublishRtmp": { -# # "enable": <Enable>, -# # "audioOnly": <AudioOnly>, -# # "height": <OutputHeight>, -# # "width": <OutputHeight>, -# # "fps": <OutputFps>, -# # "kbps": <OutputKbps>, -# # "url": "<URL>", -# # "streamTitle": "<StreamTitle>" -# # } -# } -# print ( rtc.UpdateApp('desmfnkw5', update_data) ) -# print ('\n\n\n') -# - -# print ( rtc.ListUser( 'd7rqwfxqd','test' ) ) -# print ('\n\n\n') -# -# print ( rtc.KickUser( 'd7rqwfxqd', 'test' ,'test' ) ) -# print ('\n\n\n') -# -# print ( rtc.ListActiveRoom( 'd7rqwfxqd' ) ) -# print ('\n\n\n') - -# roomAccess = { -# "appId": "d7rqwfxqd" , -# "roomName": "RoomName" , -# "userId": "UserID" , -# "expireAt": "1524056400" , -# "permission": "user" -# } -# print (RtcRoomToken ( access_key, secret_key, roomAccess ) ) -# print ('\n\n\n') - - -# access_key='gwd_gV4gPKZZsmEOvAuNU1AcumicmuHooTfu64q5' -# secret_key='9G4isTkVuj5ITPqH1ajhljJMTc2k4m-hZh5r5ZsK' -# -# roomAccess = { -# "appId": "desobxqpx" , -# "roomName": "lfxl" , -# "userId": "1111" , -# "expireAt": "" , -# "permission": "" -# } -# print (RtcRoomToken ( access_key, secret_key, roomAccess ) ) -# +# 需要填写你的 Access Key 和 Secret Key +access_key = '...' +secret_key = '...' +assert access_key != '...' and secret_key != '...' , '你必须填写你自己七牛账号的密钥，密钥地址：https://developer.qiniu.com/kodo/kb/1334/the-access-key-secret-key-encryption-key-safe-use-instructions' +# 构建鉴权对象 +q = QiniuMacAuth(access_key, secret_key ) +# 构建直播连麦管理对象 +rtc = RtcServer(q) +# 创建一个APP +# 首先需要写好创建APP的各个参数。参数如下 +create_data={ + "hub": 'python_test_hub', # Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 + "title": 'python_test_app', # Title: app 的名称，可选，注意，Title 不是唯一标识，重复 create 动作将生成多个 app。 + # "maxUsers": MaxUsers, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 + # "noAutoCloseRoom": NoAutoCloseRoom, # NoAutoCloseRoom: bool 类型，可选，禁止自动关闭房间。默认为 false ，即用户退出房间后，房间会被主动清理释放。 + # "noAutoCreateRoom": NoAutoCreateRoom, # NoAutoCreateRoom: bool 类型，可选，禁止自动创建房间。默认为 false ，即不需要主动调用接口创建即可加入房间。 + # "noAutoKickUser": NoAutoKickUser # NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ，即同一个身份的 client (app/room/user) ，新的连麦请求可以成功，旧连接被关闭。 +} +# 然后运行 rtc.CreateApp(<创建APP相关参数的字典变量>) +print ( rtc.CreateApp(create_data) ) +print ('\n\n\n') + + +# 查询一个APP +# 查询某一个具体的APP的相关信息的方法为 print ( rtc.GetApp(<AppID>) ) ，其中 AppID是类似 'desls83s2' 这样在创建时由七牛自动生成的数字字母乱序组合的字符串 +# 如果不指定具体的AppID，直接运行 print ( rtc.GetApp() ) ，那么就会列举出该账号下所有的APP +print ( rtc.GetApp() ) +print ('\n\n\n') + + +# 删除一个APP +# 使用方法为：rtc.DeleteApp(<AppID>)，例如： rtc.DeleteApp('desls83s2') +print ( rtc.DeleteApp('<AppID>:必填') ) +print ('\n\n\n') + + +# 更新一个APP的相关参数 +# 首先需要写好更新的APP的各个参数。参数如下： +update_data={ + "hub": "python_new_hub", # Hub: 绑定的直播 hub，可选，用于合流后 rtmp 推流。 + "title": "python_new_app", # Title: app 的名称， 可选。 + # "maxUsers": <MaxUsers>, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 + # "noAutoCloseRoom": <NoAutoCloseRoom>, # NoAutoCloseRoom: bool 指针类型，可选，true 表示禁止自动关闭房间。 + # "noAutoCreateRoom": <NoAutoCreateRoom>, # NoAutoCreateRoom: bool 指针指型，可选，true 表示禁止自动创建房间。 + # "noAutoKickUser": <NoAutoKickUser>, # NoAutoKickUser: bool 类型，可选，禁止自动踢人。 + # "mergePublishRtmp": { # MergePublishRtmp: 连麦合流转推 RTMP 的配置，可选择。其详细配置包括如下 + # "enable": <Enable>, # Enable: 布尔类型，用于开启和关闭所有房间的合流功能。 + # "audioOnly": <AudioOnly>, # AudioOnly: 布尔类型，可选，指定是否只合成音频。 + # "height": <OutputHeight>, # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 + # "width": <OutputHeight>, # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 + # "fps": <OutputFps>, # OutputFps: int64，可选，指定合流输出的帧率，默认为 25 fps 。 + # "kbps": <OutputKbps>, # OutputKbps: int64，可选，指定合流输出的码率，默认为 1000 。 + # "url": "<URL>", # URL: 合流后转推旁路直播的地址，可选，支持魔法变量配置按照连麦房间号生成不同的推流地址。如果是转推到七牛直播云，不建议使用该配置。 + # "streamTitle": "<StreamTitle>" # StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号生成不同的流名。例如，配置 Hub 为 qn-zhibo ，配置 StreamTitle 为 $(roomName) ，则房间 meeting-001 的合流将会被转推到 rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。详细配置细则，请咨询七牛技术支持。 + # } +} +# 使用方法为：rtc.UpdateApp('<AppID>:必填', update_data)，例如：AppID 是形如 desmfnkw5 的字符串 +print ( rtc.UpdateApp('<AppID>:必填', update_data) ) +print ('\n\n\n') + + +# 列举一个APP下面，某个房间的所有用户 +print ( rtc.ListUser( '<AppID>:必填','<房间名>:必填' ) ) +print ('\n\n\n') + + +# 踢出一个APP下面，某个房间的某个用户 +print ( rtc.KickUser( '<AppID>:必填', '<房间名>:必填' ,'<客户ID>:必填' ) ) +print ('\n\n\n') + + +# 列举一个APP下面，所有的房间 +print ( rtc.ListActiveRoom( '<AppID>:必填' ) ) +print ('\n\n\n') + + + +# 计算房间管理鉴权。连麦用户终端通过房间管理鉴权获取七牛连麦服务 +# 首先需要写好房间鉴权的各个参数。参数如下： +roomAccess = { + "appId": "<AppID>:必填" , # AppID: 房间所属帐号的 app 。 + "roomName": "<房间名>:必填" , # RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ + "userId": "<用户名>:必填" , # UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ + "expireAt": int( time.time()) + 3600 , # ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间，token 将在该时间后失效。 + "permission": "user" # 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时，拥有将其他用户移除出房间等特权. + } +# 获得房间管理鉴权的方法：print (RtcRoomToken ( access_key, secret_key, roomAccess ) ) +print (RtcRoomToken ( access_key, secret_key, roomAccess ) ) From eebd6ae73aacbebdd6c456df2befe03dab55b6e3 Mon Sep 17 00:00:00 2001 From: zhangyunchuan <zhangyunchuan@qiniu.com> Date: Thu, 19 Apr 2018 11:07:56 +0800 Subject: [PATCH 275/478] 1st edition --- qiniu/services/pili/rtc_server_manager.py | 73 +++-------------------- 1 file changed, 9 insertions(+), 64 deletions(-) diff --git a/qiniu/services/pili/rtc_server_manager.py b/qiniu/services/pili/rtc_server_manager.py index 07a64c4e..6bc60a4f 100644 --- a/qiniu/services/pili/rtc_server_manager.py +++ b/qiniu/services/pili/rtc_server_manager.py @@ -5,6 +5,12 @@ class RtcServer(object): """ + 直播连麦管理类 + 主要涉及了直播连麦管理及操作接口的实现，具体的接口规格可以参考： + https://github.com/pili-engineering/QNRTC-Server/blob/master/docs/api.md#41-listuser #这个是内部文档，等外部文档发布了，这一行要换成外部文档 + + Attributes: + auth: 账号管理密钥对，Auth对象 """ def __init__(self, auth): @@ -351,81 +357,20 @@ def RtcRoomToken(access_key, secret_key, roomAccess ): # 3. 将AccessKey与以上两者拼接得到房间鉴权 roomToken = "<AccessKey>" + ":" + encodedSign + ":" + encodedRoomAccess """ - def b(data): - return bytes(data) - - def urlsafe_base64_encode(data): - ret = base64.urlsafe_b64encode(b(data)) - return b(ret) - - def pre_token(data, SecretKey): - data = b(data) - SecretKey = b(SecretKey) - hashed = hmac.new(SecretKey, data, hashlib.sha1) - return urlsafe_base64_encode(hashed.digest()) - - roomAccessString = json.dumps(roomAccess) byte_result = bytes(roomAccessString, 'utf-8' ) - encodedRoomAccess = base64.urlsafe_b64encode( byte_result ) + encodedRoomAccess = base64.urlsafe_b64encode( byte_result ) - encodedSign = pre_token( encodedRoomAccess, secret_key ) - - #roomToken = bytes(access_key, 'utf-8') + bytes(':','utf-8') + encodedSign + bytes(':','utf-8') + encodedRoomAccess + sign = hmac.new( bytes(secret_key, 'utf-8') , encodedRoomAccess, hashlib.sha1).digest() + encodedSign = base64.urlsafe_b64encode( sign ) roomToken = access_key + ':' + str(encodedSign, encoding = "utf-8") + ':' + str(encodedRoomAccess, encoding = "utf-8") - print ( access_key ) - print ( encodedSign ) - print ( encodedRoomAccess ) return roomToken - # - # roomAccessString = json.dumps(roomAccess) - # byte_result = bytes(roomAccessString, 'utf-8' ) - # encodedRoomAccess = base64.urlsafe_b64encode( byte_result ) - # - # - # sign = hmac.new( bytes(secret_key, 'utf-8') , encodedRoomAccess, hashlib.sha1).digest() - # encodedSign = base64.urlsafe_b64encode( sign ) - # #roomToken = bytes(access_key, 'utf-8') + bytes(':','utf-8') + encodedSign + bytes(':','utf-8') + encodedRoomAccess - # roomToken = access_key + ':' + str(encodedSign, encoding = "utf-8") + ':' + str(encodedRoomAccess, encoding = "utf-8") - # - # print ( access_key ) - # print ( encodedSign ) - # print ( encodedRoomAccess ) - # return roomToken - - - - - - - - - # roomAccessString = json.dumps(roomAccess) - # byte_result = bytes(roomAccessString, 'utf-8' ) - # encodedRoomAccess = base64.urlsafe_b64encode( byte_result ) - # print ( type(encodedRoomAccess ) ) - # #print (encodedRoomAccess) - # - # sign = hmac.new( bytes( secret_key, 'utf-8' ) , encodedRoomAccess, hashlib.sha1).digest() - # encodedSign = base64.urlsafe_b64encode( sign ) - # #roomToken = bytes(access_key, 'utf-8') + bytes(':','utf-8') + encodedSign + bytes(':','utf-8') + encodedRoomAccess - # roomToken = access_key + ':' + str(encodedSign, encoding = "utf-8") + ':' + str(encodedRoomAccess, encoding = "utf-8") - # - # print ( access_key ) - # print ( encodedSign ) - # print ( encodedRoomAccess ) - # return roomToken - - - - - From be7dfb313a0f6eeba142e605293b4cee02cc9827 Mon Sep 17 00:00:00 2001 From: zhangyunchuan <zhangyunchuan@qiniu.com> Date: Tue, 8 May 2018 15:47:59 +0800 Subject: [PATCH 276/478] =?UTF-8?q?=E5=85=A8=E9=83=A8=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/rtc_server.py | 7 +------ qiniu/services/pili/rtc_server_manager.py | 20 -------------------- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/examples/rtc_server.py b/examples/rtc_server.py index 92476559..12ffeef8 100644 --- a/examples/rtc_server.py +++ b/examples/rtc_server.py @@ -4,7 +4,6 @@ from qiniu import RtcServer, RtcRoomToken import time -# UID 1380668373 PILI—VDN内部测试账号 # 需要填写你的 Access Key 和 Secret Key access_key = '...' @@ -25,8 +24,6 @@ "hub": 'python_test_hub', # Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 "title": 'python_test_app', # Title: app 的名称，可选，注意，Title 不是唯一标识，重复 create 动作将生成多个 app。 # "maxUsers": MaxUsers, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 - # "noAutoCloseRoom": NoAutoCloseRoom, # NoAutoCloseRoom: bool 类型，可选，禁止自动关闭房间。默认为 false ，即用户退出房间后，房间会被主动清理释放。 - # "noAutoCreateRoom": NoAutoCreateRoom, # NoAutoCreateRoom: bool 类型，可选，禁止自动创建房间。默认为 false ，即不需要主动调用接口创建即可加入房间。 # "noAutoKickUser": NoAutoKickUser # NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ，即同一个身份的 client (app/room/user) ，新的连麦请求可以成功，旧连接被关闭。 } # 然后运行 rtc.CreateApp(<创建APP相关参数的字典变量>) @@ -37,7 +34,7 @@ # 查询一个APP # 查询某一个具体的APP的相关信息的方法为 print ( rtc.GetApp(<AppID>) ) ，其中 AppID是类似 'desls83s2' 这样在创建时由七牛自动生成的数字字母乱序组合的字符串 # 如果不指定具体的AppID，直接运行 print ( rtc.GetApp() ) ，那么就会列举出该账号下所有的APP -print ( rtc.GetApp() ) +print ( rtc.GetApp('<AppID>:可选填') ) print ('\n\n\n') @@ -53,8 +50,6 @@ "hub": "python_new_hub", # Hub: 绑定的直播 hub，可选，用于合流后 rtmp 推流。 "title": "python_new_app", # Title: app 的名称， 可选。 # "maxUsers": <MaxUsers>, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 - # "noAutoCloseRoom": <NoAutoCloseRoom>, # NoAutoCloseRoom: bool 指针类型，可选，true 表示禁止自动关闭房间。 - # "noAutoCreateRoom": <NoAutoCreateRoom>, # NoAutoCreateRoom: bool 指针指型，可选，true 表示禁止自动创建房间。 # "noAutoKickUser": <NoAutoKickUser>, # NoAutoKickUser: bool 类型，可选，禁止自动踢人。 # "mergePublishRtmp": { # MergePublishRtmp: 连麦合流转推 RTMP 的配置，可选择。其详细配置包括如下 # "enable": <Enable>, # Enable: 布尔类型，用于开启和关闭所有房间的合流功能。 diff --git a/qiniu/services/pili/rtc_server_manager.py b/qiniu/services/pili/rtc_server_manager.py index 6bc60a4f..69bb6a67 100644 --- a/qiniu/services/pili/rtc_server_manager.py +++ b/qiniu/services/pili/rtc_server_manager.py @@ -28,8 +28,6 @@ def CreateApp(self,data): "hub": "<Hub>", "title": "<Title>", "maxUsers": <MaxUsers>, - "noAutoCloseRoom": <NoAutoCloseRoom>, - "noAutoCreateRoom": <NoAutoCreateRoom>, "noAutoKickUser": <NoAutoKickUser> } @@ -40,10 +38,6 @@ def CreateApp(self,data): MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 - NoAutoCloseRoom: bool 类型，可选，禁止自动关闭房间。默认为 false ，即用户退出房间后，房间会被主动清理释放。 - - NoAutoCreateRoom: bool 类型，可选，禁止自动创建房间。默认为 false ，即不需要主动调用接口创建即可加入房间。 - NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ，即同一个身份的 client (app/room/user) ，新的连麦请求可以成功，旧连接被关闭。 :return: @@ -53,8 +47,6 @@ def CreateApp(self,data): "hub": "<Hub>", "title": "<Title>", "maxUsers": <MaxUsers>, - "noAutoCloseRoom": <NoAutoCloseRoom>, - "noAutoCreateRoom": <NoAutoCreateRoom>, "noAutoKickUser": <NoAutoKickUser>, "createdAt": <CreatedAt>, "updatedAt": <UpdatedAt> @@ -85,8 +77,6 @@ def GetApp(self, appid=None): "hub": "<Hub>", "title": "<Title>", "maxUsers": <MaxUsers>, - "noAutoCloseRoom": <NoAutoCloseRoom>, - "noAutoCreateRoom": <NoAutoCreateRoom>, "noAutoKickUser": <NoAutoKickUser>, "mergePublishRtmp": { "audioOnly": <AudioOnly>, @@ -117,10 +107,6 @@ def GetApp(self, appid=None): MaxUsers: int 类型，连麦房间支持的最大在线人数。 - NoAutoCloseRoom: bool 类型，禁止自动关闭房间。 - - NoAutoCreateRoom: bool 类型，禁止自动创建房间。 - NoAutoKickUser: bool 类型，禁止自动踢人。 MergePublishRtmp: 连麦合流转推 RTMP 的配置。 @@ -165,10 +151,6 @@ def UpdateApp(self, appid, data): MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 - NoAutoCloseRoom: bool 指针类型，可选，true 表示禁止自动关闭房间。 - - NoAutoCreateRoom: bool 指针指型，可选，true 表示禁止自动创建房间。 - NoAutoKickUser: bool 类型，可选，禁止自动踢人。 MergePublishRtmp: 连麦合流转推 RTMP 的配置，可选择。其详细配置包括如下 @@ -188,8 +170,6 @@ def UpdateApp(self, appid, data): "hub": "<Hub>", "title": "<Title>", "maxUsers": <MaxUsers>, - "noAutoCloseRoom": <NoAutoCloseRoom>, - "noAutoCreateRoom": <NoAutoCreateRoom>, "noAutoKickUser": <NoAutoKickUser>, "mergePublishRtmp": { "enable": <Enable>, From c6852a604f95d84b969d6ebe3fb506f5db9d9428 Mon Sep 17 00:00:00 2001 From: zhangyunchuan <zhangyunchuan@qiniu.com> Date: Wed, 9 May 2018 14:08:43 +0800 Subject: [PATCH 277/478] =?UTF-8?q?Code/Reformat=20Code=20=20#=20=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/rtc_server.py | 85 ++++++++++------------- qiniu/services/pili/rtc_server_manager.py | 49 +++++-------- 2 files changed, 54 insertions(+), 80 deletions(-) diff --git a/examples/rtc_server.py b/examples/rtc_server.py index 12ffeef8..32f2520f 100644 --- a/examples/rtc_server.py +++ b/examples/rtc_server.py @@ -1,100 +1,85 @@ -#-*- coding: utf-8 -*- +# -*- coding: utf-8 -*- from qiniu import QiniuMacAuth from qiniu import RtcServer, RtcRoomToken import time - # 需要填写你的 Access Key 和 Secret Key access_key = '...' secret_key = '...' -assert access_key != '...' and secret_key != '...' , '你必须填写你自己七牛账号的密钥，密钥地址：https://developer.qiniu.com/kodo/kb/1334/the-access-key-secret-key-encryption-key-safe-use-instructions' - +assert access_key != '...' and secret_key != '...', '你必须填写你自己七牛账号的密钥，密钥地址：https://developer.qiniu.com/kodo/kb/1334/the-access-key-secret-key-encryption-key-safe-use-instructions' # 构建鉴权对象 -q = QiniuMacAuth(access_key, secret_key ) +q = QiniuMacAuth(access_key, secret_key) # 构建直播连麦管理对象 rtc = RtcServer(q) - # 创建一个APP # 首先需要写好创建APP的各个参数。参数如下 -create_data={ - "hub": 'python_test_hub', # Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 - "title": 'python_test_app', # Title: app 的名称，可选，注意，Title 不是唯一标识，重复 create 动作将生成多个 app。 +create_data = { + "hub": 'python_test_hub', # Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 + "title": 'python_test_app', # Title: app 的名称，可选，注意，Title 不是唯一标识，重复 create 动作将生成多个 app。 # "maxUsers": MaxUsers, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 # "noAutoKickUser": NoAutoKickUser # NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ，即同一个身份的 client (app/room/user) ，新的连麦请求可以成功，旧连接被关闭。 } # 然后运行 rtc.CreateApp(<创建APP相关参数的字典变量>) -print ( rtc.CreateApp(create_data) ) +print (rtc.CreateApp(create_data)) print ('\n\n\n') - # 查询一个APP # 查询某一个具体的APP的相关信息的方法为 print ( rtc.GetApp(<AppID>) ) ，其中 AppID是类似 'desls83s2' 这样在创建时由七牛自动生成的数字字母乱序组合的字符串 # 如果不指定具体的AppID，直接运行 print ( rtc.GetApp() ) ，那么就会列举出该账号下所有的APP -print ( rtc.GetApp('<AppID>:可选填') ) +print (rtc.GetApp('<AppID>:可选填')) print ('\n\n\n') - # 删除一个APP # 使用方法为：rtc.DeleteApp(<AppID>)，例如： rtc.DeleteApp('desls83s2') -print ( rtc.DeleteApp('<AppID>:必填') ) +print (rtc.DeleteApp('<AppID>:必填')) print ('\n\n\n') - # 更新一个APP的相关参数 # 首先需要写好更新的APP的各个参数。参数如下： -update_data={ - "hub": "python_new_hub", # Hub: 绑定的直播 hub，可选，用于合流后 rtmp 推流。 - "title": "python_new_app", # Title: app 的名称， 可选。 - # "maxUsers": <MaxUsers>, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 - # "noAutoKickUser": <NoAutoKickUser>, # NoAutoKickUser: bool 类型，可选，禁止自动踢人。 - # "mergePublishRtmp": { # MergePublishRtmp: 连麦合流转推 RTMP 的配置，可选择。其详细配置包括如下 - # "enable": <Enable>, # Enable: 布尔类型，用于开启和关闭所有房间的合流功能。 - # "audioOnly": <AudioOnly>, # AudioOnly: 布尔类型，可选，指定是否只合成音频。 - # "height": <OutputHeight>, # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 - # "width": <OutputHeight>, # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 - # "fps": <OutputFps>, # OutputFps: int64，可选，指定合流输出的帧率，默认为 25 fps 。 - # "kbps": <OutputKbps>, # OutputKbps: int64，可选，指定合流输出的码率，默认为 1000 。 - # "url": "<URL>", # URL: 合流后转推旁路直播的地址，可选，支持魔法变量配置按照连麦房间号生成不同的推流地址。如果是转推到七牛直播云，不建议使用该配置。 - # "streamTitle": "<StreamTitle>" # StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号生成不同的流名。例如，配置 Hub 为 qn-zhibo ，配置 StreamTitle 为 $(roomName) ，则房间 meeting-001 的合流将会被转推到 rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。详细配置细则，请咨询七牛技术支持。 - # } +update_data = { + "hub": "python_new_hub", # Hub: 绑定的直播 hub，可选，用于合流后 rtmp 推流。 + "title": "python_new_app", # Title: app 的名称， 可选。 + # "maxUsers": <MaxUsers>, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 + # "noAutoKickUser": <NoAutoKickUser>, # NoAutoKickUser: bool 类型，可选，禁止自动踢人。 + # "mergePublishRtmp": { # MergePublishRtmp: 连麦合流转推 RTMP 的配置，可选择。其详细配置包括如下 + # "enable": <Enable>, # Enable: 布尔类型，用于开启和关闭所有房间的合流功能。 + # "audioOnly": <AudioOnly>, # AudioOnly: 布尔类型，可选，指定是否只合成音频。 + # "height": <OutputHeight>, # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 + # "width": <OutputHeight>, # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 + # "fps": <OutputFps>, # OutputFps: int64，可选，指定合流输出的帧率，默认为 25 fps 。 + # "kbps": <OutputKbps>, # OutputKbps: int64，可选，指定合流输出的码率，默认为 1000 。 + # "url": "<URL>", # URL: 合流后转推旁路直播的地址，可选，支持魔法变量配置按照连麦房间号生成不同的推流地址。如果是转推到七牛直播云，不建议使用该配置。 + # "streamTitle": "<StreamTitle>" # StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号生成不同的流名。例如，配置 Hub 为 qn-zhibo ，配置 StreamTitle 为 $(roomName) ，则房间 meeting-001 的合流将会被转推到 rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。详细配置细则，请咨询七牛技术支持。 + # } } # 使用方法为：rtc.UpdateApp('<AppID>:必填', update_data)，例如：AppID 是形如 desmfnkw5 的字符串 -print ( rtc.UpdateApp('<AppID>:必填', update_data) ) +print (rtc.UpdateApp('<AppID>:必填', update_data)) print ('\n\n\n') - # 列举一个APP下面，某个房间的所有用户 -print ( rtc.ListUser( '<AppID>:必填','<房间名>:必填' ) ) +print (rtc.ListUser('<AppID>:必填', '<房间名>:必填')) print ('\n\n\n') - # 踢出一个APP下面，某个房间的某个用户 -print ( rtc.KickUser( '<AppID>:必填', '<房间名>:必填' ,'<客户ID>:必填' ) ) +print (rtc.KickUser('<AppID>:必填', '<房间名>:必填', '<客户ID>:必填')) print ('\n\n\n') - # 列举一个APP下面，所有的房间 -print ( rtc.ListActiveRoom( '<AppID>:必填' ) ) +print (rtc.ListActiveRoom('<AppID>:必填')) print ('\n\n\n') - - # 计算房间管理鉴权。连麦用户终端通过房间管理鉴权获取七牛连麦服务 # 首先需要写好房间鉴权的各个参数。参数如下： roomAccess = { - "appId": "<AppID>:必填" , # AppID: 房间所属帐号的 app 。 - "roomName": "<房间名>:必填" , # RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ - "userId": "<用户名>:必填" , # UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ - "expireAt": int( time.time()) + 3600 , # ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间，token 将在该时间后失效。 - "permission": "user" # 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时，拥有将其他用户移除出房间等特权. - } + "appId": "<AppID>:必填", # AppID: 房间所属帐号的 app 。 + "roomName": "<房间名>:必填", # RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ + "userId": "<用户名>:必填", # UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ + "expireAt": int(time.time()) + 3600, # ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间，token 将在该时间后失效。 + "permission": "user" # 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时，拥有将其他用户移除出房间等特权. +} # 获得房间管理鉴权的方法：print (RtcRoomToken ( access_key, secret_key, roomAccess ) ) -print (RtcRoomToken ( access_key, secret_key, roomAccess ) ) - - - - +print (RtcRoomToken(access_key, secret_key, roomAccess)) diff --git a/qiniu/services/pili/rtc_server_manager.py b/qiniu/services/pili/rtc_server_manager.py index 69bb6a67..12bc20cb 100644 --- a/qiniu/services/pili/rtc_server_manager.py +++ b/qiniu/services/pili/rtc_server_manager.py @@ -13,11 +13,12 @@ class RtcServer(object): auth: 账号管理密钥对，Auth对象 """ + def __init__(self, auth): self.auth = auth self.host = 'http://rtc.qiniuapi.com' - def CreateApp(self,data): + def CreateApp(self, data): """ Host rtc.qiniuapi.com POST /v3/apps @@ -57,9 +58,7 @@ def CreateApp(self,data): } """ - - return self.__post(self.host+'/v3/apps',data,) - + return self.__post(self.host + '/v3/apps', data, ) def GetApp(self, appid=None): """ @@ -116,9 +115,9 @@ def GetApp(self, appid=None): UpdatedAt: time 类型，app 更新的时间。 """ if appid: - return self.__get(self.host+'/v3/apps/%s' % appid ) + return self.__get(self.host + '/v3/apps/%s' % appid) else: - return self.__get(self.host+'/v3/apps' ) + return self.__get(self.host + '/v3/apps') def DeleteApp(self, appid): """ @@ -134,7 +133,7 @@ def DeleteApp(self, appid): "error": "app not found" } """ - return self.__delete(self.host+'/v3/apps/%s' % appid) + return self.__delete(self.host + '/v3/apps/%s' % appid) def UpdateApp(self, appid, data): """ @@ -195,7 +194,7 @@ def UpdateApp(self, appid, data): } """ - return self.__post(self.host+'/v3/apps/%s' % appid , data,) + return self.__post(self.host + '/v3/apps/%s' % appid, data, ) def ListUser(self, AppID, RoomName): """ @@ -222,7 +221,7 @@ def ListUser(self, AppID, RoomName): "error": "app not found" } """ - return self.__get( self.host+'/v3/apps/%s/rooms/%s/users' % (AppID, RoomName)) + return self.__get(self.host + '/v3/apps/%s/rooms/%s/users' % (AppID, RoomName)) def KickUser(self, AppID, RoomName, UserID): """ @@ -252,7 +251,7 @@ def KickUser(self, AppID, RoomName, UserID): "error": "room not active" } """ - return self.__delete( self.host+'/v3/apps/%s/rooms/%s/users/%s' % (AppID, RoomName, UserID) ) + return self.__delete(self.host + '/v3/apps/%s/rooms/%s/users/%s' % (AppID, RoomName, UserID)) def ListActiveRoom(self, AppID, RoomNamePrefix=None): """ @@ -291,21 +290,21 @@ def ListActiveRoom(self, AppID, RoomNamePrefix=None): RoomName: 当前活跃的房间名。 """ if RoomNamePrefix: - return self.__get( self.host+'/v3/apps/%s/rooms?prefix=%s' % (AppID, RoomNamePrefix) ) + return self.__get(self.host + '/v3/apps/%s/rooms?prefix=%s' % (AppID, RoomNamePrefix)) else: - return self.__get( self.host+'/v3/apps/%s/rooms' % AppID ) + return self.__get(self.host + '/v3/apps/%s/rooms' % AppID) def __post(self, url, data=None): return http._post_with_qiniu_mac(url, data, self.auth) def __get(self, url, params=None): - return http._get_with_qiniu_mac(url,params,self.auth) + return http._get_with_qiniu_mac(url, params, self.auth) def __delete(self, url, params=None): - return http._delete_with_qiniu_mac(url,params,self.auth) + return http._delete_with_qiniu_mac(url, params, self.auth) -def RtcRoomToken(access_key, secret_key, roomAccess ): +def RtcRoomToken(access_key, secret_key, roomAccess): """ :arg: AppID: 房间所属帐号的 app 。 @@ -338,21 +337,11 @@ def RtcRoomToken(access_key, secret_key, roomAccess ): roomToken = "<AccessKey>" + ":" + encodedSign + ":" + encodedRoomAccess """ roomAccessString = json.dumps(roomAccess) - byte_result = bytes(roomAccessString, 'utf-8' ) - encodedRoomAccess = base64.urlsafe_b64encode( byte_result ) + byte_result = bytes(roomAccessString, 'utf-8') + encodedRoomAccess = base64.urlsafe_b64encode(byte_result) - sign = hmac.new( bytes(secret_key, 'utf-8') , encodedRoomAccess, hashlib.sha1).digest() - encodedSign = base64.urlsafe_b64encode( sign ) - roomToken = access_key + ':' + str(encodedSign, encoding = "utf-8") + ':' + str(encodedRoomAccess, encoding = "utf-8") + sign = hmac.new(bytes(secret_key, 'utf-8'), encodedRoomAccess, hashlib.sha1).digest() + encodedSign = base64.urlsafe_b64encode(sign) + roomToken = access_key + ':' + str(encodedSign, encoding="utf-8") + ':' + str(encodedRoomAccess, encoding="utf-8") return roomToken - - - - - - - - - - From 1eef2da4dc5a818ced06d1b9a81bd91a0eae4452 Mon Sep 17 00:00:00 2001 From: zhangyunchuan <zhangyunchuan@qiniu.com> Date: Wed, 9 May 2018 14:23:45 +0800 Subject: [PATCH 278/478] =?UTF-8?q?=E7=B1=BB=E5=90=8D=E4=B8=BAHelloWorld?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=EF=BC=8C=E5=87=BD=E6=95=B0=E5=90=8Dhello=5Fw?= =?UTF-8?q?orld=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/workspace.xml | 451 ++++++++++++++++++++++ examples/rtc_server.py | 25 +- qiniu/__init__.py | 2 +- qiniu/services/pili/rtc_server_manager.py | 16 +- 4 files changed, 469 insertions(+), 25 deletions(-) create mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 00000000..e4c11eb5 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,451 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="ChangeListManager"> + <list default="true" id="476ff082-c46b-43e0-8891-c14f75fd6b58" name="Default" comment=""> + <change type="MODIFICATION" beforePath="$PROJECT_DIR$/examples/rtc_server.py" afterPath="$PROJECT_DIR$/examples/rtc_server.py" /> + <change type="MODIFICATION" beforePath="$PROJECT_DIR$/qiniu/__init__.py" afterPath="$PROJECT_DIR$/qiniu/__init__.py" /> + <change type="MODIFICATION" beforePath="$PROJECT_DIR$/qiniu/services/pili/rtc_server_manager.py" afterPath="$PROJECT_DIR$/qiniu/services/pili/rtc_server_manager.py" /> + </list> + <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" /> + <option name="TRACKING_ENABLED" value="true" /> + <option name="SHOW_DIALOG" value="false" /> + <option name="HIGHLIGHT_CONFLICTS" value="true" /> + <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" /> + <option name="LAST_RESOLUTION" value="IGNORE" /> + </component> + <component name="CreatePatchCommitExecutor"> + <option name="PATCH_PATH" value="" /> + </component> + <component name="ExecutionTargetManager" SELECTED_TARGET="default_target" /> + <component name="FileEditorManager"> + <leaf> + <file leaf-file-name="rtc_server_manager.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/qiniu/services/pili/rtc_server_manager.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="488"> + <caret line="255" column="20" lean-forward="false" selection-start-line="255" selection-start-column="20" selection-end-line="255" selection-end-column="20" /> + <folding> + <element signature="e#24#46#0" expanded="true" /> + </folding> + </state> + </provider> + </entry> + </file> + <file leaf-file-name="rtc_server.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/examples/rtc_server.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="494"> + <caret line="27" column="0" lean-forward="true" selection-start-line="27" selection-start-column="0" selection-end-line="27" selection-end-column="0" /> + <folding> + <element signature="e#25#55#0" expanded="true" /> + </folding> + </state> + </provider> + </entry> + </file> + <file leaf-file-name="bucket.py" pinned="false" current-in-tab="true"> + <entry file="file://$PROJECT_DIR$/qiniu/services/storage/bucket.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="38"> + <caret line="2" column="0" lean-forward="true" selection-start-line="2" selection-start-column="0" selection-end-line="2" selection-end-column="0" /> + <folding> + <element signature="e#25#49#0" expanded="true" /> + </folding> + </state> + </provider> + </entry> + </file> + <file leaf-file-name="app.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/qiniu/services/compute/app.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="19"> + <caret line="1" column="0" lean-forward="true" selection-start-line="1" selection-start-column="0" selection-end-line="1" selection-end-column="0" /> + <folding> + <element signature="e#24#60#0" expanded="true" /> + </folding> + </state> + </provider> + </entry> + </file> + <file leaf-file-name="upload_progress_recorder.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/qiniu/services/storage/upload_progress_recorder.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="-234"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding /> + </state> + </provider> + </entry> + </file> + <file leaf-file-name="pfop.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/qiniu/services/processing/pfop.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="-44"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding /> + </state> + </provider> + </entry> + </file> + <file leaf-file-name="cmd.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/qiniu/services/processing/cmd.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="418"> + <caret line="22" column="0" lean-forward="true" selection-start-line="22" selection-start-column="0" selection-end-line="22" selection-end-column="0" /> + <folding /> + </state> + </provider> + </entry> + </file> + <file leaf-file-name="__init__.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/qiniu/__init__.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="532"> + <caret line="28" column="0" lean-forward="true" selection-start-line="28" selection-start-column="0" selection-end-line="28" selection-end-column="0" /> + <folding /> + </state> + </provider> + </entry> + </file> + <file leaf-file-name="auth.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/qiniu/auth.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="304"> + <caret line="16" column="29" lean-forward="false" selection-start-line="16" selection-start-column="29" selection-end-line="16" selection-end-column="29" /> + <folding> + <element signature="e#25#36#0" expanded="true" /> + </folding> + </state> + </provider> + </entry> + </file> + </leaf> + </component> + <component name="Git.Settings"> + <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" /> + </component> + <component name="IdeDocumentHistory"> + <option name="CHANGED_PATHS"> + <list> + <option value="$PROJECT_DIR$/qiniu/auth.py" /> + <option value="$PROJECT_DIR$/qiniu/__init__.py" /> + <option value="$PROJECT_DIR$/examples/rtc_server.py" /> + <option value="$PROJECT_DIR$/qiniu/services/pili/rtc_server_manager.py" /> + </list> + </option> + </component> + <component name="ProjectFrameBounds"> + <option name="x" value="1440" /> + <option name="y" value="-180" /> + <option name="width" value="1920" /> + <option name="height" value="1080" /> + </component> + <component name="ProjectView"> + <navigator currentView="ProjectPane" proportions="" version="1"> + <flattenPackages /> + <showMembers /> + <showModules /> + <showLibraryContents /> + <hideEmptyPackages /> + <abbreviatePackageNames /> + <autoscrollToSource /> + <autoscrollFromSource /> + <sortByType /> + <manualOrder /> + <foldersAlwaysOnTop value="true" /> + </navigator> + <panes> + <pane id="Scratches" /> + <pane id="ProjectPane"> + <subPane> + <PATH> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + </PATH> + <PATH> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="qiniu" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + </PATH> + <PATH> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="qiniu" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="services" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + </PATH> + <PATH> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="qiniu" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="services" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="storage" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + </PATH> + <PATH> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="qiniu" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="services" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="processing" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + </PATH> + <PATH> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="qiniu" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="services" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="pili" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + </PATH> + <PATH> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="qiniu" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="services" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="compute" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + </PATH> + <PATH> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="python-sdk" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="examples" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + </PATH> + </subPane> + </pane> + <pane id="Scope" /> + </panes> + </component> + <component name="PropertiesComponent"> + <property name="settings.editor.selected.configurable" value="preferences.sourceCode.Python" /> + <property name="settings.editor.splitter.proportion" value="0.2" /> + <property name="last_opened_file_path" value="$PROJECT_DIR$" /> + <property name="FullScreen" value="true" /> + </component> + <component name="ShelveChangesManager" show_recycled="false"> + <option name="remove_strategy" value="false" /> + </component> + <component name="TaskManager"> + <task active="true" id="Default" summary="Default task"> + <changelist id="476ff082-c46b-43e0-8891-c14f75fd6b58" name="Default" comment="" /> + <created>1525836017672</created> + <option name="number" value="Default" /> + <option name="presentableId" value="Default" /> + <updated>1525836017672</updated> + </task> + <servers /> + </component> + <component name="ToolWindowManager"> + <frame x="1440" y="-180" width="1920" height="1080" extended-state="0" /> + <editor active="true" /> + <layout> + <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.1864139" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" /> + <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" /> + <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" /> + <window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> + <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" /> + <window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" /> + <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> + <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> + <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" /> + <window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" /> + <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> + <window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" /> + <window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" /> + <window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" /> + <window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" /> + <window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" /> + <window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> + </layout> + </component> + <component name="VcsContentAnnotationSettings"> + <option name="myLimit" value="2678400000" /> + </component> + <component name="XDebuggerManager"> + <breakpoint-manager /> + <watches-manager /> + </component> + <component name="editorHistoryManager"> + <entry file="file://$PROJECT_DIR$/qiniu/auth.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="304"> + <caret line="16" column="29" lean-forward="false" selection-start-line="16" selection-start-column="29" selection-end-line="16" selection-end-column="29" /> + <folding> + <element signature="e#25#36#0" expanded="true" /> + </folding> + </state> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/qiniu/services/storage/upload_progress_recorder.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="-234"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding /> + </state> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/qiniu/services/processing/pfop.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="-44"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding /> + </state> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/qiniu/services/processing/cmd.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="418"> + <caret line="22" column="0" lean-forward="true" selection-start-line="22" selection-start-column="0" selection-end-line="22" selection-end-column="0" /> + <folding /> + </state> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/qiniu/services/compute/app.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="19"> + <caret line="1" column="0" lean-forward="true" selection-start-line="1" selection-start-column="0" selection-end-line="1" selection-end-column="0" /> + <folding> + <element signature="e#24#60#0" expanded="true" /> + </folding> + </state> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/qiniu/services/processing/__init__.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="0"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding /> + </state> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/qiniu/__init__.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="532"> + <caret line="28" column="0" lean-forward="true" selection-start-line="28" selection-start-column="0" selection-end-line="28" selection-end-column="0" /> + <folding /> + </state> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/qiniu/services/pili/rtc_server_manager.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="488"> + <caret line="255" column="20" lean-forward="false" selection-start-line="255" selection-start-column="20" selection-end-line="255" selection-end-column="20" /> + <folding> + <element signature="e#24#46#0" expanded="true" /> + </folding> + </state> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/examples/rtc_server.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="494"> + <caret line="27" column="0" lean-forward="true" selection-start-line="27" selection-start-column="0" selection-end-line="27" selection-end-column="0" /> + <folding> + <element signature="e#25#55#0" expanded="true" /> + </folding> + </state> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/qiniu/services/storage/bucket.py"> + <provider selected="true" editor-type-id="text-editor"> + <state relative-caret-position="38"> + <caret line="2" column="0" lean-forward="true" selection-start-line="2" selection-start-column="0" selection-end-line="2" selection-end-column="0" /> + <folding> + <element signature="e#25#49#0" expanded="true" /> + </folding> + </state> + </provider> + </entry> + </component> +</project> \ No newline at end of file diff --git a/examples/rtc_server.py b/examples/rtc_server.py index 32f2520f..6c7a8abf 100644 --- a/examples/rtc_server.py +++ b/examples/rtc_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from qiniu import QiniuMacAuth -from qiniu import RtcServer, RtcRoomToken +from qiniu import RtcServer, rtc_room_token import time # 需要填写你的 Access Key 和 Secret Key @@ -24,19 +24,16 @@ # "noAutoKickUser": NoAutoKickUser # NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ，即同一个身份的 client (app/room/user) ，新的连麦请求可以成功，旧连接被关闭。 } # 然后运行 rtc.CreateApp(<创建APP相关参数的字典变量>) -print (rtc.CreateApp(create_data)) -print ('\n\n\n') +print (rtc.create_app(create_data)) # 查询一个APP # 查询某一个具体的APP的相关信息的方法为 print ( rtc.GetApp(<AppID>) ) ，其中 AppID是类似 'desls83s2' 这样在创建时由七牛自动生成的数字字母乱序组合的字符串 # 如果不指定具体的AppID，直接运行 print ( rtc.GetApp() ) ，那么就会列举出该账号下所有的APP -print (rtc.GetApp('<AppID>:可选填')) -print ('\n\n\n') +print (rtc.get_app('<AppID>:可选填')) # 删除一个APP # 使用方法为：rtc.DeleteApp(<AppID>)，例如： rtc.DeleteApp('desls83s2') -print (rtc.DeleteApp('<AppID>:必填')) -print ('\n\n\n') +print (rtc.delete_app('<AppID>:必填')) # 更新一个APP的相关参数 # 首先需要写好更新的APP的各个参数。参数如下： @@ -57,20 +54,16 @@ # } } # 使用方法为：rtc.UpdateApp('<AppID>:必填', update_data)，例如：AppID 是形如 desmfnkw5 的字符串 -print (rtc.UpdateApp('<AppID>:必填', update_data)) -print ('\n\n\n') +print (rtc.update_app('<AppID>:必填', update_data)) # 列举一个APP下面，某个房间的所有用户 -print (rtc.ListUser('<AppID>:必填', '<房间名>:必填')) -print ('\n\n\n') +print (rtc.list_user('<AppID>:必填', '<房间名>:必填')) # 踢出一个APP下面，某个房间的某个用户 -print (rtc.KickUser('<AppID>:必填', '<房间名>:必填', '<客户ID>:必填')) -print ('\n\n\n') +print (rtc.kick_user('<AppID>:必填', '<房间名>:必填', '<客户ID>:必填')) # 列举一个APP下面，所有的房间 -print (rtc.ListActiveRoom('<AppID>:必填')) -print ('\n\n\n') +print (rtc.list_active_room('<AppID>:必填')) # 计算房间管理鉴权。连麦用户终端通过房间管理鉴权获取七牛连麦服务 # 首先需要写好房间鉴权的各个参数。参数如下： @@ -82,4 +75,4 @@ "permission": "user" # 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时，拥有将其他用户移除出房间等特权. } # 获得房间管理鉴权的方法：print (RtcRoomToken ( access_key, secret_key, roomAccess ) ) -print (RtcRoomToken(access_key, secret_key, roomAccess)) +print (rtc_room_token(access_key, secret_key, roomAccess)) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 7543850e..56fd2adb 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -25,6 +25,6 @@ from .services.compute.app import AccountClient from .services.compute.qcos_api import QcosClient -from .services.pili.rtc_server_manager import RtcServer, RtcRoomToken +from .services.pili.rtc_server_manager import RtcServer, rtc_room_token from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry diff --git a/qiniu/services/pili/rtc_server_manager.py b/qiniu/services/pili/rtc_server_manager.py index 12bc20cb..f4497542 100644 --- a/qiniu/services/pili/rtc_server_manager.py +++ b/qiniu/services/pili/rtc_server_manager.py @@ -18,7 +18,7 @@ def __init__(self, auth): self.auth = auth self.host = 'http://rtc.qiniuapi.com' - def CreateApp(self, data): + def create_app(self, data): """ Host rtc.qiniuapi.com POST /v3/apps @@ -60,7 +60,7 @@ def CreateApp(self, data): return self.__post(self.host + '/v3/apps', data, ) - def GetApp(self, appid=None): + def get_app(self, appid=None): """ Host rtc.qiniuapi.com GET /v3/apps/<AppID> @@ -119,7 +119,7 @@ def GetApp(self, appid=None): else: return self.__get(self.host + '/v3/apps') - def DeleteApp(self, appid): + def delete_app(self, appid): """ Host rtc.qiniuapi.com DELETE /v3/apps/<AppID> @@ -135,7 +135,7 @@ def DeleteApp(self, appid): """ return self.__delete(self.host + '/v3/apps/%s' % appid) - def UpdateApp(self, appid, data): + def update_app(self, appid, data): """ Host rtc.qiniuapi.com Post /v3/apps/<AppID> @@ -196,7 +196,7 @@ def UpdateApp(self, appid, data): return self.__post(self.host + '/v3/apps/%s' % appid, data, ) - def ListUser(self, AppID, RoomName): + def list_user(self, AppID, RoomName): """ Host rtc.qiniuapi.com GET /v3/apps/<AppID>/rooms/<RoomName>/users @@ -223,7 +223,7 @@ def ListUser(self, AppID, RoomName): """ return self.__get(self.host + '/v3/apps/%s/rooms/%s/users' % (AppID, RoomName)) - def KickUser(self, AppID, RoomName, UserID): + def kick_user(self, AppID, RoomName, UserID): """ Host rtc.qiniuapi.com DELETE /v3/apps/<AppID>/rooms/<RoomName>/users/<UserID> @@ -253,7 +253,7 @@ def KickUser(self, AppID, RoomName, UserID): """ return self.__delete(self.host + '/v3/apps/%s/rooms/%s/users/%s' % (AppID, RoomName, UserID)) - def ListActiveRoom(self, AppID, RoomNamePrefix=None): + def list_active_room(self, AppID, RoomNamePrefix=None): """ Host rtc.qiniuapi.com GET /v3/apps/<AppID>/rooms?prefix=<RoomNamePrefix>&offset=<Offset>&limit=<Limit> @@ -304,7 +304,7 @@ def __delete(self, url, params=None): return http._delete_with_qiniu_mac(url, params, self.auth) -def RtcRoomToken(access_key, secret_key, roomAccess): +def rtc_room_token(access_key, secret_key, roomAccess): """ :arg: AppID: 房间所属帐号的 app 。 From da40d6f7f9b21e2c950a7eac8fea465064425820 Mon Sep 17 00:00:00 2001 From: zhangyunchuan <zhangyunchuan@qiniu.com> Date: Wed, 9 May 2018 14:26:01 +0800 Subject: [PATCH 279/478] =?UTF-8?q?=E7=B1=BB=E5=90=8D=E4=B8=BAHelloWorld?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=EF=BC=8C=E5=87=BD=E6=95=B0=E5=90=8Dhello=5Fw?= =?UTF-8?q?orld=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/workspace.xml | 451 -------------------------------------------- 1 file changed, 451 deletions(-) delete mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index e4c11eb5..00000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,451 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project version="4"> - <component name="ChangeListManager"> - <list default="true" id="476ff082-c46b-43e0-8891-c14f75fd6b58" name="Default" comment=""> - <change type="MODIFICATION" beforePath="$PROJECT_DIR$/examples/rtc_server.py" afterPath="$PROJECT_DIR$/examples/rtc_server.py" /> - <change type="MODIFICATION" beforePath="$PROJECT_DIR$/qiniu/__init__.py" afterPath="$PROJECT_DIR$/qiniu/__init__.py" /> - <change type="MODIFICATION" beforePath="$PROJECT_DIR$/qiniu/services/pili/rtc_server_manager.py" afterPath="$PROJECT_DIR$/qiniu/services/pili/rtc_server_manager.py" /> - </list> - <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" /> - <option name="TRACKING_ENABLED" value="true" /> - <option name="SHOW_DIALOG" value="false" /> - <option name="HIGHLIGHT_CONFLICTS" value="true" /> - <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" /> - <option name="LAST_RESOLUTION" value="IGNORE" /> - </component> - <component name="CreatePatchCommitExecutor"> - <option name="PATCH_PATH" value="" /> - </component> - <component name="ExecutionTargetManager" SELECTED_TARGET="default_target" /> - <component name="FileEditorManager"> - <leaf> - <file leaf-file-name="rtc_server_manager.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/qiniu/services/pili/rtc_server_manager.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="488"> - <caret line="255" column="20" lean-forward="false" selection-start-line="255" selection-start-column="20" selection-end-line="255" selection-end-column="20" /> - <folding> - <element signature="e#24#46#0" expanded="true" /> - </folding> - </state> - </provider> - </entry> - </file> - <file leaf-file-name="rtc_server.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/examples/rtc_server.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="494"> - <caret line="27" column="0" lean-forward="true" selection-start-line="27" selection-start-column="0" selection-end-line="27" selection-end-column="0" /> - <folding> - <element signature="e#25#55#0" expanded="true" /> - </folding> - </state> - </provider> - </entry> - </file> - <file leaf-file-name="bucket.py" pinned="false" current-in-tab="true"> - <entry file="file://$PROJECT_DIR$/qiniu/services/storage/bucket.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="38"> - <caret line="2" column="0" lean-forward="true" selection-start-line="2" selection-start-column="0" selection-end-line="2" selection-end-column="0" /> - <folding> - <element signature="e#25#49#0" expanded="true" /> - </folding> - </state> - </provider> - </entry> - </file> - <file leaf-file-name="app.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/qiniu/services/compute/app.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="19"> - <caret line="1" column="0" lean-forward="true" selection-start-line="1" selection-start-column="0" selection-end-line="1" selection-end-column="0" /> - <folding> - <element signature="e#24#60#0" expanded="true" /> - </folding> - </state> - </provider> - </entry> - </file> - <file leaf-file-name="upload_progress_recorder.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/qiniu/services/storage/upload_progress_recorder.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="-234"> - <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> - <folding /> - </state> - </provider> - </entry> - </file> - <file leaf-file-name="pfop.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/qiniu/services/processing/pfop.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="-44"> - <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> - <folding /> - </state> - </provider> - </entry> - </file> - <file leaf-file-name="cmd.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/qiniu/services/processing/cmd.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="418"> - <caret line="22" column="0" lean-forward="true" selection-start-line="22" selection-start-column="0" selection-end-line="22" selection-end-column="0" /> - <folding /> - </state> - </provider> - </entry> - </file> - <file leaf-file-name="__init__.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/qiniu/__init__.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="532"> - <caret line="28" column="0" lean-forward="true" selection-start-line="28" selection-start-column="0" selection-end-line="28" selection-end-column="0" /> - <folding /> - </state> - </provider> - </entry> - </file> - <file leaf-file-name="auth.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/qiniu/auth.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="304"> - <caret line="16" column="29" lean-forward="false" selection-start-line="16" selection-start-column="29" selection-end-line="16" selection-end-column="29" /> - <folding> - <element signature="e#25#36#0" expanded="true" /> - </folding> - </state> - </provider> - </entry> - </file> - </leaf> - </component> - <component name="Git.Settings"> - <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" /> - </component> - <component name="IdeDocumentHistory"> - <option name="CHANGED_PATHS"> - <list> - <option value="$PROJECT_DIR$/qiniu/auth.py" /> - <option value="$PROJECT_DIR$/qiniu/__init__.py" /> - <option value="$PROJECT_DIR$/examples/rtc_server.py" /> - <option value="$PROJECT_DIR$/qiniu/services/pili/rtc_server_manager.py" /> - </list> - </option> - </component> - <component name="ProjectFrameBounds"> - <option name="x" value="1440" /> - <option name="y" value="-180" /> - <option name="width" value="1920" /> - <option name="height" value="1080" /> - </component> - <component name="ProjectView"> - <navigator currentView="ProjectPane" proportions="" version="1"> - <flattenPackages /> - <showMembers /> - <showModules /> - <showLibraryContents /> - <hideEmptyPackages /> - <abbreviatePackageNames /> - <autoscrollToSource /> - <autoscrollFromSource /> - <sortByType /> - <manualOrder /> - <foldersAlwaysOnTop value="true" /> - </navigator> - <panes> - <pane id="Scratches" /> - <pane id="ProjectPane"> - <subPane> - <PATH> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - </PATH> - <PATH> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="qiniu" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - </PATH> - <PATH> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="qiniu" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="services" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - </PATH> - <PATH> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="qiniu" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="services" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="storage" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - </PATH> - <PATH> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="qiniu" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="services" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="processing" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - </PATH> - <PATH> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="qiniu" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="services" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="pili" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - </PATH> - <PATH> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="qiniu" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="services" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="compute" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - </PATH> - <PATH> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="python-sdk" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="examples" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - </PATH> - </subPane> - </pane> - <pane id="Scope" /> - </panes> - </component> - <component name="PropertiesComponent"> - <property name="settings.editor.selected.configurable" value="preferences.sourceCode.Python" /> - <property name="settings.editor.splitter.proportion" value="0.2" /> - <property name="last_opened_file_path" value="$PROJECT_DIR$" /> - <property name="FullScreen" value="true" /> - </component> - <component name="ShelveChangesManager" show_recycled="false"> - <option name="remove_strategy" value="false" /> - </component> - <component name="TaskManager"> - <task active="true" id="Default" summary="Default task"> - <changelist id="476ff082-c46b-43e0-8891-c14f75fd6b58" name="Default" comment="" /> - <created>1525836017672</created> - <option name="number" value="Default" /> - <option name="presentableId" value="Default" /> - <updated>1525836017672</updated> - </task> - <servers /> - </component> - <component name="ToolWindowManager"> - <frame x="1440" y="-180" width="1920" height="1080" extended-state="0" /> - <editor active="true" /> - <layout> - <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.1864139" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" /> - <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" /> - <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" /> - <window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> - <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" /> - <window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" /> - <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> - <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> - <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" /> - <window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" /> - <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> - <window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" /> - <window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" /> - <window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" /> - <window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" /> - <window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" /> - <window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> - </layout> - </component> - <component name="VcsContentAnnotationSettings"> - <option name="myLimit" value="2678400000" /> - </component> - <component name="XDebuggerManager"> - <breakpoint-manager /> - <watches-manager /> - </component> - <component name="editorHistoryManager"> - <entry file="file://$PROJECT_DIR$/qiniu/auth.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="304"> - <caret line="16" column="29" lean-forward="false" selection-start-line="16" selection-start-column="29" selection-end-line="16" selection-end-column="29" /> - <folding> - <element signature="e#25#36#0" expanded="true" /> - </folding> - </state> - </provider> - </entry> - <entry file="file://$PROJECT_DIR$/qiniu/services/storage/upload_progress_recorder.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="-234"> - <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> - <folding /> - </state> - </provider> - </entry> - <entry file="file://$PROJECT_DIR$/qiniu/services/processing/pfop.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="-44"> - <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> - <folding /> - </state> - </provider> - </entry> - <entry file="file://$PROJECT_DIR$/qiniu/services/processing/cmd.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="418"> - <caret line="22" column="0" lean-forward="true" selection-start-line="22" selection-start-column="0" selection-end-line="22" selection-end-column="0" /> - <folding /> - </state> - </provider> - </entry> - <entry file="file://$PROJECT_DIR$/qiniu/services/compute/app.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="19"> - <caret line="1" column="0" lean-forward="true" selection-start-line="1" selection-start-column="0" selection-end-line="1" selection-end-column="0" /> - <folding> - <element signature="e#24#60#0" expanded="true" /> - </folding> - </state> - </provider> - </entry> - <entry file="file://$PROJECT_DIR$/qiniu/services/processing/__init__.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="0"> - <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> - <folding /> - </state> - </provider> - </entry> - <entry file="file://$PROJECT_DIR$/qiniu/__init__.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="532"> - <caret line="28" column="0" lean-forward="true" selection-start-line="28" selection-start-column="0" selection-end-line="28" selection-end-column="0" /> - <folding /> - </state> - </provider> - </entry> - <entry file="file://$PROJECT_DIR$/qiniu/services/pili/rtc_server_manager.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="488"> - <caret line="255" column="20" lean-forward="false" selection-start-line="255" selection-start-column="20" selection-end-line="255" selection-end-column="20" /> - <folding> - <element signature="e#24#46#0" expanded="true" /> - </folding> - </state> - </provider> - </entry> - <entry file="file://$PROJECT_DIR$/examples/rtc_server.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="494"> - <caret line="27" column="0" lean-forward="true" selection-start-line="27" selection-start-column="0" selection-end-line="27" selection-end-column="0" /> - <folding> - <element signature="e#25#55#0" expanded="true" /> - </folding> - </state> - </provider> - </entry> - <entry file="file://$PROJECT_DIR$/qiniu/services/storage/bucket.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="38"> - <caret line="2" column="0" lean-forward="true" selection-start-line="2" selection-start-column="0" selection-end-line="2" selection-end-column="0" /> - <folding> - <element signature="e#25#49#0" expanded="true" /> - </folding> - </state> - </provider> - </entry> - </component> -</project> \ No newline at end of file From 88aa6420a72536dc38ae7dc59efd178e6591ccdf Mon Sep 17 00:00:00 2001 From: zhangyunchuan <zhangyunchuan@qiniu.com> Date: Wed, 9 May 2018 14:40:04 +0800 Subject: [PATCH 280/478] =?UTF-8?q?=E5=87=BD=E6=95=B0=E7=9A=84=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E5=90=8D=E4=B8=BAhello=5Fworld=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/rtc_server.py | 22 +++++++-------- qiniu/services/pili/rtc_server_manager.py | 34 +++++++++++------------ 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/examples/rtc_server.py b/examples/rtc_server.py index 6c7a8abf..770e0d01 100644 --- a/examples/rtc_server.py +++ b/examples/rtc_server.py @@ -27,13 +27,13 @@ print (rtc.create_app(create_data)) # 查询一个APP -# 查询某一个具体的APP的相关信息的方法为 print ( rtc.GetApp(<AppID>) ) ，其中 AppID是类似 'desls83s2' 这样在创建时由七牛自动生成的数字字母乱序组合的字符串 -# 如果不指定具体的AppID，直接运行 print ( rtc.GetApp() ) ，那么就会列举出该账号下所有的APP -print (rtc.get_app('<AppID>:可选填')) +# 查询某一个具体的APP的相关信息的方法为 print ( rtc.GetApp(<app_id>) ) ，其中 app_id 是类似 'desls83s2' 这样在创建时由七牛自动生成的数字字母乱序组合的字符串 +# 如果不指定具体的app_id，直接运行 print ( rtc.GetApp() ) ，那么就会列举出该账号下所有的APP +print (rtc.get_app('<app_id>:可选填')) # 删除一个APP -# 使用方法为：rtc.DeleteApp(<AppID>)，例如： rtc.DeleteApp('desls83s2') -print (rtc.delete_app('<AppID>:必填')) +# 使用方法为：rtc.DeleteApp(<app_id>)，例如： rtc.DeleteApp('desls83s2') +print (rtc.delete_app('<app_id>:必填')) # 更新一个APP的相关参数 # 首先需要写好更新的APP的各个参数。参数如下： @@ -53,22 +53,22 @@ # "streamTitle": "<StreamTitle>" # StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号生成不同的流名。例如，配置 Hub 为 qn-zhibo ，配置 StreamTitle 为 $(roomName) ，则房间 meeting-001 的合流将会被转推到 rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。详细配置细则，请咨询七牛技术支持。 # } } -# 使用方法为：rtc.UpdateApp('<AppID>:必填', update_data)，例如：AppID 是形如 desmfnkw5 的字符串 -print (rtc.update_app('<AppID>:必填', update_data)) +# 使用方法为：rtc.UpdateApp('<app_id>:必填', update_data)，例如：app_id 是形如 desmfnkw5 的字符串 +print (rtc.update_app('<app_id>:必填', update_data)) # 列举一个APP下面，某个房间的所有用户 -print (rtc.list_user('<AppID>:必填', '<房间名>:必填')) +print (rtc.list_user('<app_id>:必填', '<房间名>:必填')) # 踢出一个APP下面，某个房间的某个用户 -print (rtc.kick_user('<AppID>:必填', '<房间名>:必填', '<客户ID>:必填')) +print (rtc.kick_user('<app_id>:必填', '<房间名>:必填', '<客户ID>:必填')) # 列举一个APP下面，所有的房间 -print (rtc.list_active_room('<AppID>:必填')) +print (rtc.list_active_room('<app_id>:必填')) # 计算房间管理鉴权。连麦用户终端通过房间管理鉴权获取七牛连麦服务 # 首先需要写好房间鉴权的各个参数。参数如下： roomAccess = { - "appId": "<AppID>:必填", # AppID: 房间所属帐号的 app 。 + "AppID": "<AppID>:必填", # AppID: 房间所属帐号的 app 。 "roomName": "<房间名>:必填", # RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ "userId": "<用户名>:必填", # UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ "expireAt": int(time.time()) + 3600, # ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间，token 将在该时间后失效。 diff --git a/qiniu/services/pili/rtc_server_manager.py b/qiniu/services/pili/rtc_server_manager.py index f4497542..bbd06f74 100644 --- a/qiniu/services/pili/rtc_server_manager.py +++ b/qiniu/services/pili/rtc_server_manager.py @@ -60,7 +60,7 @@ def create_app(self, data): return self.__post(self.host + '/v3/apps', data, ) - def get_app(self, appid=None): + def get_app(self, app_id=None): """ Host rtc.qiniuapi.com GET /v3/apps/<AppID> @@ -114,12 +114,12 @@ def get_app(self, appid=None): UpdatedAt: time 类型，app 更新的时间。 """ - if appid: - return self.__get(self.host + '/v3/apps/%s' % appid) + if app_id: + return self.__get(self.host + '/v3/apps/%s' % app_id) else: return self.__get(self.host + '/v3/apps') - def delete_app(self, appid): + def delete_app(self, app_id): """ Host rtc.qiniuapi.com DELETE /v3/apps/<AppID> @@ -133,9 +133,9 @@ def delete_app(self, appid): "error": "app not found" } """ - return self.__delete(self.host + '/v3/apps/%s' % appid) + return self.__delete(self.host + '/v3/apps/%s' % app_id) - def update_app(self, appid, data): + def update_app(self, app_id, data): """ Host rtc.qiniuapi.com Post /v3/apps/<AppID> @@ -194,9 +194,9 @@ def update_app(self, appid, data): } """ - return self.__post(self.host + '/v3/apps/%s' % appid, data, ) + return self.__post(self.host + '/v3/apps/%s' % app_id, data, ) - def list_user(self, AppID, RoomName): + def list_user(self, app_id, room_name): """ Host rtc.qiniuapi.com GET /v3/apps/<AppID>/rooms/<RoomName>/users @@ -221,9 +221,9 @@ def list_user(self, AppID, RoomName): "error": "app not found" } """ - return self.__get(self.host + '/v3/apps/%s/rooms/%s/users' % (AppID, RoomName)) + return self.__get(self.host + '/v3/apps/%s/rooms/%s/users' % (app_id, room_name)) - def kick_user(self, AppID, RoomName, UserID): + def kick_user(self, app_id, room_name, user_id): """ Host rtc.qiniuapi.com DELETE /v3/apps/<AppID>/rooms/<RoomName>/users/<UserID> @@ -251,9 +251,9 @@ def kick_user(self, AppID, RoomName, UserID): "error": "room not active" } """ - return self.__delete(self.host + '/v3/apps/%s/rooms/%s/users/%s' % (AppID, RoomName, UserID)) + return self.__delete(self.host + '/v3/apps/%s/rooms/%s/users/%s' % (app_id, room_name, user_id)) - def list_active_room(self, AppID, RoomNamePrefix=None): + def list_active_room(self, app_id, room_name_prefix=None): """ Host rtc.qiniuapi.com GET /v3/apps/<AppID>/rooms?prefix=<RoomNamePrefix>&offset=<Offset>&limit=<Limit> @@ -289,10 +289,10 @@ def list_active_room(self, AppID, RoomNamePrefix=None): RoomName: 当前活跃的房间名。 """ - if RoomNamePrefix: - return self.__get(self.host + '/v3/apps/%s/rooms?prefix=%s' % (AppID, RoomNamePrefix)) + if room_name_prefix: + return self.__get(self.host + '/v3/apps/%s/rooms?prefix=%s' % (app_id, room_name_prefix)) else: - return self.__get(self.host + '/v3/apps/%s/rooms' % AppID) + return self.__get(self.host + '/v3/apps/%s/rooms' % app_id) def __post(self, url, data=None): return http._post_with_qiniu_mac(url, data, self.auth) @@ -304,7 +304,7 @@ def __delete(self, url, params=None): return http._delete_with_qiniu_mac(url, params, self.auth) -def rtc_room_token(access_key, secret_key, roomAccess): +def rtc_room_token(access_key, secret_key, room_access): """ :arg: AppID: 房间所属帐号的 app 。 @@ -336,7 +336,7 @@ def rtc_room_token(access_key, secret_key, roomAccess): # 3. 将AccessKey与以上两者拼接得到房间鉴权 roomToken = "<AccessKey>" + ":" + encodedSign + ":" + encodedRoomAccess """ - roomAccessString = json.dumps(roomAccess) + roomAccessString = json.dumps(room_access) byte_result = bytes(roomAccessString, 'utf-8') encodedRoomAccess = base64.urlsafe_b64encode(byte_result) From ebc8ef454dfb82555d2a93017741ec851181c88e Mon Sep 17 00:00:00 2001 From: zhangyunchuan <zhangyunchuan@qiniu.com> Date: Wed, 9 May 2018 14:44:05 +0800 Subject: [PATCH 281/478] =?UTF-8?q?=E5=87=BD=E6=95=B0=E7=9A=84=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E5=90=8D=E4=B8=BAhello=5Fworld=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/pili/rtc_server_manager.py | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/qiniu/services/pili/rtc_server_manager.py b/qiniu/services/pili/rtc_server_manager.py index bbd06f74..7126f98e 100644 --- a/qiniu/services/pili/rtc_server_manager.py +++ b/qiniu/services/pili/rtc_server_manager.py @@ -199,13 +199,13 @@ def update_app(self, app_id, data): def list_user(self, app_id, room_name): """ Host rtc.qiniuapi.com - GET /v3/apps/<AppID>/rooms/<RoomName>/users + GET /v3/apps/<app_id>/rooms/<room_name>/users Authorization: qiniu mac :param: - AppID: 连麦房间所属的 app 。 + app_id: 连麦房间所属的 app 。 - RoomName: 操作所查询的连麦房间。 + room_name: 操作所查询的连麦房间。 :return: 200 OK @@ -226,15 +226,15 @@ def list_user(self, app_id, room_name): def kick_user(self, app_id, room_name, user_id): """ Host rtc.qiniuapi.com - DELETE /v3/apps/<AppID>/rooms/<RoomName>/users/<UserID> + DELETE /v3/apps/<app_id>/rooms/<room_name>/users/<user_id> Authorization: qiniu mac :param: - AppID: 连麦房间所属的 app 。 + app_id: 连麦房间所属的 app 。 - RoomName: 连麦房间。 + room_name: 连麦房间。 - UserID: 操作所剔除的用户。 + user_id: 操作所剔除的用户。 :return: 200 OK @@ -256,17 +256,17 @@ def kick_user(self, app_id, room_name, user_id): def list_active_room(self, app_id, room_name_prefix=None): """ Host rtc.qiniuapi.com - GET /v3/apps/<AppID>/rooms?prefix=<RoomNamePrefix>&offset=<Offset>&limit=<Limit> + GET /v3/apps/<app_id>/rooms?prefix=<room_name_prefix>&offset=<off_set>&limit=<limit> Authorization: qiniu mac :param: - AppID: 连麦房间所属的 app 。 + app_id: 连麦房间所属的 app 。 - RoomNamePrefix: 所查询房间名的前缀索引，可以为空。 + room_name_prefix: 所查询房间名的前缀索引，可以为空。 - Offset: int 类型，分页查询的位移标记。 + off_set: int 类型，分页查询的位移标记。 - Limit: int 类型，此次查询的最大长度。 + limit: int 类型，此次查询的最大长度。 :return: 200 OK From 688f69e97a6c6bba52be3a74a0fadbe884d025a7 Mon Sep 17 00:00:00 2001 From: Jemy <jemygraw@gmail.com> Date: Wed, 9 May 2018 14:56:06 +0800 Subject: [PATCH 282/478] format the code comment --- examples/rtc_server.py | 2 +- qiniu/services/pili/rtc_server_manager.py | 643 +++++++++++----------- 2 files changed, 311 insertions(+), 334 deletions(-) diff --git a/examples/rtc_server.py b/examples/rtc_server.py index 770e0d01..152cd46f 100644 --- a/examples/rtc_server.py +++ b/examples/rtc_server.py @@ -63,7 +63,7 @@ print (rtc.kick_user('<app_id>:必填', '<房间名>:必填', '<客户ID>:必填')) # 列举一个APP下面，所有的房间 -print (rtc.list_active_room('<app_id>:必填')) +print (rtc.list_active_rooms('<app_id>:必填')) # 计算房间管理鉴权。连麦用户终端通过房间管理鉴权获取七牛连麦服务 # 首先需要写好房间鉴权的各个参数。参数如下： diff --git a/qiniu/services/pili/rtc_server_manager.py b/qiniu/services/pili/rtc_server_manager.py index 7126f98e..b6d48e60 100644 --- a/qiniu/services/pili/rtc_server_manager.py +++ b/qiniu/services/pili/rtc_server_manager.py @@ -4,344 +4,321 @@ class RtcServer(object): - """ - 直播连麦管理类 - 主要涉及了直播连麦管理及操作接口的实现，具体的接口规格可以参考： + """ + 直播连麦管理类 + 主要涉及了直播连麦管理及操作接口的实现，具体的接口规格可以参考： https://github.com/pili-engineering/QNRTC-Server/blob/master/docs/api.md#41-listuser #这个是内部文档，等外部文档发布了，这一行要换成外部文档 Attributes: auth: 账号管理密钥对，Auth对象 - """ - - def __init__(self, auth): - self.auth = auth - self.host = 'http://rtc.qiniuapi.com' - - def create_app(self, data): - """ - Host rtc.qiniuapi.com - POST /v3/apps - Authorization: qiniu mac - Content-Type: application/json - - { - "hub": "<Hub>", - "title": "<Title>", - "maxUsers": <MaxUsers>, - "noAutoKickUser": <NoAutoKickUser> - } - - :param appid: - Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 - - Title: app 的名称，可选，注意，Title 不是唯一标识，重复 create 动作将生成多个 app。 - - MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 - - NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ，即同一个身份的 client (app/room/user) ，新的连麦请求可以成功，旧连接被关闭。 - - :return: - 200 OK - { - "appId": "<AppID>", - "hub": "<Hub>", - "title": "<Title>", - "maxUsers": <MaxUsers>, - "noAutoKickUser": <NoAutoKickUser>, - "createdAt": <CreatedAt>, - "updatedAt": <UpdatedAt> - } - 616 - { - "error": "hub not match" - } - """ - - return self.__post(self.host + '/v3/apps', data, ) - - def get_app(self, app_id=None): - """ - Host rtc.qiniuapi.com - GET /v3/apps/<AppID> - Authorization: qiniu mac - - :param appid: - AppID: app 的唯一标识。 可以不填写，不填写的话，默认就是输出所有app的相关信息 - - :return: - 200 OK - { - "appId": "<AppID>", - "hub": "<Hub>", - "title": "<Title>", - "maxUsers": <MaxUsers>, - "noAutoKickUser": <NoAutoKickUser>, - "mergePublishRtmp": { - "audioOnly": <AudioOnly>, - "height": <OutputHeight>, - "width": <OutputHeight>, - "fps": <OutputFps>, - "kbps": <OutputKbps>, - "url": "<URL>", - "streamTitle": "<StreamTitle>" - }, - "createdAt": <CreatedAt>, - "updatedAt": <UpdatedAt> - } - - 612 - { - "error": "app not found" - } - - #### - AppID: app 的唯一标识。 - - UID: 客户的七牛帐号。 - - Hub: 绑定的直播 hub，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 - - Title: app 的名称，注意，Title不是唯一标识。 - - MaxUsers: int 类型，连麦房间支持的最大在线人数。 - - NoAutoKickUser: bool 类型，禁止自动踢人。 - - MergePublishRtmp: 连麦合流转推 RTMP 的配置。 - - CreatedAt: time 类型，app 创建的时间。 - - UpdatedAt: time 类型，app 更新的时间。 - """ - if app_id: - return self.__get(self.host + '/v3/apps/%s' % app_id) - else: - return self.__get(self.host + '/v3/apps') - - def delete_app(self, app_id): - """ - Host rtc.qiniuapi.com - DELETE /v3/apps/<AppID> - Authorization: qiniu mac - - :return: - 200 OK - - 612 - { - "error": "app not found" - } - """ - return self.__delete(self.host + '/v3/apps/%s' % app_id) - - def update_app(self, app_id, data): - """ - Host rtc.qiniuapi.com - Post /v3/apps/<AppID> - Authorization: qiniu mac - - :param appid: - AppID: app 的唯一标识，创建的时候由系统生成。 - - Title: app 的名称， 可选。 - - Hub: 绑定的直播 hub，可选，用于合流后 rtmp 推流。 - - MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 - - NoAutoKickUser: bool 类型，可选，禁止自动踢人。 - - MergePublishRtmp: 连麦合流转推 RTMP 的配置，可选择。其详细配置包括如下 - - Enable: 布尔类型，用于开启和关闭所有房间的合流功能。 - AudioOnly: 布尔类型，可选，指定是否只合成音频。 - Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 - OutputFps: int64，可选，指定合流输出的帧率，默认为 25 fps 。 - OutputKbps: int64，可选，指定合流输出的码率，默认为 1000 。 - URL: 合流后转推旁路直播的地址，可选，支持魔法变量配置按照连麦房间号生成不同的推流地址。如果是转推到七牛直播云，不建议使用该配置。 - StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号生成不同的流名。例如，配置 Hub 为 qn-zhibo ，配置 StreamTitle 为 $(roomName) ，则房间 meeting-001 的合流将会被转推到 rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。详细配置细则，请咨询七牛技术支持。 - - :return: - 200 OK - { - "appId": "<AppID>", - "hub": "<Hub>", - "title": "<Title>", - "maxUsers": <MaxUsers>, - "noAutoKickUser": <NoAutoKickUser>, - "mergePublishRtmp": { - "enable": <Enable>, - "audioOnly": <AudioOnly>, - "height": <OutputHeight>, - "width": <OutputHeight>, - "fps": <OutputFps>, - "kbps": <OutputKbps>, - "url": "<URL>", - "streamTitle": "<StreamTitle>" - }, - "createdAt": <CreatedAt>, - "updatedAt": <UpdatedAt> - } - - 612 - { - "error": "app not found" - } - 616 - { - "error": "hub not match" - } - """ - - return self.__post(self.host + '/v3/apps/%s' % app_id, data, ) - - def list_user(self, app_id, room_name): - """ - Host rtc.qiniuapi.com - GET /v3/apps/<app_id>/rooms/<room_name>/users - Authorization: qiniu mac - - :param: - app_id: 连麦房间所属的 app 。 - - room_name: 操作所查询的连麦房间。 - - :return: - 200 OK - { - "users": [ - { - "userId": "<UserID>" - }, - ] - } - 612 - { - "error": "app not found" - } - """ - return self.__get(self.host + '/v3/apps/%s/rooms/%s/users' % (app_id, room_name)) - - def kick_user(self, app_id, room_name, user_id): - """ - Host rtc.qiniuapi.com - DELETE /v3/apps/<app_id>/rooms/<room_name>/users/<user_id> - Authorization: qiniu mac - - :param: - app_id: 连麦房间所属的 app 。 - - room_name: 连麦房间。 - - user_id: 操作所剔除的用户。 - - :return: - 200 OK - 612 - { - "error": "app not found" - } - 612 - { - "error": "user not found" - } - 615 - { - "error": "room not active" - } - """ - return self.__delete(self.host + '/v3/apps/%s/rooms/%s/users/%s' % (app_id, room_name, user_id)) - - def list_active_room(self, app_id, room_name_prefix=None): - """ - Host rtc.qiniuapi.com - GET /v3/apps/<app_id>/rooms?prefix=<room_name_prefix>&offset=<off_set>&limit=<limit> - Authorization: qiniu mac - - :param: - app_id: 连麦房间所属的 app 。 - - room_name_prefix: 所查询房间名的前缀索引，可以为空。 - - off_set: int 类型，分页查询的位移标记。 - - limit: int 类型，此次查询的最大长度。 - - :return: - 200 OK - { - "end": <IsEnd>, - "offset": <Offset>, - "rooms": [ - "<RoomName>", - ... - ] - } - 612 - { - "error": "app not found" - } - ### - IsEnd: bool 类型，分页查询是否已经查完所有房间。 - - Offset: int 类型，下次分页查询使用的位移标记。 - - RoomName: 当前活跃的房间名。 - """ - if room_name_prefix: - return self.__get(self.host + '/v3/apps/%s/rooms?prefix=%s' % (app_id, room_name_prefix)) - else: - return self.__get(self.host + '/v3/apps/%s/rooms' % app_id) - - def __post(self, url, data=None): - return http._post_with_qiniu_mac(url, data, self.auth) - - def __get(self, url, params=None): - return http._get_with_qiniu_mac(url, params, self.auth) - - def __delete(self, url, params=None): - return http._delete_with_qiniu_mac(url, params, self.auth) + """ + + def __init__(self, auth): + self.auth = auth + self.host = 'http://rtc.qiniuapi.com' + + def create_app(self, data): + """ + Host rtc.qiniuapi.com + POST /v3/apps + Authorization: qiniu mac + Content-Type: application/json + + { + "hub": "<Hub>", + "title": "<Title>", + "maxUsers": <MaxUsers>, + "noAutoKickUser": <NoAutoKickUser> + } + + :param appid: + Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 + Title: app 的名称，可选，注意，Title 不是唯一标识，重复 create 动作将生成多个 app。 + MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 + NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ，即同一个身份的 client (app/room/user) ， + 新的连麦请求可以成功，旧连接被关闭。 + + :return: + 200 OK + { + "appId": "<AppID>", + "hub": "<Hub>", + "title": "<Title>", + "maxUsers": <MaxUsers>, + "noAutoKickUser": <NoAutoKickUser>, + "createdAt": <CreatedAt>, + "updatedAt": <UpdatedAt> + } + 616 + { + "error": "hub not match" + } + """ + + return self.__post(self.host + '/v3/apps', data) + + def get_app(self, app_id=None): + """ + Host rtc.qiniuapi.com + GET /v3/apps/<AppID> + Authorization: qiniu mac + + :param appid: + AppID: app 的唯一标识。 可以不填写，不填写的话，默认就是输出所有app的相关信息 + + :return: + 200 OK + { + "appId": "<AppID>", + "hub": "<Hub>", + "title": "<Title>", + "maxUsers": <MaxUsers>, + "noAutoKickUser": <NoAutoKickUser>, + "mergePublishRtmp": { + "audioOnly": <AudioOnly>, + "height": <OutputHeight>, + "width": <OutputHeight>, + "fps": <OutputFps>, + "kbps": <OutputKbps>, + "url": "<URL>", + "streamTitle": "<StreamTitle>" + }, + "createdAt": <CreatedAt>, + "updatedAt": <UpdatedAt> + } + + 612 + { + "error": "app not found" + } + + #### + AppID: app 的唯一标识。 + UID: 客户的七牛帐号。 + Hub: 绑定的直播 hub，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 + Title: app 的名称，注意，Title不是唯一标识。 + MaxUsers: int 类型，连麦房间支持的最大在线人数。 + NoAutoKickUser: bool 类型，禁止自动踢人。 + MergePublishRtmp: 连麦合流转推 RTMP 的配置。 + CreatedAt: time 类型，app 创建的时间。 + UpdatedAt: time 类型，app 更新的时间。 + """ + if app_id: + return self.__get(self.host + '/v3/apps/%s' % app_id) + else: + return self.__get(self.host + '/v3/apps') + + def delete_app(self, app_id): + """ + Host rtc.qiniuapi.com + DELETE /v3/apps/<AppID> + Authorization: qiniu mac + + :return: + 200 OK + + 612 + { + "error": "app not found" + } + """ + return self.__delete(self.host + '/v3/apps/%s' % app_id) + + def update_app(self, app_id, data): + """ + Host rtc.qiniuapi.com + Post /v3/apps/<AppID> + Authorization: qiniu mac + + :param appid: + AppID: app 的唯一标识，创建的时候由系统生成。 + Title: app 的名称， 可选。 + Hub: 绑定的直播 hub，可选，用于合流后 rtmp 推流。 + MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 + NoAutoKickUser: bool 类型，可选，禁止自动踢人。 + MergePublishRtmp: 连麦合流转推 RTMP 的配置，可选择。其详细配置包括如下 + Enable: 布尔类型，用于开启和关闭所有房间的合流功能。 + AudioOnly: 布尔类型，可选，指定是否只合成音频。 + Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 + OutputFps: int64，可选，指定合流输出的帧率，默认为 25 fps 。 + OutputKbps: int64，可选，指定合流输出的码率，默认为 1000 。 + URL: 合流后转推旁路直播的地址，可选，支持魔法变量配置按照连麦房间号生成不同的推流地址。如果是转推到七牛直播云，不建议使用该配置。 + StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号生成不同的流名。例如，配置 Hub 为 qn-zhibo ， + 配置 StreamTitle 为 $(roomName) ，则房间 meeting-001 的合流将会被转推到 + rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。详细配置细则，请咨询七牛技术支持。 + + :return: + 200 OK + { + "appId": "<AppID>", + "hub": "<Hub>", + "title": "<Title>", + "maxUsers": <MaxUsers>, + "noAutoKickUser": <NoAutoKickUser>, + "mergePublishRtmp": { + "enable": <Enable>, + "audioOnly": <AudioOnly>, + "height": <OutputHeight>, + "width": <OutputHeight>, + "fps": <OutputFps>, + "kbps": <OutputKbps>, + "url": "<URL>", + "streamTitle": "<StreamTitle>" + }, + "createdAt": <CreatedAt>, + "updatedAt": <UpdatedAt> + } + + 612 + { + "error": "app not found" + } + 616 + { + "error": "hub not match" + } + """ + + return self.__post(self.host + '/v3/apps/%s' % app_id, data) + + def list_user(self, app_id, room_name): + """ + Host rtc.qiniuapi.com + GET /v3/apps/<app_id>/rooms/<room_name>/users + Authorization: qiniu mac + + :param: + app_id: 连麦房间所属的 app 。 + + room_name: 操作所查询的连麦房间。 + + :return: + 200 OK + { + "users": [ + { + "userId": "<UserID>" + }, + ] + } + 612 + { + "error": "app not found" + } + """ + return self.__get(self.host + '/v3/apps/%s/rooms/%s/users' % (app_id, room_name)) + + def kick_user(self, app_id, room_name, user_id): + """ + Host rtc.qiniuapi.com + DELETE /v3/apps/<app_id>/rooms/<room_name>/users/<user_id> + Authorization: qiniu mac + + :param: + app_id: 连麦房间所属的 app 。 + + room_name: 连麦房间。 + + user_id: 操作所剔除的用户。 + + :return: + 200 OK + 612 + { + "error": "app not found" + } + 612 + { + "error": "user not found" + } + 615 + { + "error": "room not active" + } + """ + return self.__delete(self.host + '/v3/apps/%s/rooms/%s/users/%s' % (app_id, room_name, user_id)) + + def list_active_rooms(self, app_id, room_name_prefix=None): + """ + Host rtc.qiniuapi.com + GET /v3/apps/<app_id>/rooms?prefix=<room_name_prefix>&offset=<off_set>&limit=<limit> + Authorization: qiniu mac + + :param: + app_id: 连麦房间所属的 app 。 + + room_name_prefix: 所查询房间名的前缀索引，可以为空。 + off_set: int 类型，分页查询的位移标记。 + limit: int 类型，此次查询的最大长度。 + + :return: + 200 OK + { + "end": <IsEnd>, + "offset": <Offset>, + "rooms": [ + "<RoomName>", + ... + ] + } + 612 + { + "error": "app not found" + } + ### + IsEnd: bool 类型，分页查询是否已经查完所有房间。 + Offset: int 类型，下次分页查询使用的位移标记。 + RoomName: 当前活跃的房间名。 + """ + if room_name_prefix: + return self.__get(self.host + '/v3/apps/%s/rooms?prefix=%s' % (app_id, room_name_prefix)) + else: + return self.__get(self.host + '/v3/apps/%s/rooms' % app_id) + + def __post(self, url, data=None): + return http._post_with_qiniu_mac(url, data, self.auth) + + def __get(self, url, params=None): + return http._get_with_qiniu_mac(url, params, self.auth) + + def __delete(self, url, params=None): + return http._delete_with_qiniu_mac(url, params, self.auth) def rtc_room_token(access_key, secret_key, room_access): - """ - :arg: - AppID: 房间所属帐号的 app 。 - - RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ - - UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ - - ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间，token 将在该时间后失效。 - - Permission: 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时，拥有将其他用户移除出房间等特权. - - :method: - # 1. 定义房间管理凭证，并对凭证字符做URL安全的Base64编码 - roomAccess = { - "appId": "<AppID>" - "roomName": "<RoomName>", - "userId": "<UserID>", - "expireAt": <ExpireAt>, - "permission": "<Permission>" - } - roomAccessString = json_to_string(roomAccess) - encodedRoomAccess = urlsafe_base64_encode(roomAccessString) - - # 2. 计算HMAC-SHA1签名，并对签名结果做URL安全的Base64编码 - sign = hmac_sha1(encodedRoomAccess, <SecretKey>) - encodedSign = urlsafe_base64_encode(sign) - - # 3. 将AccessKey与以上两者拼接得到房间鉴权 - roomToken = "<AccessKey>" + ":" + encodedSign + ":" + encodedRoomAccess - """ - roomAccessString = json.dumps(room_access) - byte_result = bytes(roomAccessString, 'utf-8') - encodedRoomAccess = base64.urlsafe_b64encode(byte_result) - - sign = hmac.new(bytes(secret_key, 'utf-8'), encodedRoomAccess, hashlib.sha1).digest() - encodedSign = base64.urlsafe_b64encode(sign) - roomToken = access_key + ':' + str(encodedSign, encoding="utf-8") + ':' + str(encodedRoomAccess, encoding="utf-8") - - return roomToken + """ + :arg: + AppID: 房间所属帐号的 app 。 + RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ + UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ + ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间，token 将在该时间后失效。 + Permission: 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时，拥有将其他用户移除出房间等特权. + :method: + # 1. 定义房间管理凭证，并对凭证字符做URL安全的Base64编码 + roomAccess = { + "appId": "<AppID>" + "roomName": "<RoomName>", + "userId": "<UserID>", + "expireAt": <ExpireAt>, + "permission": "<Permission>" + } + roomAccessString = json_to_string(roomAccess) + encodedRoomAccess = urlsafe_base64_encode(roomAccessString) + + # 2. 计算HMAC-SHA1签名，并对签名结果做URL安全的Base64编码 + sign = hmac_sha1(encodedRoomAccess, <SecretKey>) + encodedSign = urlsafe_base64_encode(sign) + + # 3. 将AccessKey与以上两者拼接得到房间鉴权 + roomToken = "<AccessKey>" + ":" + encodedSign + ":" + encodedRoomAccess + """ + roomAccessString = json.dumps(room_access) + byte_result = bytes(roomAccessString, 'utf-8') + encodedRoomAccess = base64.urlsafe_b64encode(byte_result) + + sign = hmac.new(bytes(secret_key, 'utf-8'), encodedRoomAccess, hashlib.sha1).digest() + encodedSign = base64.urlsafe_b64encode(sign) + roomToken = access_key + ':' + str(encodedSign, encoding="utf-8") + ':' + str(encodedRoomAccess, encoding="utf-8") + + return roomToken From 6e1293d2546526aea759e385b19575d9b3858ece Mon Sep 17 00:00:00 2001 From: Jemy <jemygraw@gmail.com> Date: Wed, 9 May 2018 16:16:57 +0800 Subject: [PATCH 283/478] remove comments from the code --- examples/rtc_server.py | 59 ++--- qiniu/__init__.py | 2 +- qiniu/services/pili/rtc_server_manager.py | 282 +--------------------- 3 files changed, 41 insertions(+), 302 deletions(-) diff --git a/examples/rtc_server.py b/examples/rtc_server.py index 152cd46f..c3f7980e 100644 --- a/examples/rtc_server.py +++ b/examples/rtc_server.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- from qiniu import QiniuMacAuth -from qiniu import RtcServer, rtc_room_token +from qiniu import RtcServer, get_room_token import time # 需要填写你的 Access Key 和 Secret Key -access_key = '...' -secret_key = '...' -assert access_key != '...' and secret_key != '...', '你必须填写你自己七牛账号的密钥，密钥地址：https://developer.qiniu.com/kodo/kb/1334/the-access-key-secret-key-encryption-key-safe-use-instructions' +access_key = 'xxx' +secret_key = 'xxx' # 构建鉴权对象 q = QiniuMacAuth(access_key, secret_key) @@ -18,16 +17,18 @@ # 创建一个APP # 首先需要写好创建APP的各个参数。参数如下 create_data = { - "hub": 'python_test_hub', # Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 - "title": 'python_test_app', # Title: app 的名称，可选，注意，Title 不是唯一标识，重复 create 动作将生成多个 app。 - # "maxUsers": MaxUsers, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 - # "noAutoKickUser": NoAutoKickUser # NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ，即同一个身份的 client (app/room/user) ，新的连麦请求可以成功，旧连接被关闭。 + "hub": 'python_test_hub', # Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 + "title": 'python_test_app', # Title: app 的名称，可选，注意，Title 不是唯一标识，重复 create 动作将生成多个 app。 + # "maxUsers": MaxUsers, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 + # "noAutoKickUser": NoAutoKickUser # NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ， + # 即同一个身份的 client (app/room/user) ，新的连麦请求可以成功，旧连接被关闭。 } # 然后运行 rtc.CreateApp(<创建APP相关参数的字典变量>) print (rtc.create_app(create_data)) # 查询一个APP -# 查询某一个具体的APP的相关信息的方法为 print ( rtc.GetApp(<app_id>) ) ，其中 app_id 是类似 'desls83s2' 这样在创建时由七牛自动生成的数字字母乱序组合的字符串 +# 查询某一个具体的APP的相关信息的方法为 print ( rtc.GetApp(<app_id>) ) ，其中 app_id 是类似 'desls83s2' +# 这样在创建时由七牛自动生成的数字字母乱序组合的字符串 # 如果不指定具体的app_id，直接运行 print ( rtc.GetApp() ) ，那么就会列举出该账号下所有的APP print (rtc.get_app('<app_id>:可选填')) @@ -38,20 +39,20 @@ # 更新一个APP的相关参数 # 首先需要写好更新的APP的各个参数。参数如下： update_data = { - "hub": "python_new_hub", # Hub: 绑定的直播 hub，可选，用于合流后 rtmp 推流。 - "title": "python_new_app", # Title: app 的名称， 可选。 - # "maxUsers": <MaxUsers>, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 - # "noAutoKickUser": <NoAutoKickUser>, # NoAutoKickUser: bool 类型，可选，禁止自动踢人。 - # "mergePublishRtmp": { # MergePublishRtmp: 连麦合流转推 RTMP 的配置，可选择。其详细配置包括如下 - # "enable": <Enable>, # Enable: 布尔类型，用于开启和关闭所有房间的合流功能。 - # "audioOnly": <AudioOnly>, # AudioOnly: 布尔类型，可选，指定是否只合成音频。 - # "height": <OutputHeight>, # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 - # "width": <OutputHeight>, # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 - # "fps": <OutputFps>, # OutputFps: int64，可选，指定合流输出的帧率，默认为 25 fps 。 - # "kbps": <OutputKbps>, # OutputKbps: int64，可选，指定合流输出的码率，默认为 1000 。 - # "url": "<URL>", # URL: 合流后转推旁路直播的地址，可选，支持魔法变量配置按照连麦房间号生成不同的推流地址。如果是转推到七牛直播云，不建议使用该配置。 - # "streamTitle": "<StreamTitle>" # StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号生成不同的流名。例如，配置 Hub 为 qn-zhibo ，配置 StreamTitle 为 $(roomName) ，则房间 meeting-001 的合流将会被转推到 rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。详细配置细则，请咨询七牛技术支持。 - # } + "hub": "python_new_hub", # Hub: 绑定的直播 hub，可选，用于合流后 rtmp 推流。 + "title": "python_new_app", # Title: app 的名称， 可选。 + # "maxUsers": <MaxUsers>, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 + # "noAutoKickUser": <NoAutoKickUser>, # NoAutoKickUser: bool 类型，可选，禁止自动踢人。 + # "mergePublishRtmp": { # MergePublishRtmp: 连麦合流转推 RTMP 的配置，可选择。其详细配置包括如下 + # "enable": <Enable>, # Enable: 布尔类型，用于开启和关闭所有房间的合流功能。 + # "audioOnly": <AudioOnly>, # AudioOnly: 布尔类型，可选，指定是否只合成音频。 + # "height": <OutputHeight>, # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 + # "width": <OutputHeight>, # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 + # "fps": <OutputFps>, # OutputFps: int64，可选，指定合流输出的帧率，默认为 25 fps 。 + # "kbps": <OutputKbps>, # OutputKbps: int64，可选，指定合流输出的码率，默认为 1000 。 + # "url": "<URL>", # URL: 合流后转推旁路直播的地址，可选，支持魔法变量配置按照连麦房间号生成不同的推流地址。如果是转推到七牛直播云，不建议使用该配置。 + # "streamTitle": "<StreamTitle>" # StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号生成不同的流名。例如，配置 Hub 为 qn-zhibo ，配置 StreamTitle 为 $(roomName) ，则房间 meeting-001 的合流将会被转推到 rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。详细配置细则，请咨询七牛技术支持。 + # } } # 使用方法为：rtc.UpdateApp('<app_id>:必填', update_data)，例如：app_id 是形如 desmfnkw5 的字符串 print (rtc.update_app('<app_id>:必填', update_data)) @@ -68,11 +69,11 @@ # 计算房间管理鉴权。连麦用户终端通过房间管理鉴权获取七牛连麦服务 # 首先需要写好房间鉴权的各个参数。参数如下： roomAccess = { - "AppID": "<AppID>:必填", # AppID: 房间所属帐号的 app 。 - "roomName": "<房间名>:必填", # RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ - "userId": "<用户名>:必填", # UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ - "expireAt": int(time.time()) + 3600, # ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间，token 将在该时间后失效。 - "permission": "user" # 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时，拥有将其他用户移除出房间等特权. + "AppID": "<AppID>:必填", # AppID: 房间所属帐号的 app 。 + "roomName": "<房间名>:必填", # RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ + "userId": "<用户名>:必填", # UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ + "expireAt": int(time.time()) + 3600, # ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间，token 将在该时间后失效。 + "permission": "user" # 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时，拥有将其他用户移除出房间等特权. } # 获得房间管理鉴权的方法：print (RtcRoomToken ( access_key, secret_key, roomAccess ) ) -print (rtc_room_token(access_key, secret_key, roomAccess)) +print (get_room_token(access_key, secret_key, roomAccess)) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 56fd2adb..64d5ad58 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -25,6 +25,6 @@ from .services.compute.app import AccountClient from .services.compute.qcos_api import QcosClient -from .services.pili.rtc_server_manager import RtcServer, rtc_room_token +from .services.pili.rtc_server_manager import RtcServer, get_room_token from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry diff --git a/qiniu/services/pili/rtc_server_manager.py b/qiniu/services/pili/rtc_server_manager.py index b6d48e60..ed5b9b00 100644 --- a/qiniu/services/pili/rtc_server_manager.py +++ b/qiniu/services/pili/rtc_server_manager.py @@ -1,14 +1,13 @@ # -*- coding: utf-8 -*- -from qiniu import http -import json, hashlib, hmac, base64 +from qiniu.utils import urlsafe_base64_encode +from qiniu import http, Auth +import json class RtcServer(object): """ 直播连麦管理类 - 主要涉及了直播连麦管理及操作接口的实现，具体的接口规格可以参考： - https://github.com/pili-engineering/QNRTC-Server/blob/master/docs/api.md#41-listuser #这个是内部文档，等外部文档发布了，这一行要换成外部文档 - + 主要涉及了直播连麦管理及操作接口的实现，具体的接口规格可以参考官方文档 https://developer.qiniu.com Attributes: auth: 账号管理密钥对，Auth对象 @@ -19,258 +18,27 @@ def __init__(self, auth): self.host = 'http://rtc.qiniuapi.com' def create_app(self, data): - """ - Host rtc.qiniuapi.com - POST /v3/apps - Authorization: qiniu mac - Content-Type: application/json - - { - "hub": "<Hub>", - "title": "<Title>", - "maxUsers": <MaxUsers>, - "noAutoKickUser": <NoAutoKickUser> - } - - :param appid: - Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 - Title: app 的名称，可选，注意，Title 不是唯一标识，重复 create 动作将生成多个 app。 - MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 - NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ，即同一个身份的 client (app/room/user) ， - 新的连麦请求可以成功，旧连接被关闭。 - - :return: - 200 OK - { - "appId": "<AppID>", - "hub": "<Hub>", - "title": "<Title>", - "maxUsers": <MaxUsers>, - "noAutoKickUser": <NoAutoKickUser>, - "createdAt": <CreatedAt>, - "updatedAt": <UpdatedAt> - } - 616 - { - "error": "hub not match" - } - """ - return self.__post(self.host + '/v3/apps', data) def get_app(self, app_id=None): - """ - Host rtc.qiniuapi.com - GET /v3/apps/<AppID> - Authorization: qiniu mac - - :param appid: - AppID: app 的唯一标识。 可以不填写，不填写的话，默认就是输出所有app的相关信息 - - :return: - 200 OK - { - "appId": "<AppID>", - "hub": "<Hub>", - "title": "<Title>", - "maxUsers": <MaxUsers>, - "noAutoKickUser": <NoAutoKickUser>, - "mergePublishRtmp": { - "audioOnly": <AudioOnly>, - "height": <OutputHeight>, - "width": <OutputHeight>, - "fps": <OutputFps>, - "kbps": <OutputKbps>, - "url": "<URL>", - "streamTitle": "<StreamTitle>" - }, - "createdAt": <CreatedAt>, - "updatedAt": <UpdatedAt> - } - - 612 - { - "error": "app not found" - } - - #### - AppID: app 的唯一标识。 - UID: 客户的七牛帐号。 - Hub: 绑定的直播 hub，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 - Title: app 的名称，注意，Title不是唯一标识。 - MaxUsers: int 类型，连麦房间支持的最大在线人数。 - NoAutoKickUser: bool 类型，禁止自动踢人。 - MergePublishRtmp: 连麦合流转推 RTMP 的配置。 - CreatedAt: time 类型，app 创建的时间。 - UpdatedAt: time 类型，app 更新的时间。 - """ if app_id: return self.__get(self.host + '/v3/apps/%s' % app_id) else: return self.__get(self.host + '/v3/apps') def delete_app(self, app_id): - """ - Host rtc.qiniuapi.com - DELETE /v3/apps/<AppID> - Authorization: qiniu mac - - :return: - 200 OK - - 612 - { - "error": "app not found" - } - """ return self.__delete(self.host + '/v3/apps/%s' % app_id) def update_app(self, app_id, data): - """ - Host rtc.qiniuapi.com - Post /v3/apps/<AppID> - Authorization: qiniu mac - - :param appid: - AppID: app 的唯一标识，创建的时候由系统生成。 - Title: app 的名称， 可选。 - Hub: 绑定的直播 hub，可选，用于合流后 rtmp 推流。 - MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 - NoAutoKickUser: bool 类型，可选，禁止自动踢人。 - MergePublishRtmp: 连麦合流转推 RTMP 的配置，可选择。其详细配置包括如下 - Enable: 布尔类型，用于开启和关闭所有房间的合流功能。 - AudioOnly: 布尔类型，可选，指定是否只合成音频。 - Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 - OutputFps: int64，可选，指定合流输出的帧率，默认为 25 fps 。 - OutputKbps: int64，可选，指定合流输出的码率，默认为 1000 。 - URL: 合流后转推旁路直播的地址，可选，支持魔法变量配置按照连麦房间号生成不同的推流地址。如果是转推到七牛直播云，不建议使用该配置。 - StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号生成不同的流名。例如，配置 Hub 为 qn-zhibo ， - 配置 StreamTitle 为 $(roomName) ，则房间 meeting-001 的合流将会被转推到 - rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。详细配置细则，请咨询七牛技术支持。 - - :return: - 200 OK - { - "appId": "<AppID>", - "hub": "<Hub>", - "title": "<Title>", - "maxUsers": <MaxUsers>, - "noAutoKickUser": <NoAutoKickUser>, - "mergePublishRtmp": { - "enable": <Enable>, - "audioOnly": <AudioOnly>, - "height": <OutputHeight>, - "width": <OutputHeight>, - "fps": <OutputFps>, - "kbps": <OutputKbps>, - "url": "<URL>", - "streamTitle": "<StreamTitle>" - }, - "createdAt": <CreatedAt>, - "updatedAt": <UpdatedAt> - } - - 612 - { - "error": "app not found" - } - 616 - { - "error": "hub not match" - } - """ - return self.__post(self.host + '/v3/apps/%s' % app_id, data) def list_user(self, app_id, room_name): - """ - Host rtc.qiniuapi.com - GET /v3/apps/<app_id>/rooms/<room_name>/users - Authorization: qiniu mac - - :param: - app_id: 连麦房间所属的 app 。 - - room_name: 操作所查询的连麦房间。 - - :return: - 200 OK - { - "users": [ - { - "userId": "<UserID>" - }, - ] - } - 612 - { - "error": "app not found" - } - """ return self.__get(self.host + '/v3/apps/%s/rooms/%s/users' % (app_id, room_name)) def kick_user(self, app_id, room_name, user_id): - """ - Host rtc.qiniuapi.com - DELETE /v3/apps/<app_id>/rooms/<room_name>/users/<user_id> - Authorization: qiniu mac - - :param: - app_id: 连麦房间所属的 app 。 - - room_name: 连麦房间。 - - user_id: 操作所剔除的用户。 - - :return: - 200 OK - 612 - { - "error": "app not found" - } - 612 - { - "error": "user not found" - } - 615 - { - "error": "room not active" - } - """ return self.__delete(self.host + '/v3/apps/%s/rooms/%s/users/%s' % (app_id, room_name, user_id)) def list_active_rooms(self, app_id, room_name_prefix=None): - """ - Host rtc.qiniuapi.com - GET /v3/apps/<app_id>/rooms?prefix=<room_name_prefix>&offset=<off_set>&limit=<limit> - Authorization: qiniu mac - - :param: - app_id: 连麦房间所属的 app 。 - - room_name_prefix: 所查询房间名的前缀索引，可以为空。 - off_set: int 类型，分页查询的位移标记。 - limit: int 类型，此次查询的最大长度。 - - :return: - 200 OK - { - "end": <IsEnd>, - "offset": <Offset>, - "rooms": [ - "<RoomName>", - ... - ] - } - 612 - { - "error": "app not found" - } - ### - IsEnd: bool 类型，分页查询是否已经查完所有房间。 - Offset: int 类型，下次分页查询使用的位移标记。 - RoomName: 当前活跃的房间名。 - """ if room_name_prefix: return self.__get(self.host + '/v3/apps/%s/rooms?prefix=%s' % (app_id, room_name_prefix)) else: @@ -286,39 +54,9 @@ def __delete(self, url, params=None): return http._delete_with_qiniu_mac(url, params, self.auth) -def rtc_room_token(access_key, secret_key, room_access): - """ - :arg: - AppID: 房间所属帐号的 app 。 - RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ - UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ - ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间，token 将在该时间后失效。 - Permission: 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时，拥有将其他用户移除出房间等特权. - :method: - # 1. 定义房间管理凭证，并对凭证字符做URL安全的Base64编码 - roomAccess = { - "appId": "<AppID>" - "roomName": "<RoomName>", - "userId": "<UserID>", - "expireAt": <ExpireAt>, - "permission": "<Permission>" - } - roomAccessString = json_to_string(roomAccess) - encodedRoomAccess = urlsafe_base64_encode(roomAccessString) - - # 2. 计算HMAC-SHA1签名，并对签名结果做URL安全的Base64编码 - sign = hmac_sha1(encodedRoomAccess, <SecretKey>) - encodedSign = urlsafe_base64_encode(sign) - - # 3. 将AccessKey与以上两者拼接得到房间鉴权 - roomToken = "<AccessKey>" + ":" + encodedSign + ":" + encodedRoomAccess - """ - roomAccessString = json.dumps(room_access) - byte_result = bytes(roomAccessString, 'utf-8') - encodedRoomAccess = base64.urlsafe_b64encode(byte_result) - - sign = hmac.new(bytes(secret_key, 'utf-8'), encodedRoomAccess, hashlib.sha1).digest() - encodedSign = base64.urlsafe_b64encode(sign) - roomToken = access_key + ':' + str(encodedSign, encoding="utf-8") + ':' + str(encodedRoomAccess, encoding="utf-8") - - return roomToken +def get_room_token(access_key, secret_key, room_access): + auth = Auth(access_key, secret_key) + room_access_str = json.dumps(room_access) + encoded_room_access = urlsafe_base64_encode(room_access_str) + room_token = auth.token_with_data(encoded_room_access) + return room_token From 10973f8410a8caaa424a2dcb0c2befa8a357ed33 Mon Sep 17 00:00:00 2001 From: Jemy <jemygraw@gmail.com> Date: Wed, 9 May 2018 18:12:24 +0800 Subject: [PATCH 284/478] fix get rtc room token error fix the comment style --- examples/rtc_server.py | 19 +++++++++++++------ qiniu/services/pili/rtc_server_manager.py | 3 +-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/examples/rtc_server.py b/examples/rtc_server.py index c3f7980e..77610beb 100644 --- a/examples/rtc_server.py +++ b/examples/rtc_server.py @@ -17,7 +17,7 @@ # 创建一个APP # 首先需要写好创建APP的各个参数。参数如下 create_data = { - "hub": 'python_test_hub', # Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub 与 app 必须属于同一个七牛账户。 + "hub": 'python_test_hub', # Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub与app 必须属于同一个七牛账户。 "title": 'python_test_app', # Title: app 的名称，可选，注意，Title 不是唯一标识，重复 create 动作将生成多个 app。 # "maxUsers": MaxUsers, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 # "noAutoKickUser": NoAutoKickUser # NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ， @@ -50,8 +50,13 @@ # "width": <OutputHeight>, # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 # "fps": <OutputFps>, # OutputFps: int64，可选，指定合流输出的帧率，默认为 25 fps 。 # "kbps": <OutputKbps>, # OutputKbps: int64，可选，指定合流输出的码率，默认为 1000 。 - # "url": "<URL>", # URL: 合流后转推旁路直播的地址，可选，支持魔法变量配置按照连麦房间号生成不同的推流地址。如果是转推到七牛直播云，不建议使用该配置。 - # "streamTitle": "<StreamTitle>" # StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号生成不同的流名。例如，配置 Hub 为 qn-zhibo ，配置 StreamTitle 为 $(roomName) ，则房间 meeting-001 的合流将会被转推到 rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。详细配置细则，请咨询七牛技术支持。 + # "url": "<URL>", # URL: 合流后转推旁路直播的地址，可选，支持魔法变量配置按照连麦房间号生成不同 + # 的推流地址。如果是转推到七牛直播云，不建议使用该配置。 + + # "streamTitle": "<StreamTitle>" # StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号 + # 生成不同的流名。例如，配置 Hub 为 qn-zhibo ，配置 StreamTitle 为 $(roomName) ， + # 则房间 meeting-001 的合流将会被转推到 rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。 + # 详细配置细则，请咨询七牛技术支持。 # } } # 使用方法为：rtc.UpdateApp('<app_id>:必填', update_data)，例如：app_id 是形如 desmfnkw5 的字符串 @@ -69,11 +74,13 @@ # 计算房间管理鉴权。连麦用户终端通过房间管理鉴权获取七牛连麦服务 # 首先需要写好房间鉴权的各个参数。参数如下： roomAccess = { - "AppID": "<AppID>:必填", # AppID: 房间所属帐号的 app 。 + "appId": "<AppID>:必填", # AppID: 房间所属帐号的 app 。 "roomName": "<房间名>:必填", # RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ "userId": "<用户名>:必填", # UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ - "expireAt": int(time.time()) + 3600, # ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间，token 将在该时间后失效。 - "permission": "user" # 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时，拥有将其他用户移除出房间等特权. + "expireAt": int(time.time()) + 3600, # ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间， + # token 将在该时间后失效。 + "permission": "user" # 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时， + # 拥有将其他用户移除出房间等特权. } # 获得房间管理鉴权的方法：print (RtcRoomToken ( access_key, secret_key, roomAccess ) ) print (get_room_token(access_key, secret_key, roomAccess)) diff --git a/qiniu/services/pili/rtc_server_manager.py b/qiniu/services/pili/rtc_server_manager.py index ed5b9b00..01c244a3 100644 --- a/qiniu/services/pili/rtc_server_manager.py +++ b/qiniu/services/pili/rtc_server_manager.py @@ -57,6 +57,5 @@ def __delete(self, url, params=None): def get_room_token(access_key, secret_key, room_access): auth = Auth(access_key, secret_key) room_access_str = json.dumps(room_access) - encoded_room_access = urlsafe_base64_encode(room_access_str) - room_token = auth.token_with_data(encoded_room_access) + room_token = auth.token_with_data(room_access_str) return room_token From d6406e23af430c2c52b81ecf131e67ab5dc60019 Mon Sep 17 00:00:00 2001 From: Jemy <jemygraw@gmail.com> Date: Wed, 9 May 2018 21:17:03 +0800 Subject: [PATCH 285/478] remove useless space and imports --- examples/rtc_server.py | 22 +++++++++++----------- qiniu/services/pili/rtc_server_manager.py | 1 - 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/examples/rtc_server.py b/examples/rtc_server.py index 77610beb..5e3d5d84 100644 --- a/examples/rtc_server.py +++ b/examples/rtc_server.py @@ -24,17 +24,17 @@ # 即同一个身份的 client (app/room/user) ，新的连麦请求可以成功，旧连接被关闭。 } # 然后运行 rtc.CreateApp(<创建APP相关参数的字典变量>) -print (rtc.create_app(create_data)) +print(rtc.create_app(create_data)) # 查询一个APP -# 查询某一个具体的APP的相关信息的方法为 print ( rtc.GetApp(<app_id>) ) ，其中 app_id 是类似 'desls83s2' +# 查询某一个具体的APP的相关信息的方法为 print( rtc.GetApp(<app_id>) ) ，其中 app_id 是类似 'desls83s2' # 这样在创建时由七牛自动生成的数字字母乱序组合的字符串 -# 如果不指定具体的app_id，直接运行 print ( rtc.GetApp() ) ，那么就会列举出该账号下所有的APP -print (rtc.get_app('<app_id>:可选填')) +# 如果不指定具体的app_id，直接运行 print( rtc.GetApp() ) ，那么就会列举出该账号下所有的APP +print(rtc.get_app('<app_id>:可选填')) # 删除一个APP # 使用方法为：rtc.DeleteApp(<app_id>)，例如： rtc.DeleteApp('desls83s2') -print (rtc.delete_app('<app_id>:必填')) +print(rtc.delete_app('<app_id>:必填')) # 更新一个APP的相关参数 # 首先需要写好更新的APP的各个参数。参数如下： @@ -60,16 +60,16 @@ # } } # 使用方法为：rtc.UpdateApp('<app_id>:必填', update_data)，例如：app_id 是形如 desmfnkw5 的字符串 -print (rtc.update_app('<app_id>:必填', update_data)) +print(rtc.update_app('<app_id>:必填', update_data)) # 列举一个APP下面，某个房间的所有用户 -print (rtc.list_user('<app_id>:必填', '<房间名>:必填')) +print(rtc.list_user('<app_id>:必填', '<房间名>:必填')) # 踢出一个APP下面，某个房间的某个用户 -print (rtc.kick_user('<app_id>:必填', '<房间名>:必填', '<客户ID>:必填')) +print(rtc.kick_user('<app_id>:必填', '<房间名>:必填', '<客户ID>:必填')) # 列举一个APP下面，所有的房间 -print (rtc.list_active_rooms('<app_id>:必填')) +print(rtc.list_active_rooms('<app_id>:必填')) # 计算房间管理鉴权。连麦用户终端通过房间管理鉴权获取七牛连麦服务 # 首先需要写好房间鉴权的各个参数。参数如下： @@ -82,5 +82,5 @@ "permission": "user" # 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时， # 拥有将其他用户移除出房间等特权. } -# 获得房间管理鉴权的方法：print (RtcRoomToken ( access_key, secret_key, roomAccess ) ) -print (get_room_token(access_key, secret_key, roomAccess)) +# 获得房间管理鉴权的方法：print(RtcRoomToken ( access_key, secret_key, roomAccess ) ) +print(get_room_token(access_key, secret_key, roomAccess)) diff --git a/qiniu/services/pili/rtc_server_manager.py b/qiniu/services/pili/rtc_server_manager.py index 01c244a3..ba12bcb1 100644 --- a/qiniu/services/pili/rtc_server_manager.py +++ b/qiniu/services/pili/rtc_server_manager.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from qiniu.utils import urlsafe_base64_encode from qiniu import http, Auth import json From 3e925988ea6ee8b13d49a13b17fa5ff7b6c8f255 Mon Sep 17 00:00:00 2001 From: Jemy <jemygraw@gmail.com> Date: Thu, 10 May 2018 09:18:34 +0800 Subject: [PATCH 286/478] prepare to publish v7.2.1 --- CHANGELOG.md | 3 +++ qiniu/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26f2b5d2..98f07427 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +# 7.2.1 (2018-05-10) +* 增加连麦rtc服务端API功能 + # 7.2.0（2017-11-23） * 修复put_data不支持file like object的问题 * 增加空间写错时，抛出异常提示客户的功能 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 64d5ad58..4268673f 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.0' +__version__ = '7.2.1' from .auth import Auth, QiniuMacAuth From 5f7bd8c31eefe30427ea774c520f83d9f6254d4a Mon Sep 17 00:00:00 2001 From: Jemy <jemygraw@gmail.com> Date: Thu, 10 May 2018 16:25:21 +0800 Subject: [PATCH 287/478] add the module to the setup script --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f686b5cf..1190c4f3 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ 'qiniu.services.processing', 'qiniu.services.compute', 'qiniu.services.cdn', + 'qiniu.services.pili', ] From 69ab1318b31e421f80ce7c556f9ab866b65b628d Mon Sep 17 00:00:00 2001 From: Jemy <jemygraw@gmail.com> Date: Thu, 10 May 2018 16:26:37 +0800 Subject: [PATCH 288/478] prepare to publish v7.2.2 --- CHANGELOG.md | 2 +- qiniu/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98f07427..148fc69f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -# 7.2.1 (2018-05-10) +# 7.2.2 (2018-05-10) * 增加连麦rtc服务端API功能 # 7.2.0（2017-11-23） diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 4268673f..5fbca695 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.1' +__version__ = '7.2.2' from .auth import Auth, QiniuMacAuth From 5eb0a0e364f2cef57c3004ec56a6b1af792953d2 Mon Sep 17 00:00:00 2001 From: Jemy Graw <jemygraw@gmail.com> Date: Tue, 15 May 2018 15:08:06 +0800 Subject: [PATCH 289/478] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b200edbb..2439b8eb 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Qiniu Resource Storage SDK for Python +# Qiniu Cloud SDK for Python [![@qiniu on weibo](http://img.shields.io/badge/weibo-%40qiniutek-blue.svg)](http://weibo.com/qiniutek) [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE) From ccdc0a0364ed4d2ebfb38906648879ed4be087b0 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Tue, 15 May 2018 15:51:46 +0800 Subject: [PATCH 290/478] fix cmd error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 打水印参数漏了 --- examples/pfop_watermark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/pfop_watermark.py b/examples/pfop_watermark.py index da5cd43a..72b7de7d 100755 --- a/examples/pfop_watermark.py +++ b/examples/pfop_watermark.py @@ -18,7 +18,7 @@ base64URL = urlsafe_base64_encode('http://developer.qiniu.com/resource/logo-2.jpg') # 视频水印参数 -fops = 'avthumb/mp4/'+base64URL +fops = 'avthumb/mp4/wmImage/'+base64URL # 可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') @@ -28,4 +28,4 @@ ops.append(fops) ret, info = pfop.execute(key, ops, 1) print(info) -assert ret['persistentId'] \ No newline at end of file +assert ret['persistentId'] From 63434df31667198878492afb6946b69a24036692 Mon Sep 17 00:00:00 2001 From: longbai <slongbai@gmail.com> Date: Thu, 17 May 2018 08:57:29 +0800 Subject: [PATCH 291/478] format --- examples/batch.py | 2 +- examples/batch_copy.py | 6 ++++- examples/batch_move.py | 6 ++++- examples/batch_rename.py | 4 ++- examples/cdn_bandwidth.py | 3 ++- examples/delete_afte_days.py | 2 -- examples/fops.py | 2 +- examples/pfop_vframe.py | 4 +-- examples/pfop_watermark.py | 9 ++++--- examples/rtc_server.py | 3 ++- examples/timestamp_url.py | 5 ++-- examples/update_cdn_sslcert.py | 15 ++++++----- examples/upload_callback.py | 9 +++---- examples/upload_pfops.py | 8 +++--- examples/upload_token.py | 19 +++++++------- examples/upload_with_zone.py | 8 ++++-- qiniu/__init__.py | 2 +- qiniu/auth.py | 38 +++++++++++++++++++++------ qiniu/http.py | 47 +++++++++++++++++++++++++--------- qiniu/main.py | 9 +++++-- qiniu/utils.py | 3 ++- qiniu/zone.py | 22 ++++++++++++---- 22 files changed, 152 insertions(+), 74 deletions(-) diff --git a/examples/batch.py b/examples/batch.py index dd05bde9..276e6ea8 100755 --- a/examples/batch.py +++ b/examples/batch.py @@ -4,7 +4,7 @@ from qiniu import Auth from qiniu import BucketManager from qiniu import build_batch_copy -from qiniu import build_batch_move,build_batch_rename +from qiniu import build_batch_move, build_batch_rename access_key = '...' secret_key = '...' diff --git a/examples/batch_copy.py b/examples/batch_copy.py index ccb8781f..f12225a3 100644 --- a/examples/batch_copy.py +++ b/examples/batch_copy.py @@ -21,6 +21,10 @@ target_bucket_name = '' # force为true时强制同名覆盖, 字典的键为原文件，值为目标文件 -ops = build_batch_copy(src_bucket_name, {'src_key1': 'target_key1', 'src_key2': 'target_key2'}, target_bucket_name, force='true') +ops = build_batch_copy(src_bucket_name, + {'src_key1': 'target_key1', + 'src_key2': 'target_key2'}, + target_bucket_name, + force='true') ret, info = bucket.batch(ops) print(info) diff --git a/examples/batch_move.py b/examples/batch_move.py index b333e295..c4f51076 100644 --- a/examples/batch_move.py +++ b/examples/batch_move.py @@ -22,6 +22,10 @@ target_bucket_name = '' # force为true时强制同名覆盖, 字典的键为原文件，值为目标文件 -ops = build_batch_move(src_bucket_name, {'src_key1': 'target_key1', 'src_key2': 'target_key2'}, target_bucket_name, force='true') +ops = build_batch_move(src_bucket_name, + {'src_key1': 'target_key1', + 'src_key2': 'target_key2'}, + target_bucket_name, + force='true') ret, info = bucket.batch(ops) print(info) diff --git a/examples/batch_rename.py b/examples/batch_rename.py index dc7c35e5..12e7c806 100644 --- a/examples/batch_rename.py +++ b/examples/batch_rename.py @@ -21,6 +21,8 @@ # force为true时强制同名覆盖, 字典的键为原文件，值为目标文件 -ops = build_batch_rename(bucket_name, {'src_key1': 'target_key1', 'src_key2': 'target_key2'}, force='true') +ops = build_batch_rename( + bucket_name, { + 'src_key1': 'target_key1', 'src_key2': 'target_key2'}, force='true') ret, info = bucket.batch(ops) print(info) diff --git a/examples/cdn_bandwidth.py b/examples/cdn_bandwidth.py index a98c0423..c4c4c0ce 100644 --- a/examples/cdn_bandwidth.py +++ b/examples/cdn_bandwidth.py @@ -24,7 +24,8 @@ 'b.example.com' ] -ret, info = cdn_manager.get_bandwidth_data(urls, startDate, endDate, granularity) +ret, info = cdn_manager.get_bandwidth_data( + urls, startDate, endDate, granularity) print(ret) print(info) diff --git a/examples/delete_afte_days.py b/examples/delete_afte_days.py index bfffb378..00401f65 100755 --- a/examples/delete_afte_days.py +++ b/examples/delete_afte_days.py @@ -22,5 +22,3 @@ ret, info = bucket.delete_after_days(bucket_name, key, days) print(info) - - diff --git a/examples/fops.py b/examples/fops.py index 90983a23..f61de679 100755 --- a/examples/fops.py +++ b/examples/fops.py @@ -19,7 +19,7 @@ # 可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') -fops = fops+'|saveas/'+saveas_key +fops = fops + '|saveas/' + saveas_key ops = [] pfop = PersistentFop(q, bucket_name, pipeline) diff --git a/examples/pfop_vframe.py b/examples/pfop_vframe.py index e18564ec..381e5e98 100755 --- a/examples/pfop_vframe.py +++ b/examples/pfop_vframe.py @@ -19,11 +19,11 @@ # 可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') -fops = fops+'|saveas/'+saveas_key +fops = fops + '|saveas/' + saveas_key pfop = PersistentFop(q, bucket, pipeline) ops = [] ops.append(fops) ret, info = pfop.execute(key, ops, 1) print(info) -assert ret['persistentId'] is not None \ No newline at end of file +assert ret['persistentId'] is not None diff --git a/examples/pfop_watermark.py b/examples/pfop_watermark.py index da5cd43a..c671607e 100755 --- a/examples/pfop_watermark.py +++ b/examples/pfop_watermark.py @@ -15,17 +15,18 @@ pipeline = 'pipeline_name' # 需要添加水印的图片UrlSafeBase64,可以参考http://developer.qiniu.com/code/v6/api/dora-api/av/video-watermark.html -base64URL = urlsafe_base64_encode('http://developer.qiniu.com/resource/logo-2.jpg') +base64URL = urlsafe_base64_encode( + 'http://developer.qiniu.com/resource/logo-2.jpg') # 视频水印参数 -fops = 'avthumb/mp4/'+base64URL +fops = 'avthumb/mp4/' + base64URL # 可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') -fops = fops+'|saveas/'+saveas_key +fops = fops + '|saveas/' + saveas_key ops = [] pfop = PersistentFop(q, bucket, pipeline) ops.append(fops) ret, info = pfop.execute(key, ops, 1) print(info) -assert ret['persistentId'] \ No newline at end of file +assert ret['persistentId'] diff --git a/examples/rtc_server.py b/examples/rtc_server.py index 5e3d5d84..84a39bb3 100644 --- a/examples/rtc_server.py +++ b/examples/rtc_server.py @@ -77,7 +77,8 @@ "appId": "<AppID>:必填", # AppID: 房间所属帐号的 app 。 "roomName": "<房间名>:必填", # RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ "userId": "<用户名>:必填", # UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ - "expireAt": int(time.time()) + 3600, # ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间， + # ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间， + "expireAt": int(time.time()) + 3600, # token 将在该时间后失效。 "permission": "user" # 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时， # 拥有将其他用户移除出房间等特权. diff --git a/examples/timestamp_url.py b/examples/timestamp_url.py index d1e548b3..0b543a78 100644 --- a/examples/timestamp_url.py +++ b/examples/timestamp_url.py @@ -19,9 +19,10 @@ query_string = '' # 截止日期的时间戳,秒为单位，3600为当前时间一小时之后过期 -deadline = int(time.time())+3600 +deadline = int(time.time()) + 3600 -timestamp_url = create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline) +timestamp_url = create_timestamp_anti_leech_url( + host, file_name, query_string, encrypt_key, deadline) print(timestamp_url) diff --git a/examples/update_cdn_sslcert.py b/examples/update_cdn_sslcert.py index dbdffca5..06652712 100644 --- a/examples/update_cdn_sslcert.py +++ b/examples/update_cdn_sslcert.py @@ -14,16 +14,17 @@ privatekey = "ssl/www.qiniu.com/privkey.pem" ca = "ssl/www.qiniu.com/fullchain.pem" -domain_name='www.qiniu.com' +domain_name = 'www.qiniu.com' -with open(privatekey,'r') as f: - privatekey_str=f.read() +with open(privatekey, 'r') as f: + privatekey_str = f.read() -with open(ca,'r') as f: - ca_str=f.read() +with open(ca, 'r') as f: + ca_str = f.read() -ret, info = domain_manager.create_sslcert(domain_name, domain_name, privatekey_str, ca_str) +ret, info = domain_manager.create_sslcert( + domain_name, domain_name, privatekey_str, ca_str) print(ret['certID']) -ret, info = domain_manager.put_httpsconf(domain_name, ret['certID'],False) +ret, info = domain_manager.put_httpsconf(domain_name, ret['certID'], False) print(info) diff --git a/examples/upload_callback.py b/examples/upload_callback.py index 1cc8cd38..d8a0a788 100755 --- a/examples/upload_callback.py +++ b/examples/upload_callback.py @@ -12,11 +12,11 @@ key = 'my-python-logo.png' -#上传文件到七牛后， 七牛将文件名和文件大小回调给业务服务器。 +# 上传文件到七牛后， 七牛将文件名和文件大小回调给业务服务器。 policy = { - 'callbackUrl': 'http://your.domain.com/callback.php', - 'callbackBody': 'filename=$(fname)&filesize=$(fsize)' - } + 'callbackUrl': 'http://your.domain.com/callback.php', + 'callbackBody': 'filename=$(fname)&filesize=$(fsize)' +} token = q.upload_token(bucket_name, key, 3600, policy) @@ -26,4 +26,3 @@ print(info) assert ret['key'] == key assert ret['hash'] == etag(localfile) - diff --git a/examples/upload_pfops.py b/examples/upload_pfops.py index 06670a60..7cc2b9e1 100755 --- a/examples/upload_pfops.py +++ b/examples/upload_pfops.py @@ -21,13 +21,13 @@ # 通过添加'|saveas'参数，指定处理后的文件保存的bucket和key，不指定默认保存在当前空间，bucket_saved为目标bucket，bucket_saved为目标key saveas_key = urlsafe_base64_encode('bucket_saved:bucket_saved') -fops = fops+'|saveas/'+saveas_key +fops = fops + '|saveas/' + saveas_key # 在上传策略中指定fobs和pipeline policy = { - 'persistentOps': fops, - 'persistentPipeline': pipeline - } + 'persistentOps': fops, + 'persistentPipeline': pipeline +} token = q.upload_token(bucket_name, key, 3600, policy) diff --git a/examples/upload_token.py b/examples/upload_token.py index 911319c4..69ba24e3 100644 --- a/examples/upload_token.py +++ b/examples/upload_token.py @@ -3,30 +3,29 @@ from qiniu import Auth -#需要填写你的 Access Key 和 Secret Key +# 需要填写你的 Access Key 和 Secret Key access_key = '' secret_key = '' -#构建鉴权对象 +# 构建鉴权对象 q = Auth(access_key, secret_key) -#要上传的空间 +# 要上传的空间 bucket_name = '' -#上传到七牛后保存的文件名 +# 上传到七牛后保存的文件名 key = '' -#生成上传 Token，可以指定过期时间等 +# 生成上传 Token，可以指定过期时间等 # 上传策略示例 # https://developer.qiniu.com/kodo/manual/1206/put-policy policy = { - # 'callbackUrl':'https://requestb.in/1c7q2d31', - # 'callbackBody':'filename=$(fname)&filesize=$(fsize)' - # 'persistentOps':'imageView2/1/w/200/h/200' - } + # 'callbackUrl':'https://requestb.in/1c7q2d31', + # 'callbackBody':'filename=$(fname)&filesize=$(fsize)' + # 'persistentOps':'imageView2/1/w/200/h/200' +} token = q.upload_token(bucket_name, key, 3600, policy) print(token) - diff --git a/examples/upload_with_zone.py b/examples/upload_with_zone.py index 5151a970..0c8e56de 100644 --- a/examples/upload_with_zone.py +++ b/examples/upload_with_zone.py @@ -16,7 +16,7 @@ bucket_name = 'Bucket_Name' # 上传到七牛后保存的文件名 -key = 'my-python-logo.png'; +key = 'my-python-logo.png' # 生成上传 Token，可以指定过期时间等 token = q.upload_token(bucket_name, key, 3600) @@ -25,7 +25,11 @@ localfile = 'stat.py' # 指定固定的zone -zone = Zone(up_host='uptest.qiniu.com', up_host_backup='uptest.qiniu.com', io_host='iovip.qbox.me', scheme='http') +zone = Zone( + up_host='uptest.qiniu.com', + up_host_backup='uptest.qiniu.com', + io_host='iovip.qbox.me', + scheme='http') set_default(default_zone=zone) ret, info = put_file(token, key, localfile) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 8278c1f7..f229397a 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -19,7 +19,7 @@ from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ build_batch_stat, build_batch_delete from .services.storage.uploader import put_data, put_file, put_stream -from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url,DomainManager +from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url, DomainManager from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save from .services.compute.app import AccountClient diff --git a/qiniu/auth.py b/qiniu/auth.py index 00c629ff..c25ad6d0 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -69,7 +69,8 @@ def token(self, data): def token_with_data(self, data): data = urlsafe_base64_encode(data) - return '{0}:{1}:{2}'.format(self.__access_key, self.__token(data), data) + return '{0}:{1}:{2}'.format( + self.__access_key, self.__token(data), data) def token_of_request(self, url, body=None, content_type=None): """带请求体的签名（本质上是管理凭证的签名） @@ -124,7 +125,13 @@ def private_download_url(self, url, expires=3600): token = self.token(url) return '{0}&token={1}'.format(url, token) - def upload_token(self, bucket, key=None, expires=3600, policy=None, strict_policy=True): + def upload_token( + self, + bucket, + key=None, + expires=3600, + policy=None, + strict_policy=True): """生成上传凭证 Args: @@ -157,7 +164,12 @@ def __upload_token(self, policy): data = json.dumps(policy, separators=(',', ':')) return self.token_with_data(data) - def verify_callback(self, origin_authorization, url, body, content_type='application/x-www-form-urlencoded'): + def verify_callback( + self, + origin_authorization, + url, + body, + content_type='application/x-www-form-urlencoded'): """回调验证 Args: @@ -186,7 +198,8 @@ def __init__(self, auth): def __call__(self, r): if r.body is not None and r.headers['Content-Type'] == 'application/x-www-form-urlencoded': - token = self.auth.token_of_request(r.url, r.body, 'application/x-www-form-urlencoded') + token = self.auth.token_of_request( + r.url, r.body, 'application/x-www-form-urlencoded') else: token = self.auth.token_of_request(r.url) r.headers['Authorization'] = 'QBox {0}'.format(token) @@ -215,7 +228,14 @@ def __token(self, data): hashed = hmac.new(self.__secret_key, data, sha1) return urlsafe_base64_encode(hashed.digest()) - def token_of_request(self, method, host, url, qheaders, content_type=None, body=None): + def token_of_request( + self, + method, + host, + url, + qheaders, + content_type=None, + body=None): """ <Method> <PathWithRawQuery> Host: <Host> @@ -236,7 +256,9 @@ def token_of_request(self, method, host, url, qheaders, content_type=None, body= path_with_query = path if query != '': path_with_query = ''.join([path_with_query, '?', query]) - data = ''.join(["%s %s" % (method, path_with_query), "\n", "Host: %s" % host, "\n"]) + data = ''.join(["%s %s" % + (method, path_with_query), "\n", "Host: %s" % + host, "\n"]) if content_type: data += "Content-Type: %s" % (content_type) + "\n" @@ -253,7 +275,7 @@ def qiniu_headers(self, headers): res = "" for key in headers: if key.startswith(self.qiniu_header_prefix): - res += key+": %s\n" % (headers.get(key)) + res += key + ": %s\n" % (headers.get(key)) return res @staticmethod @@ -272,6 +294,6 @@ def __call__(self, r): r.url, self.auth.qiniu_headers(r.headers), r.headers.get('Content-Type', None), r.body - ) + ) r.headers['Authorization'] = 'Qiniu {0}'.format(token) return r diff --git a/qiniu/http.py b/qiniu/http.py index b44a207d..80596700 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -12,7 +12,8 @@ _sys_info = '{0}; {1}'.format(platform.system(), platform.machine()) _python_ver = platform.python_version() -USER_AGENT = 'QiniuPython/{0} ({1}; ) Python/{2}'.format(__version__, _sys_info, _python_ver) +USER_AGENT = 'QiniuPython/{0} ({1}; ) Python/{2}'.format( + __version__, _sys_info, _python_ver) _session = None _headers = {'User-Agent': USER_AGENT} @@ -29,7 +30,8 @@ def __return_wrapper(resp): def _init(): session = requests.Session() adapter = requests.adapters.HTTPAdapter( - pool_connections=config.get_default('connection_pool'), pool_maxsize=config.get_default('connection_pool'), + pool_connections=config.get_default('connection_pool'), + pool_maxsize=config.get_default('connection_pool'), max_retries=config.get_default('connection_retries')) session.mount('http://', adapter) global _session @@ -51,6 +53,7 @@ def _post(url, data, files, auth, headers=None): return None, ResponseInfo(None, e) return __return_wrapper(r) + def _put(url, data, files, auth, headers=None): if _session is None: _init() @@ -66,11 +69,15 @@ def _put(url, data, files, auth, headers=None): return None, ResponseInfo(None, e) return __return_wrapper(r) + def _get(url, params, auth): try: r = requests.get( - url, params=params, auth=qiniu.auth.RequestsAuth(auth) if auth is not None else None, - timeout=config.get_default('connection_timeout'), headers=_headers) + url, + params=params, + auth=qiniu.auth.RequestsAuth(auth) if auth is not None else None, + timeout=config.get_default('connection_timeout'), + headers=_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) @@ -100,19 +107,27 @@ def _post_with_auth(url, data, auth): def _post_with_auth_and_headers(url, data, auth, headers): return _post(url, data, None, qiniu.auth.RequestsAuth(auth), headers) + def _put_with_auth(url, data, auth): return _put(url, data, None, qiniu.auth.RequestsAuth(auth)) + def _put_with_auth_and_headers(url, data, auth, headers): return _put(url, data, None, qiniu.auth.RequestsAuth(auth), headers) def _post_with_qiniu_mac(url, data, auth): - qn_auth = qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None + qn_auth = qiniu.auth.QiniuMacRequestsAuth( + auth) if auth is not None else None timeout = config.get_default('connection_timeout') try: - r = requests.post(url, json=data, auth=qn_auth, timeout=timeout, headers=_headers) + r = requests.post( + url, + json=data, + auth=qn_auth, + timeout=timeout, + headers=_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) @@ -121,8 +136,11 @@ def _post_with_qiniu_mac(url, data, auth): def _get_with_qiniu_mac(url, params, auth): try: r = requests.get( - url, params=params, auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, - timeout=config.get_default('connection_timeout'), headers=_headers) + url, + params=params, + auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, + timeout=config.get_default('connection_timeout'), + headers=_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) @@ -131,8 +149,11 @@ def _get_with_qiniu_mac(url, params, auth): def _delete_with_qiniu_mac(url, params, auth): try: r = requests.delete( - url, params=params, auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, - timeout=config.get_default('connection_timeout'), headers=_headers) + url, + params=params, + auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, + timeout=config.get_default('connection_timeout'), + headers=_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) @@ -191,9 +212,11 @@ def connect_failed(self): def __str__(self): if is_py2: - return ', '.join(['%s:%s' % item for item in self.__dict__.items()]).encode('utf-8') + return ', '.join( + ['%s:%s' % item for item in self.__dict__.items()]).encode('utf-8') elif is_py3: - return ', '.join(['%s:%s' % item for item in self.__dict__.items()]) + return ', '.join(['%s:%s' % + item for item in self.__dict__.items()]) def __repr__(self): return self.__str__() diff --git a/qiniu/main.py b/qiniu/main.py index 7f8653e8..6f0b81a0 100755 --- a/qiniu/main.py +++ b/qiniu/main.py @@ -11,9 +11,14 @@ def main(): sub_parsers = parser.add_subparsers() parser_etag = sub_parsers.add_parser( - 'etag', description='calculate the etag of the file', help='etag [file...]') + 'etag', + description='calculate the etag of the file', + help='etag [file...]') parser_etag.add_argument( - 'etag_files', metavar='N', nargs='+', help='the file list for calculate') + 'etag_files', + metavar='N', + nargs='+', + help='the file list for calculate') args = parser.parse_args() diff --git a/qiniu/utils.py b/qiniu/utils.py index 716f4c6d..8b7eaa66 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -168,5 +168,6 @@ def rfc_from_timestamp(timestamp): timestamp: 整型Unix时间戳（单位秒） """ last_modified_date = datetime.utcfromtimestamp(timestamp) - last_modified_str = last_modified_date.strftime('%a, %d %b %Y %H:%M:%S GMT') + last_modified_str = last_modified_date.strftime( + '%a, %d %b %Y %H:%M:%S GMT') return last_modified_str diff --git a/qiniu/zone.py b/qiniu/zone.py index 349a597d..b817f212 100644 --- a/qiniu/zone.py +++ b/qiniu/zone.py @@ -19,8 +19,14 @@ class Zone(object): up_host_backup: 备用上传地址 """ - def __init__(self, up_host=None, up_host_backup=None, io_host=None, host_cache={}, scheme="http", - home_dir=os.getcwd()): + def __init__( + self, + up_host=None, + up_host_backup=None, + io_host=None, + host_cache={}, + scheme="http", + home_dir=os.getcwd()): """初始化Zone类""" self.up_host, self.up_host_backup, self.io_host = up_host, up_host_backup, io_host self.host_cache = host_cache @@ -58,7 +64,10 @@ def unmarshal_up_token(self, up_token): raise ValueError('invalid up_token') ak = token[0] - policy = compat.json.loads(compat.s(utils.urlsafe_base64_decode(token[2]))) + policy = compat.json.loads( + compat.s( + utils.urlsafe_base64_decode( + token[2]))) scope = policy["scope"] bucket = scope @@ -84,7 +93,8 @@ def get_bucket_hosts(self, ak, bucket): hosts[self.scheme]['up'].append(self.scheme + "://" + self.up_host) if self.up_host_backup is not None: - hosts[self.scheme]['up'].append(self.scheme + "://" + self.up_host_backup) + hosts[self.scheme]['up'].append( + self.scheme + "://" + self.up_host_backup) if self.io_host is not None: hosts[self.scheme]['io'].append(self.scheme + "://" + self.io_host) @@ -98,7 +108,9 @@ def get_bucket_hosts(self, ak, bucket): try: scheme_hosts = hosts[self.scheme] except KeyError: - raise KeyError("Please check your BUCKET_NAME! The UpHosts is %s" % hosts) + raise KeyError( + "Please check your BUCKET_NAME! The UpHosts is %s" % + hosts) bucket_hosts = { 'upHosts': scheme_hosts['up'], 'ioHosts': scheme_hosts['io'], From 3cc81313e45b65ae3442ccbb0042794dffe18826 Mon Sep 17 00:00:00 2001 From: bernieyangmh <berniey@163.com> Date: Mon, 12 Nov 2018 10:59:19 +0800 Subject: [PATCH 292/478] add qvmupload --- examples/upload_with_zone.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/upload_with_zone.py b/examples/upload_with_zone.py index 0c8e56de..218b946a 100644 --- a/examples/upload_with_zone.py +++ b/examples/upload_with_zone.py @@ -24,7 +24,10 @@ # 要上传文件的本地路径 localfile = 'stat.py' -# 指定固定的zone +# 指定固定的zone 自行指定上传域名及空间源站域名，可用于qvm云主机的内网上传，选择服务端或客户端优化的域名，或http\https上传 +# https://developer.qiniu.com/qvm/manual/4269/qvm-kodo +# https://developer.qiniu.com/kodo/manual/1671/region-endpoint + zone = Zone( up_host='uptest.qiniu.com', up_host_backup='uptest.qiniu.com', From 8d151f00d663d72ddba082660e2eee4001615c37 Mon Sep 17 00:00:00 2001 From: bernieyangmh <berniey@163.com> Date: Mon, 12 Nov 2018 10:59:19 +0800 Subject: [PATCH 293/478] add qvmupload --- examples/upload_with_qvmzone.py | 40 +++++++++++++++++++++++++++++++++ examples/upload_with_zone.py | 5 ++++- 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 examples/upload_with_qvmzone.py diff --git a/examples/upload_with_qvmzone.py b/examples/upload_with_qvmzone.py new file mode 100644 index 00000000..54f8b603 --- /dev/null +++ b/examples/upload_with_qvmzone.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth, put_file, etag, urlsafe_base64_encode +import qiniu.config +from qiniu import Zone, set_default + +# 需要填写你的 Access Key 和 Secret Key +access_key = '...' +secret_key = '...' + +# 构建鉴权对象 +q = Auth(access_key, secret_key) + +# 要上传的空间 +bucket_name = 'Bucket_Name' + +# 上传到七牛后保存的文件名 +key = 'my-python-logo.png' + +# 生成上传 Token，可以指定过期时间等 +token = q.upload_token(bucket_name, key, 3600) + +# 要上传文件的本地路径 +localfile = 'stat.py' + +# up_host, 指定上传域名，注意不同区域的qvm上传域名不同 +# https://developer.qiniu.com/qvm/manual/4269/qvm-kodo + +zone = Zone( + up_host='free-qvm-z1-zz.qiniup.com', + up_host_backup='free-qvm-z1-zz.qiniup.com', + io_host='iovip.qbox.me', + scheme='http') +set_default(default_zone=zone) + +ret, info = put_file(token, key, localfile) +print(info) +assert ret['key'] == key +assert ret['hash'] == etag(localfile) diff --git a/examples/upload_with_zone.py b/examples/upload_with_zone.py index 0c8e56de..218b946a 100644 --- a/examples/upload_with_zone.py +++ b/examples/upload_with_zone.py @@ -24,7 +24,10 @@ # 要上传文件的本地路径 localfile = 'stat.py' -# 指定固定的zone +# 指定固定的zone 自行指定上传域名及空间源站域名，可用于qvm云主机的内网上传，选择服务端或客户端优化的域名，或http\https上传 +# https://developer.qiniu.com/qvm/manual/4269/qvm-kodo +# https://developer.qiniu.com/kodo/manual/1671/region-endpoint + zone = Zone( up_host='uptest.qiniu.com', up_host_backup='uptest.qiniu.com', From 5d59aba9e65b866f5c299ee78cc2e8977985214e Mon Sep 17 00:00:00 2001 From: bernieyangmh <berniey@163.com> Date: Fri, 22 Feb 2019 17:26:51 +0800 Subject: [PATCH 294/478] add rs and region --- examples/batch_stat.py | 2 - examples/bucket_info.py | 19 ++++ examples/change_status.py | 29 +++++ examples/list_buckets.py | 25 +++++ examples/mk_bucket.py | 20 ++++ examples/upload_with_zone.py | 32 +++--- qiniu/config.py | 16 +-- qiniu/region.py | 161 ++++++++++++++++++++++++++++ qiniu/services/storage/bucket.py | 40 +++++++ qiniu/services/storage/uploader.py | 20 +++- qiniu/zone.py | 164 +---------------------------- 11 files changed, 341 insertions(+), 187 deletions(-) create mode 100644 examples/bucket_info.py create mode 100644 examples/change_status.py create mode 100644 examples/list_buckets.py create mode 100644 examples/mk_bucket.py create mode 100644 qiniu/region.py diff --git a/examples/batch_stat.py b/examples/batch_stat.py index 4ab7592c..84eea4f3 100644 --- a/examples/batch_stat.py +++ b/examples/batch_stat.py @@ -5,11 +5,9 @@ https://developer.qiniu.com/kodo/api/1250/batch """ - from qiniu import build_batch_stat, Auth, BucketManager access_key = '' - secret_key = '' q = Auth(access_key, secret_key) diff --git a/examples/bucket_info.py b/examples/bucket_info.py new file mode 100644 index 00000000..14ea4bbd --- /dev/null +++ b/examples/bucket_info.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +# 需要填写你的 Access Key 和 Secret Key +access_key = '' +secret_key = '' + +# 空间名 +bucket_name = 'bucket_name' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +ret, info = bucket.bucket_info(bucket_name) +print(info) diff --git a/examples/change_status.py b/examples/change_status.py new file mode 100644 index 00000000..ae0a6145 --- /dev/null +++ b/examples/change_status.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +# 需要填写你的 Access Key 和 Secret Key +access_key = '' +secret_key = '' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +# 空间名 +bucket_name = 'bucket_name' + +# 文件名 +key = 'file_name' + +# 条件匹配，只有匹配上才会执行修改操作 +# cond可以填空，一个或多个 +cond = {"fsize": "186371", + "putTime": "14899798962573916", + "hash": "FiRxWzeeD6ofGTpwTZub5Fx1ozvi", + "mime": "image/png"} + +ret, info = bucket.change_status(bucket_name, key, '0', cond) +print(info) diff --git a/examples/list_buckets.py b/examples/list_buckets.py new file mode 100644 index 00000000..471322ef --- /dev/null +++ b/examples/list_buckets.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +# 需要填写你的 Access Key 和 Secret Key +access_key = '' +secret_key = '' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +# 指定需要列举的区域，填空字符串返回全部空间，为减少响应时间建议填写 +# z0:只返回华东区域的空间 +# z1:只返回华北区域的空间 +# z2:只返回华南区域的空间 +# na0:只返回北美区域的空间 +# as0:只返回东南亚区域的空间 +region = "z0" + +ret, info = bucket.list_bucket(region) +print(info) +print(ret) diff --git a/examples/mk_bucket.py b/examples/mk_bucket.py new file mode 100644 index 00000000..09035d79 --- /dev/null +++ b/examples/mk_bucket.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +# 需要填写你的 Access Key 和 Secret Key +access_key = '...' +secret_key = '...' + +bucket_name = 'Bucket_Name' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +region = "z0" + +ret, info = bucket.mkbucketv2(bucket_name, region) +print(info) diff --git a/examples/upload_with_zone.py b/examples/upload_with_zone.py index 0c8e56de..c8650d39 100644 --- a/examples/upload_with_zone.py +++ b/examples/upload_with_zone.py @@ -1,38 +1,42 @@ # -*- coding: utf-8 -*- # flake8: noqa -from qiniu import Auth, put_file, etag, urlsafe_base64_encode -import qiniu.config +from qiniu import Auth, put_file from qiniu import Zone, set_default # 需要填写你的 Access Key 和 Secret Key -access_key = '...' -secret_key = '...' +access_key = '' +secret_key = '' # 构建鉴权对象 q = Auth(access_key, secret_key) # 要上传的空间 -bucket_name = 'Bucket_Name' +bucket_name = 'bucket_name' # 上传到七牛后保存的文件名 -key = 'my-python-logo.png' +key = 'a.jpg' # 生成上传 Token，可以指定过期时间等 token = q.upload_token(bucket_name, key, 3600) # 要上传文件的本地路径 -localfile = 'stat.py' +localfile = '/Users/abc/Documents/a.jpg' + +# 指定固定域名的zone,不同区域uphost域名见下文档 +# https://developer.qiniu.com/kodo/manual/1671/region-endpoint +# 未指定或上传错误，sdk会根据token自动查询对应的上传域名 +# *.qiniup.com 支持https上传 +# 备用*.qiniu.com域名 不支持https上传 +# 要求https上传时，如果客户指定的两个host都错误，且sdk自动查询的第一个*.qiniup.com上传域名因意外不可用导致访问到备用*.qiniu.com会报ssl错误 +# 建议https上传时查看上面文档，指定正确的host -# 指定固定的zone zone = Zone( - up_host='uptest.qiniu.com', - up_host_backup='uptest.qiniu.com', - io_host='iovip.qbox.me', - scheme='http') + up_host='https://up.qiniup.com', + up_host_backup='https://upload.qiniup.com', + io_host='http://iovip.qbox.me', + scheme='https') set_default(default_zone=zone) ret, info = put_file(token, key, localfile) print(info) -assert ret['key'] == key -assert ret['hash'] == etag(localfile) diff --git a/qiniu/config.py b/qiniu/config.py index 9b827962..694749eb 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -2,9 +2,10 @@ from qiniu import zone -RS_HOST = 'http://rs.qbox.me' # 管理操作Host -RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host +RS_HOST = 'http://rs.qiniu.com' # 管理操作Host +RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host API_HOST = 'http://api.qiniu.com' # 数据处理操作Host +UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 @@ -13,9 +14,10 @@ 'default_rs_host': RS_HOST, 'default_rsf_host': RSF_HOST, 'default_api_host': API_HOST, - 'connection_timeout': 30, # 链接超时为时间为30s - 'connection_retries': 3, # 链接重试次数为3次 - 'connection_pool': 10, # 链接池个数为10 + 'default_uc_host': UC_HOST, + 'connection_timeout': 30, # 链接超时为时间为30s + 'connection_retries': 3, # 链接重试次数为3次 + 'connection_pool': 10, # 链接池个数为10 } @@ -25,7 +27,7 @@ def get_default(key): def set_default( default_zone=None, connection_retries=None, connection_pool=None, - connection_timeout=None, default_rs_host=None, + connection_timeout=None, default_rs_host=None, default_uc_host=None, default_rsf_host=None, default_api_host=None): if default_zone: _config['default_zone'] = default_zone @@ -35,6 +37,8 @@ def set_default( _config['default_rsf_host'] = default_rsf_host if default_api_host: _config['default_api_host'] = default_api_host + if default_uc_host: + _config['default_uc_host'] = default_api_host if connection_retries: _config['connection_retries'] = connection_retries if connection_pool: diff --git a/qiniu/region.py b/qiniu/region.py new file mode 100644 index 00000000..c87c02c5 --- /dev/null +++ b/qiniu/region.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- + +import os +import time +import requests +from qiniu import compat +from qiniu import utils + +UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host + + +class Region(object): + """七牛上传区域类 + + 该类主要内容上传区域地址。 + + """ + + def __init__( + self, + up_host=None, + up_host_backup=None, + io_host=None, + host_cache={}, + scheme="http", + home_dir=os.getcwd()): + """初始化Zone类""" + self.up_host, self.up_host_backup, self.io_host = up_host, up_host_backup, io_host + self.host_cache = host_cache + self.scheme = scheme + self.home_dir = home_dir + + def get_up_host_by_token(self, up_token): + ak, bucket = self.unmarshal_up_token(up_token) + up_hosts = self.get_up_host(ak, bucket) + return up_hosts[0] + + def get_up_host_backup_by_token(self, up_token): + ak, bucket = self.unmarshal_up_token(up_token) + up_hosts = self.get_up_host(ak, bucket) + if (len(up_hosts) <= 1): + up_host = up_hosts[0] + else: + up_host = up_hosts[1] + return up_host + + def get_io_host(self, ak, bucket): + if self.io_host: + return self.io_host + bucket_hosts = self.get_bucket_hosts(ak, bucket) + io_hosts = bucket_hosts['ioHosts'] + return io_hosts[0] + + def get_up_host(self, ak, bucket): + bucket_hosts = self.get_bucket_hosts(ak, bucket) + up_hosts = bucket_hosts['upHosts'] + return up_hosts + + def unmarshal_up_token(self, up_token): + token = up_token.split(':') + if (len(token) != 3): + raise ValueError('invalid up_token') + + ak = token[0] + policy = compat.json.loads( + compat.s( + utils.urlsafe_base64_decode( + token[2]))) + + scope = policy["scope"] + bucket = scope + if (':' in scope): + bucket = scope.split(':')[0] + + return ak, bucket + + def get_bucket_hosts(self, ak, bucket): + key = self.scheme + ":" + ak + ":" + bucket + + bucket_hosts = self.get_bucket_hosts_to_cache(key) + if (len(bucket_hosts) > 0): + return bucket_hosts + + hosts = {} + hosts.update({self.scheme: {}}) + + hosts[self.scheme].update({'up': []}) + hosts[self.scheme].update({'io': []}) + + if self.up_host is not None: + hosts[self.scheme]['up'].append(self.scheme + "://" + self.up_host) + + if self.up_host_backup is not None: + hosts[self.scheme]['up'].append( + self.scheme + "://" + self.up_host_backup) + + if self.io_host is not None: + hosts[self.scheme]['io'].append(self.scheme + "://" + self.io_host) + + if len(hosts[self.scheme]) == 0 or self.io_host is None: + hosts = compat.json.loads(self.bucket_hosts(ak, bucket)) + else: + # 1 year + hosts['ttl'] = int(time.time()) + 31536000 + try: + scheme_hosts = hosts[self.scheme] + except KeyError: + raise KeyError( + "Please check your BUCKET_NAME! The UpHosts is %s" % + hosts) + bucket_hosts = { + 'upHosts': scheme_hosts['up'], + 'ioHosts': scheme_hosts['io'], + 'deadline': int(time.time()) + hosts['ttl'] + } + + self.set_bucket_hosts_to_cache(key, bucket_hosts) + return bucket_hosts + + def get_bucket_hosts_to_cache(self, key): + ret = [] + if (len(self.host_cache) == 0): + self.host_cache_from_file() + + if (not (key in self.host_cache)): + return ret + + if (self.host_cache[key]['deadline'] > time.time()): + ret = self.host_cache[key] + + return ret + + def set_bucket_hosts_to_cache(self, key, val): + self.host_cache[key] = val + self.host_cache_to_file() + return + + def host_cache_from_file(self): + path = self.host_cache_file_path() + if not os.path.isfile(path): + return None + with open(path, 'r') as f: + bucket_hosts = compat.json.load(f) + self.host_cache = bucket_hosts + f.close() + return + + def host_cache_file_path(self): + return os.path.join(self.home_dir, ".qiniu_pythonsdk_hostscache.json") + + def host_cache_to_file(self): + path = self.host_cache_file_path() + with open(path, 'w') as f: + compat.json.dump(self.host_cache, f) + f.close() + + def bucket_hosts(self, ak, bucket): + url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) + ret = requests.get(url) + data = compat.json.dumps(ret.json(), separators=(',', ':')) + return data diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 30e0d100..cb8da771 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -3,6 +3,7 @@ from qiniu import config from qiniu import http from qiniu.utils import urlsafe_base64_encode, entry +import json class BucketManager(object): @@ -225,6 +226,25 @@ def change_type(self, bucket, key, storage_type): resource = entry(bucket, key) return self.__rs_do('chtype', resource, 'type/{0}'.format(storage_type)) + def change_status(self, bucket, key, status, cond): + """修改文件的状态 + + 修改文件的存储类型为可用或禁用： + + Args: + bucket: 待操作资源所在空间 + key: 待操作资源文件名 + storage_type: 待操作资源存储类型，0为启用，1为禁用 + """ + resource = entry(bucket, key) + if cond and isinstance(cond, dict): + condstr = "" + for k,v in cond.items(): + condstr+="{0}={1}&".format(k, v) + condstr = urlsafe_base64_encode(condstr[:-1]) + return self.__rs_do('chstatus', resource, 'status/{0}'.format(status), 'cond', condstr) + return self.__rs_do('chstatus', resource, 'status/{0}'.format(status)) + def batch(self, operations): """批量操作: @@ -295,6 +315,26 @@ def mkbucketv2(self, bucket_name, region): bucket_name = urlsafe_base64_encode(bucket_name) return self.__rs_do('mkbucketv2', bucket_name, 'region', region) + def list_bucket(self, region): + """ + 列举存储空间列表 + + Args: + """ + return self.__uc_do('v3/buckets?region={0}'.format(region)) + + def bucket_info(self, bucket_name): + """ + 获取存储空间信息 + + Args: + bucket_name: 存储空间名 + """ + return self.__post('v2/bucketInfo?bucket={}'.format(bucket_name), ) + + def __uc_do(self, operation, *args): + return self.__server_do(config.get_default('default_uc_host'), operation, *args) + def __rs_do(self, operation, *args): return self.__server_do(config.get_default('default_rs_host'), operation, *args) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 4e1a2157..babd8a7b 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -92,7 +92,10 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None fields['key'] = key fields['token'] = up_token - url = config.get_default('default_zone').get_up_host_by_token(up_token) + '/' + if config.get_default('default_zone').up_host: + url = config.get_default('default_zone').up_host + else: + url = config.get_default('default_zone').get_up_host_by_token(up_token) # name = key if key else file_name fname = file_name @@ -106,7 +109,10 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) if r is None and info.need_retry(): if info.connect_failed: - url = config.get_default('default_zone').get_up_host_backup_by_token(up_token) + '/' + if config.get_default('default_zone').up_host_backup: + url = config.get_default('default_zone').up_host_backup + else: + url = config.get_default('default_zone').get_up_host_backup_by_token(up_token) if hasattr(data, 'read') is False: pass elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()): @@ -190,7 +196,10 @@ def recovery_from_record(self): def upload(self): """上传操作""" self.blockStatus = [] - host = config.get_default('default_zone').get_up_host_by_token(self.up_token) + if config.get_default('default_zone').up_host: + host = config.get_default('default_zone').up_host + else: + host = config.get_default('default_zone').get_up_host_by_token(self.up_token) offset = self.recovery_from_record() for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset): length = len(block) @@ -199,7 +208,10 @@ def upload(self): if ret is None and not info.need_retry(): return ret, info if info.connect_failed(): - host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token) + if config.get_default('default_zone').up_host_backup: + host = config.get_default('default_zone').up_host_backup + else: + host = config.get_default('default_zone').get_up_host_backup_by_token(up_token) if info.need_retry() or crc != ret['crc32']: ret, info = self.make_block(block, length, host) if ret is None or crc != ret['crc32']: diff --git a/qiniu/zone.py b/qiniu/zone.py index b817f212..acb34d00 100644 --- a/qiniu/zone.py +++ b/qiniu/zone.py @@ -1,165 +1,7 @@ # -*- coding: utf-8 -*- -import os -import time -import requests -from qiniu import compat -from qiniu import utils +from qiniu.region import Region -UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host - -class Zone(object): - """七牛上传区域类 - - 该类主要内容上传区域地址。 - - Attributes: - up_host: 首选上传地址 - up_host_backup: 备用上传地址 - """ - - def __init__( - self, - up_host=None, - up_host_backup=None, - io_host=None, - host_cache={}, - scheme="http", - home_dir=os.getcwd()): - """初始化Zone类""" - self.up_host, self.up_host_backup, self.io_host = up_host, up_host_backup, io_host - self.host_cache = host_cache - self.scheme = scheme - self.home_dir = home_dir - - def get_up_host_by_token(self, up_token): - ak, bucket = self.unmarshal_up_token(up_token) - up_hosts = self.get_up_host(ak, bucket) - return up_hosts[0] - - def get_up_host_backup_by_token(self, up_token): - ak, bucket = self.unmarshal_up_token(up_token) - up_hosts = self.get_up_host(ak, bucket) - - if (len(up_hosts) <= 1): - up_host = up_hosts[0] - else: - up_host = up_hosts[1] - return up_host - - def get_io_host(self, ak, bucket): - bucket_hosts = self.get_bucket_hosts(ak, bucket) - io_hosts = bucket_hosts['ioHosts'] - return io_hosts[0] - - def get_up_host(self, ak, bucket): - bucket_hosts = self.get_bucket_hosts(ak, bucket) - up_hosts = bucket_hosts['upHosts'] - return up_hosts - - def unmarshal_up_token(self, up_token): - token = up_token.split(':') - if (len(token) != 3): - raise ValueError('invalid up_token') - - ak = token[0] - policy = compat.json.loads( - compat.s( - utils.urlsafe_base64_decode( - token[2]))) - - scope = policy["scope"] - bucket = scope - if (':' in scope): - bucket = scope.split(':')[0] - - return ak, bucket - - def get_bucket_hosts(self, ak, bucket): - key = self.scheme + ":" + ak + ":" + bucket - - bucket_hosts = self.get_bucket_hosts_to_cache(key) - if (len(bucket_hosts) > 0): - return bucket_hosts - - hosts = {} - hosts.update({self.scheme: {}}) - - hosts[self.scheme].update({'up': []}) - hosts[self.scheme].update({'io': []}) - - if self.up_host is not None: - hosts[self.scheme]['up'].append(self.scheme + "://" + self.up_host) - - if self.up_host_backup is not None: - hosts[self.scheme]['up'].append( - self.scheme + "://" + self.up_host_backup) - - if self.io_host is not None: - hosts[self.scheme]['io'].append(self.scheme + "://" + self.io_host) - - if len(hosts[self.scheme]) == 0 or self.io_host is None: - # print(hosts) - hosts = compat.json.loads(self.bucket_hosts(ak, bucket)) - else: - # 1 year - hosts['ttl'] = int(time.time()) + 31536000 - try: - scheme_hosts = hosts[self.scheme] - except KeyError: - raise KeyError( - "Please check your BUCKET_NAME! The UpHosts is %s" % - hosts) - bucket_hosts = { - 'upHosts': scheme_hosts['up'], - 'ioHosts': scheme_hosts['io'], - 'deadline': int(time.time()) + hosts['ttl'] - } - - self.set_bucket_hosts_to_cache(key, bucket_hosts) - - return bucket_hosts - - def get_bucket_hosts_to_cache(self, key): - ret = [] - if (len(self.host_cache) == 0): - self.host_cache_from_file() - - if (not (key in self.host_cache)): - return ret - - if (self.host_cache[key]['deadline'] > time.time()): - ret = self.host_cache[key] - - return ret - - def set_bucket_hosts_to_cache(self, key, val): - self.host_cache[key] = val - self.host_cache_to_file() - return - - def host_cache_from_file(self): - path = self.host_cache_file_path() - if not os.path.isfile(path): - return None - with open(path, 'r') as f: - bucket_hosts = compat.json.load(f) - self.host_cache = bucket_hosts - f.close() - return - - def host_cache_file_path(self): - return os.path.join(self.home_dir, ".qiniu_pythonsdk_hostscache.json") - - def host_cache_to_file(self): - path = self.host_cache_file_path() - with open(path, 'w') as f: - compat.json.dump(self.host_cache, f) - f.close() - - def bucket_hosts(self, ak, bucket): - url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) - ret = requests.get(url) - data = compat.json.dumps(ret.json(), separators=(',', ':')) - return data +class Zone(Region): + pass From 6759aff7b26742bca79e0c91f432513173fee8fc Mon Sep 17 00:00:00 2001 From: bernieyangmh <berniey@163.com> Date: Fri, 22 Feb 2019 17:34:26 +0800 Subject: [PATCH 295/478] add rs and region --- examples/change_mime.py | 5 +++-- examples/change_status.py | 15 ++++++++------- examples/list_buckets.py | 9 +++++---- examples/upload.py | 2 +- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/examples/change_mime.py b/examples/change_mime.py index 5db61f8a..ad4294d2 100644 --- a/examples/change_mime.py +++ b/examples/change_mime.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# flake8: noqa - +""" +改变文件的mimeType +""" from qiniu import Auth from qiniu import BucketManager diff --git a/examples/change_status.py b/examples/change_status.py index ae0a6145..1cf526df 100644 --- a/examples/change_status.py +++ b/examples/change_status.py @@ -1,22 +1,23 @@ # -*- coding: utf-8 -*- -# flake8: noqa - +""" +改变文件状态，可用或不可用 +""" from qiniu import Auth from qiniu import BucketManager # 需要填写你的 Access Key 和 Secret Key -access_key = '' -secret_key = '' +access_key = 'C3S6li9F4Iq9toDkq3tzoHz_tRwS2LWNE9aUjIJy' +secret_key = 'VsjH9zF3XdFfeo6K6y-nyEU5ia62NTW4KM7xcwt4' q = Auth(access_key, secret_key) bucket = BucketManager(q) # 空间名 -bucket_name = 'bucket_name' +bucket_name = 'bernie' # 文件名 -key = 'file_name' +key = '233.jpg' # 条件匹配，只有匹配上才会执行修改操作 # cond可以填空，一个或多个 @@ -25,5 +26,5 @@ "hash": "FiRxWzeeD6ofGTpwTZub5Fx1ozvi", "mime": "image/png"} -ret, info = bucket.change_status(bucket_name, key, '0', cond) +ret, info = bucket.change_status(bucket_name, key, '1', cond) print(info) diff --git a/examples/list_buckets.py b/examples/list_buckets.py index 471322ef..5fb4faca 100644 --- a/examples/list_buckets.py +++ b/examples/list_buckets.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# flake8: noqa - +""" +列举账号下的空间 +""" from qiniu import Auth from qiniu import BucketManager @@ -12,13 +13,13 @@ bucket = BucketManager(q) -# 指定需要列举的区域，填空字符串返回全部空间，为减少响应时间建议填写 +# 指定需要列举的区域，填空字符串返回全部空间，为减少响应时间建议不为空 # z0:只返回华东区域的空间 # z1:只返回华北区域的空间 # z2:只返回华南区域的空间 # na0:只返回北美区域的空间 # as0:只返回东南亚区域的空间 -region = "z0" +region = "as0" ret, info = bucket.list_bucket(region) print(info) diff --git a/examples/upload.py b/examples/upload.py index 29a071e7..690ea046 100755 --- a/examples/upload.py +++ b/examples/upload.py @@ -13,7 +13,7 @@ q = Auth(access_key, secret_key) # 要上传的空间 -bucket_name = 'if-bc' +bucket_name = '' # 上传到七牛后保存的文件名 key = 'my-python-七牛.png' From 199a87f3618a26b3405f201e4af2a806385e0ba3 Mon Sep 17 00:00:00 2001 From: bernieyangmh <berniey@163.com> Date: Mon, 25 Feb 2019 14:47:43 +0800 Subject: [PATCH 296/478] adjust format --- examples/batch_copy.py | 3 +++ examples/batch_delete.py | 2 ++ examples/batch_move.py | 2 ++ examples/batch_rename.py | 2 ++ examples/batch_stat.py | 2 ++ examples/cdn_bandwidth.py | 2 ++ examples/cdn_flux.py | 2 ++ examples/cdn_log.py | 2 ++ examples/change_mime.py | 2 ++ examples/change_status.py | 2 ++ examples/create_bucket.py | 2 ++ examples/list_buckets.py | 2 ++ examples/prefetch_to_cdn.py | 1 + examples/refresh_dirs.py | 2 ++ examples/refresh_urls.py | 2 ++ examples/rtc_server.py | 1 + examples/timestamp_url.py | 1 + examples/update_cdn_sslcert.py | 2 ++ qiniu/auth.py | 35 +++++++++++++++--------------- qiniu/services/storage/bucket.py | 5 ++--- qiniu/services/storage/uploader.py | 2 +- 21 files changed, 54 insertions(+), 22 deletions(-) diff --git a/examples/batch_copy.py b/examples/batch_copy.py index f12225a3..6fd15b63 100644 --- a/examples/batch_copy.py +++ b/examples/batch_copy.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +# flake8: noqa + + """ 批量拷贝文件 diff --git a/examples/batch_delete.py b/examples/batch_delete.py index 89ea1180..c74dd951 100644 --- a/examples/batch_delete.py +++ b/examples/batch_delete.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 批量删除文件 diff --git a/examples/batch_move.py b/examples/batch_move.py index c4f51076..3375e2ea 100644 --- a/examples/batch_move.py +++ b/examples/batch_move.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 批量移动文件 diff --git a/examples/batch_rename.py b/examples/batch_rename.py index 12e7c806..75a48289 100644 --- a/examples/batch_rename.py +++ b/examples/batch_rename.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 批量重命名文件 diff --git a/examples/batch_stat.py b/examples/batch_stat.py index 84eea4f3..9ad9d7b0 100644 --- a/examples/batch_stat.py +++ b/examples/batch_stat.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 批量查询文件信息 diff --git a/examples/cdn_bandwidth.py b/examples/cdn_bandwidth.py index c4c4c0ce..ad4e97a8 100644 --- a/examples/cdn_bandwidth.py +++ b/examples/cdn_bandwidth.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 查询指定域名指定时间段内的带宽 """ diff --git a/examples/cdn_flux.py b/examples/cdn_flux.py index c88517e3..bb42efc1 100644 --- a/examples/cdn_flux.py +++ b/examples/cdn_flux.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 查询指定域名指定时间段内的流量 """ diff --git a/examples/cdn_log.py b/examples/cdn_log.py index 2ebb7271..aee1e5c8 100644 --- a/examples/cdn_log.py +++ b/examples/cdn_log.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 获取指定域名指定时间内的日志链接 """ diff --git a/examples/change_mime.py b/examples/change_mime.py index ad4294d2..770405cc 100644 --- a/examples/change_mime.py +++ b/examples/change_mime.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 改变文件的mimeType """ diff --git a/examples/change_status.py b/examples/change_status.py index 1cf526df..b0d90704 100644 --- a/examples/change_status.py +++ b/examples/change_status.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 改变文件状态，可用或不可用 """ diff --git a/examples/create_bucket.py b/examples/create_bucket.py index e77099d7..4b0b02c4 100644 --- a/examples/create_bucket.py +++ b/examples/create_bucket.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 创建存储空间 """ diff --git a/examples/list_buckets.py b/examples/list_buckets.py index 5fb4faca..6c7f7c0c 100644 --- a/examples/list_buckets.py +++ b/examples/list_buckets.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 列举账号下的空间 """ diff --git a/examples/prefetch_to_cdn.py b/examples/prefetch_to_cdn.py index 8bdaa0a7..fbab188c 100644 --- a/examples/prefetch_to_cdn.py +++ b/examples/prefetch_to_cdn.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# flake8: noqa """ 预取资源到cdn节点 diff --git a/examples/refresh_dirs.py b/examples/refresh_dirs.py index 1db73015..5ce41438 100644 --- a/examples/refresh_dirs.py +++ b/examples/refresh_dirs.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + import qiniu from qiniu import CdnManager diff --git a/examples/refresh_urls.py b/examples/refresh_urls.py index 43b62baa..b4194275 100644 --- a/examples/refresh_urls.py +++ b/examples/refresh_urls.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + import qiniu from qiniu import CdnManager diff --git a/examples/rtc_server.py b/examples/rtc_server.py index 84a39bb3..94b9ebb9 100644 --- a/examples/rtc_server.py +++ b/examples/rtc_server.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# flake8: noqa from qiniu import QiniuMacAuth from qiniu import RtcServer, get_room_token diff --git a/examples/timestamp_url.py b/examples/timestamp_url.py index 0b543a78..0873a181 100644 --- a/examples/timestamp_url.py +++ b/examples/timestamp_url.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# flake8: noqa """ 获取一个配置时间戳防盗链的url diff --git a/examples/update_cdn_sslcert.py b/examples/update_cdn_sslcert.py index 06652712..40152f56 100644 --- a/examples/update_cdn_sslcert.py +++ b/examples/update_cdn_sslcert.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 更新cdn证书(可配合let's encrypt 等完成自动证书更新) """ diff --git a/qiniu/auth.py b/qiniu/auth.py index c25ad6d0..fe58f930 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -9,34 +9,33 @@ from .compat import urlparse, json, b from .utils import urlsafe_base64_encode - # 上传策略，参数规格详见 # https://developer.qiniu.com/kodo/manual/1206/put-policy _policy_fields = set([ - 'callbackUrl', # 回调URL - 'callbackBody', # 回调Body - 'callbackHost', # 回调URL指定的Host + 'callbackUrl', # 回调URL + 'callbackBody', # 回调Body + 'callbackHost', # 回调URL指定的Host 'callbackBodyType', # 回调Body的Content-Type 'callbackFetchKey', # 回调FetchKey模式开关 - 'returnUrl', # 上传端的303跳转URL - 'returnBody', # 上传端简单反馈获取的Body + 'returnUrl', # 上传端的303跳转URL + 'returnBody', # 上传端简单反馈获取的Body - 'endUser', # 回调时上传端标识 - 'saveKey', # 自定义资源名 - 'insertOnly', # 插入模式开关 + 'endUser', # 回调时上传端标识 + 'saveKey', # 自定义资源名 + 'insertOnly', # 插入模式开关 - 'detectMime', # MimeType侦测开关 - 'mimeLimit', # MimeType限制 - 'fsizeLimit', # 上传文件大小限制 - 'fsizeMin', # 上传文件最少字节数 + 'detectMime', # MimeType侦测开关 + 'mimeLimit', # MimeType限制 + 'fsizeLimit', # 上传文件大小限制 + 'fsizeMin', # 上传文件最少字节数 - 'persistentOps', # 持久化处理操作 + 'persistentOps', # 持久化处理操作 'persistentNotifyUrl', # 持久化处理结果通知URL - 'persistentPipeline', # 持久化处理独享队列 - 'deleteAfterDays', # 文件多少天后自动删除 - 'fileType', # 文件的存储类型，0为普通存储，1为低频存储 - 'isPrefixalScope' # 指定上传文件必须使用的前缀 + 'persistentPipeline', # 持久化处理独享队列 + 'deleteAfterDays', # 文件多少天后自动删除 + 'fileType', # 文件的存储类型，0为普通存储，1为低频存储 + 'isPrefixalScope' # 指定上传文件必须使用的前缀 ]) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index cb8da771..2d12860e 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -3,7 +3,6 @@ from qiniu import config from qiniu import http from qiniu.utils import urlsafe_base64_encode, entry -import json class BucketManager(object): @@ -239,8 +238,8 @@ def change_status(self, bucket, key, status, cond): resource = entry(bucket, key) if cond and isinstance(cond, dict): condstr = "" - for k,v in cond.items(): - condstr+="{0}={1}&".format(k, v) + for k, v in cond.items(): + condstr += "{0}={1}&".format(k, v) condstr = urlsafe_base64_encode(condstr[:-1]) return self.__rs_do('chstatus', resource, 'status/{0}'.format(status), 'cond', condstr) return self.__rs_do('chstatus', resource, 'status/{0}'.format(status)) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index babd8a7b..1922e4c2 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -211,7 +211,7 @@ def upload(self): if config.get_default('default_zone').up_host_backup: host = config.get_default('default_zone').up_host_backup else: - host = config.get_default('default_zone').get_up_host_backup_by_token(up_token) + host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token) if info.need_retry() or crc != ret['crc32']: ret, info = self.make_block(block, length, host) if ret is None or crc != ret['crc32']: From 7c752190b92a7ffbae867e88845be95686d1be63 Mon Sep 17 00:00:00 2001 From: bernieyangmh <berniey@163.com> Date: Mon, 25 Feb 2019 15:08:48 +0800 Subject: [PATCH 297/478] see env --- test_qiniu.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_qiniu.py b/test_qiniu.py index 1a42e6fb..544dbc38 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -107,6 +107,8 @@ def test_verify_callback(self): class BucketTestCase(unittest.TestCase): + print(access_key) + print(secret_key) q = Auth(access_key, secret_key) bucket = BucketManager(q) From 321b1a42b9da496c0e6b8767f118418681011f7f Mon Sep 17 00:00:00 2001 From: bernieyangmh <berniey@163.com> Date: Mon, 25 Feb 2019 16:00:17 +0800 Subject: [PATCH 298/478] add global secrue --- .travis.yml | 20 ++++++++++---------- test_qiniu.py | 2 -- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 45775088..f1927ac1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: false language: python python: -- '2.6.9' +- 2.6.9 - '2.7' - '3.4' - '3.5' @@ -17,13 +17,13 @@ before_script: - export QINIU_TEST_ENV="travis" - export PYTHONPATH="$PYTHONPATH:." script: -- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 .; fi +- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 + .; fi - py.test --cov qiniu -- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then ocular --data-file .coverage; fi -deploy: - provider: pypi - user: qiniusdk - password: - secure: N2u9xzhncbziIhoDdpaCcr7D3lW/N7AOIZDpx+M5QW0lPqIXkZDioOTZ7b4QNwx/XFMu6tdeK79A2Wg7T9/8VfEWDd2bYL7a1J7spoFJi9k3HVHHiFBmg7vXr1OGn3D51xqsrq3Kh9uRP150a5CA2qxYabKb6b6dn5QhOTPhfFY= - on: - tags: true +- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then ocular --data-file .coverage; + fi + +env: + global: + secure: McZuxM4UAKabtGvCi+t1F/Spb/3Yzb6O7hEk0JLwJEYCnl7hkfV1ogAgjjYdHwkNPjOwUaz3rpdmahz64ohtpucPsIyQjgK7tigTM+UgdAcg77RflB50yJ3yCnJOHMxVRF0RNLZqFeuf3GkfnOyzZFynN+LmM5n+0/iIuC4LXgs= + QINIU_ACCESS_KEY=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5 \ No newline at end of file diff --git a/test_qiniu.py b/test_qiniu.py index 544dbc38..1a42e6fb 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -107,8 +107,6 @@ def test_verify_callback(self): class BucketTestCase(unittest.TestCase): - print(access_key) - print(secret_key) q = Auth(access_key, secret_key) bucket = BucketManager(q) From ef80b62fd3e9c249a905260d5d59e4ff4337e232 Mon Sep 17 00:00:00 2001 From: bernieyangmh <berniey@163.com> Date: Mon, 25 Feb 2019 16:20:32 +0800 Subject: [PATCH 299/478] version to 7.2.3 --- CHANGELOG.md | 6 ++++++ examples/change_status.py | 4 ++-- examples/list_buckets.py | 2 +- qiniu/__init__.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 148fc69f..66aa1c37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +# 7.2.3 (2019-02-25) +* 新增region类，zone继承 +* 上传可以指定上传域名 +* 新增上传指定上传空间和qvm指定上传内网的例子 +* 新增列举账号空间，创建空间，查询空间信息，改变文件状态接口，并提供例子 + # 7.2.2 (2018-05-10) * 增加连麦rtc服务端API功能 diff --git a/examples/change_status.py b/examples/change_status.py index b0d90704..5f1acdd5 100644 --- a/examples/change_status.py +++ b/examples/change_status.py @@ -8,8 +8,8 @@ from qiniu import BucketManager # 需要填写你的 Access Key 和 Secret Key -access_key = 'C3S6li9F4Iq9toDkq3tzoHz_tRwS2LWNE9aUjIJy' -secret_key = 'VsjH9zF3XdFfeo6K6y-nyEU5ia62NTW4KM7xcwt4' +access_key = '' +secret_key = '' q = Auth(access_key, secret_key) diff --git a/examples/list_buckets.py b/examples/list_buckets.py index 6c7f7c0c..e83589f7 100644 --- a/examples/list_buckets.py +++ b/examples/list_buckets.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # flake8: noqa - + """ 列举账号下的空间 """ diff --git a/qiniu/__init__.py b/qiniu/__init__.py index f229397a..f123ff70 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.2' +__version__ = '7.2.3' from .auth import Auth, QiniuMacAuth From 462b1891d54762257d0bad365048f68c1ee61fe6 Mon Sep 17 00:00:00 2001 From: Cai Xiaohua <caixiaohua@qiniu.com> Date: Wed, 20 Mar 2019 22:53:18 +0800 Subject: [PATCH 300/478] turn json null to json empty object Change-Id: I3e566bb6f4c425922c5940983c427988eee30565 Signed-off-by: Cai Xiaohua <caixiaohua@qiniu.com> --- qiniu/http.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qiniu/http.py b/qiniu/http.py index 80596700..14a422f4 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -24,6 +24,8 @@ def __return_wrapper(resp): return None, ResponseInfo(resp) resp.encoding = 'utf-8' ret = resp.json(encoding='utf-8') if resp.text != '' else {} + if ret is None: # json null + ret = {} return ret, ResponseInfo(resp) From d3dcc507a04795a6e930f5959d2678439ad3371f Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Mon, 1 Apr 2019 19:28:54 +0800 Subject: [PATCH 301/478] Update __init__.py add Region --- qiniu/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index f123ff70..fc7365e1 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -15,6 +15,7 @@ from .config import set_default from .zone import Zone +from .region import Region from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ build_batch_stat, build_batch_delete From a69fbef4e3e6ea1ebe09f4610a5b18bb2c17de59 Mon Sep 17 00:00:00 2001 From: longbai <slongbai@gmail.com> Date: Wed, 3 Apr 2019 21:49:17 +0800 Subject: [PATCH 302/478] 7.2.4 --- CHANGELOG.md | 3 +++ qiniu/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66aa1c37..0ba182a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +# 7.2.4 (2019-04-01) +* 默认导入region类 + # 7.2.3 (2019-02-25) * 新增region类，zone继承 * 上传可以指定上传域名 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index fc7365e1..4ff86c51 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.3' +__version__ = '7.2.4' from .auth import Auth, QiniuMacAuth From f582d6463db7c154a273e881909a7237bdd0ed48 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Thu, 6 Jun 2019 22:08:01 +0800 Subject: [PATCH 303/478] add sms_ch --- CHANGELOG.md | 3 + examples/sms_test.py | 98 ++++++++++++++++ qiniu/__init__.py | 5 +- qiniu/auth.py | 6 +- qiniu/http.py | 46 +++++++- qiniu/services/sms/__init__.py | 0 qiniu/services/sms/sms.py | 207 +++++++++++++++++++++++++++++++++ 7 files changed, 356 insertions(+), 9 deletions(-) create mode 100644 examples/sms_test.py create mode 100644 qiniu/services/sms/__init__.py create mode 100644 qiniu/services/sms/sms.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ba182a3..4562d660 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +# 7.2.5 (2019-06-06) +* 添加sms + # 7.2.4 (2019-04-01) * 默认导入region类 diff --git a/examples/sms_test.py b/examples/sms_test.py new file mode 100644 index 00000000..05d5b957 --- /dev/null +++ b/examples/sms_test.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import QiniuMacAuth +from qiniu import Sms + + +access_key = 'bjtWBQXrcxgo7HWwlC_bgHg81j352_GhgBGZPeOW' +secret_key = 'pCav6rTslxP2SIFg0XJmAw53D9PjWEcuYWVdUqAf' + +# 初始化Auth状态 +q = QiniuMacAuth(access_key, secret_key) + +# 初始化Sms +sms = Sms(q) + +""" +#创建签名 +signature = 'abs' +source = 'website' +req, info = sms.createSignature(signature, source) +print(req,info) +""" + +""" +#查询签名 +audit_status = '' +page = 1 +page_size = 20 +req, info = sms.querySignature(audit_status, page, page_size) +print(req, info) +""" + +""" +编辑签名 +id = 1136530250662940672 +signature = 'sssss' +req, info = sms.updateSignature(id, signature) +print(req, info) +""" + +""" +#删除签名 +signature_id= 1136530250662940672 +req, info = sms.deleteSignature(signature_id) +print(req, info) +""" + +""" +#创建模版 +name = '06-062-test' +template = '${test}' +type = 'notification' +description = '就测试啊' +signature_id = '1131464448834277376' +req, info = sms.createTemplate(name, template, type, description, signature_id) +print(req, info) +""" + +""" +#查询模版 +audit_status = '' +page = 1 +page_size = 20 +req, info = sms.queryTemplate(audit_status, page, page_size) +print(req, info) +""" + +""" +#编辑模版 +template_id = '1136589777022226432' +name = '06-06-test' +template = 'hi,你好' +description = '就测试啊' +signature_id = '1131464448834277376' +req, info = sms.updateTemplate(template_id, name, template, description, signature_id) +print(info) +""" + +""" +#删除模版 +template_id = '1136589777022226432' +req, info = sms.deleteTemplate(template_id) +print(req, info) +""" + +""" +#发送短信 +""" +template_id = '' +mobiles = [] +parameters = {} +req, info = sms.sendMessage(template_id, mobiles, parameters) +print(req, info) + + + + diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 4ff86c51..61854789 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.4' +__version__ = '7.2.5' from .auth import Auth, QiniuMacAuth @@ -25,7 +25,6 @@ from .services.processing.cmd import build_op, pipe_cmd, op_save from .services.compute.app import AccountClient from .services.compute.qcos_api import QcosClient - +from .services.sms.sms import Sms from .services.pili.rtc_server_manager import RtcServer, get_room_token - from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry diff --git a/qiniu/auth.py b/qiniu/auth.py index fe58f930..d8d8ddd0 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -3,9 +3,7 @@ import hmac import time from hashlib import sha1 - from requests.auth import AuthBase - from .compat import urlparse, json, b from .utils import urlsafe_base64_encode @@ -266,8 +264,8 @@ def token_of_request( data += "\n" if content_type and content_type != "application/octet-stream" and body: - data += body.decode(encoding='UTF-8') - + # data += body.decode(encoding='UTF-8') + data += body return '{0}:{1}'.format(self.__access_key, self.__token(data)) def qiniu_headers(self, headers): diff --git a/qiniu/http.py b/qiniu/http.py index 14a422f4..ff0089e7 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -9,6 +9,7 @@ import qiniu.auth from . import __version__ + _sys_info = '{0}; {1}'.format(platform.system(), platform.machine()) _python_ver = platform.python_version() @@ -24,7 +25,7 @@ def __return_wrapper(resp): return None, ResponseInfo(resp) resp.encoding = 'utf-8' ret = resp.json(encoding='utf-8') if resp.text != '' else {} - if ret is None: # json null + if ret is None: # json null ret = {} return ret, ResponseInfo(resp) @@ -110,6 +111,10 @@ def _post_with_auth_and_headers(url, data, auth, headers): return _post(url, data, None, qiniu.auth.RequestsAuth(auth), headers) +def _post_with_qiniu_mac_and_headers(url, data, auth, headers): + return _post(url, data, None, qiniu.auth.QiniuMacRequestsAuth(auth), headers) + + def _put_with_auth(url, data, auth): return _put(url, data, None, qiniu.auth.RequestsAuth(auth)) @@ -118,11 +123,14 @@ def _put_with_auth_and_headers(url, data, auth, headers): return _put(url, data, None, qiniu.auth.RequestsAuth(auth), headers) +def _put_with_qiniu_mac_and_headers(url, data, auth, headers): + return _put(url, data, None, qiniu.auth.QiniuMacRequestsAuth(auth), headers) + + def _post_with_qiniu_mac(url, data, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth( auth) if auth is not None else None timeout = config.get_default('connection_timeout') - try: r = requests.post( url, @@ -148,6 +156,23 @@ def _get_with_qiniu_mac(url, params, auth): return __return_wrapper(r) +def _get_with_qiniu_mac_and_headers(url, params, auth, headers): + try: + post_headers = _headers.copy() + if headers is not None: + for k, v in headers.items(): + post_headers.update({k: v}) + r = requests.get( + url, + params=params, + auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, + timeout=config.get_default('connection_timeout'), + headers=post_headers) + except Exception as e: + return None, ResponseInfo(None, e) + return __return_wrapper(r) + + def _delete_with_qiniu_mac(url, params, auth): try: r = requests.delete( @@ -161,6 +186,23 @@ def _delete_with_qiniu_mac(url, params, auth): return __return_wrapper(r) +def _delete_with_qiniu_mac_and_headers(url, params, auth, headers): + try: + post_headers = _headers.copy() + if headers is not None: + for k, v in headers.items(): + post_headers.update({k: v}) + r = requests.delete( + url, + params=params, + auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, + timeout=config.get_default('connection_timeout'), + headers=post_headers) + except Exception as e: + return None, ResponseInfo(None, e) + return __return_wrapper(r) + + class ResponseInfo(object): """七牛HTTP请求返回信息类 diff --git a/qiniu/services/sms/__init__.py b/qiniu/services/sms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qiniu/services/sms/sms.py b/qiniu/services/sms/sms.py new file mode 100644 index 00000000..310c9fe7 --- /dev/null +++ b/qiniu/services/sms/sms.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- + +from qiniu import http +import json + + +class Sms(object): + def __init__(self, auth): + self.auth = auth + self.server = 'https://sms.qiniuapi.com' + + def createSignature(self, signature, source, pics=None): + """ + *创建签名 + *signature: string类型，必填，【长度限制8个字符内】超过长度会报错 + *source: string类型，必填，申请签名时必须指定签名来源。取值范围为： + enterprises_and_institutions 企事业单位的全称或简称 + website 工信部备案网站的全称或简称 + app APP应用的全称或简称 + public_number_or_small_program 公众号或小程序的全称或简称 + store_name 电商平台店铺名的全称或简称 + trade_name 商标名的全称或简称 + *pics: 签名对应的资质证明图片进行 base64 编码格式转换后的字符串 + * @ return: 类型array + { + "signature_id": < signature_id > + } + """ + req = {} + req['signature'] = signature + req['source'] = source + if pics: + req['pics'] = pics + body = json.dumps(req) + url = '{0}/v1/signature'.format(self.server) + return self.__post(url, body) + + def querySignature(self, audit_status=None, page=1, page_size=20): + """ + 查询签名 + * audit_status: 审核状态 string 类型，可选，取值范围为: "passed"(通过), "rejected"(未通过), "reviewing"(审核中) + * page:页码 int 类型， + * page_size: 分页大小 int 类型，可选， 默认为20 + *@return: 类型array { + "items": [{ + "id": string, + "signature": string, + "source": string, + "audit_status": string, + "reject_reason": string, + "created_at": int64, + "updated_at": int64 + }...], + "total": int, + "page": int, + "page_size": int, + } + """ + url = '{}/v1/signature'.format(self.server) + if audit_status: + url = '{}?audit_status={}&page={}&page_size={}'.format(url, audit_status, page, page_size) + else: + url = '{}?page={}&page_size={}'.format(url, page, page_size) + return self.__get(url) + + def updateSignature(self, id, signature): + """ + 编辑签名 + * id 签名id : string 类型，必填， + * signature: string 类型，必填， + request 类型array { + "signature": string + } + :return: + """ + url = '{}/v1/signature/{}'.format(self.server, id) + req = {} + req['signature'] = signature + body = json.dumps(req) + return self.__put(url, body) + + def deleteSignature(self, id): + + """ + 删除辑签名 + * id 签名id : string 类型，必填， + * @retrun : 请求成功 HTTP 状态码为 200 + + """ + url = '{}/v1/signature/{}'.format(self.server, id) + return self.__delete(url) + + def createTemplate(self, name, template, type, description, signature_id): + """ + 创建模版 + :param name: 模板名称 string 类型 ，必填 + :param template: 模板内容 string 类型，必填 + :param type: 模板类型 string 类型，必填， + 取值范围为: notification (通知类短信), verification (验证码短信), marketing (营销类短信) + :param description: 申请理由简述 string 类型，必填 + :param signature_id: 已经审核通过的签名 string 类型，必填 + :return: 类型 array { + "template_id": string + } + """ + url = '{}/v1/template'.format(self.server) + req = {} + req['name'] = name + req['template'] = template + req['type'] = type + req['description'] = description + req['signature_id'] = signature_id + body = json.dumps(req) + return self.__post(url, body) + + def queryTemplate(self, audit_status, page=1, page_size=20): + """ + 查询模版 + :param audit_status: 审核状态, 取值范围为: passed (通过), rejected (未通过), reviewing (审核中) + :param page: 页码。默认为 1 + :param page_size: 分页大小。默认为 20 + :return:{ + "items": [{ + "id": string, + "name": string, + "template": string, + "audit_status": string, + "reject_reason": string, + "type": string, + "signature_id": string, // 模版绑定的签名ID + "signature_text": string, // 模版绑定的签名内容 + "created_at": int64, + "updated_at": int64 + }...], + "total": int, + "page": int, + "page_size": int + } + """ + url = '{}/v1/template'.format(self.server) + if audit_status: + url = '{}?audit_status={}&page={}&page_size={}'.format(url, audit_status, page, page_size) + else: + url = '{}?page={}&page_size={}'.format(url, page, page_size) + return self.__get(url) + + def updateTemplate(self, id, name, template, description, signature_id): + """ + 更新模版 + :param id: template_id + :param name: 模板名称 string 类型 ，必填 + :param template: 模板内容 string 类型，必填 + :param description: 申请理由简述 string 类型，必填 + :param signature_id: 已经审核通过的签名 string 类型，必填 + :return: 请求成功 HTTP 状态码为 200 + """ + url = '{}/v1/template/{}'.format(self.server, id) + req = {} + req['name'] = name + req['template'] = template + req['description'] = description + req['signature_id'] = signature_id + body = json.dumps(req) + return self.__put(url, body) + + def deleteTemplate(self, id): + """ + 删除模版 + :param id: template_id + :return: 请求成功 HTTP 状态码为 200 + """ + url = '{}/v1/template/{}'.format(self.server, id) + return self.__delete(url) + + def sendMessage(self, template_id, mobiles, parameters): + """ + 发送短信 + :param template_id: 模板 ID + :param mobiles: 手机号 + :param parameters: 自定义魔法变量，变量设置在创建模板时，参数template指定 + :return:{ + "job_id": string + } + """ + url = '{}/v1/message'.format(self.server) + req = {} + req['template_id'] = template_id + req['mobiles'] = mobiles + req['parameters'] = parameters + body = json.dumps(req) + return self.__post(url, body) + + def __post(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._post_with_qiniu_mac_and_headers(url, data, self.auth, headers) + + def __get(self, url, params=None): + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + return http._get_with_qiniu_mac_and_headers(url, params, self.auth, headers) + + def __put(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._put_with_qiniu_mac_and_headers(url, data, self.auth, headers) + + def __delete(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._delete_with_qiniu_mac_and_headers(url, data, self.auth, headers) From 2547fb75575d13912d8fc0cfe4118384259ffedd Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Mon, 10 Jun 2019 09:48:27 +0800 Subject: [PATCH 304/478] add sms_ch,rm ak,sk --- examples/sms_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/sms_test.py b/examples/sms_test.py index 05d5b957..bb7c1a51 100644 --- a/examples/sms_test.py +++ b/examples/sms_test.py @@ -5,8 +5,8 @@ from qiniu import Sms -access_key = 'bjtWBQXrcxgo7HWwlC_bgHg81j352_GhgBGZPeOW' -secret_key = 'pCav6rTslxP2SIFg0XJmAw53D9PjWEcuYWVdUqAf' +access_key = '' +secret_key = '' # 初始化Auth状态 q = QiniuMacAuth(access_key, secret_key) From f047e99b926db9dd3805974b3ac6355d81d45c21 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Wed, 26 Jun 2019 11:36:16 +0800 Subject: [PATCH 305/478] Update setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1190c4f3..a71d6857 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ 'qiniu.services.processing', 'qiniu.services.compute', 'qiniu.services.cdn', + 'qiniu.services.sms', 'qiniu.services.pili', ] From 2d1e3b19a707eea1c34c97977f8480e9d7fccd29 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Wed, 26 Jun 2019 11:36:45 +0800 Subject: [PATCH 306/478] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 61854789..8f009b11 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.5' +__version__ = '7.2.6' from .auth import Auth, QiniuMacAuth From 8f138c064efae9d95e8ba279010a83a48f2b06bc Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Wed, 26 Jun 2019 11:37:25 +0800 Subject: [PATCH 307/478] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4562d660..484ff61f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +# 7.2.6(2019-06-26) +* 添加sms + # 7.2.5 (2019-06-06) * 添加sms From c6b1a99bd9182481eef9e17880ddeb2ccac02d4b Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Fri, 8 Nov 2019 14:51:31 +0800 Subject: [PATCH 308/478] Update bucket.py --- qiniu/services/storage/bucket.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 2d12860e..20148202 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -302,17 +302,15 @@ def delete_after_days(self, bucket, key, days): resource = entry(bucket, key) return self.__rs_do('deleteAfterDays', resource, days) - def mkbucketv2(self, bucket_name, region): + def mkbucketv3(self, bucket_name, region): """ - 创建存储空间 - https://developer.qiniu.com/kodo/api/1382/mkbucketv2 + 创建存储空间，全局唯一，其他账号有同名空间就无法创建 Args: bucket_name: 存储空间名 region: 存储区域 """ - bucket_name = urlsafe_base64_encode(bucket_name) - return self.__rs_do('mkbucketv2', bucket_name, 'region', region) + return self.__rs_do('mkbucketv3', bucket_name, 'region', region) def list_bucket(self, region): """ From ffd2a3c09322cdf5c7e0977fd4054d5f7565088d Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Wed, 4 Dec 2019 12:02:11 +0800 Subject: [PATCH 309/478] Update auth.py --- qiniu/auth.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index d8d8ddd0..3511209e 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -264,8 +264,10 @@ def token_of_request( data += "\n" if content_type and content_type != "application/octet-stream" and body: - # data += body.decode(encoding='UTF-8') - data += body + if isinstance(body,bytes): + data += body.decode(encoding='UTF-8') + else: + data += body return '{0}:{1}'.format(self.__access_key, self.__token(data)) def qiniu_headers(self, headers): From 0c8cca4da40f176f1f3c6ac384cf99852f7ca3b5 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Tue, 10 Mar 2020 18:56:18 +0800 Subject: [PATCH 310/478] fix bucket_info --- qiniu/services/storage/bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 20148202..ada1a2ca 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -327,7 +327,7 @@ def bucket_info(self, bucket_name): Args: bucket_name: 存储空间名 """ - return self.__post('v2/bucketInfo?bucket={}'.format(bucket_name), ) + return self.__uc_do('v2/bucketInfo?bucket={}'.format(bucket_name), ) def __uc_do(self, operation, *args): return self.__server_do(config.get_default('default_uc_host'), operation, *args) From cfdf5960f20c6794b7edad25bd3919baf0e48a7b Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Tue, 10 Mar 2020 19:20:54 +0800 Subject: [PATCH 311/478] fix bucket_info --- CHANGELOG.md | 3 +++ qiniu/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 484ff61f..82a46d4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +# 7.2.7(2020-03-10) +* fix bucket_info + # 7.2.6(2019-06-26) * 添加sms diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 8f009b11..688b1eba 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.6' +__version__ = '7.2.7' from .auth import Auth, QiniuMacAuth From 804eedf00d0b974c13adf2ea3dd1f769b6f79c75 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Fri, 27 Mar 2020 15:26:55 +0800 Subject: [PATCH 312/478] add restoreAr --- CHANGELOG.md | 2 ++ examples/restorear.py | 17 +++++++++++++++++ qiniu/__init__.py | 2 +- qiniu/services/storage/bucket.py | 16 +++++++++++++++- 4 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 examples/restorear.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 82a46d4c..60e4135d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ # Changelog +# 7.2.8(2020-03-27) +* add restoreAr # 7.2.7(2020-03-10) * fix bucket_info diff --git a/examples/restorear.py b/examples/restorear.py new file mode 100644 index 00000000..794c41f3 --- /dev/null +++ b/examples/restorear.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth +from qiniu import BucketManager + + +access_key = '' +secret_key = '' + +q = Auth(access_key, secret_key) +bucket = BucketManager(q) +bucket_name = '13' +key = 'fb8539c39f65d74b4e70db9133c1e9d5.mp4' +ret,info = bucket.restoreAr(bucket_name,key,3) +print(ret) +print(info) + diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 688b1eba..7b6b55f1 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.7' +__version__ = '7.2.8' from .auth import Auth, QiniuMacAuth diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index ada1a2ca..ce769b1a 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -220,11 +220,25 @@ def change_type(self, bucket, key, storage_type): Args: bucket: 待操作资源所在空间 key: 待操作资源文件名 - storage_type: 待操作资源存储类型，0为普通存储，1为低频存储 + storage_type: 待操作资源存储类型，0为普通存储，1为低频存储，2 为归档存储 """ resource = entry(bucket, key) return self.__rs_do('chtype', resource, 'type/{0}'.format(storage_type)) + def restoreAr(self, bucket, key, freezeAfter_days): + """解冻归档存储文件 + + 修改文件的存储类型为普通存储或者是低频存储，参考文档： + https://developer.qiniu.com/kodo/api/6380/restore-archive + + Args: + bucket: 待操作资源所在空间 + key: 待操作资源文件名 + freezeAfter_days: 解冻有效时长，取值范围 1～7 + """ + resource = entry(bucket, key) + return self.__rs_do('restoreAr', resource, 'freezeAfterDays/{0}'.format(freezeAfter_days)) + def change_status(self, bucket, key, status, cond): """修改文件的状态 From 6efed9280d49b1f579e03758445bc04a4ba94bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=92=9F=E6=A5=9A=E5=90=9B?= <zhongchujun@zhongchujundeMacBook-Pro.local> Date: Thu, 23 Jul 2020 12:06:39 +0800 Subject: [PATCH 313/478] test travis --- .travis.yml | 15 +++++++-------- qiniu/auth.py | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index f1927ac1..248d7584 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,9 @@ sudo: false language: python python: -- 2.6.9 -- '2.7' -- '3.4' -- '3.5' +- "2.7" +- "3.4" +- "3.5" install: - pip install flake8 - pip install pytest @@ -17,13 +16,13 @@ before_script: - export QINIU_TEST_ENV="travis" - export PYTHONPATH="$PYTHONPATH:." script: -- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 - .; fi +- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 .; + fi - py.test --cov qiniu - if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then ocular --data-file .coverage; fi env: global: - secure: McZuxM4UAKabtGvCi+t1F/Spb/3Yzb6O7hEk0JLwJEYCnl7hkfV1ogAgjjYdHwkNPjOwUaz3rpdmahz64ohtpucPsIyQjgK7tigTM+UgdAcg77RflB50yJ3yCnJOHMxVRF0RNLZqFeuf3GkfnOyzZFynN+LmM5n+0/iIuC4LXgs= - QINIU_ACCESS_KEY=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5 \ No newline at end of file + - secure: McZuxM4UAKabtGvCi+t1F/Spb/3Yzb6O7hEk0JLwJEYCnl7hkfV1ogAgjjYdHwkNPjOwUaz3rpdmahz64ohtpucPsIyQjgK7tigTM+UgdAcg77RflB50yJ3yCnJOHMxVRF0RNLZqFeuf3GkfnOyzZFynN+LmM5n+0/iIuC4LXgs= + - QINIU_ACCESS_KEY=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5 \ No newline at end of file diff --git a/qiniu/auth.py b/qiniu/auth.py index 3511209e..1374d4a6 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -264,7 +264,7 @@ def token_of_request( data += "\n" if content_type and content_type != "application/octet-stream" and body: - if isinstance(body,bytes): + if isinstance(body, bytes): data += body.decode(encoding='UTF-8') else: data += body From aeab3c771d34643fc736a126fc5e83956cb1396f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=92=9F=E6=A5=9A=E5=90=9B?= <zhongchujun@zhongchujundeMacBook-Pro.local> Date: Thu, 23 Jul 2020 14:59:35 +0800 Subject: [PATCH 314/478] remove the 2.6 version --- .travis.yml | 11 +++++++++-- README.md | 8 +++++--- examples/sms_test.py | 6 +++--- test_qiniu.py | 2 +- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 248d7584..757d4525 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,19 +10,26 @@ install: - pip install pytest-cov - pip install requests - pip install scrutinizer-ocular +- pip install codecov + before_script: - export QINIU_TEST_BUCKET="pythonsdk" - export QINIU_TEST_DOMAIN="pythonsdk.qiniudn.com" - export QINIU_TEST_ENV="travis" - export PYTHONPATH="$PYTHONPATH:." + script: - if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 .; fi - py.test --cov qiniu - if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then ocular --data-file .coverage; fi +- coverage run test_qiniu.py env: global: - - secure: McZuxM4UAKabtGvCi+t1F/Spb/3Yzb6O7hEk0JLwJEYCnl7hkfV1ogAgjjYdHwkNPjOwUaz3rpdmahz64ohtpucPsIyQjgK7tigTM+UgdAcg77RflB50yJ3yCnJOHMxVRF0RNLZqFeuf3GkfnOyzZFynN+LmM5n+0/iIuC4LXgs= - - QINIU_ACCESS_KEY=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5 \ No newline at end of file + - secure: "McZuxM4UAKabtGvCi+t1F/Spb/3Yzb6O7hEk0JLwJEYCnl7hkfV1ogAgjjYdHwkNPjOwUaz3rpdmahz64ohtpucPsIyQjgK7tigTM+UgdAcg77RflB50yJ3yCnJOHMxVRF0RNLZqFeuf3GkfnOyzZFynN+LmM5n+0/iIuC4LXgs=" + - QINIU_ACCESS_KEY=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5 + +after_success: + - codecov \ No newline at end of file diff --git a/README.md b/README.md index 2439b8eb..4c81e19a 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,12 @@ [![@qiniu on weibo](http://img.shields.io/badge/weibo-%40qiniutek-blue.svg)](http://weibo.com/qiniutek) [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE) [![Build Status](https://travis-ci.org/qiniu/python-sdk.svg)](https://travis-ci.org/qiniu/python-sdk) +[![GitHub release](https://img.shields.io/github/v/tag/qiniu/python-sdk.svg?label=release)](https://github.com/qiniu/python-sdk/releases) [![Latest Stable Version](https://img.shields.io/pypi/v/qiniu.svg)](https://pypi.python.org/pypi/qiniu) [![Download Times](https://img.shields.io/pypi/dm/qiniu.svg)](https://pypi.python.org/pypi/qiniu) [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/qiniu/python-sdk/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/qiniu/python-sdk/?branch=master) -[![Code Coverage](https://scrutinizer-ci.com/g/qiniu/python-sdk/badges/coverage.png?b=master)](https://scrutinizer-ci.com/g/qiniu/python-sdk/?branch=master) +[![Coverage Status](https://codecov.io/gh/qiniu/python-sdk/branch/master/graph/badge.svg)](https://codecov.io/gh/qiniu/python-sdk) + ## 安装 通过pip @@ -19,8 +21,8 @@ $ pip install qiniu | Qiniu SDK版本 | Python 版本 | |:--------------------:|:---------------------------:| -| 7.x | 2.6, 2.7, 3.3, 3.4, 3.5| -| 6.x | 2.6, 2.7 | +| 7.x | 2.7, 3.3, 3.4, 3.5| +| 6.x | 2.7 | ## 使用方法 diff --git a/examples/sms_test.py b/examples/sms_test.py index bb7c1a51..3f72cf4d 100644 --- a/examples/sms_test.py +++ b/examples/sms_test.py @@ -3,10 +3,10 @@ from qiniu import QiniuMacAuth from qiniu import Sms +import os - -access_key = '' -secret_key = '' +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') # 初始化Auth状态 q = QiniuMacAuth(access_key, secret_key) diff --git a/test_qiniu.py b/test_qiniu.py index 1a42e6fb..ad23ef80 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -413,7 +413,7 @@ class DownloadTestCase(unittest.TestCase): def test_private_url(self): private_bucket = 'private-res' private_key = 'gogopher.jpg' - base_url = 'http://%s/%s' % (private_bucket + '.qiniudn.com', private_key) + base_url = 'http://%s/%s' % (private_bucket + '.qiniupkg.com', private_key) private_url = self.q.private_download_url(base_url, expires=3600) print(private_url) r = requests.get(private_url) From 1fde5f0e827882d52994898631072a0d778b4ddd Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Mon, 3 Aug 2020 17:02:06 +0800 Subject: [PATCH 315/478] add no json response processing and update _get --- qiniu/http.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/qiniu/http.py b/qiniu/http.py index ff0089e7..a562b2f8 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -9,7 +9,6 @@ import qiniu.auth from . import __version__ - _sys_info = '{0}; {1}'.format(platform.system(), platform.machine()) _python_ver = platform.python_version() @@ -74,8 +73,10 @@ def _put(url, data, files, auth, headers=None): def _get(url, params, auth): + if _session is None: + _init() try: - r = requests.get( + r = _session.get( url, params=params, auth=qiniu.auth.RequestsAuth(auth) if auth is not None else None, @@ -232,11 +233,14 @@ def __init__(self, response, exception=None): self.req_id = response.headers.get('X-Reqid') self.x_log = response.headers.get('X-Log') if self.status_code >= 400: - ret = response.json() if response.text != '' else None - if ret is None or ret['error'] is None: - self.error = 'unknown' + if 600 > self.status_code >= 499: + self.error = response.text else: - self.error = ret['error'] + ret = response.json() if response.text != '' else None + if ret is None or ret['error'] is None: + self.error = 'unknown' + else: + self.error = ret['error'] if self.req_id is None and self.status_code == 200: self.error = 'server is not qiniu' From 6f1fdd0b0469f3cc2a649b17e18410e462607a03 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Mon, 3 Aug 2020 17:09:24 +0800 Subject: [PATCH 316/478] add .qiniu_pythonsdk_hostscache.json file ptah parameter and update fetch returns info --- qiniu/region.py | 51 ++++++++++++--------- qiniu/services/storage/bucket.py | 24 +++++----- qiniu/services/storage/uploader.py | 72 ++++++++++++++++-------------- test_qiniu.py | 11 +++-- 4 files changed, 89 insertions(+), 69 deletions(-) diff --git a/qiniu/region.py b/qiniu/region.py index c87c02c5..f143795b 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -22,37 +22,42 @@ def __init__( up_host_backup=None, io_host=None, host_cache={}, - scheme="http", - home_dir=os.getcwd()): + home_dir=None, + scheme="http"): """初始化Zone类""" - self.up_host, self.up_host_backup, self.io_host = up_host, up_host_backup, io_host + self.up_host, self.up_host_backup, self.io_host, self.home_dir = up_host, up_host_backup, io_host, home_dir self.host_cache = host_cache self.scheme = scheme - self.home_dir = home_dir - def get_up_host_by_token(self, up_token): + def get_up_host_by_token(self, up_token, home_dir): ak, bucket = self.unmarshal_up_token(up_token) - up_hosts = self.get_up_host(ak, bucket) + if home_dir is None: + home_dir = os.getcwd() + up_hosts = self.get_up_host(ak, bucket, home_dir) return up_hosts[0] - def get_up_host_backup_by_token(self, up_token): + def get_up_host_backup_by_token(self, up_token, home_dir): ak, bucket = self.unmarshal_up_token(up_token) - up_hosts = self.get_up_host(ak, bucket) + if home_dir is None: + home_dir = os.getcwd() + up_hosts = self.get_up_host(ak, bucket, home_dir) if (len(up_hosts) <= 1): up_host = up_hosts[0] else: up_host = up_hosts[1] return up_host - def get_io_host(self, ak, bucket): + def get_io_host(self, ak, bucket, home_dir): if self.io_host: return self.io_host - bucket_hosts = self.get_bucket_hosts(ak, bucket) + if home_dir is None: + home_dir = os.getcwd() + bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) io_hosts = bucket_hosts['ioHosts'] return io_hosts[0] - def get_up_host(self, ak, bucket): - bucket_hosts = self.get_bucket_hosts(ak, bucket) + def get_up_host(self, ak, bucket, home_dir): + bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) up_hosts = bucket_hosts['upHosts'] return up_hosts @@ -74,10 +79,10 @@ def unmarshal_up_token(self, up_token): return ak, bucket - def get_bucket_hosts(self, ak, bucket): + def get_bucket_hosts(self, ak, bucket, home_dir): key = self.scheme + ":" + ak + ":" + bucket - bucket_hosts = self.get_bucket_hosts_to_cache(key) + bucket_hosts = self.get_bucket_hosts_to_cache(key, home_dir) if (len(bucket_hosts) > 0): return bucket_hosts @@ -113,14 +118,14 @@ def get_bucket_hosts(self, ak, bucket): 'ioHosts': scheme_hosts['io'], 'deadline': int(time.time()) + hosts['ttl'] } - - self.set_bucket_hosts_to_cache(key, bucket_hosts) + home_dir = "" + self.set_bucket_hosts_to_cache(key, bucket_hosts, home_dir) return bucket_hosts - def get_bucket_hosts_to_cache(self, key): + def get_bucket_hosts_to_cache(self, key, home_dir): ret = [] if (len(self.host_cache) == 0): - self.host_cache_from_file() + self.host_cache_from_file(home_dir) if (not (key in self.host_cache)): return ret @@ -130,12 +135,14 @@ def get_bucket_hosts_to_cache(self, key): return ret - def set_bucket_hosts_to_cache(self, key, val): + def set_bucket_hosts_to_cache(self, key, val, home_dir): self.host_cache[key] = val - self.host_cache_to_file() + self.host_cache_to_file(home_dir) return - def host_cache_from_file(self): + def host_cache_from_file(self, home_dir): + if home_dir is not None: + self.home_dir = home_dir path = self.host_cache_file_path() if not os.path.isfile(path): return None @@ -148,7 +155,7 @@ def host_cache_from_file(self): def host_cache_file_path(self): return os.path.join(self.home_dir, ".qiniu_pythonsdk_hostscache.json") - def host_cache_to_file(self): + def host_cache_to_file(self, home_dir): path = self.host_cache_file_path() with open(path, 'w') as f: compat.json.dump(self.host_cache, f) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index ce769b1a..74eb52b7 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -161,25 +161,28 @@ def copy(self, bucket, key, bucket_to, key_to, force='false'): to = entry(bucket_to, key_to) return self.__rs_do('copy', resource, to, 'force/{0}'.format(force)) - def fetch(self, url, bucket, key=None): + def fetch(self, url, bucket, key=None, hostscache_dir=None): """抓取文件: 从指定URL抓取资源，并将该资源存储到指定空间中，具体规格参考： http://developer.qiniu.com/docs/v6/api/reference/rs/fetch.html Args: - url: 指定的URL - bucket: 目标资源空间 - key: 目标资源文件名 + url: 指定的URL + bucket: 目标资源空间 + key: 目标资源文件名 + hostscache_dir： host请求 缓存文件保存位置 Returns: - 一个dict变量，成功返回NULL，失败返回{"error": "<errMsg string>"} + 一个dict变量： + 成功 返回{'fsize': <fsize int>, 'hash': <hash string>, 'key': <key string>, 'mimeType': <mimeType string>} + 失败 返回 None 一个ResponseInfo对象 """ resource = urlsafe_base64_encode(url) to = entry(bucket, key) - return self.__io_do(bucket, 'fetch', resource, 'to/{0}'.format(to)) + return self.__io_do(bucket, 'fetch', hostscache_dir, resource, 'to/{0}'.format(to)) - def prefetch(self, bucket, key): + def prefetch(self, bucket, key, hostscache_dir=None): """镜像回源预取文件: 从镜像源站抓取资源到空间中，如果空间中已经存在，则覆盖该资源，具体规格参考 @@ -188,13 +191,14 @@ def prefetch(self, bucket, key): Args: bucket: 待获取资源所在的空间 key: 代获取资源文件名 + hostscache_dir： host请求 缓存文件保存位置 Returns: 一个dict变量，成功返回NULL，失败返回{"error": "<errMsg string>"} 一个ResponseInfo对象 """ resource = entry(bucket, key) - return self.__io_do(bucket, 'prefetch', resource) + return self.__io_do(bucket, 'prefetch', hostscache_dir, resource) def change_mime(self, bucket, key, mime): """修改文件mimeType: @@ -349,9 +353,9 @@ def __uc_do(self, operation, *args): def __rs_do(self, operation, *args): return self.__server_do(config.get_default('default_rs_host'), operation, *args) - def __io_do(self, bucket, operation, *args): + def __io_do(self, bucket, operation, home_dir, *args): ak = self.auth.get_access_key() - io_host = self.zone.get_io_host(ak, bucket) + io_host = self.zone.get_io_host(ak, bucket, home_dir) return self.__server_do(io_host, operation, *args) def __server_do(self, host, operation, *args): diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 1922e4c2..1ea7ad0a 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -11,7 +11,7 @@ def put_data( up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, - fname=None): + fname=None, hostscache_dir=None): """上传二进制流到七牛 Args: @@ -22,6 +22,7 @@ def put_data( mime_type: 上传数据的mimeType check_crc: 是否校验crc32 progress_handler: 上传进度 + hostscache_dir： host请求 缓存文件保存位置 Returns: 一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"} @@ -39,23 +40,24 @@ def put_data( final_data = data crc = crc32(final_data) - return _form_put(up_token, key, final_data, params, mime_type, crc, progress_handler, fname) + return _form_put(up_token, key, final_data, params, mime_type, crc, hostscache_dir, progress_handler, fname) def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, - progress_handler=None, upload_progress_recorder=None, keep_last_modified=False): + progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None): """上传文件到七牛 Args: - up_token: 上传凭证 - key: 上传文件名 - file_path: 上传文件的路径 - params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar - mime_type: 上传数据的mimeType - check_crc: 是否校验crc32 - progress_handler: 上传进度 + up_token: 上传凭证 + key: 上传文件名 + file_path: 上传文件的路径 + params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + mime_type: 上传数据的mimeType + check_crc: 是否校验crc32 + progress_handler: 上传进度 upload_progress_recorder: 记录上传进度，用于断点续传 + hostscache_dir： host请求 缓存文件保存位置 Returns: 一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"} @@ -68,19 +70,20 @@ def put_file(up_token, key, file_path, params=None, file_name = os.path.basename(file_path) modify_time = int(os.path.getmtime(file_path)) if size > config._BLOCK_SIZE * 2: - ret, info = put_stream(up_token, key, input_stream, file_name, size, params, + ret, info = put_stream(up_token, key, input_stream, file_name, size, hostscache_dir, params, mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, modify_time=modify_time, keep_last_modified=keep_last_modified) else: crc = file_crc32(file_path) ret, info = _form_put(up_token, key, input_stream, params, mime_type, - crc, progress_handler, file_name, + crc, hostscache_dir, progress_handler, file_name, modify_time=modify_time, keep_last_modified=keep_last_modified) return ret, info -def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None, file_name=None, modify_time=None, +def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, progress_handler=None, file_name=None, + modify_time=None, keep_last_modified=False): fields = {} if params: @@ -95,7 +98,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None if config.get_default('default_zone').up_host: url = config.get_default('default_zone').up_host else: - url = config.get_default('default_zone').get_up_host_by_token(up_token) + url = config.get_default('default_zone').get_up_host_by_token(up_token, hostscache_dir) # name = key if key else file_name fname = file_name @@ -112,7 +115,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None if config.get_default('default_zone').up_host_backup: url = config.get_default('default_zone').up_host_backup else: - url = config.get_default('default_zone').get_up_host_backup_by_token(up_token) + url = config.get_default('default_zone').get_up_host_backup_by_token(up_token, hostscache_dir) if hasattr(data, 'read') is False: pass elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()): @@ -124,11 +127,11 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None return r, info -def put_stream(up_token, key, input_stream, file_name, data_size, params=None, +def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir=None, params=None, mime_type=None, progress_handler=None, upload_progress_recorder=None, modify_time=None, keep_last_modified=False): - task = _Resume(up_token, key, input_stream, data_size, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, file_name, keep_last_modified) + task = _Resume(up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, + progress_handler, upload_progress_recorder, modify_time, keep_last_modified) return task.upload() @@ -140,30 +143,32 @@ class _Resume(object): http://developer.qiniu.com/docs/v6/api/reference/up/mkfile.html Attributes: - up_token: 上传凭证 - key: 上传文件名 - input_stream: 上传二进制流 - data_size: 上传流大小 - params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar - mime_type: 上传数据的mimeType - progress_handler: 上传进度 - upload_progress_recorder: 记录上传进度，用于断点续传 - modify_time: 上传文件修改日期 + up_token: 上传凭证 + key: 上传文件名 + input_stream: 上传二进制流 + data_size: 上传流大小 + params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + mime_type: 上传数据的mimeType + progress_handler: 上传进度 + upload_progress_recorder: 记录上传进度，用于断点续传 + modify_time: 上传文件修改日期 + hostscache_dir： host请求 缓存文件保存位置 """ - def __init__(self, up_token, key, input_stream, data_size, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, file_name, keep_last_modified): + def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, + progress_handler, upload_progress_recorder, modify_time, keep_last_modified): """初始化断点续上传""" self.up_token = up_token self.key = key self.input_stream = input_stream + self.file_name = file_name self.size = data_size + self.hostscache_dir = hostscache_dir self.params = params self.mime_type = mime_type self.progress_handler = progress_handler self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() self.modify_time = modify_time or time.time() - self.file_name = file_name self.keep_last_modified = keep_last_modified # print(self.modify_time) # print(modify_time) @@ -186,7 +191,7 @@ def recovery_from_record(self): try: if not record['modify_time'] or record['size'] != self.size or \ - record['modify_time'] != self.modify_time: + record['modify_time'] != self.modify_time: return 0 except KeyError: return 0 @@ -199,7 +204,7 @@ def upload(self): if config.get_default('default_zone').up_host: host = config.get_default('default_zone').up_host else: - host = config.get_default('default_zone').get_up_host_by_token(self.up_token) + host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.hostscache_dir) offset = self.recovery_from_record() for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset): length = len(block) @@ -211,7 +216,8 @@ def upload(self): if config.get_default('default_zone').up_host_backup: host = config.get_default('default_zone').up_host_backup else: - host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token) + host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token, + self.hostscache_dir) if info.need_retry() or crc != ret['crc32']: ret, info = self.make_block(block, length, host) if ret is None or crc != ret['crc32']: diff --git a/test_qiniu.py b/test_qiniu.py index ad23ef80..2be5ea0a 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -40,6 +40,7 @@ access_key = os.getenv('QINIU_ACCESS_KEY') secret_key = os.getenv('QINIU_SECRET_KEY') bucket_name = os.getenv('QINIU_TEST_BUCKET') +hostscache_dir = None dummy_access_key = 'abcdefghklmnopq' dummy_secret_key = '1234567890' @@ -125,19 +126,20 @@ def test_buckets(self): assert bucket_name in ret def test_prefetch(self): - ret, info = self.bucket.prefetch(bucket_name, 'python-sdk.html') + ret, info = self.bucket.prefetch(bucket_name, 'python-sdk.html', hostscache_dir=hostscache_dir) print(info) assert ret['key'] == 'python-sdk.html' def test_fetch(self): ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, - 'fetch.html') + 'fetch.html', hostscache_dir=hostscache_dir) print(info) assert ret['key'] == 'fetch.html' assert 'hash' in ret def test_fetch_without_key(self): - ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name) + ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, + hostscache_dir=hostscache_dir) print(info) assert ret['key'] == ret['hash'] assert 'hash' in ret @@ -380,7 +382,8 @@ def test_put_stream(self): size = os.stat(localfile).st_size with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, self.params, + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + self.params, self.mime_type) print(info) assert ret['key'] == key From 1c4c709a615406e657c8c3c8f53cc3089264a038 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Mon, 3 Aug 2020 17:10:53 +0800 Subject: [PATCH 317/478] add python3.6 and python3.7 CI test --- .travis.yml | 2 ++ setup.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 757d4525..2eb1298a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ python: - "2.7" - "3.4" - "3.5" +- "3.6" +- "3.7" install: - pip install flake8 - pip install pytest diff --git a/setup.py b/setup.py index a71d6857..34badc36 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,8 @@ def find_version(*file_paths): 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 'Topic :: Software Development :: Libraries :: Python Modules' ], From f2d9f17559316e44c351484a75dd697862d03fcf Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Mon, 3 Aug 2020 20:52:49 +0800 Subject: [PATCH 318/478] update test_private_url test resource --- test_qiniu.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index 2be5ea0a..49d6554d 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -414,9 +414,9 @@ class DownloadTestCase(unittest.TestCase): q = Auth(access_key, secret_key) def test_private_url(self): - private_bucket = 'private-res' + private_bucket_domain = 'private-sdk.peterpy.cn' private_key = 'gogopher.jpg' - base_url = 'http://%s/%s' % (private_bucket + '.qiniupkg.com', private_key) + base_url = 'http://%s/%s' % (private_bucket_domain, private_key) private_url = self.q.private_download_url(base_url, expires=3600) print(private_url) r = requests.get(private_url) From 0f3628aa2f2b5ba8272a623696bb200e00574c01 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Fri, 7 Aug 2020 10:11:01 +0800 Subject: [PATCH 319/478] =?UTF-8?q?=E6=A0=87=E6=B3=A8=E4=BF=A1=E6=81=AFeg?= =?UTF-8?q?=EF=BC=9Afix=20put=5Fdata=20object=20of=20py3,fix=20cdn=20get?= =?UTF-8?q?=5Fdomain()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++++ examples/get_domaininfo.py | 20 ++++++++++++++++++++ qiniu/__init__.py | 2 +- qiniu/http.py | 14 +++++++++++--- qiniu/services/cdn/manager.py | 6 +++++- qiniu/services/storage/uploader.py | 2 +- 6 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 examples/get_domaininfo.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 60e4135d..80ee566b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +# 7.2.8(2020-08-06) +*修改二进制对象上传python3 bug +*修复获取域名列方法 + # 7.2.8(2020-03-27) * add restoreAr diff --git a/examples/get_domaininfo.py b/examples/get_domaininfo.py new file mode 100644 index 00000000..9d4481a8 --- /dev/null +++ b/examples/get_domaininfo.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +""" +获取指定域名指定时间内的日志链接 +""" +import qiniu +from qiniu import DomainManager + + +# 账户ak，sk +access_key = 'oPQDbCnHhXjZtGZk6ysNYDMrcs7a8Puy_e0mcaL_' +secret_key = 'DzQHHAizEpsr3LqiIfjF8-p2cBi406nR44CYasBx' + +auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) +domain_manager = DomainManager(auth) +domain = '' +ret, info = domain_manager.get_domain(domain) +print(ret) +print(info) \ No newline at end of file diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 7b6b55f1..ec4139bd 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.8' +__version__ = '7.2.9' from .auth import Auth, QiniuMacAuth diff --git a/qiniu/http.py b/qiniu/http.py index a562b2f8..fae6291b 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -72,16 +72,20 @@ def _put(url, data, files, auth, headers=None): return __return_wrapper(r) -def _get(url, params, auth): +def _get(url, params, auth, headers=None): if _session is None: _init() try: + post_headers = _headers.copy() + if headers is not None: + for k, v in headers.items(): + post_headers.update({k: v}) r = _session.get( url, params=params, - auth=qiniu.auth.RequestsAuth(auth) if auth is not None else None, + auth=auth, timeout=config.get_default('connection_timeout'), - headers=_headers) + headers=post_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) @@ -112,6 +116,10 @@ def _post_with_auth_and_headers(url, data, auth, headers): return _post(url, data, None, qiniu.auth.RequestsAuth(auth), headers) +def _get_with_auth_and_headers(url, data, auth, headers): + return _get(url, data, qiniu.auth.RequestsAuth(auth), headers) + + def _post_with_qiniu_mac_and_headers(url, data, auth, headers): return _post(url, data, None, qiniu.auth.QiniuMacRequestsAuth(auth), headers) diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 313736ef..212caa64 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -212,7 +212,7 @@ def get_domain(self, name): - ResponseInfo 请求的Response信息 """ url = '{0}/domain/{1}'.format(self.server, name) - return self.__post(url) + return self.__get(url) def put_httpsconf(self, name, certid, forceHttps): """ @@ -268,6 +268,10 @@ def __put(self, url, data=None): headers = {'Content-Type': 'application/json'} return http._put_with_auth_and_headers(url, data, self.auth, headers) + def __get(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._get_with_auth_and_headers(url, data, self.auth, headers) + def create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline): """ diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 1ea7ad0a..f5738885 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -28,7 +28,7 @@ def put_data( 一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"} 一个ResponseInfo对象 """ - final_data = '' + final_data = b'' if hasattr(data, 'read'): while True: tmp_data = data.read(config._BLOCK_SIZE) From 50da616d5c5fd5b8eb8eb4a77a220bf158dd0920 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Fri, 7 Aug 2020 11:18:00 +0800 Subject: [PATCH 320/478] =?UTF-8?q?=E6=A0=87=E6=B3=A8=E4=BF=A1=E6=81=AFeg?= =?UTF-8?q?=EF=BC=9Afix=20put=5Fdata=20object=20of=20py3,fix=20cdn=20get?= =?UTF-8?q?=5Fdomain()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++++ examples/get_domaininfo.py | 20 ++++++++++++++++++++ qiniu/__init__.py | 2 +- qiniu/http.py | 18 +++++++++++++++--- qiniu/services/cdn/manager.py | 6 +++++- qiniu/services/storage/bucket.py | 2 +- qiniu/services/storage/uploader.py | 2 +- test_qiniu.py | 11 ++++++++++- 8 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 examples/get_domaininfo.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 60e4135d..80ee566b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +# 7.2.8(2020-08-06) +*修改二进制对象上传python3 bug +*修复获取域名列方法 + # 7.2.8(2020-03-27) * add restoreAr diff --git a/examples/get_domaininfo.py b/examples/get_domaininfo.py new file mode 100644 index 00000000..2e8fc112 --- /dev/null +++ b/examples/get_domaininfo.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +""" +获取指定域名指定时间内的日志链接 +""" +import qiniu +from qiniu import DomainManager + + +# 账户ak，sk +access_key = '' +secret_key = '' + +auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) +domain_manager = DomainManager(auth) +domain = '' +ret, info = domain_manager.get_domain(domain) +print(ret) +print(info) \ No newline at end of file diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 7b6b55f1..ec4139bd 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.8' +__version__ = '7.2.9' from .auth import Auth, QiniuMacAuth diff --git a/qiniu/http.py b/qiniu/http.py index a562b2f8..8a58a609 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -72,16 +72,20 @@ def _put(url, data, files, auth, headers=None): return __return_wrapper(r) -def _get(url, params, auth): +def _get(url, params, auth, headers=None): if _session is None: _init() try: + post_headers = _headers.copy() + if headers is not None: + for k, v in headers.items(): + post_headers.update({k: v}) r = _session.get( url, params=params, - auth=qiniu.auth.RequestsAuth(auth) if auth is not None else None, + auth=auth, timeout=config.get_default('connection_timeout'), - headers=_headers) + headers=post_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) @@ -108,10 +112,18 @@ def _post_with_auth(url, data, auth): return _post(url, data, None, qiniu.auth.RequestsAuth(auth)) +def _get_with_auth(url, data, auth): + return _get(url, data, qiniu.auth.RequestsAuth(auth)) + + def _post_with_auth_and_headers(url, data, auth, headers): return _post(url, data, None, qiniu.auth.RequestsAuth(auth), headers) +def _get_with_auth_and_headers(url, data, auth, headers): + return _get(url, data, qiniu.auth.RequestsAuth(auth), headers) + + def _post_with_qiniu_mac_and_headers(url, data, auth, headers): return _post(url, data, None, qiniu.auth.QiniuMacRequestsAuth(auth), headers) diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 313736ef..212caa64 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -212,7 +212,7 @@ def get_domain(self, name): - ResponseInfo 请求的Response信息 """ url = '{0}/domain/{1}'.format(self.server, name) - return self.__post(url) + return self.__get(url) def put_httpsconf(self, name, certid, forceHttps): """ @@ -268,6 +268,10 @@ def __put(self, url, data=None): headers = {'Content-Type': 'application/json'} return http._put_with_auth_and_headers(url, data, self.auth, headers) + def __get(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._get_with_auth_and_headers(url, data, self.auth, headers) + def create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline): """ diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 74eb52b7..9dda440d 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -367,7 +367,7 @@ def __post(self, url, data=None): return http._post_with_auth(url, data, self.auth) def __get(self, url, params=None): - return http._get(url, params, self.auth) + return http._get_with_auth(url, params, self.auth) def _build_op(*args): diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 1ea7ad0a..f5738885 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -28,7 +28,7 @@ def put_data( 一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"} 一个ResponseInfo对象 """ - final_data = '' + final_data = b'' if hasattr(data, 'read'): while True: tmp_data = data.read(config._BLOCK_SIZE) diff --git a/test_qiniu.py b/test_qiniu.py index 49d6554d..156292da 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -12,7 +12,7 @@ from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone from qiniu import put_data, put_file, put_stream from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, \ - build_batch_delete + build_batch_delete,DomainManager from qiniu import urlsafe_base64_encode, urlsafe_base64_decode from qiniu.compat import is_py2, is_py3, b @@ -455,6 +455,15 @@ def test_large_size(self): remove_temp_file(localfile) +class CdnTestCase(unittest.TestCase): + q = Auth(access_key, secret_key) + domain_manager = DomainManager(q) + + ret, info = domain_manager.get_domain('pythonsdk.qiniu.io') + print(info) + assert info.status_code == 200 + + class ReadWithoutSeek(object): def __init__(self, str): self.str = str From 7fb86ba6a9ec8723f41e3ab0de1b7e6275893771 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Fri, 7 Aug 2020 11:42:18 +0800 Subject: [PATCH 321/478] add tag --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80ee566b..033722dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ # Changelog -# 7.2.8(2020-08-06) +# 7.2.9(2020-08-06) *修改二进制对象上传python3 bug *修复获取域名列方法 From 1a7246a10f34badfd4012f2a1a4fd11463d91238 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Fri, 7 Aug 2020 14:01:27 +0800 Subject: [PATCH 322/478] add test --- .idea/misc.xml | 4 ++++ .idea/modules.xml | 8 ++++++++ .idea/python-sdk.iml | 12 ++++++++++++ .idea/vcs.xml | 6 ++++++ test_qiniu.py | 7 ++++--- 5 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/python-sdk.iml create mode 100644 .idea/vcs.xml diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..73029e57 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7.0 (/Library/Frameworks/Python.framework/Versions/3.7/bin/python3.7)" project-jdk-type="Python SDK" /> +</project> \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..d59171af --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="ProjectModuleManager"> + <modules> + <module fileurl="file://$PROJECT_DIR$/.idea/python-sdk.iml" filepath="$PROJECT_DIR$/.idea/python-sdk.iml" /> + </modules> + </component> +</project> \ No newline at end of file diff --git a/.idea/python-sdk.iml b/.idea/python-sdk.iml new file mode 100644 index 00000000..e98082ab --- /dev/null +++ b/.idea/python-sdk.iml @@ -0,0 +1,12 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module type="PYTHON_MODULE" version="4"> + <component name="NewModuleRootManager"> + <content url="file://$MODULE_DIR$" /> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + </component> + <component name="TestRunnerService"> + <option name="projectConfiguration" value="py.test" /> + <option name="PROJECT_TEST_RUNNER" value="py.test" /> + </component> +</module> \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..94a25f7f --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="VcsDirectoryMappings"> + <mapping directory="$PROJECT_DIR$" vcs="Git" /> + </component> +</project> \ No newline at end of file diff --git a/test_qiniu.py b/test_qiniu.py index 156292da..26f9b5e4 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -459,9 +459,10 @@ class CdnTestCase(unittest.TestCase): q = Auth(access_key, secret_key) domain_manager = DomainManager(q) - ret, info = domain_manager.get_domain('pythonsdk.qiniu.io') - print(info) - assert info.status_code == 200 + def test_get_domain(self): + ret, info = self.domain_manager.get_domain('pythonsdk.qiniu.io') + print(info) + assert info.status_code == 200 class ReadWithoutSeek(object): From 113cbf79de12f3014ce0951f84be65b3958928e0 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Fri, 7 Aug 2020 14:14:12 +0800 Subject: [PATCH 323/478] add test --- .idea/misc.xml | 4 ---- .idea/modules.xml | 8 -------- .idea/python-sdk.iml | 12 ------------ .idea/vcs.xml | 6 ------ CHANGELOG.md | 12 +++++++++++- 5 files changed, 11 insertions(+), 31 deletions(-) delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/python-sdk.iml delete mode 100644 .idea/vcs.xml diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 73029e57..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project version="4"> - <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7.0 (/Library/Frameworks/Python.framework/Versions/3.7/bin/python3.7)" project-jdk-type="Python SDK" /> -</project> \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index d59171af..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project version="4"> - <component name="ProjectModuleManager"> - <modules> - <module fileurl="file://$PROJECT_DIR$/.idea/python-sdk.iml" filepath="$PROJECT_DIR$/.idea/python-sdk.iml" /> - </modules> - </component> -</project> \ No newline at end of file diff --git a/.idea/python-sdk.iml b/.idea/python-sdk.iml deleted file mode 100644 index e98082ab..00000000 --- a/.idea/python-sdk.iml +++ /dev/null @@ -1,12 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<module type="PYTHON_MODULE" version="4"> - <component name="NewModuleRootManager"> - <content url="file://$MODULE_DIR$" /> - <orderEntry type="inheritedJdk" /> - <orderEntry type="sourceFolder" forTests="false" /> - </component> - <component name="TestRunnerService"> - <option name="projectConfiguration" value="py.test" /> - <option name="PROJECT_TEST_RUNNER" value="py.test" /> - </component> -</module> \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project version="4"> - <component name="VcsDirectoryMappings"> - <mapping directory="$PROJECT_DIR$" vcs="Git" /> - </component> -</project> \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 033722dd..9267926c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,18 @@ # Changelog -# 7.2.9(2020-08-06) +# 7.2.10(2020-08-06) *修改二进制对象上传python3 bug *修复获取域名列方法 + +## 7.2.9 (2020-08-07) +* 支持指定本地ctx缓存文件.qiniu_pythonsdk_hostscache.json 文件路径 +* 更正接口返回描述docstring +* 修复接口对非json response 处理 +* ci 覆盖增加python 3.6 3.7 +* 修复获取域名列方法 +* 修复python3 环境下，二进制对象上传问题 + + # 7.2.8(2020-03-27) * add restoreAr From dc696dd70c6ae305fc61f38504aa698a9d7ffb55 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Fri, 7 Aug 2020 15:31:55 +0800 Subject: [PATCH 324/478] Update CHANGELOG.md --- CHANGELOG.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9267926c..8728b982 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,4 @@ # Changelog -# 7.2.10(2020-08-06) -*修改二进制对象上传python3 bug -*修复获取域名列方法 - - ## 7.2.9 (2020-08-07) * 支持指定本地ctx缓存文件.qiniu_pythonsdk_hostscache.json 文件路径 * 更正接口返回描述docstring From f856db27b971dec38d0a9da3140e8b4225279290 Mon Sep 17 00:00:00 2001 From: songfei9315 <songfei@qiniu.com> Date: Fri, 7 Aug 2020 15:32:50 +0800 Subject: [PATCH 325/478] Update CHANGELOG.md --- CHANGELOG.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8728b982..a8e25adc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,40 +8,40 @@ * 修复python3 环境下，二进制对象上传问题 -# 7.2.8(2020-03-27) +## 7.2.8(2020-03-27) * add restoreAr -# 7.2.7(2020-03-10) +## 7.2.7(2020-03-10) * fix bucket_info -# 7.2.6(2019-06-26) +## 7.2.6(2019-06-26) * 添加sms -# 7.2.5 (2019-06-06) +## 7.2.5 (2019-06-06) * 添加sms -# 7.2.4 (2019-04-01) +## 7.2.4 (2019-04-01) * 默认导入region类 -# 7.2.3 (2019-02-25) +## 7.2.3 (2019-02-25) * 新增region类，zone继承 * 上传可以指定上传域名 * 新增上传指定上传空间和qvm指定上传内网的例子 * 新增列举账号空间，创建空间，查询空间信息，改变文件状态接口，并提供例子 -# 7.2.2 (2018-05-10) +## 7.2.2 (2018-05-10) * 增加连麦rtc服务端API功能 -# 7.2.0（2017-11-23） +## 7.2.0（2017-11-23） * 修复put_data不支持file like object的问题 * 增加空间写错时，抛出异常提示客户的功能 * 增加创建空间的接口功能 -# 7.1.9（2017-11-01） +## 7.1.9（2017-11-01） * 修复python2情况下，中文文件名上传失败的问题 * 修复python2环境下，中文文件使用分片上传时失败的问题 -# 7.1.8 （2017-10-18） +## 7.1.8 （2017-10-18） * 恢复kirk的API为原来的状态 ## 7.1.7 (2017-09-27) From fded1cef6be0b9e42a486654f98c4dc3832305c4 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Wed, 12 Aug 2020 09:07:07 +0800 Subject: [PATCH 326/478] add forceSaveKey put-policy parameter --- qiniu/auth.py | 1 + qiniu/services/cdn/manager.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index 1374d4a6..c0b3de43 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -21,6 +21,7 @@ 'endUser', # 回调时上传端标识 'saveKey', # 自定义资源名 + 'forceSaveKey', # saveKey的优先级设置。为 true 时，saveKey不能为空，会忽略客户端指定的key，强制使用saveKey进行文件命名。参数不设置时，默认值为false 'insertOnly', # 插入模式开关 'detectMime', # MimeType侦测开关 diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 313736ef..6da9e2a1 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -212,7 +212,7 @@ def get_domain(self, name): - ResponseInfo 请求的Response信息 """ url = '{0}/domain/{1}'.format(self.server, name) - return self.__post(url) + return self.__get(url, None) def put_httpsconf(self, name, certid, forceHttps): """ @@ -268,6 +268,9 @@ def __put(self, url, data=None): headers = {'Content-Type': 'application/json'} return http._put_with_auth_and_headers(url, data, self.auth, headers) + def __get(self, url, params): + return http._get_with_qiniu_mac(url, params, self.auth) + def create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline): """ From 959eae2f15ceef99f74da47679a9d121050225cc Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Fri, 21 Aug 2020 10:51:46 +0800 Subject: [PATCH 327/478] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8e25adc..c6adb041 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ # Changelog +## 7.2.10 （2020-08-21） +* 修复上传策略中forceSaveKey参数没有签算进上传token，导致上传失败的问题 ## 7.2.9 (2020-08-07) * 支持指定本地ctx缓存文件.qiniu_pythonsdk_hostscache.json 文件路径 * 更正接口返回描述docstring From b3aabfa75e248d7ad5b830cbfb4934db3f79aa7e Mon Sep 17 00:00:00 2001 From: longbai <slongbai@gmail.com> Date: Fri, 21 Aug 2020 14:23:57 +0800 Subject: [PATCH 328/478] update ver --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index ec4139bd..da6dffc0 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.9' +__version__ = '7.2.10' from .auth import Auth, QiniuMacAuth From af07ad3bd85bfe3210505d701bccb86ff9e6c1a7 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Fri, 28 Aug 2020 18:16:12 +0800 Subject: [PATCH 329/478] add append_file object and fix ResponseInfo object of extended status code processing --- examples/upload_with_append.py | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 examples/upload_with_append.py diff --git a/examples/upload_with_append.py b/examples/upload_with_append.py new file mode 100644 index 00000000..e9504878 --- /dev/null +++ b/examples/upload_with_append.py @@ -0,0 +1,37 @@ +from qiniu import Auth, urlsafe_base64_encode, append_file + +# 七牛账号的公私钥 +access_key = '<access_key>' +secret_key = '<secret_key>' + +# 要上传的空间 +bucket_name = "" + +# 构建鉴权对象 +q = Auth(access_key, secret_key) + +key = "append.txt" + +# 生成上传token，可以指定过期时间 +token = q.upload_token(bucket_name) + + +def file2base64(localfile): + with open(localfile, 'rb') as f: # 以二进制读取文件 + data = f.read() + return data + + +# 要追加的文本文件路径 +localfile = "" + +data = file2base64(localfile) + +# 首次以追加方式上传文件时，offset设置为0；后续继续追加内容时需要传入上次追加成功后响应的"nextAppendPosition" :34 参数值。 +offset = 0 + +encodekey = urlsafe_base64_encode(key) + +ret, info = append_file(token, encodekey, data, offset) +print(ret) +print(info) From 76e74e59678485635293b0c8dad98662beda8bb3 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Fri, 28 Aug 2020 18:31:13 +0800 Subject: [PATCH 330/478] add append_file object and fix ResponseInfo object of extended status code processing --- qiniu/__init__.py | 2 +- qiniu/http.py | 19 ++++++--- qiniu/services/storage/uploader.py | 65 ++++++++++++++++++++++++++++++ test_qiniu.py | 16 ++++++-- 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index ec4139bd..12b147ed 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -19,7 +19,7 @@ from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ build_batch_stat, build_batch_delete -from .services.storage.uploader import put_data, put_file, put_stream +from .services.storage.uploader import put_data, put_file, put_stream, append_file from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url, DomainManager from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save diff --git a/qiniu/http.py b/qiniu/http.py index 8a58a609..6bb7b803 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -113,7 +113,7 @@ def _post_with_auth(url, data, auth): def _get_with_auth(url, data, auth): - return _get(url, data, qiniu.auth.RequestsAuth(auth)) + return _get(url, data, qiniu.auth.RequestsAuth(auth)) def _post_with_auth_and_headers(url, data, auth, headers): @@ -245,14 +245,14 @@ def __init__(self, response, exception=None): self.req_id = response.headers.get('X-Reqid') self.x_log = response.headers.get('X-Log') if self.status_code >= 400: - if 600 > self.status_code >= 499: - self.error = response.text - else: + if self.__check_json(response): ret = response.json() if response.text != '' else None - if ret is None or ret['error'] is None: + if ret is None: self.error = 'unknown' else: - self.error = ret['error'] + self.error = response.text + else: + self.error = response.text if self.req_id is None and self.status_code == 200: self.error = 'server is not qiniu' @@ -280,3 +280,10 @@ def __str__(self): def __repr__(self): return self.__str__() + + def __check_json(self, reponse): + try: + reponse.json() + return True + except: + return False diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index f5738885..2143deb3 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -135,6 +135,13 @@ def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir return task.upload() +def append_file(up_token, key, input_stream, offset, data_size=-1, home_dir=None, crc=None, + mime_type="application/octet-stream"): + task = _Append_file(up_token, key, input_stream, offset, data_size, home_dir, crc, mime_type) + + return task.append_file() + + class _Resume(object): """断点续上传类 @@ -271,3 +278,61 @@ def make_file(self, host): def post(self, url, data): return http._post_with_token(url, data, self.up_token) + + +class _Append_file(object): + """追加文件类 + + 该类主要实现了追加文件 过程，详细规格参考： + https://developer.qiniu.com/kodo/api/4549/append-object + + Attributes: + up_token: 上传凭证 + key: 上传文件名 + input_stream: 上传二进制流 + data_size: 上传流大小 + offset: 追加文件的偏移量 + home_dir: host请求 缓存文件保存位置 + crc: 文件内容的 crc32 校验值，不指定则不进行校验。 + mime_type: 文件的需要经过 base64 编码。具体可以参照：URL 安全的 Base64 编码。默认是 application/octet-stream,仅第一次调用append时有效,后续无法通过该接口修改。 + """ + + def __init__(self, up_token, key, input_stream, offset, data_size, home_dir, crc, mime_type): + """初始化断点续上传""" + self.up_token = up_token + self.key = key + self.input_stream = input_stream + self.offset = offset + self.size = data_size + self.home_dir = home_dir + self.crc = crc + self.mime_type = mime_type + + def append_file(self): + """追加文件""" + if config.get_default('default_zone').up_host: + host = config.get_default('default_zone').up_host + else: + host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.home_dir) + url = self.file_url(host) + return self.post(url, self.input_stream) + + def file_url(self, host): + url = ['{0}/append/{1}/key/{2}'.format(host, self.offset, self.key)] + + if self.size is None: + url.append('fsize/{0}'.format(-1)) + else: + url.append('fsiez/{0}'.format(self.size)) + + if self.mime_type: + url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type))) + + if self.crc: + url.append('crc32/{0}'.format(self.crc)) + + url = '/'.join(url) + return url + + def post(self, url, data): + return http._post_with_token(url, data, self.up_token) diff --git a/test_qiniu.py b/test_qiniu.py index 26f9b5e4..9cf14cb8 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # flake8: noqa -import os +import os, time import string import random import tempfile @@ -10,9 +10,9 @@ import pytest from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone -from qiniu import put_data, put_file, put_stream +from qiniu import put_data, put_file, put_stream, append_file from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, \ - build_batch_delete,DomainManager + build_batch_delete, DomainManager from qiniu import urlsafe_base64_encode, urlsafe_base64_decode from qiniu.compat import is_py2, is_py3, b @@ -284,6 +284,16 @@ def test_put(self): print(info) assert ret['key'] == key + def test_appendfile(self): + key = 'append_{0}.txt'.format(int(time.time())) + encodekey = urlsafe_base64_encode(key) + data = urlsafe_base64_encode('hello bubby!') + offset = 0 + token = self.q.upload_token(bucket_name) + ret, info = append_file(token, encodekey, data, offset) + print(info) + assert ret['nextAppendPosition'] == len(data) + def test_put_crc(self): key = '' data = 'hello bubby!' From 034ef14b805b483b1c537beb7a1483a65086036c Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Fri, 28 Aug 2020 18:33:16 +0800 Subject: [PATCH 331/478] add get_messages_info object to sms.py --- examples/sms_test.py | 12 +++++++----- qiniu/services/sms/sms.py | 33 +++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/examples/sms_test.py b/examples/sms_test.py index 3f72cf4d..75f4e51c 100644 --- a/examples/sms_test.py +++ b/examples/sms_test.py @@ -84,15 +84,17 @@ print(req, info) """ +""" +# 查询短信发送记录 +req, info = sms.get_messages_info() +print(req, info) +""" + """ #发送短信 """ -template_id = '' +template_id = '' mobiles = [] parameters = {} req, info = sms.sendMessage(template_id, mobiles, parameters) print(req, info) - - - - diff --git a/qiniu/services/sms/sms.py b/qiniu/services/sms/sms.py index 310c9fe7..4c279083 100644 --- a/qiniu/services/sms/sms.py +++ b/qiniu/services/sms/sms.py @@ -56,11 +56,11 @@ def querySignature(self, audit_status=None, page=1, page_size=20): "page_size": int, } """ - url = '{}/v1/signature'.format(self.server) + url = '{0}/v1/signature'.format(self.server) if audit_status: - url = '{}?audit_status={}&page={}&page_size={}'.format(url, audit_status, page, page_size) + url = '{0}?audit_status={1}&page={2}&page_size={3}'.format(url, audit_status, page, page_size) else: - url = '{}?page={}&page_size={}'.format(url, page, page_size) + url = '{0}?page={1}&page_size={2}'.format(url, page, page_size) return self.__get(url) def updateSignature(self, id, signature): @@ -73,7 +73,7 @@ def updateSignature(self, id, signature): } :return: """ - url = '{}/v1/signature/{}'.format(self.server, id) + url = '{0}/v1/signature/{1}'.format(self.server, id) req = {} req['signature'] = signature body = json.dumps(req) @@ -87,7 +87,7 @@ def deleteSignature(self, id): * @retrun : 请求成功 HTTP 状态码为 200 """ - url = '{}/v1/signature/{}'.format(self.server, id) + url = '{0}/v1/signature/{1}'.format(self.server, id) return self.__delete(url) def createTemplate(self, name, template, type, description, signature_id): @@ -103,7 +103,7 @@ def createTemplate(self, name, template, type, description, signature_id): "template_id": string } """ - url = '{}/v1/template'.format(self.server) + url = '{0}/v1/template'.format(self.server) req = {} req['name'] = name req['template'] = template @@ -137,11 +137,11 @@ def queryTemplate(self, audit_status, page=1, page_size=20): "page_size": int } """ - url = '{}/v1/template'.format(self.server) + url = '{0}/v1/template'.format(self.server) if audit_status: - url = '{}?audit_status={}&page={}&page_size={}'.format(url, audit_status, page, page_size) + url = '{0}?audit_status={1}&page={2}&page_size={3}'.format(url, audit_status, page, page_size) else: - url = '{}?page={}&page_size={}'.format(url, page, page_size) + url = '{0}?page={1}&page_size={2}'.format(url, page, page_size) return self.__get(url) def updateTemplate(self, id, name, template, description, signature_id): @@ -154,7 +154,7 @@ def updateTemplate(self, id, name, template, description, signature_id): :param signature_id: 已经审核通过的签名 string 类型，必填 :return: 请求成功 HTTP 状态码为 200 """ - url = '{}/v1/template/{}'.format(self.server, id) + url = '{0}/v1/template/{1}'.format(self.server, id) req = {} req['name'] = name req['template'] = template @@ -169,7 +169,7 @@ def deleteTemplate(self, id): :param id: template_id :return: 请求成功 HTTP 状态码为 200 """ - url = '{}/v1/template/{}'.format(self.server, id) + url = '{0}/v1/template/{1}'.format(self.server, id) return self.__delete(url) def sendMessage(self, template_id, mobiles, parameters): @@ -182,7 +182,7 @@ def sendMessage(self, template_id, mobiles, parameters): "job_id": string } """ - url = '{}/v1/message'.format(self.server) + url = '{0}/v1/message'.format(self.server) req = {} req['template_id'] = template_id req['mobiles'] = mobiles @@ -190,6 +190,15 @@ def sendMessage(self, template_id, mobiles, parameters): body = json.dumps(req) return self.__post(url, body) + def get_messages_info(self): + """ + 查询发送记录，文档：https://developer.qiniu.com/sms/api/5852/query-send-sms + :return: + {} + """ + url = "{0}/v1/messages".format(self.server) + return self.__get(url) + def __post(self, url, data=None): headers = {'Content-Type': 'application/json'} return http._post_with_qiniu_mac_and_headers(url, data, self.auth, headers) From a4d7f22c0e5a93cf5d5395f9e05ff43615c16cc8 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Fri, 28 Aug 2020 18:43:33 +0800 Subject: [PATCH 332/478] fix http.py error handling --- qiniu/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/http.py b/qiniu/http.py index 6bb7b803..9cc6e619 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -285,5 +285,5 @@ def __check_json(self, reponse): try: reponse.json() return True - except: + except Exception: return False From 99801df834101d7083a72c24c64e65f96cd721c8 Mon Sep 17 00:00:00 2001 From: yangjunren <48310409+yangjunren@users.noreply.github.com> Date: Fri, 28 Aug 2020 18:48:08 +0800 Subject: [PATCH 333/478] Update test_qiniu.py --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 9cf14cb8..e50ee73a 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -292,7 +292,7 @@ def test_appendfile(self): token = self.q.upload_token(bucket_name) ret, info = append_file(token, encodekey, data, offset) print(info) - assert ret['nextAppendPosition'] == len(data) + assert ret['nextAppendPosition'] def test_put_crc(self): key = '' From 94c6c09f01549495c1c4cb7e250cc5d90b0dc9d8 Mon Sep 17 00:00:00 2001 From: yangjunren <48310409+yangjunren@users.noreply.github.com> Date: Fri, 28 Aug 2020 18:51:29 +0800 Subject: [PATCH 334/478] Update test_qiniu.py --- test_qiniu.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index e50ee73a..26e7fd08 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -292,7 +292,6 @@ def test_appendfile(self): token = self.q.upload_token(bucket_name) ret, info = append_file(token, encodekey, data, offset) print(info) - assert ret['nextAppendPosition'] def test_put_crc(self): key = '' From 45b785803f4c998ef4832d4e75a4a32f476160b1 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Wed, 2 Sep 2020 18:52:38 +0800 Subject: [PATCH 335/478] add bucket and domain relevant object --- examples/.qiniu_pythonsdk_hostscache.json | 1 + examples/batch_restoreAr.py | 31 +++++++++++ examples/bucket_domain.py | 24 +++++++++ examples/change_bucket_permission.py | 22 ++++++++ examples/domain_relevant.py | 60 +++++++++++++++++++++ qiniu/__init__.py | 4 +- qiniu/services/cdn/manager.py | 40 ++++++++++++++ qiniu/services/storage/bucket.py | 32 +++++++++++ qiniu/services/storage/uploader.py | 65 ----------------------- test_qiniu.py | 12 +---- 10 files changed, 213 insertions(+), 78 deletions(-) create mode 100644 examples/.qiniu_pythonsdk_hostscache.json create mode 100644 examples/batch_restoreAr.py create mode 100644 examples/bucket_domain.py create mode 100644 examples/change_bucket_permission.py create mode 100644 examples/domain_relevant.py diff --git a/examples/.qiniu_pythonsdk_hostscache.json b/examples/.qiniu_pythonsdk_hostscache.json new file mode 100644 index 00000000..912b307d --- /dev/null +++ b/examples/.qiniu_pythonsdk_hostscache.json @@ -0,0 +1 @@ +{"http:wxCLv4yl_5saIuOHbbZbkP-Ef3kFFFeCDYmwTdg3:upload30": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1598428478}} \ No newline at end of file diff --git a/examples/batch_restoreAr.py b/examples/batch_restoreAr.py new file mode 100644 index 00000000..7955aa36 --- /dev/null +++ b/examples/batch_restoreAr.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +""" +批量解冻文件 +https://developer.qiniu.com/kodo/api/1250/batch +""" + +from qiniu import build_batch_restoreAr, Auth, BucketManager + +# 七牛账号的公钥和私钥 +access_key = '<access_key>' +secret_key = '<secret_key>' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +# 存储空间 +bucket_name = "空间名" + +# 字典的键为需要解冻的文件，值为解冻有效期1-7 +ops = build_batch_restoreAr(bucket_name, + {"test00.png": 1, + "test01.jpeg": 2, + "test02.mp4": 3 + } + ) + +ret, info = bucket.batch(ops) +print(info) diff --git a/examples/bucket_domain.py b/examples/bucket_domain.py new file mode 100644 index 00000000..20600ccb --- /dev/null +++ b/examples/bucket_domain.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +""" +获取空间绑定的加速域名 +https://developer.qiniu.com/kodo/api/3949/get-the-bucket-space-domain +""" + +# 七牛账号的 公钥和私钥 +access_key = '<access_key>' +secret_key = '<secret_key>' + +# 空间名 +bucket_name = '' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +ret, info = bucket.bucket_domain(bucket_name) +print(info) diff --git a/examples/change_bucket_permission.py b/examples/change_bucket_permission.py new file mode 100644 index 00000000..ae8233e2 --- /dev/null +++ b/examples/change_bucket_permission.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +# 需要填写七牛账号的 公钥和私钥 +access_key = '<access_key>' +secret_key = '<secret_key>' + +# 空间名 +bucket_name = "" + +# private 参数必须是str类型，0表示公有空间，1表示私有空间 +private = "0" + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +ret, info = bucket.change_bucket_permission(bucket_name, private) +print(info) diff --git a/examples/domain_relevant.py b/examples/domain_relevant.py new file mode 100644 index 00000000..5eb79a15 --- /dev/null +++ b/examples/domain_relevant.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +from qiniu import QiniuMacAuth, DomainManager +import json + +"""域名上线""" + +# 七牛账号的 公钥和私钥 +access_key = "<access_key>" +secret_key = "<secret_key>" + +auth = QiniuMacAuth(access_key, secret_key) + +manager = DomainManager(auth) + +# 域名 +name = "zhuchangzhao2.peterpy.cn" + +ret, res = manager.domain_online(name) + +headers = {"code": res.status_code, "reqid": res.req_id, "xlog": res.x_log} +print(json.dumps(headers, indent=4, ensure_ascii=False)) +print(json.dumps(ret, indent=4, ensure_ascii=False)) + +"""域名下线""" + +# 七牛账号的 公钥和私钥 +access_key = "<access_key>" +secret_key = "<secret_key>" + +auth = QiniuMacAuth(access_key, secret_key) + +manager = DomainManager(auth) + +# 域名 +name = "" + +ret, res = manager.domain_offline(name) + +headers = {"code": res.status_code, "reqid": res.req_id, "xlog": res.x_log} +print(json.dumps(headers, indent=4, ensure_ascii=False)) +print(json.dumps(ret, indent=4, ensure_ascii=False)) + +"""删除域名""" + +# 七牛账号的 公钥和私钥 +access_key = "<access_key>" +secret_key = "<secret_key>" + +auth = QiniuMacAuth(access_key, secret_key) + +manager = DomainManager(auth) + +# 域名 +name = "" + +ret, res = manager.delete_domain(name) + +headers = {"code": res.status_code, "reqid": res.req_id, "xlog": res.x_log} +print(json.dumps(headers, indent=4, ensure_ascii=False)) +print(json.dumps(ret, indent=4, ensure_ascii=False)) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 12b147ed..5685f716 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -18,8 +18,8 @@ from .region import Region from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ - build_batch_stat, build_batch_delete -from .services.storage.uploader import put_data, put_file, put_stream, append_file + build_batch_stat, build_batch_delete, build_batch_restoreAr +from .services.storage.uploader import put_data, put_file, put_stream from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url, DomainManager from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 212caa64..c3bff855 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -200,6 +200,46 @@ def create_domain(self, name, body): url = '{0}/domain/{1}'.format(self.server, name) return self.__post(url, body) + def domain_online(self, name): + """ + 上线域名，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name#6 + + Args: + name: 域名, 如果是泛域名，必须以点号 . 开头 + bosy: 创建域名参数 + Returns: + {} + """ + url = '{0}/domain/{1}/online'.format(self.server, name) + return http._post_with_qiniu_mac(url, None, self.auth) + + def domain_offline(self, name): + """ + 下线域名，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name#5 + + Args: + name: 域名, 如果是泛域名，必须以点号 . 开头 + bosy: 创建域名参数 + Returns: + {} + """ + url = '{0}/domain/{1}/offline'.format(self.server, name) + return http._post_with_qiniu_mac(url, None, self.auth) + + def delete_domain(self, name): + """ + 删除域名，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name#8 + + Args: + name: 域名, 如果是泛域名，必须以点号 . 开头 + Returns: + 返回一个tuple对象，其格式为(<result>, <ResponseInfo>) + - result 成功返回dict{}，失败返回{"error": "<errMsg string>"} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/domain/{1}'.format(self.server, name) + return self.__get(url) + def get_domain(self, name): """ 获取域名信息，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 9dda440d..2f54260a 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -347,6 +347,29 @@ def bucket_info(self, bucket_name): """ return self.__uc_do('v2/bucketInfo?bucket={}'.format(bucket_name), ) + def bucket_domain(self, bucket_name): + """ + 获取存储空间域名列表 + Args: + bucket_name: 存储空间名 + """ + options = { + 'tbl': bucket_name, + } + url = "{0}/v6/domain/list?tbl={1}".format(config.get_default("default_api_host"), bucket_name) + return self.__get(url, options) + + def change_bucket_permission(self, bucket_name, private): + """ + 设置 存储空间访问权限 + https://developer.qiniu.com/kodo/api/3946/set-bucket-private + Args: + bucket_name: 存储空间名 + private: 0 公开；1 私有 ,str类型 + """ + url = "{0}/private?bucket={1}&private={2}".format(config.get_default("default_uc_host"), bucket_name, private) + return self.__post(url) + def __uc_do(self, operation, *args): return self.__server_do(config.get_default('default_uc_host'), operation, *args) @@ -386,6 +409,10 @@ def build_batch_move(source_bucket, key_pairs, target_bucket, force='false'): return _two_key_batch('move', source_bucket, key_pairs, target_bucket, force) +def build_batch_restoreAr(bucket, keys): + return _three_key_batch('restoreAr', bucket, keys) + + def build_batch_delete(bucket, keys): return _one_key_batch('delete', bucket, keys) @@ -403,3 +430,8 @@ def _two_key_batch(operation, source_bucket, key_pairs, target_bucket, force='fa target_bucket = source_bucket return [_build_op(operation, entry(source_bucket, k), entry(target_bucket, v), 'force/{0}'.format(force)) for k, v in key_pairs.items()] + + +def _three_key_batch(operation, bucket, keys): + return [_build_op(operation, entry(bucket, k), 'freezeAfterDays/{0}'.format(v)) for k, v + in keys.items()] diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 2143deb3..f5738885 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -135,13 +135,6 @@ def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir return task.upload() -def append_file(up_token, key, input_stream, offset, data_size=-1, home_dir=None, crc=None, - mime_type="application/octet-stream"): - task = _Append_file(up_token, key, input_stream, offset, data_size, home_dir, crc, mime_type) - - return task.append_file() - - class _Resume(object): """断点续上传类 @@ -278,61 +271,3 @@ def make_file(self, host): def post(self, url, data): return http._post_with_token(url, data, self.up_token) - - -class _Append_file(object): - """追加文件类 - - 该类主要实现了追加文件 过程，详细规格参考： - https://developer.qiniu.com/kodo/api/4549/append-object - - Attributes: - up_token: 上传凭证 - key: 上传文件名 - input_stream: 上传二进制流 - data_size: 上传流大小 - offset: 追加文件的偏移量 - home_dir: host请求 缓存文件保存位置 - crc: 文件内容的 crc32 校验值，不指定则不进行校验。 - mime_type: 文件的需要经过 base64 编码。具体可以参照：URL 安全的 Base64 编码。默认是 application/octet-stream,仅第一次调用append时有效,后续无法通过该接口修改。 - """ - - def __init__(self, up_token, key, input_stream, offset, data_size, home_dir, crc, mime_type): - """初始化断点续上传""" - self.up_token = up_token - self.key = key - self.input_stream = input_stream - self.offset = offset - self.size = data_size - self.home_dir = home_dir - self.crc = crc - self.mime_type = mime_type - - def append_file(self): - """追加文件""" - if config.get_default('default_zone').up_host: - host = config.get_default('default_zone').up_host - else: - host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.home_dir) - url = self.file_url(host) - return self.post(url, self.input_stream) - - def file_url(self, host): - url = ['{0}/append/{1}/key/{2}'.format(host, self.offset, self.key)] - - if self.size is None: - url.append('fsize/{0}'.format(-1)) - else: - url.append('fsiez/{0}'.format(self.size)) - - if self.mime_type: - url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type))) - - if self.crc: - url.append('crc32/{0}'.format(self.crc)) - - url = '/'.join(url) - return url - - def post(self, url, data): - return http._post_with_token(url, data, self.up_token) diff --git a/test_qiniu.py b/test_qiniu.py index 9cf14cb8..3cdaf6fc 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -10,7 +10,7 @@ import pytest from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone -from qiniu import put_data, put_file, put_stream, append_file +from qiniu import put_data, put_file, put_stream from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, \ build_batch_delete, DomainManager from qiniu import urlsafe_base64_encode, urlsafe_base64_decode @@ -284,16 +284,6 @@ def test_put(self): print(info) assert ret['key'] == key - def test_appendfile(self): - key = 'append_{0}.txt'.format(int(time.time())) - encodekey = urlsafe_base64_encode(key) - data = urlsafe_base64_encode('hello bubby!') - offset = 0 - token = self.q.upload_token(bucket_name) - ret, info = append_file(token, encodekey, data, offset) - print(info) - assert ret['nextAppendPosition'] == len(data) - def test_put_crc(self): key = '' data = 'hello bubby!' From d3080581396741103b331002eb8df109b6af5cc1 Mon Sep 17 00:00:00 2001 From: yangjunren <48310409+yangjunren@users.noreply.github.com> Date: Thu, 3 Sep 2020 17:52:28 +0800 Subject: [PATCH 336/478] Delete upload_with_append.py --- examples/upload_with_append.py | 37 ---------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 examples/upload_with_append.py diff --git a/examples/upload_with_append.py b/examples/upload_with_append.py deleted file mode 100644 index e9504878..00000000 --- a/examples/upload_with_append.py +++ /dev/null @@ -1,37 +0,0 @@ -from qiniu import Auth, urlsafe_base64_encode, append_file - -# 七牛账号的公私钥 -access_key = '<access_key>' -secret_key = '<secret_key>' - -# 要上传的空间 -bucket_name = "" - -# 构建鉴权对象 -q = Auth(access_key, secret_key) - -key = "append.txt" - -# 生成上传token，可以指定过期时间 -token = q.upload_token(bucket_name) - - -def file2base64(localfile): - with open(localfile, 'rb') as f: # 以二进制读取文件 - data = f.read() - return data - - -# 要追加的文本文件路径 -localfile = "" - -data = file2base64(localfile) - -# 首次以追加方式上传文件时，offset设置为0；后续继续追加内容时需要传入上次追加成功后响应的"nextAppendPosition" :34 参数值。 -offset = 0 - -encodekey = urlsafe_base64_encode(key) - -ret, info = append_file(token, encodekey, data, offset) -print(ret) -print(info) From 1d611134ebf7799cdeafe5114e829ce72eb2459d Mon Sep 17 00:00:00 2001 From: yangjunren <48310409+yangjunren@users.noreply.github.com> Date: Thu, 3 Sep 2020 18:06:14 +0800 Subject: [PATCH 337/478] Modify the README.md API usage demo link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c81e19a..6e020bd9 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ $ py.test ## 常见问题 - 第二个参数info保留了请求响应的信息，失败情况下ret 为none, 将info可以打印出来，提交给我们。 -- API 的使用 demo 可以参考 [单元测试](https://github.com/qiniu/python-sdk/blob/master/test_qiniu.py)。 +- API 的使用 demo 可以参考 [examples示例](https://github.com/qiniu/python-sdk/tree/master/examples)。 - 如果碰到`ImportError: No module named requests.auth` 请安装 `requests` 。 ## 代码贡献 From 9b4538fb2758ebbc052ea1b5f009ec055a4553e5 Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Wed, 23 Sep 2020 12:08:32 +0800 Subject: [PATCH 338/478] Update __init__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 准备发版本7.3.0 新增功能： 短信：查询短信发送记录 cdn: 上线域名 domain_online 方法、下线域名 domain_offline 方法 和 删除域名 delete_domain 方法 存储：批量解冻 build_batch_restoreAr 方法、获取空间列表 bucket_domain 方法 和 修改空间访问权限 change_bucket_permission 方法 --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index d2c3cd61..ea229947 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.10' +__version__ = '7.3.0' from .auth import Auth, QiniuMacAuth From 00c1a67f6ecb39bf890be48828fb81cd8264357f Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Wed, 23 Sep 2020 12:20:18 +0800 Subject: [PATCH 339/478] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 准备发版本7.3.0 --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6adb041..73f9ca54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,14 @@ # Changelog + +## 7.3.0 (2020-09-23) +新增 +* sms[云短信]：新增查询短信发送记录方法：get_messages_info +* cdn: 新增上线域名 domain_online 方法、下线域名 domain_offline 方法和删除域名 delete_domain 方法 +* 对象存储：新增批量解冻build_batch_restoreAr方法、获取空间列表bucket_domain方法和修改空间访问权限change_bucket_permission方法 + +修复 +* 修复ResponseInfo对扩展码错误处理问题 + ## 7.2.10 （2020-08-21） * 修复上传策略中forceSaveKey参数没有签算进上传token，导致上传失败的问题 ## 7.2.9 (2020-08-07) From 007e02ae3f9a20cf16e68a14cd72f1e892156bef Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Fri, 11 Dec 2020 14:10:35 +0800 Subject: [PATCH 340/478] update qiniu/http.py to be compatible with python3.9 json.loads() method --- qiniu/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/http.py b/qiniu/http.py index 9cc6e619..55a53fc0 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -23,7 +23,7 @@ def __return_wrapper(resp): if resp.status_code != 200 or resp.headers.get('X-Reqid') is None: return None, ResponseInfo(resp) resp.encoding = 'utf-8' - ret = resp.json(encoding='utf-8') if resp.text != '' else {} + ret = resp.json() if resp.text != '' else {} if ret is None: # json null ret = {} return ret, ResponseInfo(resp) From 05a5e3c36cf5bcd7c384a038295183ead52cc6ea Mon Sep 17 00:00:00 2001 From: yangjunren <48310409+yangjunren@users.noreply.github.com> Date: Fri, 11 Dec 2020 14:45:42 +0800 Subject: [PATCH 341/478] Update test_qiniu.py --- test_qiniu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index 3cdaf6fc..653a2369 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -330,7 +330,7 @@ def test_putWithoutKey(self): def test_withoutRead_withoutSeek_retry(self): key = 'retry' data = 'hello retry!' - set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) + set_default(default_zone=Zone('http://a', 'http://upload.qiniup.com')) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) @@ -393,7 +393,7 @@ def test_big_file(self): token = self.q.upload_token(bucket_name, key) localfile = create_temp_file(4 * 1024 * 1024 + 1) progress_handler = lambda progress, total: progress - qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) + qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniup.com')) ret, info = put_file(token, key, localfile, self.params, self.mime_type, progress_handler=progress_handler) print(info) assert ret['key'] == key @@ -402,7 +402,7 @@ def test_big_file(self): def test_retry(self): localfile = __file__ key = 'test_file_r_retry' - qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) + qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniup.com')) token = self.q.upload_token(bucket_name, key) ret, info = put_file(token, key, localfile, self.params, self.mime_type) print(info) From 71cf09cc04060524b4835a9b5d45a8ae3a4483c6 Mon Sep 17 00:00:00 2001 From: yangjunren <48310409+yangjunren@users.noreply.github.com> Date: Fri, 11 Dec 2020 15:16:18 +0800 Subject: [PATCH 342/478] Update config.py --- qiniu/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/config.py b/qiniu/config.py index 694749eb..bc036940 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -38,7 +38,7 @@ def set_default( if default_api_host: _config['default_api_host'] = default_api_host if default_uc_host: - _config['default_uc_host'] = default_api_host + _config['default_uc_host'] = default_uc_host if connection_retries: _config['connection_retries'] = connection_retries if connection_pool: From 3825b0766c91221583373f5cb31237d1c49cf7ae Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Mon, 14 Dec 2020 10:40:06 +0800 Subject: [PATCH 343/478] add python3.8 3.9 CI test --- .travis.yml | 2 ++ CHANGELOG.md | 4 ++++ README.md | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2eb1298a..f49f61c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,8 @@ python: - "3.5" - "3.6" - "3.7" +- "3.8" +- "3.9" install: - pip install flake8 - pip install pytest diff --git a/CHANGELOG.md b/CHANGELOG.md index a8e25adc..84118469 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 7.3.0 (2020-12-14) +* 修复python3.9 环境下，json.loads() 处理 +* ci 覆盖增加python 3.8 3.9 + ## 7.2.9 (2020-08-07) * 支持指定本地ctx缓存文件.qiniu_pythonsdk_hostscache.json 文件路径 * 更正接口返回描述docstring diff --git a/README.md b/README.md index 6e020bd9..b2eb13c4 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ $ pip install qiniu | Qiniu SDK版本 | Python 版本 | |:--------------------:|:---------------------------:| -| 7.x | 2.7, 3.3, 3.4, 3.5| +| 7.x | 2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9| | 6.x | 2.7 | ## 使用方法 From 701f3e07c8f7fd39e41da82d9924942de3099209 Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Wed, 6 Jan 2021 13:31:30 +0800 Subject: [PATCH 344/478] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index ea229947..5178f0b4 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.3.0' +__version__ = '7.3.1' from .auth import Auth, QiniuMacAuth From e92b477b1c23c3fbfaa0ed30bb7567b1a1d0e4a4 Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Wed, 6 Jan 2021 13:33:20 +0800 Subject: [PATCH 345/478] Update CHANGELOG.md --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73f9ca54..673aeb55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,15 @@ # Changelog +## 7.3.1 (2021=01-06) +* 修复ResponseInfo对扩展码错误处理问题 +* 增加python v3.7，v3.8，v3,9 ci测试 + ## 7.3.0 (2020-09-23) 新增 * sms[云短信]：新增查询短信发送记录方法：get_messages_info * cdn: 新增上线域名 domain_online 方法、下线域名 domain_offline 方法和删除域名 delete_domain 方法 * 对象存储：新增批量解冻build_batch_restoreAr方法、获取空间列表bucket_domain方法和修改空间访问权限change_bucket_permission方法 -修复 -* 修复ResponseInfo对扩展码错误处理问题 - ## 7.2.10 （2020-08-21） * 修复上传策略中forceSaveKey参数没有签算进上传token，导致上传失败的问题 ## 7.2.9 (2020-08-07) From 830037b5746906d4530194f6ee9ddcc14a551853 Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Wed, 6 Jan 2021 13:33:37 +0800 Subject: [PATCH 346/478] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 673aeb55..1ccca691 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 7.3.1 (2021=01-06) +## 7.3.1 (2021-01-06) * 修复ResponseInfo对扩展码错误处理问题 * 增加python v3.7，v3.8，v3,9 ci测试 From b1ef99e7cb616c6c0ce97870aca919c865848026 Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Wed, 6 Jan 2021 13:34:55 +0800 Subject: [PATCH 347/478] Update CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ccca691..76f5bfe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Changelog ## 7.3.1 (2021-01-06) -* 修复ResponseInfo对扩展码错误处理问题 -* 增加python v3.7，v3.8，v3,9 ci测试 +* 修复 ResponseInfo 对扩展码错误处理问题 +* 增加python v3.7，v3.8，v3.9 版本 CI 测试 ## 7.3.0 (2020-09-23) 新增 From db11d919e0db433d6f12507a58a149191d7ccd35 Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Wed, 6 Jan 2021 13:35:19 +0800 Subject: [PATCH 348/478] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76f5bfe4..59f95d2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 7.3.1 (2021-01-06) * 修复 ResponseInfo 对扩展码错误处理问题 -* 增加python v3.7，v3.8，v3.9 版本 CI 测试 +* 增加 python v3.7，v3.8，v3.9 版本 CI 测试 ## 7.3.0 (2020-09-23) 新增 From 050b66ff5ad9222fc94caf1eec1fef9bc1009557 Mon Sep 17 00:00:00 2001 From: Bachue Zhou <bachue.shu@gmail.com> Date: Tue, 18 May 2021 16:11:44 +0800 Subject: [PATCH 349/478] migrate to github action --- .github/workflows/ci-test.yml | 39 +++++++++++++++++++++++++++++++++++ .travis.yml | 39 ----------------------------------- 2 files changed, 39 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/ci-test.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml new file mode 100644 index 00000000..f331727d --- /dev/null +++ b/.github/workflows/ci-test.yml @@ -0,0 +1,39 @@ +on: [push] +name: Run Test Cases +jobs: + test: + strategy: + fail-fast: false + max-parallel: 1 + matrix: + python_version: ['2.7', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9'] + runs-on: ubuntu-18.04 + steps: + - name: Checkout repo + uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python_version }} + architecture: x64 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest pytest-cov requests scrutinizer-ocular codecov + - name: Run cases + env: + QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} + QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} + QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }} + QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }} + QINIU_TEST_ENV: "travis" + PYTHONPATH: "$PYTHONPATH:." + run: | + set -e + flake8 --show-source --max-line-length=160 . + py.test --cov qiniu + ocular --data-file .coverage + coverage run test_qiniu.py + codecov diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f49f61c3..00000000 --- a/.travis.yml +++ /dev/null @@ -1,39 +0,0 @@ -sudo: false -language: python -python: -- "2.7" -- "3.4" -- "3.5" -- "3.6" -- "3.7" -- "3.8" -- "3.9" -install: -- pip install flake8 -- pip install pytest -- pip install pytest-cov -- pip install requests -- pip install scrutinizer-ocular -- pip install codecov - -before_script: -- export QINIU_TEST_BUCKET="pythonsdk" -- export QINIU_TEST_DOMAIN="pythonsdk.qiniudn.com" -- export QINIU_TEST_ENV="travis" -- export PYTHONPATH="$PYTHONPATH:." - -script: -- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 .; - fi -- py.test --cov qiniu -- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then ocular --data-file .coverage; - fi -- coverage run test_qiniu.py - -env: - global: - - secure: "McZuxM4UAKabtGvCi+t1F/Spb/3Yzb6O7hEk0JLwJEYCnl7hkfV1ogAgjjYdHwkNPjOwUaz3rpdmahz64ohtpucPsIyQjgK7tigTM+UgdAcg77RflB50yJ3yCnJOHMxVRF0RNLZqFeuf3GkfnOyzZFynN+LmM5n+0/iIuC4LXgs=" - - QINIU_ACCESS_KEY=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5 - -after_success: - - codecov \ No newline at end of file From 6d275d9279261d4576c18da97db32300f26e61a9 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Tue, 18 May 2021 12:47:56 +0800 Subject: [PATCH 350/478] add resumeable upload v2 apis --- qiniu/auth.py | 14 ++- qiniu/config.py | 2 + qiniu/services/storage/uploader.py | 149 +++++++++++++++++++++++++---- test_qiniu.py | 10 +- 4 files changed, 147 insertions(+), 28 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index c0b3de43..cc9a3183 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -195,12 +195,16 @@ def __init__(self, auth): self.auth = auth def __call__(self, r): - if r.body is not None and r.headers['Content-Type'] == 'application/x-www-form-urlencoded': - token = self.auth.token_of_request( - r.url, r.body, 'application/x-www-form-urlencoded') + if isinstance(self.auth, str): + r.headers['Authorization'] = 'UpToken {0}'.format(self.auth) else: - token = self.auth.token_of_request(r.url) - r.headers['Authorization'] = 'QBox {0}'.format(token) + if r.body is not None and r.headers['Content-Type'] == 'application/x-www-form-urlencoded': + token = self.auth.token_of_request( + r.url, r.body, 'application/x-www-form-urlencoded') + else: + token = self.auth.token_of_request(r.url) + r.headers['Authorization'] = 'QBox {0}'.format(token) + return r diff --git a/qiniu/config.py b/qiniu/config.py index bc036940..083bda12 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -8,6 +8,8 @@ UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 +_BLOCK_MIN_SIZE = 1024 * 1024 #v2: 断点续传分片最小值 +_BLOCK_MAX_SIZE = 1024 * 1024 * 1024 #v2：断点续传分片最大值 _config = { 'default_zone': zone.Zone(), diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index f5738885..a721d624 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -1,8 +1,12 @@ # -*- coding: utf-8 -*- +import hashlib +import json import os import time +from requests.api import post + from qiniu import config from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter, rfc_from_timestamp from qiniu import http @@ -153,10 +157,11 @@ class _Resume(object): upload_progress_recorder: 记录上传进度，用于断点续传 modify_time: 上传文件修改日期 hostscache_dir： host请求 缓存文件保存位置 + version 分片上传版本 目前支持v1/v2版本 默认v1 """ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, keep_last_modified): + progress_handler, upload_progress_recorder, modify_time, keep_last_modified, version='v1'): """初始化断点续上传""" self.up_token = up_token self.key = key @@ -170,6 +175,7 @@ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() self.modify_time = modify_time or time.time() self.keep_last_modified = keep_last_modified + self.version = version # print(self.modify_time) # print(modify_time) @@ -177,11 +183,13 @@ def record_upload_progress(self, offset): record_data = { 'size': self.size, 'offset': offset, - 'contexts': [block['ctx'] for block in self.blockStatus] } + if self.version == 'v1': + record_data['contexts'] = [block['ctx'] for block in self.blockStatus] + elif self.version == 'v2': + record_data['contexts'] = self.blockStatus if self.modify_time: record_data['modify_time'] = self.modify_time - # print(record_data) self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) def recovery_from_record(self): @@ -195,21 +203,39 @@ def recovery_from_record(self): return 0 except KeyError: return 0 - self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] + if self.version == 'v1': + self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] + elif self.version == 'v2': + self.blockStatus = record['contexts'] return record['offset'] - def upload(self): + def upload(self, bucket_name=None, part_size=None, metadata=None): """上传操作""" self.blockStatus = [] - if config.get_default('default_zone').up_host: - host = config.get_default('default_zone').up_host - else: - host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.hostscache_dir) + self.recovery_index = 1 + encode_object_name = self.key or '~' + host = self.get_up_host() offset = self.recovery_from_record() - for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset): + if self.version == 'v1': + part_size_ = config._BLOCK_SIZE + elif self.version == 'v2': + part_size_ = self.set_part_size(part_size) + if offset > 0 and self.blockStatus != []: + self.recovery_index = self.blockStatus[-1]['partNumber'] + 1 + else: + self.recovery_index = 1 + init_url = self.block_url_v2(host, bucket_name, encode_object_name) + upload_id, expire = self.init_upload_task(init_url) + for index, block in enumerate(_file_iter(self.input_stream, part_size_, offset)): length = len(block) - crc = crc32(block) - ret, info = self.make_block(block, length, host) + if self.version == 'v1': + crc = crc32(block) + ret, info = self.make_block(block, length, host) + elif self.version == 'v2': + index_ = index + self.recovery_index + md = hashlib.md5(block).hexdigest() + url = init_url + '/%s/%d' % (upload_id, index_) + ret, info = self.make_block_v2(block, url) if ret is None and not info.need_retry(): return ret, info if info.connect_failed(): @@ -217,29 +243,82 @@ def upload(self): host = config.get_default('default_zone').up_host_backup else: host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token, - self.hostscache_dir) - if info.need_retry() or crc != ret['crc32']: - ret, info = self.make_block(block, length, host) - if ret is None or crc != ret['crc32']: - return ret, info + self.hostscache_dir) + if self.version == 'v1': + if info.need_retry() or crc != ret['crc32']: + ret, info = self.make_block(block, length, host) + if ret is None or crc != ret['crc32']: + return ret, info + elif self.version == 'v2': + if info.need_retry() or md != ret['md5']: + url = self.block_url_v2(host, bucket_name, encode_object_name) + '/%s/%d' % (upload_id, index + 1) + ret, info = self.make_block_v2(block, url) + if ret is None or md != ret['md5']: + return ret, info + del ret['md5'] + ret['partNumber'] = index_ self.blockStatus.append(ret) offset += length self.record_upload_progress(offset) if (callable(self.progress_handler)): - self.progress_handler(((len(self.blockStatus) - 1) * config._BLOCK_SIZE) + length, self.size) - return self.make_file(host) + self.progress_handler(((len(self.blockStatus) - 1) * part_size_) + len(block), self.size) + if self.version == 'v1': + return self.make_file(host) + elif self.version == 'v2': + make_file_url = self.block_url_v2(host, bucket_name, encode_object_name) + '/%s' % upload_id + return self.make_file_v2(self.blockStatus, make_file_url, self.mime_type, metadata, self.params) + + + def make_file_v2(self, block_status, url, file_name=None, mime_type=None, metadata=None, customVars=None): + """completeMultipartUpload""" + parts = self.get_parts(block_status) + headers = { + 'Content-Type': 'application/json', + } + data = { + 'parts': parts, + 'fname': file_name, + 'mimeType': mime_type, + 'metadata': metadata, + 'customVars': customVars + } + ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) + # print("\n\n resp is: %s" % ret) + return ret, info + + + def get_up_host(self): + if config.get_default('default_zone').up_host: + host = config.get_default('default_zone').up_host + else: + host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.hostscache_dir) + return host + def make_block(self, block, block_size, host): """创建块""" url = self.block_url(host, block_size) return self.post(url, block) + + def make_block_v2(self, block, url): + headers = { + 'Content-Type': 'application/octet-stream', + 'Content-MD5': hashlib.md5(block).hexdigest(), + } + return self.put(url, block, headers) + + def block_url(self, host, size): return '{0}/mkblk/{1}'.format(host, size) + + def block_url_v2(self, host, bucket_name, encode_object_name): + return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, urlsafe_base64_encode(encode_object_name)) + + def file_url(self, host): url = ['{0}/mkfile/{1}'.format(host, self.size)] - if self.mime_type: url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type))) @@ -262,6 +341,7 @@ def file_url(self, host): # print url return url + def make_file(self, host): """创建文件""" url = self.file_url(host) @@ -269,5 +349,34 @@ def make_file(self, host): self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) return self.post(url, body) + + def init_upload_task(self, url): + body, resp = self.post(url, '') + if body is not None: + return body['uploadId'], body['expireAt'] + else: + return None, None + + def post(self, url, data): return http._post_with_token(url, data, self.up_token) + + + def post_with_headers(self, url, data, headers): + return http._post_with_auth_and_headers(url=url, data=data, auth=self.up_token, headers=headers) + + + def put(self, url, data, headers): + return http._put_with_auth_and_headers(url, data, self.up_token, headers) + + + def set_part_size(self, part_size): + return part_size if part_size > config._BLOCK_MIN_SIZE \ + and part_size < config._BLOCK_MAX_SIZE \ + else config._BLOCK_SIZE + + + def get_parts(self, block_status): + return sorted(block_status, key=lambda i: i['partNumber']) + + diff --git a/test_qiniu.py b/test_qiniu.py index 653a2369..87dc243d 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -37,9 +37,13 @@ StringIO = io.StringIO urlopen = urllib.request.urlopen -access_key = os.getenv('QINIU_ACCESS_KEY') -secret_key = os.getenv('QINIU_SECRET_KEY') -bucket_name = os.getenv('QINIU_TEST_BUCKET') +# access_key = os.getenv('QINIU_ACCESS_KEY') +# secret_key = os.getenv('QINIU_SECRET_KEY') +# bucket_name = os.getenv('QINIU_TEST_BUCKET') + +access_key = "qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_" +secret_key = "3sSWVQQ_HvD6pVJSjfEsRQMl9ZRnNRf0-G5iomNV" +bucket_name = "z0-bucket" hostscache_dir = None dummy_access_key = 'abcdefghklmnopq' From e112a2eb8b7ec795f3820470fe3fa0fa52b85138 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 10:43:25 +0800 Subject: [PATCH 351/478] =?UTF-8?q?=E8=A7=84=E8=8C=83sdk=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=20=E6=B7=BB=E5=8A=A0=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/.gitignore | 8 ++ .../inspectionProfiles/profiles_settings.xml | 6 ++ .idea/misc.xml | 4 + .idea/modules.xml | 8 ++ .idea/python-sdk.iml | 17 +++++ .idea/vcs.xml | 6 ++ .qiniu_pythonsdk_hostscache.json | 1 + qiniu/auth.py | 14 ++-- qiniu/http.py | 7 ++ qiniu/services/storage/uploader.py | 73 +++++++++++-------- test_qiniu.py | 32 +++++--- venv/pyvenv.cfg | 3 + 12 files changed, 129 insertions(+), 50 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/python-sdk.iml create mode 100644 .idea/vcs.xml create mode 100644 .qiniu_pythonsdk_hostscache.json create mode 100644 venv/pyvenv.cfg diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..73f69e09 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 00000000..105ce2da --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ +<component name="InspectionProjectProfileManager"> + <settings> + <option name="USE_PROJECT_PROFILE" value="false" /> + <version value="1.0" /> + </settings> +</component> \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..f7614b94 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (python-sdk)" project-jdk-type="Python SDK" /> +</project> \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..d59171af --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="ProjectModuleManager"> + <modules> + <module fileurl="file://$PROJECT_DIR$/.idea/python-sdk.iml" filepath="$PROJECT_DIR$/.idea/python-sdk.iml" /> + </modules> + </component> +</project> \ No newline at end of file diff --git a/.idea/python-sdk.iml b/.idea/python-sdk.iml new file mode 100644 index 00000000..159b5442 --- /dev/null +++ b/.idea/python-sdk.iml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module type="PYTHON_MODULE" version="4"> + <component name="NewModuleRootManager"> + <content url="file://$MODULE_DIR$"> + <excludeFolder url="file://$MODULE_DIR$/venv" /> + </content> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + </component> + <component name="PyDocumentationSettings"> + <option name="format" value="GOOGLE" /> + <option name="myDocStringFormat" value="Google" /> + </component> + <component name="TestRunnerService"> + <option name="PROJECT_TEST_RUNNER" value="pytest" /> + </component> +</module> \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..94a25f7f --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="VcsDirectoryMappings"> + <mapping directory="$PROJECT_DIR$" vcs="Git" /> + </component> +</project> \ No newline at end of file diff --git a/.qiniu_pythonsdk_hostscache.json b/.qiniu_pythonsdk_hostscache.json new file mode 100644 index 00000000..ec805b5e --- /dev/null +++ b/.qiniu_pythonsdk_hostscache.json @@ -0,0 +1 @@ +{"http:qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_:z0-bucket": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1621392204}, "http:jUbAZJctNS3focY5OscUKU7ip6T47pqftVDDMtvo:pythonsdk": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1621408813}} \ No newline at end of file diff --git a/qiniu/auth.py b/qiniu/auth.py index cc9a3183..c0b3de43 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -195,16 +195,12 @@ def __init__(self, auth): self.auth = auth def __call__(self, r): - if isinstance(self.auth, str): - r.headers['Authorization'] = 'UpToken {0}'.format(self.auth) + if r.body is not None and r.headers['Content-Type'] == 'application/x-www-form-urlencoded': + token = self.auth.token_of_request( + r.url, r.body, 'application/x-www-form-urlencoded') else: - if r.body is not None and r.headers['Content-Type'] == 'application/x-www-form-urlencoded': - token = self.auth.token_of_request( - r.url, r.body, 'application/x-www-form-urlencoded') - else: - token = self.auth.token_of_request(r.url) - r.headers['Authorization'] = 'QBox {0}'.format(token) - + token = self.auth.token_of_request(r.url) + r.headers['Authorization'] = 'QBox {0}'.format(token) return r diff --git a/qiniu/http.py b/qiniu/http.py index 55a53fc0..5168c836 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -103,6 +103,9 @@ def __call__(self, r): def _post_with_token(url, data, token): return _post(url, data, None, _TokenAuth(token)) +def _post_with_token_and_headers(url, data, token, headers): + return _post(url, data, None, _TokenAuth(token), headers) + def _post_file(url, data, files): return _post(url, data, files, None) @@ -132,6 +135,10 @@ def _put_with_auth(url, data, auth): return _put(url, data, None, qiniu.auth.RequestsAuth(auth)) +def _put_with_token_and_headers(url, data, auth, headers): + return _put(url, data, None, _TokenAuth(auth), headers) + + def _put_with_auth_and_headers(url, data, auth, headers): return _put(url, data, None, qiniu.auth.RequestsAuth(auth), headers) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index a721d624..0454af81 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -133,9 +133,11 @@ def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir=None, params=None, mime_type=None, progress_handler=None, - upload_progress_recorder=None, modify_time=None, keep_last_modified=False): + upload_progress_recorder=None, modify_time=None, keep_last_modified=False, + part_size=None, version=None, bucket_name=None): task = _Resume(up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, keep_last_modified) + progress_handler, upload_progress_recorder, modify_time, keep_last_modified, + part_size, version, bucket_name) return task.upload() @@ -158,11 +160,15 @@ class _Resume(object): modify_time: 上传文件修改日期 hostscache_dir： host请求 缓存文件保存位置 version 分片上传版本 目前支持v1/v2版本 默认v1 + part_size 分片上传v2必传字段 分片大小范围为1 MB - 1 GB + bucket_name 分片上传v2字段必传字段 空间名称 """ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, keep_last_modified, version='v1'): + progress_handler, upload_progress_recorder, modify_time, keep_last_modified, part_size=None, + version='v1', bucket_name=None): """初始化断点续上传""" + # self.auth = auth_obj self.up_token = up_token self.key = key self.input_stream = input_stream @@ -175,7 +181,9 @@ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() self.modify_time = modify_time or time.time() self.keep_last_modified = keep_last_modified - self.version = version + self.version = version or 'v1' + self.part_size = part_size + self.bucket_name = bucket_name # print(self.modify_time) # print(modify_time) @@ -187,11 +195,13 @@ def record_upload_progress(self, offset): if self.version == 'v1': record_data['contexts'] = [block['ctx'] for block in self.blockStatus] elif self.version == 'v2': - record_data['contexts'] = self.blockStatus + if self.expiredAt > time.time(): + record_data['etags'] = self.blockStatus if self.modify_time: record_data['modify_time'] = self.modify_time self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) + def recovery_from_record(self): record = self.upload_progress_recorder.get_upload_record(self.file_name, self.key) if not record: @@ -206,34 +216,34 @@ def recovery_from_record(self): if self.version == 'v1': self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] elif self.version == 'v2': - self.blockStatus = record['contexts'] + self.blockStatus = record['etags'] return record['offset'] - def upload(self, bucket_name=None, part_size=None, metadata=None): + + def upload(self, metadata=None): """上传操作""" self.blockStatus = [] self.recovery_index = 1 - encode_object_name = self.key or '~' + self.expiredAt = 1 host = self.get_up_host() offset = self.recovery_from_record() if self.version == 'v1': - part_size_ = config._BLOCK_SIZE + self.part_size = config._BLOCK_SIZE elif self.version == 'v2': - part_size_ = self.set_part_size(part_size) if offset > 0 and self.blockStatus != []: self.recovery_index = self.blockStatus[-1]['partNumber'] + 1 else: self.recovery_index = 1 - init_url = self.block_url_v2(host, bucket_name, encode_object_name) - upload_id, expire = self.init_upload_task(init_url) - for index, block in enumerate(_file_iter(self.input_stream, part_size_, offset)): + init_url = self.block_url_v2(host, self.bucket_name) + upload_id, self.expiredAt = self.init_upload_task(init_url) + else: + raise ValueError("version must choose v1 or v2 !") + for index, block in enumerate(_file_iter(self.input_stream, self.part_size, offset)): length = len(block) if self.version == 'v1': - crc = crc32(block) ret, info = self.make_block(block, length, host) elif self.version == 'v2': index_ = index + self.recovery_index - md = hashlib.md5(block).hexdigest() url = init_url + '/%s/%d' % (upload_id, index_) ret, info = self.make_block_v2(block, url) if ret is None and not info.need_retry(): @@ -245,15 +255,15 @@ def upload(self, bucket_name=None, part_size=None, metadata=None): host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token, self.hostscache_dir) if self.version == 'v1': - if info.need_retry() or crc != ret['crc32']: + if info.need_retry(): ret, info = self.make_block(block, length, host) - if ret is None or crc != ret['crc32']: + if ret is None: return ret, info elif self.version == 'v2': - if info.need_retry() or md != ret['md5']: - url = self.block_url_v2(host, bucket_name, encode_object_name) + '/%s/%d' % (upload_id, index + 1) + if info.need_retry(): + url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (upload_id, index + 1) ret, info = self.make_block_v2(block, url) - if ret is None or md != ret['md5']: + if ret is None: return ret, info del ret['md5'] ret['partNumber'] = index_ @@ -261,12 +271,12 @@ def upload(self, bucket_name=None, part_size=None, metadata=None): offset += length self.record_upload_progress(offset) if (callable(self.progress_handler)): - self.progress_handler(((len(self.blockStatus) - 1) * part_size_) + len(block), self.size) + self.progress_handler(((len(self.blockStatus) - 1) * self.part_size) + len(block), self.size) if self.version == 'v1': return self.make_file(host) elif self.version == 'v2': - make_file_url = self.block_url_v2(host, bucket_name, encode_object_name) + '/%s' % upload_id - return self.make_file_v2(self.blockStatus, make_file_url, self.mime_type, metadata, self.params) + make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % upload_id + return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, self.mime_type, metadata, self.params) def make_file_v2(self, block_status, url, file_name=None, mime_type=None, metadata=None, customVars=None): @@ -283,7 +293,6 @@ def make_file_v2(self, block_status, url, file_name=None, mime_type=None, metada 'customVars': customVars } ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) - # print("\n\n resp is: %s" % ret) return ret, info @@ -313,8 +322,9 @@ def block_url(self, host, size): return '{0}/mkblk/{1}'.format(host, size) - def block_url_v2(self, host, bucket_name, encode_object_name): - return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, urlsafe_base64_encode(encode_object_name)) + def block_url_v2(self, host, bucket_name): + encode_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '～' + return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, encode_object_name) def file_url(self, host): @@ -338,7 +348,6 @@ def file_url(self, host): "x-qn-meta-!Last-Modified/{0}".format(urlsafe_base64_encode(rfc_from_timestamp(self.modify_time)))) url = '/'.join(url) - # print url return url @@ -363,16 +372,16 @@ def post(self, url, data): def post_with_headers(self, url, data, headers): - return http._post_with_auth_and_headers(url=url, data=data, auth=self.up_token, headers=headers) + return http._post_with_token_and_headers(url, data, self.up_token, headers) def put(self, url, data, headers): - return http._put_with_auth_and_headers(url, data, self.up_token, headers) + return http._put_with_token_and_headers(url, data, self.up_token, headers) - def set_part_size(self, part_size): - return part_size if part_size > config._BLOCK_MIN_SIZE \ - and part_size < config._BLOCK_MAX_SIZE \ + def set_part_size(self): + return self.part_size if self.part_size > config._BLOCK_MIN_SIZE \ + and self.part_size < config._BLOCK_MAX_SIZE \ else config._BLOCK_SIZE diff --git a/test_qiniu.py b/test_qiniu.py index 87dc243d..a48af3db 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -4,6 +4,8 @@ import string import random import tempfile +from imp import reload + import requests import unittest @@ -37,15 +39,12 @@ StringIO = io.StringIO urlopen = urllib.request.urlopen -# access_key = os.getenv('QINIU_ACCESS_KEY') -# secret_key = os.getenv('QINIU_SECRET_KEY') -# bucket_name = os.getenv('QINIU_TEST_BUCKET') - -access_key = "qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_" -secret_key = "3sSWVQQ_HvD6pVJSjfEsRQMl9ZRnNRf0-G5iomNV" -bucket_name = "z0-bucket" +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') +bucket_name = os.getenv('QINIU_TEST_BUCKET') hostscache_dir = None + dummy_access_key = 'abcdefghklmnopq' dummy_secret_key = '1234567890' dummy_auth = Auth(dummy_access_key, dummy_secret_key) @@ -384,14 +383,29 @@ def test_put_stream(self): localfile = __file__ key = 'test_file_r' size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, self.params, - self.mime_type) - print(info) + self.mime_type, part_size=None, version=None, bucket_name=None) assert ret['key'] == key + + def test_put_stream_v2(self): + localfile = __file__ + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + self.params, + self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + print("\n\n\n\n\n", info) + assert ret['key'] == key + + def test_big_file(self): key = 'big' token = self.q.upload_token(bucket_name, key) diff --git a/venv/pyvenv.cfg b/venv/pyvenv.cfg new file mode 100644 index 00000000..c4f8eb35 --- /dev/null +++ b/venv/pyvenv.cfg @@ -0,0 +1,3 @@ +home = /usr/local/bin +include-system-site-packages = false +version = 3.7.7 From 7f2f0893b835e7f2482259556925841c8263ef08 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 12:16:35 +0800 Subject: [PATCH 352/478] =?UTF-8?q?=E8=A7=84=E8=8C=83=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/config.py | 4 +- qiniu/http.py | 1 + qiniu/services/storage/uploader.py | 66 +++++++++--------------------- test_qiniu.py | 1 - 4 files changed, 23 insertions(+), 49 deletions(-) diff --git a/qiniu/config.py b/qiniu/config.py index 083bda12..2bb825c5 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -8,8 +8,8 @@ UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 -_BLOCK_MIN_SIZE = 1024 * 1024 #v2: 断点续传分片最小值 -_BLOCK_MAX_SIZE = 1024 * 1024 * 1024 #v2：断点续传分片最大值 +_BLOCK_MIN_SIZE = 1024 * 1024 # v2:断点续传分片最小值 +_BLOCK_MAX_SIZE = 1024 * 1024 * 1024 # v2断点续传分片最大值 _config = { 'default_zone': zone.Zone(), diff --git a/qiniu/http.py b/qiniu/http.py index 5168c836..c28028fb 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -103,6 +103,7 @@ def __call__(self, r): def _post_with_token(url, data, token): return _post(url, data, None, _TokenAuth(token)) + def _post_with_token_and_headers(url, data, token, headers): return _post(url, data, None, _TokenAuth(token), headers) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 0454af81..5cdf23d6 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -5,8 +5,6 @@ import os import time -from requests.api import post - from qiniu import config from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter, rfc_from_timestamp from qiniu import http @@ -69,7 +67,6 @@ def put_file(up_token, key, file_path, params=None, """ ret = {} size = os.stat(file_path).st_size - # fname = os.path.basename(file_path) with open(file_path, 'rb') as input_stream: file_name = os.path.basename(file_path) modify_time = int(os.path.getmtime(file_path)) @@ -165,10 +162,9 @@ class _Resume(object): """ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, keep_last_modified, part_size=None, - version='v1', bucket_name=None): + progress_handler, upload_progress_recorder, modify_time, keep_last_modified, part_size=None, + version=None, bucket_name=None): """初始化断点续上传""" - # self.auth = auth_obj self.up_token = up_token self.key = key self.input_stream = input_stream @@ -184,8 +180,6 @@ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache self.version = version or 'v1' self.part_size = part_size self.bucket_name = bucket_name - # print(self.modify_time) - # print(modify_time) def record_upload_progress(self, offset): record_data = { @@ -206,10 +200,9 @@ def recovery_from_record(self): record = self.upload_progress_recorder.get_upload_record(self.file_name, self.key) if not record: return 0 - try: if not record['modify_time'] or record['size'] != self.size or \ - record['modify_time'] != self.modify_time: + record['modify_time'] != self.modify_time: return 0 except KeyError: return 0 @@ -253,7 +246,7 @@ def upload(self, metadata=None): host = config.get_default('default_zone').up_host_backup else: host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token, - self.hostscache_dir) + self.hostscache_dir) if self.version == 'v1': if info.need_retry(): ret, info = self.make_block(block, length, host) @@ -279,22 +272,22 @@ def upload(self, metadata=None): return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, self.mime_type, metadata, self.params) - def make_file_v2(self, block_status, url, file_name=None, mime_type=None, metadata=None, customVars=None): - """completeMultipartUpload""" - parts = self.get_parts(block_status) - headers = { - 'Content-Type': 'application/json', - } - data = { - 'parts': parts, - 'fname': file_name, - 'mimeType': mime_type, - 'metadata': metadata, - 'customVars': customVars - } - ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) - return ret, info - + def make_file_v2(self, block_status, url, file_name=None, mime_type=None, + metadata=None, customVars=None): + """completeMultipartUpload""" + parts = self.get_parts(block_status) + headers = { + 'Content-Type': 'application/json', + } + data = { + 'parts': parts, + 'fname': file_name, + 'mimeType': mime_type, + 'metadata': metadata, + 'customVars': customVars + } + ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) + return ret, info def get_up_host(self): if config.get_default('default_zone').up_host: @@ -303,13 +296,11 @@ def get_up_host(self): host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.hostscache_dir) return host - def make_block(self, block, block_size, host): """创建块""" url = self.block_url(host, block_size) return self.post(url, block) - def make_block_v2(self, block, url): headers = { 'Content-Type': 'application/octet-stream', @@ -317,16 +308,13 @@ def make_block_v2(self, block, url): } return self.put(url, block, headers) - def block_url(self, host, size): return '{0}/mkblk/{1}'.format(host, size) - def block_url_v2(self, host, bucket_name): encode_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '～' return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, encode_object_name) - def file_url(self, host): url = ['{0}/mkfile/{1}'.format(host, self.size)] if self.mime_type: @@ -350,7 +338,6 @@ def file_url(self, host): url = '/'.join(url) return url - def make_file(self, host): """创建文件""" url = self.file_url(host) @@ -358,7 +345,6 @@ def make_file(self, host): self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) return self.post(url, body) - def init_upload_task(self, url): body, resp = self.post(url, '') if body is not None: @@ -366,26 +352,14 @@ def init_upload_task(self, url): else: return None, None - def post(self, url, data): return http._post_with_token(url, data, self.up_token) - def post_with_headers(self, url, data, headers): return http._post_with_token_and_headers(url, data, self.up_token, headers) - def put(self, url, data, headers): return http._put_with_token_and_headers(url, data, self.up_token, headers) - - def set_part_size(self): - return self.part_size if self.part_size > config._BLOCK_MIN_SIZE \ - and self.part_size < config._BLOCK_MAX_SIZE \ - else config._BLOCK_SIZE - - def get_parts(self, block_status): return sorted(block_status, key=lambda i: i['partNumber']) - - diff --git a/test_qiniu.py b/test_qiniu.py index a48af3db..f29caccd 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -402,7 +402,6 @@ def test_put_stream_v2(self): ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, self.params, self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) - print("\n\n\n\n\n", info) assert ret['key'] == key From 3c354fa5b9a80a0668c9b36ac5786d99c5911ed7 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 12:22:12 +0800 Subject: [PATCH 353/478] =?UTF-8?q?=E8=A7=84=E8=8C=83=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 5cdf23d6..2d985a2f 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -195,7 +195,6 @@ def record_upload_progress(self, offset): record_data['modify_time'] = self.modify_time self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) - def recovery_from_record(self): record = self.upload_progress_recorder.get_upload_record(self.file_name, self.key) if not record: @@ -212,7 +211,6 @@ def recovery_from_record(self): self.blockStatus = record['etags'] return record['offset'] - def upload(self, metadata=None): """上传操作""" self.blockStatus = [] @@ -269,8 +267,8 @@ def upload(self, metadata=None): return self.make_file(host) elif self.version == 'v2': make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % upload_id - return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, self.mime_type, metadata, self.params) - + return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, + self.mime_type, metadata, self.params) def make_file_v2(self, block_status, url, file_name=None, mime_type=None, metadata=None, customVars=None): From 88e87fde4c52cdc5648ce7412798856c1da1d80b Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 12:33:41 +0800 Subject: [PATCH 354/478] =?UTF-8?q?=E8=A7=84=E8=8C=83=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .qiniu_pythonsdk_hostscache.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .qiniu_pythonsdk_hostscache.json diff --git a/.qiniu_pythonsdk_hostscache.json b/.qiniu_pythonsdk_hostscache.json deleted file mode 100644 index ec805b5e..00000000 --- a/.qiniu_pythonsdk_hostscache.json +++ /dev/null @@ -1 +0,0 @@ -{"http:qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_:z0-bucket": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1621392204}, "http:jUbAZJctNS3focY5OscUKU7ip6T47pqftVDDMtvo:pythonsdk": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1621408813}} \ No newline at end of file From bd296977f5bda79f55eddcad19fe2e227f1fac05 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 13:45:17 +0800 Subject: [PATCH 355/478] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E6=96=87=E4=BB=B6=20=E8=A7=84=E8=8C=83=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 2d985a2f..efca8cf0 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -211,7 +211,7 @@ def recovery_from_record(self): self.blockStatus = record['etags'] return record['offset'] - def upload(self, metadata=None): + def upload(self): """上传操作""" self.blockStatus = [] self.recovery_index = 1 @@ -232,6 +232,7 @@ def upload(self, metadata=None): for index, block in enumerate(_file_iter(self.input_stream, self.part_size, offset)): length = len(block) if self.version == 'v1': + crc = crc32(block) ret, info = self.make_block(block, length, host) elif self.version == 'v2': index_ = index + self.recovery_index @@ -246,9 +247,9 @@ def upload(self, metadata=None): host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token, self.hostscache_dir) if self.version == 'v1': - if info.need_retry(): + if info.need_retry() or crc != ret['crc32']: ret, info = self.make_block(block, length, host) - if ret is None: + if ret is None or crc != ret['crc32']: return ret, info elif self.version == 'v2': if info.need_retry(): @@ -268,10 +269,9 @@ def upload(self, metadata=None): elif self.version == 'v2': make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % upload_id return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, - self.mime_type, metadata, self.params) + self.mime_type, self.params) - def make_file_v2(self, block_status, url, file_name=None, mime_type=None, - metadata=None, customVars=None): + def make_file_v2(self, block_status, url, file_name=None, mime_type=None, customVars=None): """completeMultipartUpload""" parts = self.get_parts(block_status) headers = { @@ -281,7 +281,6 @@ def make_file_v2(self, block_status, url, file_name=None, mime_type=None, 'parts': parts, 'fname': file_name, 'mimeType': mime_type, - 'metadata': metadata, 'customVars': customVars } ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) @@ -310,8 +309,8 @@ def block_url(self, host, size): return '{0}/mkblk/{1}'.format(host, size) def block_url_v2(self, host, bucket_name): - encode_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '～' - return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, encode_object_name) + encoded_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '～' + return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, encoded_object_name) def file_url(self, host): url = ['{0}/mkfile/{1}'.format(host, self.size)] From 7735c9c8e3f7971cb7462d8ca55535c03a28ef34 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 15:39:05 +0800 Subject: [PATCH 356/478] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E6=96=87=E4=BB=B6=20=E4=BF=AE=E6=94=B9bucket=5Ftest=E7=94=A8?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 ++ .idea/.gitignore | 8 -------- .idea/inspectionProfiles/profiles_settings.xml | 6 ------ .idea/misc.xml | 4 ---- .idea/modules.xml | 8 -------- .idea/python-sdk.iml | 17 ----------------- .idea/vcs.xml | 6 ------ test_qiniu.py | 5 ++--- 8 files changed, 4 insertions(+), 52 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/python-sdk.iml delete mode 100644 .idea/vcs.xml diff --git a/.gitignore b/.gitignore index 05a1b20a..93665221 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,5 @@ nosetests.xml .mr.developer.cfg .project .pydevproject +/.idea +/.venv \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 73f69e09..00000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2da..00000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ -<component name="InspectionProjectProfileManager"> - <settings> - <option name="USE_PROJECT_PROFILE" value="false" /> - <version value="1.0" /> - </settings> -</component> \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index f7614b94..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project version="4"> - <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (python-sdk)" project-jdk-type="Python SDK" /> -</project> \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index d59171af..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project version="4"> - <component name="ProjectModuleManager"> - <modules> - <module fileurl="file://$PROJECT_DIR$/.idea/python-sdk.iml" filepath="$PROJECT_DIR$/.idea/python-sdk.iml" /> - </modules> - </component> -</project> \ No newline at end of file diff --git a/.idea/python-sdk.iml b/.idea/python-sdk.iml deleted file mode 100644 index 159b5442..00000000 --- a/.idea/python-sdk.iml +++ /dev/null @@ -1,17 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<module type="PYTHON_MODULE" version="4"> - <component name="NewModuleRootManager"> - <content url="file://$MODULE_DIR$"> - <excludeFolder url="file://$MODULE_DIR$/venv" /> - </content> - <orderEntry type="inheritedJdk" /> - <orderEntry type="sourceFolder" forTests="false" /> - </component> - <component name="PyDocumentationSettings"> - <option name="format" value="GOOGLE" /> - <option name="myDocStringFormat" value="Google" /> - </component> - <component name="TestRunnerService"> - <option name="PROJECT_TEST_RUNNER" value="pytest" /> - </component> -</module> \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project version="4"> - <component name="VcsDirectoryMappings"> - <mapping directory="$PROJECT_DIR$" vcs="Git" /> - </component> -</project> \ No newline at end of file diff --git a/test_qiniu.py b/test_qiniu.py index f29caccd..608b25fd 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -116,12 +116,11 @@ class BucketTestCase(unittest.TestCase): def test_list(self): ret, eof, info = self.bucket.list(bucket_name, limit=4) - print(info) assert eof is False assert len(ret.get('items')) == 4 ret, eof, info = self.bucket.list(bucket_name, limit=1000) - print(info) - assert eof is True + print(ret, eof, info) + assert eof is False def test_buckets(self): ret, info = self.bucket.buckets() From b39ccdb9a4e0188bf7efe1758d8347a09baef932 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 15:47:28 +0800 Subject: [PATCH 357/478] =?UTF-8?q?=E5=88=A0=E9=99=A4venv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- venv/pyvenv.cfg | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 venv/pyvenv.cfg diff --git a/venv/pyvenv.cfg b/venv/pyvenv.cfg deleted file mode 100644 index c4f8eb35..00000000 --- a/venv/pyvenv.cfg +++ /dev/null @@ -1,3 +0,0 @@ -home = /usr/local/bin -include-system-site-packages = false -version = 3.7.7 From ade640baba2a4369665f3f4826ab36ff9cca7271 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 17:25:17 +0800 Subject: [PATCH 358/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9recovery=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index efca8cf0..32e24bb3 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -189,8 +189,8 @@ def record_upload_progress(self, offset): if self.version == 'v1': record_data['contexts'] = [block['ctx'] for block in self.blockStatus] elif self.version == 'v2': - if self.expiredAt > time.time(): - record_data['etags'] = self.blockStatus + record_data['etags'] = self.blockStatus + record_data['expired_at'] = self.expiredAt if self.modify_time: record_data['modify_time'] = self.modify_time self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) @@ -206,8 +206,13 @@ def recovery_from_record(self): except KeyError: return 0 if self.version == 'v1': + if not record.__contains__('contexts') or len(record['contexts']) == 0: + return 0 self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] elif self.version == 'v2': + if not record.__contains__('etags') or len(record['etags']) == 0 or \ + not record.__contains__('expired_at') or float(record['expired_at']) < time.time(): + return 0 self.blockStatus = record['etags'] return record['offset'] @@ -215,7 +220,7 @@ def upload(self): """上传操作""" self.blockStatus = [] self.recovery_index = 1 - self.expiredAt = 1 + self.expiredAt = None host = self.get_up_host() offset = self.recovery_from_record() if self.version == 'v1': From a5f3335332e2a3da709d2804a18e4fc9e4c8c11f Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 17:52:19 +0800 Subject: [PATCH 359/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9recovery=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 32e24bb3..6ffbd772 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -191,6 +191,7 @@ def record_upload_progress(self, offset): elif self.version == 'v2': record_data['etags'] = self.blockStatus record_data['expired_at'] = self.expiredAt + record_data['upload_id'] = self.uploadId if self.modify_time: record_data['modify_time'] = self.modify_time self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) @@ -209,29 +210,35 @@ def recovery_from_record(self): if not record.__contains__('contexts') or len(record['contexts']) == 0: return 0 self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] + return record['offset'] elif self.version == 'v2': if not record.__contains__('etags') or len(record['etags']) == 0 or \ - not record.__contains__('expired_at') or float(record['expired_at']) < time.time(): - return 0 + not record.__contains__('expired_at') or float(record['expired_at']) < time.time() or \ + not record.__contains__('upload_id'): + return 0, None, None self.blockStatus = record['etags'] - return record['offset'] + return record['offset'], record['upload_id'], record['expired_at'] + def upload(self): """上传操作""" self.blockStatus = [] self.recovery_index = 1 self.expiredAt = None + self.uploadId = None host = self.get_up_host() - offset = self.recovery_from_record() if self.version == 'v1': + offset = self.recovery_from_record() self.part_size = config._BLOCK_SIZE elif self.version == 'v2': - if offset > 0 and self.blockStatus != []: + offset, self.uploadId, self.expiredAt = self.recovery_from_record() + if offset > 0 and self.blockStatus != [] and self.uploadId is not None \ + and self.expiredAt is not None: self.recovery_index = self.blockStatus[-1]['partNumber'] + 1 else: self.recovery_index = 1 - init_url = self.block_url_v2(host, self.bucket_name) - upload_id, self.expiredAt = self.init_upload_task(init_url) + init_url = self.block_url_v2(host, self.bucket_name) + self.uploadId, self.expiredAt = self.init_upload_task(init_url) else: raise ValueError("version must choose v1 or v2 !") for index, block in enumerate(_file_iter(self.input_stream, self.part_size, offset)): @@ -241,7 +248,7 @@ def upload(self): ret, info = self.make_block(block, length, host) elif self.version == 'v2': index_ = index + self.recovery_index - url = init_url + '/%s/%d' % (upload_id, index_) + url = init_url + '/%s/%d' % (self.uploadId, index_) ret, info = self.make_block_v2(block, url) if ret is None and not info.need_retry(): return ret, info @@ -258,7 +265,7 @@ def upload(self): return ret, info elif self.version == 'v2': if info.need_retry(): - url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (upload_id, index + 1) + url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (self.uploadId, index + 1) ret, info = self.make_block_v2(block, url) if ret is None: return ret, info @@ -272,7 +279,7 @@ def upload(self): if self.version == 'v1': return self.make_file(host) elif self.version == 'v2': - make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % upload_id + make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % self.uploadId return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, self.mime_type, self.params) From 82125e0ab961fc88db3e69f1af7e4c959884cfea Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 17:54:47 +0800 Subject: [PATCH 360/478] =?UTF-8?q?=E8=A7=84=E8=8C=83=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 6ffbd772..2ab1a82c 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -219,7 +219,6 @@ def recovery_from_record(self): self.blockStatus = record['etags'] return record['offset'], record['upload_id'], record['expired_at'] - def upload(self): """上传操作""" self.blockStatus = [] From 9bb676a6f99dfa3ef839741c1f99cfc649431470 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 18:03:34 +0800 Subject: [PATCH 361/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 2ab1a82c..c8b8747b 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -74,7 +74,8 @@ def put_file(up_token, key, file_path, params=None, ret, info = put_stream(up_token, key, input_stream, file_name, size, hostscache_dir, params, mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, - modify_time=modify_time, keep_last_modified=keep_last_modified) + modify_time=modify_time, keep_last_modified=keep_last_modified, + part_size=None, version=None, bucket_name=None) else: crc = file_crc32(file_path) ret, info = _form_put(up_token, key, input_stream, params, mime_type, @@ -199,13 +200,22 @@ def record_upload_progress(self, offset): def recovery_from_record(self): record = self.upload_progress_recorder.get_upload_record(self.file_name, self.key) if not record: - return 0 + if self.version == 'v1': + return 0 + elif self.version == 'v2': + return 0, None, None try: if not record['modify_time'] or record['size'] != self.size or \ record['modify_time'] != self.modify_time: - return 0 + if self.version == 'v1': + return 0 + elif self.version == 'v2': + return 0, None, None except KeyError: - return 0 + if self.version == 'v1': + return 0 + elif self.version == 'v2': + return 0, None, None if self.version == 'v1': if not record.__contains__('contexts') or len(record['contexts']) == 0: return 0 From 2af1e26a287872c28b6b536774484de93aab1ba2 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 18:07:00 +0800 Subject: [PATCH 362/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index c8b8747b..75f2f87b 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -47,7 +47,8 @@ def put_data( def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, - progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None): + progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None, + part_size=None, version=None, bucket_name=None): """上传文件到七牛 Args: @@ -75,7 +76,7 @@ def put_file(up_token, key, file_path, params=None, mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, modify_time=modify_time, keep_last_modified=keep_last_modified, - part_size=None, version=None, bucket_name=None) + part_size=part_size, version=version, bucket_name=bucket_name) else: crc = file_crc32(file_path) ret, info = _form_put(up_token, key, input_stream, params, mime_type, From fc472a9f3fc859c70031f5cab235cabb5a75b1ca Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 18:36:31 +0800 Subject: [PATCH 363/478] =?UTF-8?q?=E8=AE=BE=E7=BD=AEput=5Ffile=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/config.py | 7 ++++++- qiniu/services/storage/uploader.py | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/qiniu/config.py b/qiniu/config.py index 2bb825c5..b09a197c 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -10,6 +10,7 @@ _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 _BLOCK_MIN_SIZE = 1024 * 1024 # v2:断点续传分片最小值 _BLOCK_MAX_SIZE = 1024 * 1024 * 1024 # v2断点续传分片最大值 +_Upload_Threshold = 1024 * 1024 * 8 # put_file上传方式的临界默认值 _config = { 'default_zone': zone.Zone(), @@ -30,7 +31,7 @@ def get_default(key): def set_default( default_zone=None, connection_retries=None, connection_pool=None, connection_timeout=None, default_rs_host=None, default_uc_host=None, - default_rsf_host=None, default_api_host=None): + default_rsf_host=None, default_api_host=None, default_upload_threshold=None): if default_zone: _config['default_zone'] = default_zone if default_rs_host: @@ -47,3 +48,7 @@ def set_default( _config['connection_pool'] = connection_pool if connection_timeout: _config['connection_timeout'] = connection_timeout + if default_upload_threshold: + _config['default_upload_threshold'] = default_upload_threshold + else: + _config['default_upload_threshold'] = _Upload_Threshold diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 75f2f87b..370cbcb1 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -48,7 +48,7 @@ def put_data( def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None, - part_size=None, version=None, bucket_name=None): + part_size=config._BLOCK_SIZE, version=None, bucket_name=None): """上传文件到七牛 Args: @@ -61,6 +61,9 @@ def put_file(up_token, key, file_path, params=None, progress_handler: 上传进度 upload_progress_recorder: 记录上传进度，用于断点续传 hostscache_dir： host请求 缓存文件保存位置 + version 分片上传版本 目前支持v1/v2版本 默认v1 + part_size 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB + bucket_name 分片上传v2字段必传字段 空间名称 Returns: 一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"} @@ -71,7 +74,7 @@ def put_file(up_token, key, file_path, params=None, with open(file_path, 'rb') as input_stream: file_name = os.path.basename(file_path) modify_time = int(os.path.getmtime(file_path)) - if size > config._BLOCK_SIZE * 2: + if size > config.get_default('default_upload_threshold'): ret, info = put_stream(up_token, key, input_stream, file_name, size, hostscache_dir, params, mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, From d5e0a5f0d9767c4c6f170b791b411be5ccbe98e0 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 18:43:55 +0800 Subject: [PATCH 364/478] =?UTF-8?q?=E6=B7=BB=E5=8A=A0config=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qiniu/config.py b/qiniu/config.py index b09a197c..7302e3c6 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -10,7 +10,6 @@ _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 _BLOCK_MIN_SIZE = 1024 * 1024 # v2:断点续传分片最小值 _BLOCK_MAX_SIZE = 1024 * 1024 * 1024 # v2断点续传分片最大值 -_Upload_Threshold = 1024 * 1024 * 8 # put_file上传方式的临界默认值 _config = { 'default_zone': zone.Zone(), @@ -21,6 +20,7 @@ 'connection_timeout': 30, # 链接超时为时间为30s 'connection_retries': 3, # 链接重试次数为3次 'connection_pool': 10, # 链接池个数为10 + 'default_upload_threshold': 1024 * 1024 * 8 # put_file上传方式的临界默认值 } @@ -50,5 +50,3 @@ def set_default( _config['connection_timeout'] = connection_timeout if default_upload_threshold: _config['default_upload_threshold'] = default_upload_threshold - else: - _config['default_upload_threshold'] = _Upload_Threshold From 3078056a77ee6b16f622388f043c2d9de41e06cd Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 18:55:45 +0800 Subject: [PATCH 365/478] =?UTF-8?q?=E8=AE=BE=E7=BD=AEpart=5Fsize=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 370cbcb1..6c370eb6 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -48,7 +48,7 @@ def put_data( def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None, - part_size=config._BLOCK_SIZE, version=None, bucket_name=None): + part_size=None, version=None, bucket_name=None): """上传文件到七牛 Args: @@ -183,7 +183,7 @@ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache self.modify_time = modify_time or time.time() self.keep_last_modified = keep_last_modified self.version = version or 'v1' - self.part_size = part_size + self.part_size = part_size or config._BLOCK_SIZE self.bucket_name = bucket_name def record_upload_progress(self, offset): From fb32b52a48b51caa69d357492b81064f803cac7d Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 21:11:41 +0800 Subject: [PATCH 366/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .qiniu_pythonsdk_hostscache.json | 1 + qiniu/services/storage/uploader.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .qiniu_pythonsdk_hostscache.json diff --git a/.qiniu_pythonsdk_hostscache.json b/.qiniu_pythonsdk_hostscache.json new file mode 100644 index 00000000..d08d61fa --- /dev/null +++ b/.qiniu_pythonsdk_hostscache.json @@ -0,0 +1 @@ +{"http:qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_:z0-bucket": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1621515757}} \ No newline at end of file diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 6c370eb6..5350e639 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -261,7 +261,7 @@ def upload(self): ret, info = self.make_block(block, length, host) elif self.version == 'v2': index_ = index + self.recovery_index - url = init_url + '/%s/%d' % (self.uploadId, index_) + url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (self.uploadId, index_) ret, info = self.make_block_v2(block, url) if ret is None and not info.need_retry(): return ret, info From f1af3d7ce59c8ff91426c1336b040a7c8840e8b5 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 19 May 2021 21:51:26 +0800 Subject: [PATCH 367/478] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E4=B8=B4=E6=97=B6=E6=96=87=E4=BB=B6=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .qiniu_pythonsdk_hostscache.json | 1 - qiniu/services/storage/uploader.py | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 .qiniu_pythonsdk_hostscache.json diff --git a/.qiniu_pythonsdk_hostscache.json b/.qiniu_pythonsdk_hostscache.json deleted file mode 100644 index d08d61fa..00000000 --- a/.qiniu_pythonsdk_hostscache.json +++ /dev/null @@ -1 +0,0 @@ -{"http:qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_:z0-bucket": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1621515757}} \ No newline at end of file diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 5350e639..09e9de03 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -309,6 +309,9 @@ def make_file_v2(self, block_status, url, file_name=None, mime_type=None, custom 'customVars': customVars } ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) + if ret is not None and ret != {}: + if ret['hash'] and ret['key']: + self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) return ret, info def get_up_host(self): From 47920849bd371383e0204f573e1146cd755d684d Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Thu, 20 May 2021 14:18:04 +0800 Subject: [PATCH 368/478] =?UTF-8?q?bucket=5Fname=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 09e9de03..d68aa5d8 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import base64 import hashlib import json import os @@ -239,6 +239,7 @@ def upload(self): self.recovery_index = 1 self.expiredAt = None self.uploadId = None + self.get_bucket() host = self.get_up_host() if self.version == 'v1': offset = self.recovery_from_record() @@ -388,3 +389,11 @@ def put(self, url, data, headers): def get_parts(self, block_status): return sorted(block_status, key=lambda i: i['partNumber']) + + def get_bucket(self): + if self.bucket_name is None: + encoded_policy = self.up_token.split(':')[-1] + decode_policy = base64.urlsafe_b64decode(encoded_policy) + dict_policy = json.loads(decode_policy) + if dict_policy != {}: + self.bucket_name = dict_policy['scope'].split(':')[0] \ No newline at end of file From 8fc98ed6a58df7e214af1af4713ca7cf320e7537 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Thu, 20 May 2021 14:23:47 +0800 Subject: [PATCH 369/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=BC=E5=BC=8F:?= =?UTF-8?q?=E5=B0=91=E4=B8=80=E8=A1=8C=E7=A9=BA=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index d68aa5d8..c443bcf1 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -396,4 +396,4 @@ def get_bucket(self): decode_policy = base64.urlsafe_b64decode(encoded_policy) dict_policy = json.loads(decode_policy) if dict_policy != {}: - self.bucket_name = dict_policy['scope'].split(':')[0] \ No newline at end of file + self.bucket_name = dict_policy['scope'].split(':')[0] From cf64b213a702b1298c46ec16f644cde0d883e72f Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Thu, 20 May 2021 14:39:02 +0800 Subject: [PATCH 370/478] =?UTF-8?q?loads=E8=AE=BE=E7=BD=AEutf-8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index c443bcf1..53d0b473 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -394,6 +394,6 @@ def get_bucket(self): if self.bucket_name is None: encoded_policy = self.up_token.split(':')[-1] decode_policy = base64.urlsafe_b64decode(encoded_policy) - dict_policy = json.loads(decode_policy) + dict_policy = json.loads(decode_policy, encoding='utf-8') if dict_policy != {}: self.bucket_name = dict_policy['scope'].split(':')[0] From ee68d285c4544f34ffb67ac649c8822668be9974 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Thu, 20 May 2021 16:24:05 +0800 Subject: [PATCH 371/478] get_bucket utf-8 --- qiniu/services/storage/uploader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 53d0b473..3ca45fc5 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,9 +391,10 @@ def get_parts(self, block_status): return sorted(block_status, key=lambda i: i['partNumber']) def get_bucket(self): - if self.bucket_name is None: + if self.bucket_name is None or self.bucket_name == '': encoded_policy = self.up_token.split(':')[-1] decode_policy = base64.urlsafe_b64decode(encoded_policy) + decode_policy = decode_policy.decode('utf-8') dict_policy = json.loads(decode_policy, encoding='utf-8') if dict_policy != {}: self.bucket_name = dict_policy['scope'].split(':')[0] From 9615b1096f24b5dffd4c61520f78eb9ecdf9e3d6 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Thu, 20 May 2021 17:11:59 +0800 Subject: [PATCH 372/478] =?UTF-8?q?=E6=B7=BB=E5=8A=A02m=204m=2010m?= =?UTF-8?q?=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/config.py | 4 ++-- test_qiniu.py | 48 +++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/qiniu/config.py b/qiniu/config.py index 7302e3c6..1ce10a92 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -7,7 +7,7 @@ API_HOST = 'http://api.qiniu.com' # 数据处理操作Host UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host -_BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 +_BLOCK_SIZE = 1024 * 1024 * 4 # 断点续传分块大小，该参数为接口规格，暂不支持修改 _BLOCK_MIN_SIZE = 1024 * 1024 # v2:断点续传分片最小值 _BLOCK_MAX_SIZE = 1024 * 1024 * 1024 # v2断点续传分片最大值 @@ -20,7 +20,7 @@ 'connection_timeout': 30, # 链接超时为时间为30s 'connection_retries': 3, # 链接重试次数为3次 'connection_pool': 10, # 链接池个数为10 - 'default_upload_threshold': 1024 * 1024 * 8 # put_file上传方式的临界默认值 + 'default_upload_threshold': 2 * _BLOCK_SIZE # put_file上传方式的临界默认值 } diff --git a/test_qiniu.py b/test_qiniu.py index 608b25fd..933b6022 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -391,18 +391,47 @@ def test_put_stream(self): assert ret['key'] == key - def test_put_stream_v2(self): - localfile = __file__ + def test_put_2m_stream_v2(self): + file = CreateTestFile(2) + localfile = file.create_file() key = 'test_file_r' size = os.stat(localfile).st_size set_default(default_zone=Zone('http://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, + self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + assert ret['key'] == key + os.remove(localfile) + + def test_put_4m_stream_v2(self): + file = CreateTestFile(4) + localfile = file.create_file() + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, self.params, self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) assert ret['key'] == key + os.remove(localfile) + def test_put_10m_stream_v2(self): + file = CreateTestFile(10) + localfile = file.create_file() + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, + self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + assert ret['key'] == key + os.remove(localfile) def test_big_file(self): key = 'big' @@ -490,5 +519,18 @@ def read(self): print(self.str) +class CreateTestFile: + def __init__(self, size): + self.size = size + + def create_file(self): + file_path = '%dm.text' % self.size + file = open(file_path, 'w') + file.seek(1024 * 1024 * self.size) + file.write('\x00') + file.close() + return file_path + + if __name__ == '__main__': unittest.main() From b804b1ca4121b6288fe13e39eca7ebffa3c62b36 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Thu, 20 May 2021 17:35:44 +0800 Subject: [PATCH 373/478] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qiniu/config.py b/qiniu/config.py index 1ce10a92..a137a67f 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -8,8 +8,6 @@ UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续传分块大小，该参数为接口规格，暂不支持修改 -_BLOCK_MIN_SIZE = 1024 * 1024 # v2:断点续传分片最小值 -_BLOCK_MAX_SIZE = 1024 * 1024 * 1024 # v2断点续传分片最大值 _config = { 'default_zone': zone.Zone(), From fc03833c16810b1bb21b4fc7efee3673edb19430 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Thu, 20 May 2021 17:53:25 +0800 Subject: [PATCH 374/478] =?UTF-8?q?=E5=88=87=E6=8D=A2=E7=94=9F=E6=88=90?= =?UTF-8?q?=E4=B8=B4=E6=97=B6=E6=96=87=E4=BB=B6=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index 933b6022..2645a8dc 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -392,8 +392,7 @@ def test_put_stream(self): def test_put_2m_stream_v2(self): - file = CreateTestFile(2) - localfile = file.create_file() + localfile = create_temp_file(2 * 1024 * 1024 + 1) key = 'test_file_r' size = os.stat(localfile).st_size set_default(default_zone=Zone('http://upload.qiniup.com')) @@ -403,11 +402,10 @@ def test_put_2m_stream_v2(self): self.params, self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) assert ret['key'] == key - os.remove(localfile) + remove_temp_file(localfile) def test_put_4m_stream_v2(self): - file = CreateTestFile(4) - localfile = file.create_file() + localfile = create_temp_file(4 * 1024 * 1024 + 1) key = 'test_file_r' size = os.stat(localfile).st_size set_default(default_zone=Zone('http://upload.qiniup.com')) @@ -417,11 +415,10 @@ def test_put_4m_stream_v2(self): self.params, self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) assert ret['key'] == key - os.remove(localfile) + remove_temp_file(localfile) def test_put_10m_stream_v2(self): - file = CreateTestFile(10) - localfile = file.create_file() + localfile = create_temp_file(10 * 1024 * 1024 + 1) key = 'test_file_r' size = os.stat(localfile).st_size set_default(default_zone=Zone('http://upload.qiniup.com')) @@ -431,7 +428,7 @@ def test_put_10m_stream_v2(self): self.params, self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) assert ret['key'] == key - os.remove(localfile) + remove_temp_file(localfile) def test_big_file(self): key = 'big' @@ -519,18 +516,5 @@ def read(self): print(self.str) -class CreateTestFile: - def __init__(self, size): - self.size = size - - def create_file(self): - file_path = '%dm.text' % self.size - file = open(file_path, 'w') - file.seek(1024 * 1024 * self.size) - file.write('\x00') - file.close() - return file_path - - if __name__ == '__main__': unittest.main() From 3302c576f8b3bbc2e592dbce37e270e57f3b9879 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Thu, 20 May 2021 18:13:19 +0800 Subject: [PATCH 375/478] =?UTF-8?q?=E7=94=A8=E4=BE=8Bpart=5Fsiz=3D4m?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index 2645a8dc..2ca77383 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -400,7 +400,7 @@ def test_put_2m_stream_v2(self): token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) assert ret['key'] == key remove_temp_file(localfile) @@ -413,7 +413,7 @@ def test_put_4m_stream_v2(self): token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) assert ret['key'] == key remove_temp_file(localfile) @@ -426,7 +426,7 @@ def test_put_10m_stream_v2(self): token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) assert ret['key'] == key remove_temp_file(localfile) From 5013303a2ee4b361545dbf4650d5057a6a88a4e3 Mon Sep 17 00:00:00 2001 From: Bachue Zhou <bachue.shu@gmail.com> Date: Thu, 20 May 2021 20:12:53 +0800 Subject: [PATCH 376/478] optimize test case --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 2ca77383..0ea46c79 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -405,7 +405,7 @@ def test_put_2m_stream_v2(self): remove_temp_file(localfile) def test_put_4m_stream_v2(self): - localfile = create_temp_file(4 * 1024 * 1024 + 1) + localfile = create_temp_file(4 * 1024 * 1024) key = 'test_file_r' size = os.stat(localfile).st_size set_default(default_zone=Zone('http://upload.qiniup.com')) From 4920e3e3178f9593a06b1441135c7f81c19021aa Mon Sep 17 00:00:00 2001 From: Bachue Zhou <bachue.shu@gmail.com> Date: Thu, 20 May 2021 20:54:23 +0800 Subject: [PATCH 377/478] bump version to v7.4.0 --- CHANGELOG.md | 5 ++++- qiniu/__init__.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59f95d2e..92ecd531 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 7.4.0 (2021-05-21) +* 支持分片上传 v2 + ## 7.3.1 (2021-01-06) * 修复 ResponseInfo 对扩展码错误处理问题 * 增加 python v3.7，v3.8，v3.9 版本 CI 测试 @@ -77,7 +80,7 @@ ## 7.1.2 (2017-03-24) ### 增加 -* 增加设置文件生命周期的接口 +* 增加设置文件生命周期的接口 ## 7.1.1 (2017-02-03) ### 增加 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 5178f0b4..dffdaf7e 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.3.1' +__version__ = '7.4.0' from .auth import Auth, QiniuMacAuth From e9afbe2324a56cebd1abaf4f613756f5733b08a1 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Fri, 21 May 2021 10:40:41 +0800 Subject: [PATCH 378/478] =?UTF-8?q?python3.9=20json.loads=E4=B8=8D?= =?UTF-8?q?=E6=94=AF=E6=8C=81encoding=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 3ca45fc5..68dfe279 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -395,6 +395,6 @@ def get_bucket(self): encoded_policy = self.up_token.split(':')[-1] decode_policy = base64.urlsafe_b64decode(encoded_policy) decode_policy = decode_policy.decode('utf-8') - dict_policy = json.loads(decode_policy, encoding='utf-8') + dict_policy = json.loads(decode_policy) if dict_policy != {}: self.bucket_name = dict_policy['scope'].split(':')[0] From 4e61f5159f005743864f47a970036946accce460 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Fri, 21 May 2021 10:43:06 +0800 Subject: [PATCH 379/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 608b25fd..9d95c7d7 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -400,7 +400,7 @@ def test_put_stream_v2(self): token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=None) assert ret['key'] == key From 153046225a26759a05df16181020df16deac1b15 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Fri, 21 May 2021 11:08:41 +0800 Subject: [PATCH 380/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 9d95c7d7..608b25fd 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -400,7 +400,7 @@ def test_put_stream_v2(self): token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=None) + self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) assert ret['key'] == key From 8fd99e2366ff80a3821d70f2c0a5b575bd08c682 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Fri, 21 May 2021 15:06:43 +0800 Subject: [PATCH 381/478] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E7=AD=96=E7=95=A5=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/auth.py | 16 +++++++++++++++- qiniu/services/storage/uploader.py | 9 ++------- test_qiniu.py | 11 +++++++++++ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index c0b3de43..d260df8a 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import base64 import hmac import time from hashlib import sha1 @@ -158,6 +158,20 @@ def upload_token( return self.__upload_token(args) + @staticmethod + def up_token_decode(up_token): + up_token_list = up_token.split(':') + ak = up_token_list[0] + sign = base64.urlsafe_b64decode(up_token_list[1]) + decode_policy = base64.urlsafe_b64decode(up_token_list[2]) + decode_policy = decode_policy.decode('utf-8') + dict_policy = json.loads(decode_policy) + if dict_policy != {}: + bucket_name = dict_policy['scope'].split(':')[0] + else: + bucket_name = None + return ak, sign, bucket_name + def __upload_token(self, policy): data = json.dumps(policy, separators=(',', ':')) return self.token_with_data(data) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 68dfe279..0cb03fac 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -5,7 +5,7 @@ import os import time -from qiniu import config +from qiniu import config, Auth from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter, rfc_from_timestamp from qiniu import http from .upload_progress_recorder import UploadProgressRecorder @@ -392,9 +392,4 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - encoded_policy = self.up_token.split(':')[-1] - decode_policy = base64.urlsafe_b64decode(encoded_policy) - decode_policy = decode_policy.decode('utf-8') - dict_policy = json.loads(decode_policy) - if dict_policy != {}: - self.bucket_name = dict_policy['scope'].split(':')[0] + self.bucket_name = Auth(None, None).up_token_decode(self.up_token)[-1] diff --git a/test_qiniu.py b/test_qiniu.py index 608b25fd..eb499c45 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -403,6 +403,17 @@ def test_put_stream_v2(self): self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) assert ret['key'] == key + def test_put_stream_v2_without_bucket_name(self): + localfile = __file__ + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + self.params, + self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=None) + assert ret['key'] == key def test_big_file(self): key = 'big' From 5a4176320f2b02e1fbf9fb58b40e8fa074a358f5 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Fri, 21 May 2021 15:16:14 +0800 Subject: [PATCH 382/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/auth.py | 6 +----- qiniu/services/storage/uploader.py | 5 +++-- test_qiniu.py | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index d260df8a..b386115b 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -166,11 +166,7 @@ def up_token_decode(up_token): decode_policy = base64.urlsafe_b64decode(up_token_list[2]) decode_policy = decode_policy.decode('utf-8') dict_policy = json.loads(decode_policy) - if dict_policy != {}: - bucket_name = dict_policy['scope'].split(':')[0] - else: - bucket_name = None - return ak, sign, bucket_name + return ak, sign, dict_policy def __upload_token(self, policy): data = json.dumps(policy, separators=(',', ':')) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 0cb03fac..f5c6e76d 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import base64 import hashlib import json import os @@ -392,4 +391,6 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - self.bucket_name = Auth(None, None).up_token_decode(self.up_token)[-1] + pulicy = Auth(None, None).up_token_decode(self.up_token) + if pulicy != {}: + self.bucket_name = pulicy['scope'].split(':')[0] diff --git a/test_qiniu.py b/test_qiniu.py index eb499c45..8696e712 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -412,7 +412,7 @@ def test_put_stream_v2_without_bucket_name(self): token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=None) + self.mime_type, part_size=1024 * 1024 * 10, version='v2') assert ret['key'] == key def test_big_file(self): From dddfcefac91137bd943eeacc98bd2bd1b16bd4d3 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Fri, 21 May 2021 15:19:15 +0800 Subject: [PATCH 383/478] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E7=BC=A9=E8=BF=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index f5c6e76d..644b7b04 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -393,4 +393,4 @@ def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': pulicy = Auth(None, None).up_token_decode(self.up_token) if pulicy != {}: - self.bucket_name = pulicy['scope'].split(':')[0] + self.bucket_name = pulicy['scope'].split(':')[0] From 369618ca7086183b34ea8d59761d8d9ae08e6610 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Fri, 21 May 2021 15:23:35 +0800 Subject: [PATCH 384/478] ak,sk --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 644b7b04..74db6f49 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,6 +391,6 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - pulicy = Auth(None, None).up_token_decode(self.up_token) + pulicy = Auth(os.getenv('QINIU_ACCESS_KEY'), os.getenv('QINIU_SECRET_KEY')).up_token_decode(self.up_token) if pulicy != {}: self.bucket_name = pulicy['scope'].split(':')[0] From 13e1d7c7cf848beac25253bc13ad9abbb5ce5cce Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Fri, 21 May 2021 15:27:41 +0800 Subject: [PATCH 385/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 74db6f49..a8bf47c0 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,6 +391,7 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - pulicy = Auth(os.getenv('QINIU_ACCESS_KEY'), os.getenv('QINIU_SECRET_KEY')).up_token_decode(self.up_token) + pulicy = Auth(os.getenv('QINIU_ACCESS_KEY'), + os.getenv('QINIU_SECRET_KEY')).up_token_decode(self.up_token)[-1] if pulicy != {}: self.bucket_name = pulicy['scope'].split(':')[0] From 119d80b63f81100f7eca4c1606206d1ad416e1ea Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Mon, 24 May 2021 10:24:30 +0800 Subject: [PATCH 386/478] =?UTF-8?q?=E8=B0=83=E7=94=A8=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index a8bf47c0..1bc5ae6a 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,7 +391,6 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - pulicy = Auth(os.getenv('QINIU_ACCESS_KEY'), - os.getenv('QINIU_SECRET_KEY')).up_token_decode(self.up_token)[-1] + pulicy = Auth.up_token_decode(self.up_token)[-1] if pulicy != {}: self.bucket_name = pulicy['scope'].split(':')[0] From 2841baa62c4a5119647becdb97c7b10a500cad58 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Mon, 24 May 2021 16:23:39 +0800 Subject: [PATCH 387/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9issue=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20iunput=5Fstream=E8=AF=BB=E5=AE=8C=E5=90=8Eseek(0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- qiniu/utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 1bc5ae6a..36e69662 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,6 +391,6 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - pulicy = Auth.up_token_decode(self.up_token)[-1] + _, _, pulicy = Auth.up_token_decode(self.up_token)[-1] if pulicy != {}: self.bucket_name = pulicy['scope'].split(':')[0] diff --git a/qiniu/utils.py b/qiniu/utils.py index 8b7eaa66..07bcdeb6 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -90,6 +90,7 @@ def _file_iter(input_stream, size, offset=0): while d: yield d d = input_stream.read(size) + input_stream.seek(0) def _sha1(data): From 5897beb2e21f702b6ce8c2d61f831e9184692912 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Mon, 24 May 2021 16:25:17 +0800 Subject: [PATCH 388/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9issue=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20=E8=A7=84=E8=8C=83get=5Fbucket=E5=8F=96=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 36e69662..d9e8af44 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,6 +391,6 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - _, _, pulicy = Auth.up_token_decode(self.up_token)[-1] + _, _, pulicy = Auth.up_token_decode(self.up_token) if pulicy != {}: self.bucket_name = pulicy['scope'].split(':')[0] From 332168b212772c24cf8eafa6371e972a3db1d0bb Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Tue, 25 May 2021 15:23:31 +0800 Subject: [PATCH 389/478] pulicy->policy --- qiniu/services/storage/uploader.py | 6 +++--- test_qiniu.py | 34 ++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index d9e8af44..56626718 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,6 +391,6 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - _, _, pulicy = Auth.up_token_decode(self.up_token) - if pulicy != {}: - self.bucket_name = pulicy['scope'].split(':')[0] + _, _, policy = Auth.up_token_decode(self.up_token) + if policy != {}: + self.bucket_name = policy['scope'].split(':')[0] diff --git a/test_qiniu.py b/test_qiniu.py index 8696e712..8db8c5ea 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -390,18 +390,44 @@ def test_put_stream(self): self.mime_type, part_size=None, version=None, bucket_name=None) assert ret['key'] == key + def test_put_2m_stream_v2(self): + localfile = create_temp_file(2 * 1024 * 1024 + 1) + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, + self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) + assert ret['key'] == key + remove_temp_file(localfile) - def test_put_stream_v2(self): - localfile = __file__ + def test_put_4m_stream_v2(self): + localfile = create_temp_file(4 * 1024 * 1024) key = 'test_file_r' size = os.stat(localfile).st_size set_default(default_zone=Zone('http://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, + self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) + assert ret['key'] == key + remove_temp_file(localfile) + + def test_put_10m_stream_v2(self): + localfile = create_temp_file(10 * 1024 * 1024 + 1) + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) assert ret['key'] == key + remove_temp_file(localfile) def test_put_stream_v2_without_bucket_name(self): localfile = __file__ From 627188ec61c5f40b5bdb57569d3aa163918eae97 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Tue, 25 May 2021 16:34:04 +0800 Subject: [PATCH 390/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 175 ++++++++++++++++++++++++++++++++++++++++++++++ qiniu/__init__.py | 30 ++++++++ qiniu/config.py | 50 +++++++++++++ 3 files changed, 255 insertions(+) create mode 100644 CHANGELOG.md create mode 100644 qiniu/__init__.py create mode 100644 qiniu/config.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..92ecd531 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,175 @@ +# Changelog + +## 7.4.0 (2021-05-21) +* 支持分片上传 v2 + +## 7.3.1 (2021-01-06) +* 修复 ResponseInfo 对扩展码错误处理问题 +* 增加 python v3.7，v3.8，v3.9 版本 CI 测试 + +## 7.3.0 (2020-09-23) +新增 +* sms[云短信]：新增查询短信发送记录方法：get_messages_info +* cdn: 新增上线域名 domain_online 方法、下线域名 domain_offline 方法和删除域名 delete_domain 方法 +* 对象存储：新增批量解冻build_batch_restoreAr方法、获取空间列表bucket_domain方法和修改空间访问权限change_bucket_permission方法 + +## 7.2.10 （2020-08-21） +* 修复上传策略中forceSaveKey参数没有签算进上传token，导致上传失败的问题 +## 7.2.9 (2020-08-07) +* 支持指定本地ctx缓存文件.qiniu_pythonsdk_hostscache.json 文件路径 +* 更正接口返回描述docstring +* 修复接口对非json response 处理 +* ci 覆盖增加python 3.6 3.7 +* 修复获取域名列方法 +* 修复python3 环境下，二进制对象上传问题 + + +## 7.2.8(2020-03-27) +* add restoreAr + +## 7.2.7(2020-03-10) +* fix bucket_info + +## 7.2.6(2019-06-26) +* 添加sms + +## 7.2.5 (2019-06-06) +* 添加sms + +## 7.2.4 (2019-04-01) +* 默认导入region类 + +## 7.2.3 (2019-02-25) +* 新增region类，zone继承 +* 上传可以指定上传域名 +* 新增上传指定上传空间和qvm指定上传内网的例子 +* 新增列举账号空间，创建空间，查询空间信息，改变文件状态接口，并提供例子 + +## 7.2.2 (2018-05-10) +* 增加连麦rtc服务端API功能 + +## 7.2.0（2017-11-23） +* 修复put_data不支持file like object的问题 +* 增加空间写错时，抛出异常提示客户的功能 +* 增加创建空间的接口功能 + +## 7.1.9（2017-11-01） +* 修复python2情况下，中文文件名上传失败的问题 +* 修复python2环境下，中文文件使用分片上传时失败的问题 + +## 7.1.8 （2017-10-18） +* 恢复kirk的API为原来的状态 + +## 7.1.7 (2017-09-27) + +* 修复从时间戳获取rfc http格式的时间字符串问题 + +## 7.1.6 (2017-09-26) + +* 给 `put_file` 功能增加保持本地文件Last Modified功能，以支持切换源站的客户CDN不回源 + +## 7.1.5 (2017-08-26) + +* 设置表单上传默认校验crc32 +* 增加PutPolicy新参数isPrefixalScope +* 修复手动指定的zone无效的问题 + +## 7.1.4 (2017-06-05) +### 修正 +* cdn功能中获取域名日志列表的参数错误 + +## 7.1.2 (2017-03-24) +### 增加 +* 增加设置文件生命周期的接口 + +## 7.1.1 (2017-02-03) +### 增加 +* 增加cdn刷新，预取，日志获取，时间戳防盗链生成功能 + +### 修正 +* 修复分片上传的upload record path遇到中文时的问题，现在使用md5来计算文件名 + +## 7.1.0 (2016-12-08) +### 增加 +* 通用计算支持 + +## 7.0.10 (2016-11-29) +### 修正 +* 去掉homedir + +## 7.0.9 (2016-10-09) +### 增加 +* 多机房接口调用支持 + +## 7.0.8 (2016-07-05) +### 修正 +* 修复表单上传大于20M文件的400错误 + +### 增加 +* copy 和 move 操作增加 force 字段，允许强制覆盖 copy 和 move +* 增加上传策略 deleteAfterDays 字段 +* 一些 demo + +## 7.0.7 (2016-05-05) +### 修正 +* 修复大于4M的文件hash计算错误的问题 +* add fname + +### 增加 +* 一些demo +* travis 直接发布 + +## 7.0.6 (2015-12-05) +### 修正 +* 2.x unicode 问题 by @hunter007 +* 上传重试判断 +* 上传时 dns劫持处理 + +### 增加 +* fsizeMin 上传策略 +* 断点上传记录 by @hokein +* 计算stream etag +* 3.5 ci 支持 + +## 7.0.5 (2015-06-25) +### 变更 +* 配置up_host 改为配置zone + +### 增加 +* fectch 支持不指定key + +## 7.0.4 (2015-05-04) +### 修正 +* 上传重试为空文件 +* 回调应该只对form data 签名。 + +## 7.0.3 (2015-03-11) +### 增加 +* 可以配置 io/rs/api/rsf host + +## 7.0.2 (2014-12-24) +### 修正 +* 内部http get当没有auth会出错 +* python3下的qiniupy 没有参数时 arg parse会抛异常 +* 增加callback policy + +## 7.0.1 (2014-11-26) +### 增加 +* setup.py从文件中读取版本号，而不是用导入方式 +* 补充及修正了一些单元测试 + +## 7.0.0 (2014-11-13) + +### 增加 +* 简化上传接口 +* 自动选择断点续上传还是直传 +* 重构代码，接口和内部结构更清晰 +* 同时支持python 2.x 和 3.x +* 支持pfop +* 支持verify callback +* 改变mime +* 代码覆盖度报告 +* policy改为dict, 便于灵活增加，并加入过期字段检查 +* 文件列表支持目录形式 + + diff --git a/qiniu/__init__.py b/qiniu/__init__.py new file mode 100644 index 00000000..dffdaf7e --- /dev/null +++ b/qiniu/__init__.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +''' +Qiniu Resource Storage SDK for Python +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For detailed document, please see: +<https://developer.qiniu.com/kodo/sdk/1242/python> +''' + +# flake8: noqa + +__version__ = '7.4.0' + +from .auth import Auth, QiniuMacAuth + +from .config import set_default +from .zone import Zone +from .region import Region + +from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ + build_batch_stat, build_batch_delete, build_batch_restoreAr +from .services.storage.uploader import put_data, put_file, put_stream +from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url, DomainManager +from .services.processing.pfop import PersistentFop +from .services.processing.cmd import build_op, pipe_cmd, op_save +from .services.compute.app import AccountClient +from .services.compute.qcos_api import QcosClient +from .services.sms.sms import Sms +from .services.pili.rtc_server_manager import RtcServer, get_room_token +from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry diff --git a/qiniu/config.py b/qiniu/config.py new file mode 100644 index 00000000..a137a67f --- /dev/null +++ b/qiniu/config.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +from qiniu import zone + +RS_HOST = 'http://rs.qiniu.com' # 管理操作Host +RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host +API_HOST = 'http://api.qiniu.com' # 数据处理操作Host +UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host + +_BLOCK_SIZE = 1024 * 1024 * 4 # 断点续传分块大小，该参数为接口规格，暂不支持修改 + +_config = { + 'default_zone': zone.Zone(), + 'default_rs_host': RS_HOST, + 'default_rsf_host': RSF_HOST, + 'default_api_host': API_HOST, + 'default_uc_host': UC_HOST, + 'connection_timeout': 30, # 链接超时为时间为30s + 'connection_retries': 3, # 链接重试次数为3次 + 'connection_pool': 10, # 链接池个数为10 + 'default_upload_threshold': 2 * _BLOCK_SIZE # put_file上传方式的临界默认值 +} + + +def get_default(key): + return _config[key] + + +def set_default( + default_zone=None, connection_retries=None, connection_pool=None, + connection_timeout=None, default_rs_host=None, default_uc_host=None, + default_rsf_host=None, default_api_host=None, default_upload_threshold=None): + if default_zone: + _config['default_zone'] = default_zone + if default_rs_host: + _config['default_rs_host'] = default_rs_host + if default_rsf_host: + _config['default_rsf_host'] = default_rsf_host + if default_api_host: + _config['default_api_host'] = default_api_host + if default_uc_host: + _config['default_uc_host'] = default_uc_host + if connection_retries: + _config['connection_retries'] = connection_retries + if connection_pool: + _config['connection_pool'] = connection_pool + if connection_timeout: + _config['connection_timeout'] = connection_timeout + if default_upload_threshold: + _config['default_upload_threshold'] = default_upload_threshold From 8f3193a2cccbe892064f61ca0b6357f6c3117573 Mon Sep 17 00:00:00 2001 From: Bachue Zhou <bachue.shu@gmail.com> Date: Tue, 25 May 2021 18:17:50 +0800 Subject: [PATCH 391/478] =?UTF-8?q?=E5=8D=87=E7=BA=A7=E5=88=B0=20v7.4.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 3 +++ qiniu/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92ecd531..98764db4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 7.4.1 (2021-05-25) +* 分片上传 v2 方法不再强制要求 bucket_name 参数 + ## 7.4.0 (2021-05-21) * 支持分片上传 v2 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index dffdaf7e..16c52eb8 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.4.0' +__version__ = '7.4.1' from .auth import Auth, QiniuMacAuth From b24e85b9158ba17abe490a23f061abe872f92c1e Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 26 May 2021 14:41:23 +0800 Subject: [PATCH 392/478] =?UTF-8?q?=E5=8E=BB=E6=8E=89bucket=5Fname?= =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 56626718..3a96fbdf 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -62,7 +62,6 @@ def put_file(up_token, key, file_path, params=None, hostscache_dir： host请求 缓存文件保存位置 version 分片上传版本 目前支持v1/v2版本 默认v1 part_size 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB - bucket_name 分片上传v2字段必传字段 空间名称 Returns: 一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"} @@ -162,7 +161,6 @@ class _Resume(object): hostscache_dir： host请求 缓存文件保存位置 version 分片上传版本 目前支持v1/v2版本 默认v1 part_size 分片上传v2必传字段 分片大小范围为1 MB - 1 GB - bucket_name 分片上传v2字段必传字段 空间名称 """ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, From d385c08d9852e3d0fc05ef36e12759821c9bce34 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 26 May 2021 15:18:10 +0800 Subject: [PATCH 393/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 3a96fbdf..03729790 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -62,6 +62,7 @@ def put_file(up_token, key, file_path, params=None, hostscache_dir： host请求 缓存文件保存位置 version 分片上传版本 目前支持v1/v2版本 默认v1 part_size 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB + bucket_name 分片上传v2字段 空间名称 Returns: 一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"} @@ -161,6 +162,7 @@ class _Resume(object): hostscache_dir： host请求 缓存文件保存位置 version 分片上传版本 目前支持v1/v2版本 默认v1 part_size 分片上传v2必传字段 分片大小范围为1 MB - 1 GB + bucket_name 分片上传v2字段 空间名称 """ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, From 21c83ed441c3ebd310a3d948f3c058fba2ca598a Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Tue, 15 Jun 2021 12:01:08 +0800 Subject: [PATCH 394/478] =?UTF-8?q?=E4=BF=AE=E6=94=B9hosts=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E8=A7=A3=E6=9E=90=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/region.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/qiniu/region.py b/qiniu/region.py index f143795b..d253e0d5 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- - +import logging import os import time +import trace + import requests from qiniu import compat from qiniu import utils @@ -127,6 +129,9 @@ def get_bucket_hosts_to_cache(self, key, home_dir): if (len(self.host_cache) == 0): self.host_cache_from_file(home_dir) + if self.host_cache == {}: + return ret + if (not (key in self.host_cache)): return ret @@ -147,8 +152,11 @@ def host_cache_from_file(self, home_dir): if not os.path.isfile(path): return None with open(path, 'r') as f: - bucket_hosts = compat.json.load(f) - self.host_cache = bucket_hosts + try: + bucket_hosts = compat.json.load(f) + self.host_cache = bucket_hosts + except Exception as e: + logging.error(e) f.close() return From 0461a4301f51f996979bb9cd525506d97f7a8135 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Tue, 15 Jun 2021 12:04:55 +0800 Subject: [PATCH 395/478] =?UTF-8?q?=E8=A7=84=E8=8C=83=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/region.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qiniu/region.py b/qiniu/region.py index d253e0d5..c1f906dc 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -2,8 +2,6 @@ import logging import os import time -import trace - import requests from qiniu import compat from qiniu import utils From 2f7f74292401a3994973de7d8af98b462476a0fb Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 1 Sep 2021 12:21:22 +0800 Subject: [PATCH 396/478] add decode except --- qiniu/http.py | 8 ++++++-- qiniu/region.py | 4 +--- qiniu/services/storage/upload_progress_recorder.py | 5 ++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/qiniu/http.py b/qiniu/http.py index c28028fb..db70bc6a 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import json +import logging import platform import requests @@ -23,8 +25,10 @@ def __return_wrapper(resp): if resp.status_code != 200 or resp.headers.get('X-Reqid') is None: return None, ResponseInfo(resp) resp.encoding = 'utf-8' - ret = resp.json() if resp.text != '' else {} - if ret is None: # json null + try: + ret = json.loads(resp.text) + except ValueError: + logging.debug("response body decode error: %s" % resp.text) ret = {} return ret, ResponseInfo(resp) diff --git a/qiniu/region.py b/qiniu/region.py index c1f906dc..3fdda809 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -11,9 +11,7 @@ class Region(object): """七牛上传区域类 - 该类主要内容上传区域地址。 - """ def __init__( @@ -171,4 +169,4 @@ def bucket_hosts(self, ak, bucket): url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) ret = requests.get(url) data = compat.json.dumps(ret.json(), separators=(',', ':')) - return data + return data \ No newline at end of file diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index 4b4af7ac..8cf70cdb 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -37,7 +37,10 @@ def get_upload_record(self, file_name, key): if not os.path.isfile(upload_record_file_path): return None with open(upload_record_file_path, 'r') as f: - json_data = json.load(f) + try: + json_data = json.load(f) + except ValueError: + json_data = None return json_data def set_upload_record(self, file_name, key, data): From 6a91117f6c5b0dc4d1c6f1aaa35007a65334f417 Mon Sep 17 00:00:00 2001 From: okbang9 <okbang9@aliyun.com> Date: Wed, 1 Sep 2021 12:31:41 +0800 Subject: [PATCH 397/478] add new line --- qiniu/region.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/region.py b/qiniu/region.py index 3fdda809..9f8db051 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -169,4 +169,4 @@ def bucket_hosts(self, ak, bucket): url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) ret = requests.get(url) data = compat.json.dumps(ret.json(), separators=(',', ':')) - return data \ No newline at end of file + return data From 9bf65007db97904308ffeefdc27b9b097049c417 Mon Sep 17 00:00:00 2001 From: Bachue Zhou <bachue.shu@gmail.com> Date: Wed, 1 Sep 2021 14:44:13 +0800 Subject: [PATCH 398/478] try to handle hosts file read error --- qiniu/services/storage/upload_progress_recorder.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index 8cf70cdb..83a0dd65 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -36,11 +36,15 @@ def get_upload_record(self, file_name, key): upload_record_file_path = os.path.join(self.record_folder, record_file_name) if not os.path.isfile(upload_record_file_path): return None - with open(upload_record_file_path, 'r') as f: - try: - json_data = json.load(f) - except ValueError: - json_data = None + try: + with open(upload_record_file_path, 'r') as f: + try: + json_data = json.load(f) + except ValueError: + json_data = None + except IOError: + json_data = None + return json_data def set_upload_record(self, file_name, key, data): From dfe78d0a4d79b8565456c5962f235e637a9250eb Mon Sep 17 00:00:00 2001 From: Bachue Zhou <bachue.shu@gmail.com> Date: Wed, 22 Sep 2021 18:07:42 +0800 Subject: [PATCH 399/478] add keylimit to upload policy --- qiniu/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/auth.py b/qiniu/auth.py index b386115b..f74e40bf 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -28,6 +28,7 @@ 'mimeLimit', # MimeType限制 'fsizeLimit', # 上传文件大小限制 'fsizeMin', # 上传文件最少字节数 + 'keylimit', # 设置允许的key列表，不超过20个，如果设置了这个字段，api必须同时提供keyname,目前只在表单和新/旧分片上传中生效(mkfile, initial-parts,upload-parts,complete-parts) 'persistentOps', # 持久化处理操作 'persistentNotifyUrl', # 持久化处理结果通知URL From 731d645b7beea2915ebd79bfa5cd9f78fe06db1f Mon Sep 17 00:00:00 2001 From: Bachue Zhou <bachue.shu@gmail.com> Date: Thu, 23 Sep 2021 09:45:39 +0800 Subject: [PATCH 400/478] add test case for keylimit upload policy --- test_qiniu.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test_qiniu.py b/test_qiniu.py index c6b1c661..9309597f 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -462,6 +462,23 @@ def test_retry(self): assert ret['key'] == key assert ret['hash'] == etag(localfile) + def test_put_stream_with_key_limits(self): + localfile = __file__ + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key, policy={'keylimit': ['test_file_d']}) + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + self.params, + self.mime_type) + assert info.status_code == 403 + token = self.q.upload_token(bucket_name, key, policy={'keylimit': ['test_file_d', 'test_file_r']}) + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + self.params, + self.mime_type) + assert info.status_code == 200 + class DownloadTestCase(unittest.TestCase): q = Auth(access_key, secret_key) From 7a6c7d3f2305022951cd4bb638f58254382a8e5c Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Thu, 23 Sep 2021 12:02:54 +0800 Subject: [PATCH 401/478] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98764db4..9f413e2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 7.5.0 (2021-09-23) +* 上传策略新增对部分字段支持 + ## 7.4.1 (2021-05-25) * 分片上传 v2 方法不再强制要求 bucket_name 参数 From 5bd1ce168067c603dab3a994b6143b9a36040920 Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Thu, 23 Sep 2021 12:04:45 +0800 Subject: [PATCH 402/478] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 16c52eb8..76db3375 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.4.1' +__version__ = '7.5.0' from .auth import Auth, QiniuMacAuth From 2466f5ea210db1ec88a021705d83ee655ea9684f Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Thu, 23 Sep 2021 12:05:25 +0800 Subject: [PATCH 403/478] Update README.md --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index b2eb13c4..1b962d06 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,6 @@ import qiniu ``` 更多参见SDK使用指南： http://developer.qiniu.com/code/v7/sdk/python.html - -### 命令行工具 -安装完后附带有命令行工具，可以计算etag -```bash -$ qiniupy etag yourfile ``` ## 测试 From 526a10cd23e0cb593d105b0b2403ae73b3597d7c Mon Sep 17 00:00:00 2001 From: Bachue Zhou <bachue.shu@gmail.com> Date: Thu, 23 Sep 2021 12:32:15 +0800 Subject: [PATCH 404/478] use badge from github action instead of travis --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1b962d06..7846505d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![@qiniu on weibo](http://img.shields.io/badge/weibo-%40qiniutek-blue.svg)](http://weibo.com/qiniutek) [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE) -[![Build Status](https://travis-ci.org/qiniu/python-sdk.svg)](https://travis-ci.org/qiniu/python-sdk) +[![Build Status](https://github.com/qiniu/python-sdk/actions/workflows/ci-test.yml/badge.svg)](https://travis-ci.org/qiniu/python-sdk) [![GitHub release](https://img.shields.io/github/v/tag/qiniu/python-sdk.svg?label=release)](https://github.com/qiniu/python-sdk/releases) [![Latest Stable Version](https://img.shields.io/pypi/v/qiniu.svg)](https://pypi.python.org/pypi/qiniu) [![Download Times](https://img.shields.io/pypi/dm/qiniu.svg)](https://pypi.python.org/pypi/qiniu) @@ -19,10 +19,10 @@ $ pip install qiniu ## 运行环境 -| Qiniu SDK版本 | Python 版本 | -|:--------------------:|:---------------------------:| -| 7.x | 2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9| -| 6.x | 2.7 | +| Qiniu SDK版本 | Python 版本 | +| :-----------: | :------------------------------------: | +| 7.x | 2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9 | +| 6.x | 2.7 | ## 使用方法 From 5bf64deeaaf7f028f2822609b2788f06b18ac199 Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Thu, 23 Sep 2021 15:02:59 +0800 Subject: [PATCH 405/478] Update auth.py --- qiniu/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index f74e40bf..3f826c77 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -28,7 +28,7 @@ 'mimeLimit', # MimeType限制 'fsizeLimit', # 上传文件大小限制 'fsizeMin', # 上传文件最少字节数 - 'keylimit', # 设置允许的key列表，不超过20个，如果设置了这个字段，api必须同时提供keyname,目前只在表单和新/旧分片上传中生效(mkfile, initial-parts,upload-parts,complete-parts) + 'keylimit', # 设置允许的key列表，不超过20个，如果设置了这个字段，上传时必须提供key 'persistentOps', # 持久化处理操作 'persistentNotifyUrl', # 持久化处理结果通知URL From c160e086d4cb5591f389f2891f6da09b3451951c Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Thu, 23 Sep 2021 19:40:20 +0800 Subject: [PATCH 406/478] Update auth.py --- qiniu/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index 3f826c77..a1680301 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -28,7 +28,7 @@ 'mimeLimit', # MimeType限制 'fsizeLimit', # 上传文件大小限制 'fsizeMin', # 上传文件最少字节数 - 'keylimit', # 设置允许的key列表，不超过20个，如果设置了这个字段，上传时必须提供key + 'keylimit', # 设置允许上传的key列表，字符串数组类型，数组长度不可超过20个，如果设置了这个字段，上传时必须提供key 'persistentOps', # 持久化处理操作 'persistentNotifyUrl', # 持久化处理结果通知URL From 3de48b399e91d8833e9889a0f6b045afc54f15de Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Mon, 13 Dec 2021 11:23:52 +0800 Subject: [PATCH 407/478] add json decode error test --- qiniu/http.py | 3 +-- test_qiniu.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/qiniu/http.py b/qiniu/http.py index db70bc6a..93600f56 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import json import logging import platform @@ -26,7 +25,7 @@ def __return_wrapper(resp): return None, ResponseInfo(resp) resp.encoding = 'utf-8' try: - ret = json.loads(resp.text) + ret = resp.json() except ValueError: logging.debug("response body decode error: %s" % resp.text) ret = {} diff --git a/test_qiniu.py b/test_qiniu.py index c6b1c661..15a27f11 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -21,6 +21,8 @@ from qiniu.services.storage.uploader import _form_put +from qiniu.http import __return_wrapper as return_wrapper + import qiniu.config if is_py2: @@ -75,6 +77,23 @@ def is_travis(): return os.environ['QINIU_TEST_ENV'] == 'travis' +class HttpTest(unittest.TestCase): + def test_json_decode_error(self): + def mock_res(): + r = requests.Response() + r.status_code = 200 + r.headers.__setitem__('X-Reqid', 'mockedReqid') + + def json_func(): + raise ValueError('%s: line %d column %d (char %d)' % ('Expecting value', 0, 0, 0)) + r.json = json_func + + return r + mocked_res = mock_res() + ret, _ = return_wrapper(mocked_res) + assert ret == {} + + class UtilsTest(unittest.TestCase): def test_urlsafe(self): a = 'hello\x96' From f2043430782e5c691e122728b72399d491ba7104 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Tue, 8 Mar 2022 14:23:09 +0800 Subject: [PATCH 408/478] fix auth Qiniu sign generate wrong raw text --- qiniu/__init__.py | 2 +- qiniu/auth.py | 34 +++++---- qiniu/utils.py | 32 +++++++++ test_qiniu.py | 173 +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 226 insertions(+), 15 deletions(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 76db3375..7b647f36 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -27,4 +27,4 @@ from .services.compute.qcos_api import QcosClient from .services.sms.sms import Sms from .services.pili.rtc_server_manager import RtcServer, get_room_token -from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry +from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry, canonical_mime_header_key diff --git a/qiniu/auth.py b/qiniu/auth.py index a1680301..a1065d06 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -5,7 +5,7 @@ from hashlib import sha1 from requests.auth import AuthBase from .compat import urlparse, json, b -from .utils import urlsafe_base64_encode +from .utils import urlsafe_base64_encode, canonical_mime_header_key # 上传策略，参数规格详见 # https://developer.qiniu.com/kodo/manual/1206/put-policy @@ -265,15 +265,21 @@ def token_of_request( path_with_query = path if query != '': path_with_query = ''.join([path_with_query, '?', query]) - data = ''.join(["%s %s" % - (method, path_with_query), "\n", "Host: %s" % - host, "\n"]) + data = ''.join([ + "%s %s" % (method, path_with_query), + "\n", + "Host: %s" % host + ]) if content_type: - data += "Content-Type: %s" % (content_type) + "\n" + data += "\n" + data += "Content-Type: %s" % content_type - data += qheaders - data += "\n" + if qheaders: + data += "\n" + data += qheaders + + data += "\n\n" if content_type and content_type != "application/octet-stream" and body: if isinstance(body, bytes): @@ -283,11 +289,13 @@ def token_of_request( return '{0}:{1}'.format(self.__access_key, self.__token(data)) def qiniu_headers(self, headers): - res = "" - for key in headers: - if key.startswith(self.qiniu_header_prefix): - res += key + ": %s\n" % (headers.get(key)) - return res + qiniu_fields = list(filter( + lambda k: k.startswith(self.qiniu_header_prefix) and len(k) > len(self.qiniu_header_prefix), + headers, + )) + return "\n".join([ + "%s: %s" % (canonical_mime_header_key(key), headers.get(key)) for key in sorted(qiniu_fields) + ]) @staticmethod def __checkKey(access_key, secret_key): @@ -300,6 +308,8 @@ def __init__(self, auth): self.auth = auth def __call__(self, r): + if r.headers.get('Content-Type', None) is None: + r.headers['Content-Type'] = 'application/x-www-form-urlencoded' token = self.auth.token_of_request( r.method, r.headers.get('Host', None), r.url, self.auth.qiniu_headers(r.headers), diff --git a/qiniu/utils.py b/qiniu/utils.py index 07bcdeb6..92a81449 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -172,3 +172,35 @@ def rfc_from_timestamp(timestamp): last_modified_str = last_modified_date.strftime( '%a, %d %b %Y %H:%M:%S GMT') return last_modified_str + + +def _valid_header_key_char(ch): + is_token_table = [ + "!", "#", "$", "%", "&", "\\", "*", "+", "-", ".", + "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", + "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", + "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", + "U", "W", "V", "X", "Y", "Z", + "^", "_", "`", + "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", + "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", + "u", "v", "w", "x", "y", "z", + "|", "~"] + return 0 <= ord(ch) < 128 and ch in is_token_table + + +def canonical_mime_header_key(field_name): + for ch in field_name: + if not _valid_header_key_char(ch): + return field_name + result = "" + upper = True + for ch in field_name: + if upper and "a" <= ch <= "z": + result += ch.upper() + elif not upper and "A" <= ch <= "Z": + result += ch.lower() + else: + result += ch + upper = ch == "-" + return result diff --git a/test_qiniu.py b/test_qiniu.py index 9309597f..4964dd3c 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -11,11 +11,11 @@ import unittest import pytest -from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone +from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone, QiniuMacAuth from qiniu import put_data, put_file, put_stream from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, \ build_batch_delete, DomainManager -from qiniu import urlsafe_base64_encode, urlsafe_base64_decode +from qiniu import urlsafe_base64_encode, urlsafe_base64_decode, canonical_mime_header_key from qiniu.compat import is_py2, is_py3, b @@ -81,6 +81,37 @@ def test_urlsafe(self): u = urlsafe_base64_encode(a) assert b(a) == urlsafe_base64_decode(u) + def test_canonical_mime_header_key(self): + field_names = [ + ":status", + ":x-test-1", + ":x-Test-2", + "content-type", + "CONTENT-LENGTH", + "oRiGin", + "ReFer", + "Last-Modified", + "acCePt-ChArsEt", + "x-test-3", + "cache-control", + ] + expect_canonical_field_names = [ + ":status", + ":x-test-1", + ":x-Test-2", + "Content-Type", + "Content-Length", + "Origin", + "Refer", + "Last-Modified", + "Accept-Charset", + "X-Test-3", + "Cache-Control", + ] + assert len(field_names) == len(expect_canonical_field_names) + for i in range(len(field_names)): + assert canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i] + class AuthTestCase(unittest.TestCase): def test_token(self): @@ -103,6 +134,144 @@ def test_token_of_request(self): token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', 'application/x-www-form-urlencoded') assert token == 'abcdefghklmnopq:svWRNcacOE-YMsc70nuIYdaa1e4=' + def test_QiniuMacRequestsAuth(self): + auth = QiniuMacAuth("ak", "sk") + test_cases = [ + { + "method": "GET", + "host": None, + "url": "", + "qheaders": { + "X-Qiniu-": "a", + "X-Qiniu": "b", + "Content-Type": "application/x-www-form-urlencoded", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "{\"name\": \"test\"}", + "except_sign_token": "ak:0i1vKClRDWFyNkcTFzwcE7PzX74=", + }, + { + "method": "GET", + "host": None, + "url": "", + "qheaders": { + "Content-Type": "application/json", + }, + "content_type": "application/json", + "body": "{\"name\": \"test\"}", + "except_sign_token": "ak:K1DI0goT05yhGizDFE5FiPJxAj4=", + }, + { + "method": "POST", + "host": None, + "url": "", + "qheaders": { + "Content-Type": "application/json", + "X-Qiniu": "b", + }, + "content_type": "application/json", + "body": "{\"name\": \"test\"}", + "except_sign_token": "ak:0ujEjW_vLRZxebsveBgqa3JyQ-w=", + }, + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com", + "qheaders": { + "X-Qiniu-": "a", + "X-Qiniu": "b", + "Content-Type": "application/x-www-form-urlencoded", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "{\"name\": \"test\"}", + "except_sign_token": "ak:GShw5NitGmd5TLoo38nDkGUofRw=", + }, + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com", + "qheaders": { + "Content-Type": "application/json", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + "X-Qiniu-": "a", + "X-Qiniu": "b", + }, + "content_type": "application/json", + "body": "{\"name\": \"test\"}", + "except_sign_token": "ak:DhNA1UCaBqSHCsQjMOLRfVn63GQ=", + }, + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com", + "qheaders": { + "Content-Type": "application/x-www-form-urlencoded", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + "X-Qiniu-": "a", + "X-Qiniu": "b", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "name=test&language=go", + "except_sign_token": "ak:KUAhrYh32P9bv0COD8ugZjDCmII=", + }, + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com", + "qheaders": { + "Content-Type": "application/x-www-form-urlencoded", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "name=test&language=go", + "except_sign_token": "ak:KUAhrYh32P9bv0COD8ugZjDCmII=", + }, + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com/mkfile/sdf.jpg", + "qheaders": { + "Content-Type": "application/x-www-form-urlencoded", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + "X-Qiniu-": "a", + "X-Qiniu": "b", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "name=test&language=go", + "except_sign_token": "ak:fkRck5_LeyfwdkyyLk-hyNwGKac=", + }, + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com/mkfile/sdf.jpg?s=er3&df", + "qheaders": { + "Content-Type": "application/x-www-form-urlencoded", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + "X-Qiniu-": "a", + "X-Qiniu": "b", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "name=test&language=go", + "except_sign_token": "ak:PUFPWsEUIpk_dzUvvxTTmwhp3p4=", + }, + ] + + for test_case in test_cases: + sign_token = auth.token_of_request( + method=test_case["method"], + host=test_case["host"], + url=test_case["url"], + qheaders=auth.qiniu_headers(test_case["qheaders"]), + content_type=test_case["content_type"], + body=test_case["body"], + ) + assert sign_token == test_case["except_sign_token"] + def test_verify_callback(self): body = 'name=sunflower.jpg&hash=Fn6qeQi4VDLQ347NiRm-RlQx_4O2&location=Shanghai&price=1500.00&uid=123' url = 'test.qiniu.com/callback' From d9d890e94a53b8d282d71ce9c9d19547232e40ec Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Thu, 10 Mar 2022 15:09:48 +0800 Subject: [PATCH 409/478] promote Qiniu sign in bucket --- qiniu/auth.py | 3 +++ qiniu/http.py | 35 +++++++++----------------------- qiniu/services/storage/bucket.py | 7 ++++--- 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index a1065d06..5c2819de 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -58,6 +58,9 @@ def __init__(self, access_key, secret_key): def get_access_key(self): return self.__access_key + def get_secret_key(self): + return self.__secret_key + def __token(self, data): data = b(data) hashed = hmac.new(self.__secret_key, data, sha1) diff --git a/qiniu/http.py b/qiniu/http.py index c28028fb..c3dc69a1 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -76,16 +76,16 @@ def _get(url, params, auth, headers=None): if _session is None: _init() try: - post_headers = _headers.copy() + get_headers = _headers.copy() if headers is not None: for k, v in headers.items(): - post_headers.update({k: v}) + get_headers.update({k: v}) r = _session.get( url, params=params, auth=auth, timeout=config.get_default('connection_timeout'), - headers=post_headers) + headers=get_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) @@ -150,31 +150,16 @@ def _put_with_qiniu_mac_and_headers(url, data, auth, headers): def _post_with_qiniu_mac(url, data, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth( - auth) if auth is not None else None - timeout = config.get_default('connection_timeout') - try: - r = requests.post( - url, - json=data, - auth=qn_auth, - timeout=timeout, - headers=_headers) - except Exception as e: - return None, ResponseInfo(None, e) - return __return_wrapper(r) + auth + ) if auth is not None else None + return _post(url, data, None, qn_auth) def _get_with_qiniu_mac(url, params, auth): - try: - r = requests.get( - url, - params=params, - auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, - timeout=config.get_default('connection_timeout'), - headers=_headers) - except Exception as e: - return None, ResponseInfo(None, e) - return __return_wrapper(r) + qn_auth = qiniu.auth.QiniuMacRequestsAuth( + auth + ) if auth is not None else None + return _get(url, params, qn_auth) def _get_with_qiniu_mac_and_headers(url, params, auth, headers): diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 2f54260a..187a5433 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from qiniu import config +from qiniu import config, QiniuMacAuth from qiniu import http from qiniu.utils import urlsafe_base64_encode, entry @@ -17,6 +17,7 @@ class BucketManager(object): def __init__(self, auth, zone=None): self.auth = auth + self.mac_auth = QiniuMacAuth(auth.get_access_key(), auth.get_secret_key()) if (zone is None): self.zone = config.get_default('default_zone') else: @@ -387,10 +388,10 @@ def __server_do(self, host, operation, *args): return self.__post(url) def __post(self, url, data=None): - return http._post_with_auth(url, data, self.auth) + return http._post_with_qiniu_mac(url, data, self.mac_auth) def __get(self, url, params=None): - return http._get_with_auth(url, params, self.auth) + return http._get_with_qiniu_mac(url, params, self.mac_auth) def _build_op(*args): From 40f0b8a8cbee8146c2584731d25d2d49863d06bf Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Mon, 28 Mar 2022 10:48:19 +0800 Subject: [PATCH 410/478] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f413e2b..7b85e8db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 7.6.0 (2022-03-28) +* 优化了错误处理机制 +* 支持 [Qiniu](https://developer.qiniu.com/kodo/1201/access-token) 签名算法 + ## 7.5.0 (2021-09-23) * 上传策略新增对部分字段支持 From e71237c570daaa2d7bad98698bfb34e1d86988dd Mon Sep 17 00:00:00 2001 From: Bai Long <slongbai@gmail.com> Date: Mon, 28 Mar 2022 11:58:34 +0800 Subject: [PATCH 411/478] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 7b647f36..5da35d0e 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.5.0' +__version__ = '7.6.0' from .auth import Auth, QiniuMacAuth From bb6f167aecf36e8f8f3a4349da997f02ef57213e Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Wed, 20 Apr 2022 11:58:08 +0800 Subject: [PATCH 412/478] support set object lifecycle --- examples/set_object_lifecycle.py | 34 +++++++++++++++++++++++ qiniu/auth.py | 2 +- qiniu/services/storage/bucket.py | 47 +++++++++++++++++++++++++++++--- test_qiniu.py | 34 +++++++++++++++++++++++ 4 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 examples/set_object_lifecycle.py diff --git a/examples/set_object_lifecycle.py b/examples/set_object_lifecycle.py new file mode 100644 index 00000000..45005458 --- /dev/null +++ b/examples/set_object_lifecycle.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +access_key = 'your_ak' +secret_key = 'your_sk' + +# 初始化 Auth +q = Auth(access_key, secret_key) + +# 初始化 BucketManager +bucket = BucketManager(q) + +# 目标空间 +bucket_name = 'your_bucket_name' +# 目标 key +key = 'path/to/key' + +# bucket_name 更新 rule +ret, info = bucket.set_object_lifecycle( + bucket=bucket_name, + key=key, + to_line_after_days=10, + to_archive_after_days=20, + to_deep_archive_after_days=30, + delete_after_days=40, + cond={ + 'hash': 'object_hash' + } +) +print(ret) +print(info) diff --git a/qiniu/auth.py b/qiniu/auth.py index 5c2819de..100500a9 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -34,7 +34,7 @@ 'persistentNotifyUrl', # 持久化处理结果通知URL 'persistentPipeline', # 持久化处理独享队列 'deleteAfterDays', # 文件多少天后自动删除 - 'fileType', # 文件的存储类型，0为普通存储，1为低频存储 + 'fileType', # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储 'isPrefixalScope' # 指定上传文件必须使用的前缀 ]) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 187a5433..46ec2045 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -219,21 +219,21 @@ def change_mime(self, bucket, key, mime): def change_type(self, bucket, key, storage_type): """修改文件的存储类型 - 修改文件的存储类型为普通存储或者是低频存储，参考文档： + 修改文件的存储类型，参考文档： https://developer.qiniu.com/kodo/api/3710/modify-the-file-type Args: bucket: 待操作资源所在空间 key: 待操作资源文件名 - storage_type: 待操作资源存储类型，0为普通存储，1为低频存储，2 为归档存储 + storage_type: 待操作资源存储类型，0为普通存储，1为低频存储，2 为归档存储，3 为深度归档 """ resource = entry(bucket, key) return self.__rs_do('chtype', resource, 'type/{0}'.format(storage_type)) def restoreAr(self, bucket, key, freezeAfter_days): - """解冻归档存储文件 + """解冻归档存储、深度归档存储文件 - 修改文件的存储类型为普通存储或者是低频存储，参考文档： + 对归档存储、深度归档存储文件，进行解冻操作参考文档： https://developer.qiniu.com/kodo/api/6380/restore-archive Args: @@ -263,6 +263,45 @@ def change_status(self, bucket, key, status, cond): return self.__rs_do('chstatus', resource, 'status/{0}'.format(status), 'cond', condstr) return self.__rs_do('chstatus', resource, 'status/{0}'.format(status)) + def set_object_lifecycle( + self, + bucket, + key, + to_line_after_days=0, + to_archive_after_days=0, + to_deep_archive_after_days=0, + delete_after_days=0, + cond=None + ): + """ + + 设置对象的生命周期 + + Args: + bucket: 目标空间 + key: 目标资源 + to_line_after_days: 多少天后将文件转为低频存储，设置为 -1 表示取消已设置的转低频存储的生命周期规则， 0 表示不修改转低频生命周期规则。 + to_archive_after_days: 多少天后将文件转为归档存储，设置为 -1 表示取消已设置的转归档存储的生命周期规则， 0 表示不修改转归档生命周期规则。 + to_deep_archive_after_days: 多少天后将文件转为深度归档存储，设置为 -1 表示取消已设置的转深度归档存储的生命周期规则， 0 表示不修改转深度归档生命周期规则 + delete_after_days: 多少天后将文件删除，设置为 -1 表示取消已设置的删除存储的生命周期规则， 0 表示不修改删除存储的生命周期规则。 + cond: 匹配条件，只有条件匹配才会设置成功，当前支持设置 hash、mime、fsize、putTime。 + + Returns: + resBody, respInfo + + """ + options = [ + 'toIAAfterDays', str(to_line_after_days), + 'toArchiveAfterDays', str(to_archive_after_days), + 'toDeepArchiveAfterDays', str(to_deep_archive_after_days), + 'deleteAfterDays', str(delete_after_days) + ] + if cond and isinstance(cond, dict): + cond_str = '&'.join(["{0}={1}".format(k, v) for k, v in cond.items()]) + options += ['cond', urlsafe_base64_encode(cond_str)] + resource = entry(bucket, key) + return self.__rs_do('lifecycle', resource, *options) + def batch(self, operations): """批量操作: diff --git a/test_qiniu.py b/test_qiniu.py index 30687f04..8e1552e0 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -460,6 +460,40 @@ def test_delete_after_days(self): ret, info = self.bucket.delete_after_days(bucket_name, key, days) assert info.status_code == 200 + def test_set_object_lifecycle(self): + key = 'test_set_object_lifecycle' + rand_string(8) + ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + assert info.status_code == 200 + ret, info = self.bucket.set_object_lifecycle( + bucket=bucket_name, + key=key, + to_line_after_days=10, + to_archive_after_days=20, + to_deep_archive_after_days=30, + delete_after_days=40 + ) + assert info.status_code == 200 + + def test_set_object_lifecycle_with_cond(self): + key = 'test_set_object_lifecycle_cond' + rand_string(8) + ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + assert info.status_code == 200 + ret, info = self.bucket.stat(bucket_name, key) + assert info.status_code == 200 + key_hash = ret['hash'] + ret, info = self.bucket.set_object_lifecycle( + bucket=bucket_name, + key=key, + to_line_after_days=10, + to_archive_after_days=20, + to_deep_archive_after_days=30, + delete_after_days=40, + cond={ + 'hash': key_hash + } + ) + assert info.status_code == 200 + class UploaderTestCase(unittest.TestCase): mime_type = "text/plain" From 072b350d5a4163d567f7cf8be4cdda5074bb9ed0 Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Fri, 29 Apr 2022 15:40:02 +0800 Subject: [PATCH 413/478] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b85e8db..349d87cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 7.7.0 (2022-04-29) +* 对象存储，新增 set_object_lifecycle （设置 Object 生命周期） API + ## 7.6.0 (2022-03-28) * 优化了错误处理机制 * 支持 [Qiniu](https://developer.qiniu.com/kodo/1201/access-token) 签名算法 From 1a81a25cd4d33261fd7f1e4484a3f12e4afaedea Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Fri, 29 Apr 2022 15:41:02 +0800 Subject: [PATCH 414/478] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 5da35d0e..bb1513e6 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.6.0' +__version__ = '7.7.0' from .auth import Auth, QiniuMacAuth From 6034ce5c03aa0e25e80eb1fec962e62e29f85f0d Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Tue, 10 May 2022 13:31:47 +0800 Subject: [PATCH 415/478] fix upload v2 with key=None --- qiniu/services/storage/uploader.py | 2 +- test_qiniu.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 03729790..c596393d 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -337,7 +337,7 @@ def block_url(self, host, size): return '{0}/mkblk/{1}'.format(host, size) def block_url_v2(self, host, bucket_name): - encoded_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '～' + encoded_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '~' return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, encoded_object_name) def file_url(self, host): diff --git a/test_qiniu.py b/test_qiniu.py index 8e1552e0..f0d64eac 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -663,6 +663,20 @@ def test_put_10m_stream_v2(self): assert ret['key'] == key remove_temp_file(localfile) + def test_put_stream_v2_without_key(self): + part_size = 1024 * 1024 * 4 + localfile = create_temp_file(part_size + 1) + key = None + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, + self.mime_type, part_size=part_size, version='v2', bucket_name=bucket_name) + assert ret['key'] == ret['hash'] + remove_temp_file(localfile) + def test_big_file(self): key = 'big' token = self.q.upload_token(bucket_name, key) From 3fbe52afb8e48d545cd106040da493d14d95306f Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Tue, 10 May 2022 14:17:52 +0800 Subject: [PATCH 416/478] change tests to use https --- test_qiniu.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index f0d64eac..2db63081 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -148,9 +148,9 @@ def test_noKey(self): Auth('', '').token('nokey') def test_token_of_request(self): - token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', '') + token = dummy_auth.token_of_request('https://www.qiniu.com?go=1', 'test', '') assert token == 'abcdefghklmnopq:cFyRVoWrE3IugPIMP5YJFTO-O-Y=' - token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', 'application/x-www-form-urlencoded') + token = dummy_auth.token_of_request('https://www.qiniu.com?go=1', 'test', 'application/x-www-form-urlencoded') assert token == 'abcdefghklmnopq:svWRNcacOE-YMsc70nuIYdaa1e4=' def test_QiniuMacRequestsAuth(self): @@ -554,7 +554,7 @@ def test_putWithoutKey(self): def test_withoutRead_withoutSeek_retry(self): key = 'retry' data = 'hello retry!' - set_default(default_zone=Zone('http://a', 'http://upload.qiniup.com')) + set_default(default_zone=Zone('http://a', 'https://upload.qiniup.com')) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) @@ -604,7 +604,7 @@ def test_put_stream(self): localfile = __file__ key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, @@ -616,7 +616,7 @@ def test_put_stream_v2_without_bucket_name(self): localfile = __file__ key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, @@ -628,7 +628,7 @@ def test_put_2m_stream_v2(self): localfile = create_temp_file(2 * 1024 * 1024 + 1) key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -641,7 +641,7 @@ def test_put_4m_stream_v2(self): localfile = create_temp_file(4 * 1024 * 1024) key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -654,7 +654,7 @@ def test_put_10m_stream_v2(self): localfile = create_temp_file(10 * 1024 * 1024 + 1) key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -668,7 +668,7 @@ def test_put_stream_v2_without_key(self): localfile = create_temp_file(part_size + 1) key = None size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -682,7 +682,7 @@ def test_big_file(self): token = self.q.upload_token(bucket_name, key) localfile = create_temp_file(4 * 1024 * 1024 + 1) progress_handler = lambda progress, total: progress - qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniup.com')) + qiniu.set_default(default_zone=Zone('http://a', 'https://upload.qiniup.com')) ret, info = put_file(token, key, localfile, self.params, self.mime_type, progress_handler=progress_handler) print(info) assert ret['key'] == key @@ -691,7 +691,7 @@ def test_big_file(self): def test_retry(self): localfile = __file__ key = 'test_file_r_retry' - qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniup.com')) + qiniu.set_default(default_zone=Zone('http://a', 'https://upload.qiniup.com')) token = self.q.upload_token(bucket_name, key) ret, info = put_file(token, key, localfile, self.params, self.mime_type) print(info) @@ -702,7 +702,7 @@ def test_put_stream_with_key_limits(self): localfile = __file__ key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key, policy={'keylimit': ['test_file_d']}) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, From bfcfc308cd12c24e7c549e40a517b4355ce245e6 Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Wed, 11 May 2022 10:53:25 +0800 Subject: [PATCH 417/478] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index bb1513e6..058ba0d1 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.7.0' +__version__ = '7.7.1' from .auth import Auth, QiniuMacAuth From 852a32e2fce16781973ce293bb3f978a130c90b6 Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Wed, 11 May 2022 10:55:37 +0800 Subject: [PATCH 418/478] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 349d87cc..676be882 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 7.7.1 (2022-05-11) +* 对象存储，修复上传不制定 key 部分情况下会上传失败问题。 + ## 7.7.0 (2022-04-29) * 对象存储，新增 set_object_lifecycle （设置 Object 生命周期） API From af1485cc22f493ae70b7d11ec7725d9b7481b9ce Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Thu, 26 May 2022 15:21:15 +0800 Subject: [PATCH 419/478] upload can add metadata --- qiniu/services/storage/uploader.py | 52 ++++++++++++++--------- test_qiniu.py | 68 ++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 19 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index c596393d..f453160b 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -12,7 +12,7 @@ def put_data( up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, - fname=None, hostscache_dir=None): + fname=None, hostscache_dir=None, metadata=None): """上传二进制流到七牛 Args: @@ -23,7 +23,8 @@ def put_data( mime_type: 上传数据的mimeType check_crc: 是否校验crc32 progress_handler: 上传进度 - hostscache_dir： host请求 缓存文件保存位置 + hostscache_dir: host请求 缓存文件保存位置 + metadata: 元数据 Returns: 一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"} @@ -41,13 +42,14 @@ def put_data( final_data = data crc = crc32(final_data) - return _form_put(up_token, key, final_data, params, mime_type, crc, hostscache_dir, progress_handler, fname) + return _form_put(up_token, key, final_data, params, mime_type, + crc, hostscache_dir, progress_handler, fname, metadata=metadata) def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None, - part_size=None, version=None, bucket_name=None): + part_size=None, version=None, bucket_name=None, metadata=None): """上传文件到七牛 Args: @@ -59,10 +61,11 @@ def put_file(up_token, key, file_path, params=None, check_crc: 是否校验crc32 progress_handler: 上传进度 upload_progress_recorder: 记录上传进度，用于断点续传 - hostscache_dir： host请求 缓存文件保存位置 - version 分片上传版本 目前支持v1/v2版本 默认v1 - part_size 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB - bucket_name 分片上传v2字段 空间名称 + hostscache_dir: host请求 缓存文件保存位置 + version: 分片上传版本 目前支持v1/v2版本 默认v1 + part_size: 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB + bucket_name: 分片上传v2字段 空间名称 + metadata: 元数据信息 Returns: 一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"} @@ -78,18 +81,17 @@ def put_file(up_token, key, file_path, params=None, mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, modify_time=modify_time, keep_last_modified=keep_last_modified, - part_size=part_size, version=version, bucket_name=bucket_name) + part_size=part_size, version=version, bucket_name=bucket_name, metadata=metadata) else: crc = file_crc32(file_path) ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, hostscache_dir, progress_handler, file_name, - modify_time=modify_time, keep_last_modified=keep_last_modified) + modify_time=modify_time, keep_last_modified=keep_last_modified, metadata=metadata) return ret, info def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, progress_handler=None, file_name=None, - modify_time=None, - keep_last_modified=False): + modify_time=None, keep_last_modified=False, metadata=None): fields = {} if params: for k, v in params.items(): @@ -114,6 +116,11 @@ def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, if modify_time and keep_last_modified: fields['x-qn-meta-!Last-Modified'] = rfc_from_timestamp(modify_time) + if metadata: + for k, v in metadata.items(): + if k.startswith('x-qn-meta-'): + fields[k] = str(v) + r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) if r is None and info.need_retry(): if info.connect_failed: @@ -135,10 +142,10 @@ def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir=None, params=None, mime_type=None, progress_handler=None, upload_progress_recorder=None, modify_time=None, keep_last_modified=False, - part_size=None, version=None, bucket_name=None): + part_size=None, version=None, bucket_name=None, metadata=None): task = _Resume(up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, progress_handler, upload_progress_recorder, modify_time, keep_last_modified, - part_size, version, bucket_name) + part_size, version, bucket_name, metadata) return task.upload() @@ -167,7 +174,7 @@ class _Resume(object): def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, progress_handler, upload_progress_recorder, modify_time, keep_last_modified, part_size=None, - version=None, bucket_name=None): + version=None, bucket_name=None, metadata=None): """初始化断点续上传""" self.up_token = up_token self.key = key @@ -184,6 +191,7 @@ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache self.version = version or 'v1' self.part_size = part_size or config._BLOCK_SIZE self.bucket_name = bucket_name + self.metadata = metadata def record_upload_progress(self, offset): record_data = { @@ -294,9 +302,9 @@ def upload(self): elif self.version == 'v2': make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % self.uploadId return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, - self.mime_type, self.params) + self.mime_type, self.params, self.metadata) - def make_file_v2(self, block_status, url, file_name=None, mime_type=None, customVars=None): + def make_file_v2(self, block_status, url, file_name=None, mime_type=None, customVars=None, metadata=None): """completeMultipartUpload""" parts = self.get_parts(block_status) headers = { @@ -306,7 +314,8 @@ def make_file_v2(self, block_status, url, file_name=None, mime_type=None, custom 'parts': parts, 'fname': file_name, 'mimeType': mime_type, - 'customVars': customVars + 'customVars': customVars, + 'metadata': metadata } ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) if ret is not None and ret != {}: @@ -354,12 +363,17 @@ def file_url(self, host): if self.params: for k, v in self.params.items(): url.append('{0}/{1}'.format(k, urlsafe_base64_encode(v))) - pass if self.modify_time and self.keep_last_modified: url.append( "x-qn-meta-!Last-Modified/{0}".format(urlsafe_base64_encode(rfc_from_timestamp(self.modify_time)))) + if self.metadata: + for k, v in self.metadata.items(): + if k.startswith('x-qn-meta-'): + url.append( + "{0}/{1}".format(k, urlsafe_base64_encode(v))) + url = '/'.join(url) return url diff --git a/test_qiniu.py b/test_qiniu.py index 2db63081..d8375c15 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -498,7 +498,12 @@ def test_set_object_lifecycle_with_cond(self): class UploaderTestCase(unittest.TestCase): mime_type = "text/plain" params = {'x:a': 'a'} + metadata = { + 'x-qn-meta-name': 'qiniu', + 'x-qn-meta-age': '18' + } q = Auth(access_key, secret_key) + bucket = BucketManager(q) def test_put(self): key = 'a\\b\\c"hello' @@ -594,11 +599,40 @@ def test_putData_without_fname2(self): print(info) assert ret is not None + def test_put_file_with_metadata(self): + localfile = __file__ + key = 'test_file_with_metadata' + + token = self.q.upload_token(bucket_name, key) + ret, info = put_file(token, key, localfile, metadata=self.metadata) + assert ret['key'] == key + assert ret['hash'] == etag(localfile) + ret, info = self.bucket.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + + def test_put_data_with_metadata(self): + key = 'put_data_with_metadata' + data = 'hello metadata!' + token = self.q.upload_token(bucket_name, key) + ret, info = put_data(token, key, data, metadata=self.metadata) + assert ret['key'] == key + ret, info = self.bucket.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + class ResumableUploaderTestCase(unittest.TestCase): mime_type = "text/plain" params = {'x:a': 'a'} + metadata = { + 'x-qn-meta-name': 'qiniu', + 'x-qn-meta-age': '18' + } q = Auth(access_key, secret_key) + bucket = BucketManager(q) def test_put_stream(self): localfile = __file__ @@ -715,6 +749,40 @@ def test_put_stream_with_key_limits(self): self.mime_type) assert info.status_code == 200 + def test_put_stream_with_metadata(self): + localfile = __file__ + key = 'test_put_stream_with_metadata' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('https://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + self.params, self.mime_type, + part_size=None, version=None, bucket_name=None, metadata=self.metadata) + assert ret['key'] == key + ret, info = self.bucket.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + + def test_put_stream_v2_with_metadata(self): + part_size = 1024 * 1024 * 4 + localfile = create_temp_file(part_size + 1) + key = 'test_put_stream_v2_with_metadata' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('https://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, self.mime_type, + part_size=part_size, version='v2', bucket_name=bucket_name, metadata=self.metadata) + assert ret['key'] == key + remove_temp_file(localfile) + ret, info = self.bucket.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + class DownloadTestCase(unittest.TestCase): q = Auth(access_key, secret_key) From c4cdd53758f0c0c9759e4686bfbe7663cc02577e Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Wed, 18 May 2022 14:45:20 +0800 Subject: [PATCH 420/478] add X-Qiniu-Date --- .github/workflows/ci-test.yml | 2 +- qiniu/auth.py | 6 ++++-- qiniu/http.py | 23 +++++++++++++++++++++-- qiniu/services/storage/bucket.py | 5 ++++- test_qiniu.py | 31 +++++++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index f331727d..2614bb3e 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -21,7 +21,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest pytest-cov requests scrutinizer-ocular codecov + pip install flake8 pytest pytest-cov freezegun requests scrutinizer-ocular codecov - name: Run cases env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} diff --git a/qiniu/auth.py b/qiniu/auth.py index 100500a9..a1466564 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -49,11 +49,12 @@ class Auth(object): __secret_key: 账号密钥对重的secretKey，详见 https://portal.qiniu.com/user/key """ - def __init__(self, access_key, secret_key): + def __init__(self, access_key, secret_key, disable_qiniu_timestamp_signature=None): """初始化Auth类""" self.__checkKey(access_key, secret_key) self.__access_key = access_key self.__secret_key = b(secret_key) + self.disable_qiniu_timestamp_signature = disable_qiniu_timestamp_signature def get_access_key(self): return self.__access_key @@ -229,11 +230,12 @@ class QiniuMacAuth(object): http://kirk-docs.qiniu.com/apidocs/#TOC_325b437b89e8465e62e958cccc25c63f """ - def __init__(self, access_key, secret_key): + def __init__(self, access_key, secret_key, disable_qiniu_timestamp_signature=None): self.qiniu_header_prefix = "X-Qiniu-" self.__checkKey(access_key, secret_key) self.__access_key = access_key self.__secret_key = b(secret_key) + self.disable_qiniu_timestamp_signature = disable_qiniu_timestamp_signature def __token(self, data): data = b(data) diff --git a/qiniu/http.py b/qiniu/http.py index b12ac8ad..6d239808 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import logging +import os import platform +from datetime import datetime import requests from requests.auth import AuthBase @@ -20,6 +22,19 @@ _headers = {'User-Agent': USER_AGENT} +def __add_auth_headers(headers, auth): + x_qiniu_date = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') + if auth.disable_qiniu_timestamp_signature is not None: + if not auth.disable_qiniu_timestamp_signature: + headers['X-Qiniu-Date'] = x_qiniu_date + elif os.getenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE'): + if os.getenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE').lower() != 'true': + headers['X-Qiniu-Date'] = x_qiniu_date + else: + headers['X-Qiniu-Date'] = x_qiniu_date + return headers + + def __return_wrapper(resp): if resp.status_code != 200 or resp.headers.get('X-Reqid') is None: return None, ResponseInfo(resp) @@ -155,14 +170,18 @@ def _post_with_qiniu_mac(url, data, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth( auth ) if auth is not None else None - return _post(url, data, None, qn_auth) + headers = __add_auth_headers({}, auth) + + return _post(url, data, None, qn_auth, headers=headers) def _get_with_qiniu_mac(url, params, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth( auth ) if auth is not None else None - return _get(url, params, qn_auth) + headers = __add_auth_headers({}, auth) + + return _get(url, params, qn_auth, headers=headers) def _get_with_qiniu_mac_and_headers(url, params, auth, headers): diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 46ec2045..507e5977 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -17,7 +17,10 @@ class BucketManager(object): def __init__(self, auth, zone=None): self.auth = auth - self.mac_auth = QiniuMacAuth(auth.get_access_key(), auth.get_secret_key()) + self.mac_auth = QiniuMacAuth( + auth.get_access_key(), + auth.get_secret_key(), + auth.disable_qiniu_timestamp_signature) if (zone is None): self.zone = config.get_default('default_zone') else: diff --git a/test_qiniu.py b/test_qiniu.py index d8375c15..513723be 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -10,6 +10,7 @@ import unittest import pytest +from freezegun import freeze_time from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone, QiniuMacAuth from qiniu import put_data, put_file, put_stream @@ -494,6 +495,36 @@ def test_set_object_lifecycle_with_cond(self): ) assert info.status_code == 200 + @freeze_time("1970-01-01") + def test_invalid_x_qiniu_date(self): + ret, info = self.bucket.stat(bucket_name, 'python-sdk.html') + assert ret is None + assert info.status_code == 403 + + @freeze_time("1970-01-01") + def test_invalid_x_qiniu_date_with_disable_date_sign(self): + q = Auth(access_key, secret_key, disable_qiniu_timestamp_signature=True) + bucket = BucketManager(q) + ret, info = bucket.stat(bucket_name, 'python-sdk.html') + assert 'hash' in ret + + @freeze_time("1970-01-01") + def test_invalid_x_qiniu_date_env(self): + os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = 'True' + ret, info = self.bucket.stat(bucket_name, 'python-sdk.html') + os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE') + assert 'hash' in ret + + @freeze_time("1970-01-01") + def test_invalid_x_qiniu_date_env_be_ignored(self): + os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = 'True' + q = Auth(access_key, secret_key, disable_qiniu_timestamp_signature=False) + bucket = BucketManager(q) + ret, info = bucket.stat(bucket_name, 'python-sdk.html') + os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE') + assert ret is None + assert info.status_code == 403 + class UploaderTestCase(unittest.TestCase): mime_type = "text/plain" From 901016a684a89edd0e10fb2032faa0d7f485bb1d Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Thu, 9 Jun 2022 10:44:11 +0800 Subject: [PATCH 421/478] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 676be882..d094496e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog + +## 7.8.0（2022-06-08） +* 对象存储，管理类 API 发送请求时增加 [X-Qiniu-Date](https://developer.qiniu.com/kodo/3924/common-request-headers) （生成请求的时间） header + ## 7.7.1 (2022-05-11) * 对象存储，修复上传不制定 key 部分情况下会上传失败问题。 From 92f44e80fbfdca7e14984c3faaf1178902f2619f Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Thu, 9 Jun 2022 10:44:50 +0800 Subject: [PATCH 422/478] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 058ba0d1..d785510b 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.7.1' +__version__ = '7.8.0' from .auth import Auth, QiniuMacAuth From 939ed7f599bbfed12104607931be2f79cd6dadf9 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Thu, 23 Jun 2022 17:58:44 +0800 Subject: [PATCH 423/478] fix not query host --- qiniu/region.py | 55 ++++++++++++++++++++++++++------ qiniu/services/storage/bucket.py | 36 +++++++++++---------- 2 files changed, 65 insertions(+), 26 deletions(-) diff --git a/qiniu/region.py b/qiniu/region.py index 9f8db051..24c351d9 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -21,9 +21,16 @@ def __init__( io_host=None, host_cache={}, home_dir=None, - scheme="http"): + scheme="http", + rs_host=None, + rsf_host=None): """初始化Zone类""" - self.up_host, self.up_host_backup, self.io_host, self.home_dir = up_host, up_host_backup, io_host, home_dir + self.up_host = up_host + self.up_host_backup = up_host_backup + self.io_host = io_host + self.rs_host = rs_host + self.rsf_host = rsf_host + self.home_dir = home_dir self.host_cache = host_cache self.scheme = scheme @@ -39,23 +46,49 @@ def get_up_host_backup_by_token(self, up_token, home_dir): if home_dir is None: home_dir = os.getcwd() up_hosts = self.get_up_host(ak, bucket, home_dir) - if (len(up_hosts) <= 1): + if len(up_hosts) <= 1: up_host = up_hosts[0] else: up_host = up_hosts[1] return up_host - def get_io_host(self, ak, bucket, home_dir): + def get_io_host(self, ak, bucket, home_dir=None): if self.io_host: return self.io_host if home_dir is None: home_dir = os.getcwd() bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) + if 'ioHosts' not in bucket_hosts: + bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir, force=True) io_hosts = bucket_hosts['ioHosts'] return io_hosts[0] + def get_rs_host(self, ak, bucket, home_dir=None): + if self.rs_host: + return self.rs_host + if home_dir is None: + home_dir = os.getcwd() + bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) + if 'rsHosts' not in bucket_hosts: + bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir, force=True) + rs_hosts = bucket_hosts['rsHosts'] + return rs_hosts[0] + + def get_rsf_host(self, ak, bucket, home_dir=None): + if self.rsf_host: + return self.rsf_host + if home_dir is None: + home_dir = os.getcwd() + bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) + if 'rsfHosts' not in bucket_hosts: + bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir, force=True) + rsf_hosts = bucket_hosts['rsfHosts'] + return rsf_hosts[0] + def get_up_host(self, ak, bucket, home_dir): bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) + if 'upHosts' not in bucket_hosts: + bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir, force=True) up_hosts = bucket_hosts['upHosts'] return up_hosts @@ -77,11 +110,11 @@ def unmarshal_up_token(self, up_token): return ak, bucket - def get_bucket_hosts(self, ak, bucket, home_dir): + def get_bucket_hosts(self, ak, bucket, home_dir, force=False): key = self.scheme + ":" + ak + ":" + bucket bucket_hosts = self.get_bucket_hosts_to_cache(key, home_dir) - if (len(bucket_hosts) > 0): + if not force and len(bucket_hosts) > 0: return bucket_hosts hosts = {} @@ -114,6 +147,8 @@ def get_bucket_hosts(self, ak, bucket, home_dir): bucket_hosts = { 'upHosts': scheme_hosts['up'], 'ioHosts': scheme_hosts['io'], + 'rsHosts': scheme_hosts['rs'], + 'rsfHosts': scheme_hosts['rsf'], 'deadline': int(time.time()) + hosts['ttl'] } home_dir = "" @@ -121,17 +156,17 @@ def get_bucket_hosts(self, ak, bucket, home_dir): return bucket_hosts def get_bucket_hosts_to_cache(self, key, home_dir): - ret = [] - if (len(self.host_cache) == 0): + ret = {} + if len(self.host_cache) == 0: self.host_cache_from_file(home_dir) if self.host_cache == {}: return ret - if (not (key in self.host_cache)): + if key not in self.host_cache: return ret - if (self.host_cache[key]['deadline'] > time.time()): + if self.host_cache[key]['deadline'] > time.time(): ret = self.host_cache[key] return ret diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 507e5977..3802ac3f 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -59,7 +59,9 @@ def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): if delimiter is not None: options['delimiter'] = delimiter - url = '{0}/list'.format(config.get_default('default_rsf_host')) + ak = self.auth.get_access_key() + rs_host = self.zone.get_rsf_host(ak, bucket) + url = '{0}/list'.format(rs_host) ret, info = self.__get(url, options) eof = False @@ -90,7 +92,7 @@ def stat(self, bucket, key): 一个ResponseInfo对象 """ resource = entry(bucket, key) - return self.__rs_do('stat', resource) + return self.__rs_do(bucket, 'stat', resource) def delete(self, bucket, key): """删除文件: @@ -107,7 +109,7 @@ def delete(self, bucket, key): 一个ResponseInfo对象 """ resource = entry(bucket, key) - return self.__rs_do('delete', resource) + return self.__rs_do(bucket, 'delete', resource) def rename(self, bucket, key, key_to, force='false'): """重命名文件: @@ -143,7 +145,7 @@ def move(self, bucket, key, bucket_to, key_to, force='false'): """ resource = entry(bucket, key) to = entry(bucket_to, key_to) - return self.__rs_do('move', resource, to, 'force/{0}'.format(force)) + return self.__rs_do(bucket, 'move', resource, to, 'force/{0}'.format(force)) def copy(self, bucket, key, bucket_to, key_to, force='false'): """复制文件: @@ -163,7 +165,7 @@ def copy(self, bucket, key, bucket_to, key_to, force='false'): """ resource = entry(bucket, key) to = entry(bucket_to, key_to) - return self.__rs_do('copy', resource, to, 'force/{0}'.format(force)) + return self.__rs_do(bucket, 'copy', resource, to, 'force/{0}'.format(force)) def fetch(self, url, bucket, key=None, hostscache_dir=None): """抓取文件: @@ -217,7 +219,7 @@ def change_mime(self, bucket, key, mime): """ resource = entry(bucket, key) encode_mime = urlsafe_base64_encode(mime) - return self.__rs_do('chgm', resource, 'mime/{0}'.format(encode_mime)) + return self.__rs_do(bucket, 'chgm', resource, 'mime/{0}'.format(encode_mime)) def change_type(self, bucket, key, storage_type): """修改文件的存储类型 @@ -231,7 +233,7 @@ def change_type(self, bucket, key, storage_type): storage_type: 待操作资源存储类型，0为普通存储，1为低频存储，2 为归档存储，3 为深度归档 """ resource = entry(bucket, key) - return self.__rs_do('chtype', resource, 'type/{0}'.format(storage_type)) + return self.__rs_do(bucket, 'chtype', resource, 'type/{0}'.format(storage_type)) def restoreAr(self, bucket, key, freezeAfter_days): """解冻归档存储、深度归档存储文件 @@ -245,7 +247,7 @@ def restoreAr(self, bucket, key, freezeAfter_days): freezeAfter_days: 解冻有效时长，取值范围 1～7 """ resource = entry(bucket, key) - return self.__rs_do('restoreAr', resource, 'freezeAfterDays/{0}'.format(freezeAfter_days)) + return self.__rs_do(bucket, 'restoreAr', resource, 'freezeAfterDays/{0}'.format(freezeAfter_days)) def change_status(self, bucket, key, status, cond): """修改文件的状态 @@ -263,8 +265,8 @@ def change_status(self, bucket, key, status, cond): for k, v in cond.items(): condstr += "{0}={1}&".format(k, v) condstr = urlsafe_base64_encode(condstr[:-1]) - return self.__rs_do('chstatus', resource, 'status/{0}'.format(status), 'cond', condstr) - return self.__rs_do('chstatus', resource, 'status/{0}'.format(status)) + return self.__rs_do(bucket, 'chstatus', resource, 'status/{0}'.format(status), 'cond', condstr) + return self.__rs_do(bucket, 'chstatus', resource, 'status/{0}'.format(status)) def set_object_lifecycle( self, @@ -303,7 +305,7 @@ def set_object_lifecycle( cond_str = '&'.join(["{0}={1}".format(k, v) for k, v in cond.items()]) options += ['cond', urlsafe_base64_encode(cond_str)] resource = entry(bucket, key) - return self.__rs_do('lifecycle', resource, *options) + return self.__rs_do(bucket, 'lifecycle', resource, *options) def batch(self, operations): """批量操作: @@ -339,7 +341,7 @@ def buckets(self): [ <Bucket1>, <Bucket2>, ... ] 一个ResponseInfo对象 """ - return self.__rs_do('buckets') + return self.__uc_do('buckets') def delete_after_days(self, bucket, key, days): """更新文件生命周期 @@ -361,7 +363,7 @@ def delete_after_days(self, bucket, key, days): days: 指定天数 """ resource = entry(bucket, key) - return self.__rs_do('deleteAfterDays', resource, days) + return self.__rs_do(bucket, 'deleteAfterDays', resource, days) def mkbucketv3(self, bucket_name, region): """ @@ -371,7 +373,7 @@ def mkbucketv3(self, bucket_name, region): bucket_name: 存储空间名 region: 存储区域 """ - return self.__rs_do('mkbucketv3', bucket_name, 'region', region) + return self.__uc_do('mkbucketv3', bucket_name, 'region', region) def list_bucket(self, region): """ @@ -416,8 +418,10 @@ def change_bucket_permission(self, bucket_name, private): def __uc_do(self, operation, *args): return self.__server_do(config.get_default('default_uc_host'), operation, *args) - def __rs_do(self, operation, *args): - return self.__server_do(config.get_default('default_rs_host'), operation, *args) + def __rs_do(self, bucket, operation, *args): + ak = self.auth.get_access_key() + rs_host = self.zone.get_rs_host(ak, bucket) + return self.__server_do(rs_host, operation, *args) def __io_do(self, bucket, operation, home_dir, *args): ak = self.auth.get_access_key() From 24585315bf97f42ff6686038df649b73429bc3bd Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Thu, 12 May 2022 16:54:03 +0800 Subject: [PATCH 424/478] fix resume upload v2 customize return body without hash and key --- qiniu/services/storage/uploader.py | 5 ++--- test_qiniu.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index f453160b..e68f823f 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -318,9 +318,8 @@ def make_file_v2(self, block_status, url, file_name=None, mime_type=None, custom 'metadata': metadata } ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) - if ret is not None and ret != {}: - if ret['hash'] and ret['key']: - self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) + if info.status_code == 200: + self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) return ret, info def get_up_host(self): diff --git a/test_qiniu.py b/test_qiniu.py index 513723be..b326df9c 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -742,6 +742,21 @@ def test_put_stream_v2_without_key(self): assert ret['key'] == ret['hash'] remove_temp_file(localfile) + def test_put_stream_v2_with_empty_return_body(self): + part_size = 1024 * 1024 * 4 + localfile = create_temp_file(part_size + 1) + key = 'test_file_empty_return_body' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('https://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key, policy={'returnBody': ' '}) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, + self.mime_type, part_size=part_size, version='v2', bucket_name=bucket_name) + assert info.status_code == 200 + assert ret == {} + remove_temp_file(localfile) + def test_big_file(self): key = 'big' token = self.q.upload_token(bucket_name, key) From 575f6d93438ecb77acf143392fb77aeb459c67eb Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Tue, 31 May 2022 17:50:19 +0800 Subject: [PATCH 425/478] bucket manager add list_domains --- examples/list_domains.py | 21 +++++++++++++++++++++ qiniu/services/storage/bucket.py | 13 +++++++++++++ test_qiniu.py | 6 ++++++ 3 files changed, 40 insertions(+) create mode 100755 examples/list_domains.py diff --git a/examples/list_domains.py b/examples/list_domains.py new file mode 100755 index 00000000..3bbff58c --- /dev/null +++ b/examples/list_domains.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth +from qiniu import BucketManager + +access_key = '...' +secret_key = '...' + +# 初始化Auth状态 +q = Auth(access_key, secret_key) + +# 初始化BucketManager +bucket = BucketManager(q) + +# 要获取域名的空间名 +bucket_name = 'Bucket_Name' + +# 获取空间绑定的域名列表 +ret, info = bucket.list_domains(bucket_name) +print(ret) +print(info) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 3802ac3f..35b6d677 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -70,6 +70,19 @@ def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): return ret, eof, info + def list_domains(self, bucket): + """获取 Bucket 空间域名 + https://developer.qiniu.com/kodo/3949/get-the-bucket-space-domain + + Args: + bucket: 空间名 + + Returns: + resBody, respInfo + resBody 为绑定的域名列表，格式：["example.com"] + """ + return self.__uc_do('v2/domains?tbl={0}'.format(bucket)) + def stat(self, bucket, key): """获取文件信息: diff --git a/test_qiniu.py b/test_qiniu.py index b326df9c..f40cd83e 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -495,6 +495,12 @@ def test_set_object_lifecycle_with_cond(self): ) assert info.status_code == 200 + def test_list_domains(self): + ret, info = self.bucket.list_domains(bucket_name) + print(info) + assert info.status_code == 200 + assert isinstance(ret, list) + @freeze_time("1970-01-01") def test_invalid_x_qiniu_date(self): ret, info = self.bucket.stat(bucket_name, 'python-sdk.html') From 286a286251c48c2025a14ea2e7a3aff6c6b7ad30 Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Wed, 20 Jul 2022 14:15:42 +0800 Subject: [PATCH 426/478] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index d785510b..7cceb8d0 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.8.0' +__version__ = '7.9.0' from .auth import Auth, QiniuMacAuth From 406ce34db876ecd7ae70f29ccf0d66c851c6d88d Mon Sep 17 00:00:00 2001 From: ZhaoMei <Mei-Zhao@users.noreply.github.com> Date: Wed, 20 Jul 2022 15:06:23 +0800 Subject: [PATCH 427/478] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d094496e..4d8c8781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 7.9.0（2022-07-20） +* 对象存储，支持使用时不配置区域信息，SDK 自动获取； +* 对象存储，新增 list_domains API 用于查询空间绑定的域名 +* 对象存储，上传 API 新增支持设置自定义元数据，详情见 put_data, put_file, put_stream API +* 解决部分已知问题 ## 7.8.0（2022-06-08） * 对象存储，管理类 API 发送请求时增加 [X-Qiniu-Date](https://developer.qiniu.com/kodo/3924/common-request-headers) （生成请求的时间） header From 1a3ec25a0ed063041e9596f40e08e46a4f3bfec4 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Fri, 29 Jul 2022 18:53:49 +0800 Subject: [PATCH 428/478] export upload recorder to qiniu --- qiniu/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 7cceb8d0..851d7729 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -20,6 +20,7 @@ from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ build_batch_stat, build_batch_delete, build_batch_restoreAr from .services.storage.uploader import put_data, put_file, put_stream +from .services.storage.upload_progress_recorder import UploadProgressRecorder from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url, DomainManager from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save From 6e4132829ce9c7adcdeaebbd56c02b3f2ae0956a Mon Sep 17 00:00:00 2001 From: xwen-winnie <657844267@qq.com> Date: Tue, 6 Sep 2022 17:00:52 +0800 Subject: [PATCH 429/478] off codecov patch --- codecov.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..3f36c50a --- /dev/null +++ b/codecov.yml @@ -0,0 +1,28 @@ +codecov: + ci: + - prow.qiniu.io # prow 里面运行需添加，其他 CI 不要 + require_ci_to_pass: no # 改为 no，否则 codecov 会等待其他 GitHub 上所有 CI 通过才会留言。 + +github_checks: #关闭github checks + annotations: false + +comment: + layout: "reach, diff, flags, files" + behavior: new # 默认是更新旧留言，改为 new，删除旧的，增加新的。 + require_changes: false # if true: only post the comment if coverage changes + require_base: no # [yes :: must have a base report to post] + require_head: yes # [yes :: must have a head report to post] + branches: # branch names that can post comment + - "master" + +coverage: + status: # 评判 pr 通过的标准 + patch: off + project: # project 统计所有代码x + default: + # basic + target: 73.5% # 总体通过标准 + threshold: 3% # 允许单次下降的幅度 + base: auto + if_not_found: success + if_ci_failed: error From 29ee67ab88067b4751214b6ac7f13cd350f6dc95 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Mon, 22 Aug 2022 17:37:17 +0800 Subject: [PATCH 430/478] support retry expired parts with upload --- .../storage/upload_progress_recorder.py | 5 +- qiniu/services/storage/uploader.py | 179 +++++++++++++----- 2 files changed, 135 insertions(+), 49 deletions(-) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index 83a0dd65..5be466d8 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -66,4 +66,7 @@ def delete_upload_record(self, file_name, key): record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() upload_record_file_path = os.path.join(self.record_folder, record_file_name) - os.remove(upload_record_file_path) + try: + os.remove(upload_record_file_path) + except OSError: + pass diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index e68f823f..81e23d29 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -182,6 +182,7 @@ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache self.file_name = file_name self.size = data_size self.hostscache_dir = hostscache_dir + self.blockStatus = [] self.params = params self.mime_type = mime_type self.progress_handler = progress_handler @@ -199,7 +200,12 @@ def record_upload_progress(self, offset): 'offset': offset, } if self.version == 'v1': - record_data['contexts'] = [block['ctx'] for block in self.blockStatus] + record_data['contexts'] = [ + { + 'ctx': block['ctx'], + 'expired_at': block['expired_at'] if 'expired_at' in block else 0 + } for block in self.blockStatus + ] elif self.version == 'v2': record_data['etags'] = self.blockStatus record_data['expired_at'] = self.expiredAt @@ -230,7 +236,11 @@ def recovery_from_record(self): if self.version == 'v1': if not record.__contains__('contexts') or len(record['contexts']) == 0: return 0 - self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] + self.blockStatus = [ + # 兼容旧版本的 ctx 持久化 + ctx if type(ctx) is dict else {'ctx': ctx, 'expired_at': 0} + for ctx in record['contexts'] + ] return record['offset'] elif self.version == 'v2': if not record.__contains__('etags') or len(record['etags']) == 0 or \ @@ -242,67 +252,126 @@ def recovery_from_record(self): def upload(self): """上传操作""" + if self.version == 'v1': + return self._upload_v1() + elif self.version == 'v2': + return self._upload_v2() + else: + raise ValueError("version must choose v1 or v2 !") + + def _upload_v1(self): self.blockStatus = [] self.recovery_index = 1 self.expiredAt = None self.uploadId = None self.get_bucket() + self.part_size = config._BLOCK_SIZE + host = self.get_up_host() - if self.version == 'v1': - offset = self.recovery_from_record() - self.part_size = config._BLOCK_SIZE - elif self.version == 'v2': - offset, self.uploadId, self.expiredAt = self.recovery_from_record() - if offset > 0 and self.blockStatus != [] and self.uploadId is not None \ - and self.expiredAt is not None: - self.recovery_index = self.blockStatus[-1]['partNumber'] + 1 + offset = self.recovery_from_record() + is_resumed = offset > 0 + + # 检查原来的分片是否过期，如有则重传该分片 + for index, block_status in enumerate(self.blockStatus): + if block_status.get('expired_at', 0) > time.time(): + self.input_stream.seek(self.part_size, os.SEEK_CUR) else: - self.recovery_index = 1 - init_url = self.block_url_v2(host, self.bucket_name) - self.uploadId, self.expiredAt = self.init_upload_task(init_url) + block = self.input_stream.read(self.part_size) + response, ok = self._make_block_with_retry(block, host) + ret, info = response + if not ok: + return ret, info + self.blockStatus[index] = ret + self.record_upload_progress(offset) + + # 从断点位置上传 + for block in _file_iter(self.input_stream, self.part_size, offset): + length = len(block) + response, ok = self._make_block_with_retry(block, host) + ret, info = response + if not ok: + return ret, info + + self.blockStatus.append(ret) + offset += length + self.record_upload_progress(offset) + if callable(self.progress_handler): + self.progress_handler(((len(self.blockStatus) - 1) * self.part_size) + len(block), self.size) + + ret, info = self.make_file(host) + if info.status_code == 200 or info.status_code == 701: + self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) + if info.status_code == 701 and is_resumed: + return self.upload() + return ret, info + + def _upload_v2(self): + self.blockStatus = [] + self.recovery_index = 1 + self.expiredAt = None + self.uploadId = None + self.get_bucket() + host = self.get_up_host() + + offset, self.uploadId, self.expiredAt = self.recovery_from_record() + is_resumed = False + if offset > 0 and self.blockStatus != [] and self.uploadId is not None \ + and self.expiredAt is not None: + self.recovery_index = self.blockStatus[-1]['partNumber'] + 1 + is_resumed = True else: - raise ValueError("version must choose v1 or v2 !") + self.recovery_index = 1 + init_url = self.block_url_v2(host, self.bucket_name) + self.uploadId, self.expiredAt = self.init_upload_task(init_url) + for index, block in enumerate(_file_iter(self.input_stream, self.part_size, offset)): length = len(block) - if self.version == 'v1': - crc = crc32(block) - ret, info = self.make_block(block, length, host) - elif self.version == 'v2': - index_ = index + self.recovery_index - url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (self.uploadId, index_) - ret, info = self.make_block_v2(block, url) + index_ = index + self.recovery_index + url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (self.uploadId, index_) + ret, info = self.make_block_v2(block, url) + if info.status_code == 612: + self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) + if info.status_code == 612 and is_resumed: + return self.upload() if ret is None and not info.need_retry(): return ret, info if info.connect_failed(): if config.get_default('default_zone').up_host_backup: host = config.get_default('default_zone').up_host_backup else: - host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token, - self.hostscache_dir) - if self.version == 'v1': - if info.need_retry() or crc != ret['crc32']: - ret, info = self.make_block(block, length, host) - if ret is None or crc != ret['crc32']: - return ret, info - elif self.version == 'v2': - if info.need_retry(): - url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (self.uploadId, index + 1) - ret, info = self.make_block_v2(block, url) - if ret is None: - return ret, info - del ret['md5'] - ret['partNumber'] = index_ + host = config.get_default('default_zone')\ + .get_up_host_backup_by_token(self.up_token, self.hostscache_dir) + + if info.need_retry(): + url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (self.uploadId, index + 1) + ret, info = self.make_block_v2(block, url) + if info.status_code == 612: + self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) + if info.status_code == 612 and is_resumed: + return self.upload() + if ret is None: + return ret, info + del ret['md5'] + ret['partNumber'] = index_ self.blockStatus.append(ret) offset += length self.record_upload_progress(offset) - if (callable(self.progress_handler)): + if callable(self.progress_handler): self.progress_handler(((len(self.blockStatus) - 1) * self.part_size) + len(block), self.size) - if self.version == 'v1': - return self.make_file(host) - elif self.version == 'v2': - make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % self.uploadId - return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, - self.mime_type, self.params, self.metadata) + + make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % self.uploadId + ret, info = self.make_file_v2( + self.blockStatus, + make_file_url, + self.file_name, + self.mime_type, + self.params, + self.metadata) + if info.status_code == 200 or info.status_code == 612: + self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) + if info.status_code == 612 and is_resumed: + return self.upload() + return ret, info def make_file_v2(self, block_status, url, file_name=None, mime_type=None, customVars=None, metadata=None): """completeMultipartUpload""" @@ -317,10 +386,7 @@ def make_file_v2(self, block_status, url, file_name=None, mime_type=None, custom 'customVars': customVars, 'metadata': metadata } - ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) - if info.status_code == 200: - self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) - return ret, info + return self.post_with_headers(url, json.dumps(data), headers=headers) def get_up_host(self): if config.get_default('default_zone').up_host: @@ -329,6 +395,24 @@ def get_up_host(self): host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.hostscache_dir) return host + def _make_block_with_retry(self, block_data, up_host): + length = len(block_data) + crc = crc32(block_data) + ret, info = self.make_block(block_data, length, up_host) + if ret is None and not info.need_retry(): + return (ret, info), False + if info.connect_failed(): + if config.get_default('default_zone').up_host_backup: + up_host = config.get_default('default_zone').up_host_backup + else: + up_host = config.get_default('default_zone') \ + .get_up_host_backup_by_token(self.up_token, self.hostscache_dir) + if info.need_retry() or crc != ret['crc32']: + ret, info = self.make_block(block_data, length, up_host) + if ret is None or crc != ret['crc32']: + return (ret, info), False + return (ret, info), True + def make_block(self, block, block_size, host): """创建块""" url = self.block_url(host, block_size) @@ -380,7 +464,6 @@ def make_file(self, host): """创建文件""" url = self.file_url(host) body = ','.join([status['ctx'] for status in self.blockStatus]) - self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) return self.post(url, body) def init_upload_task(self, url): From bb28c9a6ec7c527e79f7d54cbc5bef6c4b1d0444 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Thu, 29 Sep 2022 19:29:01 +0800 Subject: [PATCH 431/478] fix set_default rs, rsf and customize uc when query bucket region --- qiniu/config.py | 25 +++++++++++++++++++++++++ qiniu/region.py | 12 +++++++++++- test_qiniu.py | 18 ++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/qiniu/config.py b/qiniu/config.py index a137a67f..19fc305c 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -21,6 +21,22 @@ 'default_upload_threshold': 2 * _BLOCK_SIZE # put_file上传方式的临界默认值 } +_is_customized_default = { + 'default_zone': False, + 'default_rs_host': False, + 'default_rsf_host': False, + 'default_api_host': False, + 'default_uc_host': False, + 'connection_timeout': False, + 'connection_retries': False, + 'connection_pool': False, + 'default_upload_threshold': False +} + + +def is_customized_default(key): + return _is_customized_default[key] + def get_default(key): return _config[key] @@ -32,19 +48,28 @@ def set_default( default_rsf_host=None, default_api_host=None, default_upload_threshold=None): if default_zone: _config['default_zone'] = default_zone + _is_customized_default['default_zone'] = True if default_rs_host: _config['default_rs_host'] = default_rs_host + _is_customized_default['default_rs_host'] = True if default_rsf_host: _config['default_rsf_host'] = default_rsf_host + _is_customized_default['default_rsf_host'] = True if default_api_host: _config['default_api_host'] = default_api_host + _is_customized_default['default_api_host'] = True if default_uc_host: _config['default_uc_host'] = default_uc_host + _is_customized_default['default_uc_host'] = True if connection_retries: _config['connection_retries'] = connection_retries + _is_customized_default['connection_retries'] = True if connection_pool: _config['connection_pool'] = connection_pool + _is_customized_default['connection_pool'] = True if connection_timeout: _config['connection_timeout'] = connection_timeout + _is_customized_default['connection_timeout'] = True if default_upload_threshold: _config['default_upload_threshold'] = default_upload_threshold + _is_customized_default['default_upload_threshold'] = True diff --git a/qiniu/region.py b/qiniu/region.py index 24c351d9..1342d7de 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -64,8 +64,11 @@ def get_io_host(self, ak, bucket, home_dir=None): return io_hosts[0] def get_rs_host(self, ak, bucket, home_dir=None): + from .config import get_default, is_customized_default if self.rs_host: return self.rs_host + if is_customized_default('default_rs_host'): + return get_default('default_rs_host') if home_dir is None: home_dir = os.getcwd() bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) @@ -75,8 +78,11 @@ def get_rs_host(self, ak, bucket, home_dir=None): return rs_hosts[0] def get_rsf_host(self, ak, bucket, home_dir=None): + from .config import get_default, is_customized_default if self.rsf_host: return self.rsf_host + if is_customized_default('default_rsf_host'): + return get_default('default_rsf_host') if home_dir is None: home_dir = os.getcwd() bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) @@ -201,7 +207,11 @@ def host_cache_to_file(self, home_dir): f.close() def bucket_hosts(self, ak, bucket): - url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) + from .config import get_default, is_customized_default + uc_host = UC_HOST + if is_customized_default('default_uc_host'): + uc_host = get_default('default_uc_host') + url = "{0}/v1/query?ak={1}&bucket={2}".format(uc_host, ak, bucket) ret = requests.get(url) data = compat.json.dumps(ret.json(), separators=(',', ':')) return data diff --git a/test_qiniu.py b/test_qiniu.py index f40cd83e..80b99304 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -891,6 +891,24 @@ def test_get_domain(self): assert info.status_code == 200 +class RegionTestCase(unittest.TestCase): + test_rs_host = 'test.region.compatible.config.rs' + test_rsf_host = 'test.region.compatible.config.rsf' + + def test_config_compatible(self): + try: + set_default(default_rs_host=self.test_rs_host) + set_default(default_rsf_host=self.test_rsf_host) + zone = Zone() + assert zone.get_rs_host("mock_ak", "mock_bucket") == self.test_rs_host + assert zone.get_rsf_host("mock_ak", "mock_bucket") == self.test_rsf_host + finally: + set_default(default_rs_host=qiniu.config.RS_HOST) + set_default(default_rsf_host=qiniu.config.RSF_HOST) + qiniu.config._is_customized_default['default_rs_host'] = False + qiniu.config._is_customized_default['default_rsf_host'] = False + + class ReadWithoutSeek(object): def __init__(self, str): self.str = str From 9167c19c2516a364332ebfef5da350636cb66196 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Mon, 8 Aug 2022 11:43:43 +0800 Subject: [PATCH 432/478] update docs links --- README.md | 2 +- examples/pfop_watermark.py | 2 +- qiniu/http.py | 4 ++-- qiniu/services/cdn/manager.py | 16 ++++++++-------- qiniu/services/processing/pfop.py | 4 ++-- qiniu/services/storage/bucket.py | 22 +++++++++++----------- qiniu/services/storage/uploader.py | 10 +++++----- test_qiniu.py | 4 ++-- 8 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 7846505d..d5291e97 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ import qiniu ... ``` -更多参见SDK使用指南： http://developer.qiniu.com/code/v7/sdk/python.html +更多参见SDK使用指南： https://developer.qiniu.com/kodo/sdk/python ``` ## 测试 diff --git a/examples/pfop_watermark.py b/examples/pfop_watermark.py index 5994d2fa..327ac839 100755 --- a/examples/pfop_watermark.py +++ b/examples/pfop_watermark.py @@ -14,7 +14,7 @@ # 转码是使用的队列名称。 pipeline = 'pipeline_name' -# 需要添加水印的图片UrlSafeBase64,可以参考http://developer.qiniu.com/code/v6/api/dora-api/av/video-watermark.html +# 需要添加水印的图片UrlSafeBase64,可以参考 https://developer.qiniu.com/dora/api/video-watermarking base64URL = urlsafe_base64_encode( 'http://developer.qiniu.com/resource/logo-2.jpg') diff --git a/qiniu/http.py b/qiniu/http.py index 6d239808..91b01605 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -239,8 +239,8 @@ class ResponseInfo(object): Attributes: status_code: 整数变量，响应状态码 text_body: 字符串变量，响应的body - req_id: 字符串变量，七牛HTTP扩展字段，参考 http://developer.qiniu.com/docs/v6/api/reference/extended-headers.html - x_log: 字符串变量，七牛HTTP扩展字段，参考 http://developer.qiniu.com/docs/v6/api/reference/extended-headers.html + req_id: 字符串变量，七牛HTTP扩展字段，参考 https://developer.qiniu.com/kodo/3924/common-request-headers + x_log: 字符串变量，七牛HTTP扩展字段，参考 https://developer.qiniu.com/kodo/3924/common-request-headers error: 字符串变量，响应的错误内容 """ diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index c3bff855..788e0ac3 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -25,7 +25,7 @@ def __init__(self, auth): def refresh_urls(self, urls): """ - 刷新文件列表，文档 http://developer.qiniu.com/article/fusion/api/refresh.html + 刷新文件列表，文档 https://developer.qiniu.com/fusion/api/cache-refresh Args: urls: 待刷新的文件外链列表 @@ -38,7 +38,7 @@ def refresh_urls(self, urls): def refresh_dirs(self, dirs): """ - 刷新目录，文档 http://developer.qiniu.com/article/fusion/api/refresh.html + 刷新目录，文档 https://developer.qiniu.com/fusion/api/cache-refresh Args: urls: 待刷新的目录列表 @@ -51,7 +51,7 @@ def refresh_dirs(self, dirs): def refresh_urls_and_dirs(self, urls, dirs): """ - 刷新文件目录，文档 http://developer.qiniu.com/article/fusion/api/refresh.html + 刷新文件目录，文档 https://developer.qiniu.com/fusion/api/cache-refresh Args: urls: 待刷新的目录列表 @@ -73,7 +73,7 @@ def refresh_urls_and_dirs(self, urls, dirs): def prefetch_urls(self, urls): """ - 预取文件列表，文档 http://developer.qiniu.com/article/fusion/api/prefetch.html + 预取文件列表，文档 https://developer.qiniu.com/fusion/api/file-prefetching Args: urls: 待预取的文件外链列表 @@ -91,7 +91,7 @@ def prefetch_urls(self, urls): def get_bandwidth_data(self, domains, start_date, end_date, granularity): """ - 查询带宽数据，文档 http://developer.qiniu.com/article/fusion/api/traffic-bandwidth.html + 查询带宽数据，文档 https://developer.qiniu.com/fusion/api/traffic-bandwidth Args: domains: 域名列表 @@ -115,7 +115,7 @@ def get_bandwidth_data(self, domains, start_date, end_date, granularity): def get_flux_data(self, domains, start_date, end_date, granularity): """ - 查询流量数据，文档 http://developer.qiniu.com/article/fusion/api/traffic-bandwidth.html + 查询流量数据，文档 https://developer.qiniu.com/fusion/api/traffic-bandwidth Args: domains: 域名列表 @@ -139,7 +139,7 @@ def get_flux_data(self, domains, start_date, end_date, granularity): def get_log_list_data(self, domains, log_date): """ - 获取日志下载链接，文档 http://developer.qiniu.com/article/fusion/api/log.html + 获取日志下载链接，文档 https://developer.qiniu.com/fusion/api/download-the-log Args: domains: 域名列表 @@ -159,7 +159,7 @@ def get_log_list_data(self, domains, log_date): def put_httpsconf(self, name, certid, forceHttps=False): """ - 修改证书，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name#11 + 修改证书，文档 https://developer.qiniu.com/fusion/4246/the-domain-name#11 Args: domains: 域名name diff --git a/qiniu/services/processing/pfop.py b/qiniu/services/processing/pfop.py index 9829b6c9..fa414930 100644 --- a/qiniu/services/processing/pfop.py +++ b/qiniu/services/processing/pfop.py @@ -8,12 +8,12 @@ class PersistentFop(object): """持久化处理类 该类用于主动触发异步持久化操作，具体规格参考： - http://developer.qiniu.com/docs/v6/api/reference/fop/pfop/pfop.html + https://developer.qiniu.com/dora/api/persistent-data-processing-pfop Attributes: auth: 账号管理密钥对，Auth对象 bucket: 操作资源所在空间 - pipeline: 多媒体处理队列，详见 https://portal.qiniu.com/mps/pipeline + pipeline: 多媒体处理队列，详见 https://developer.qiniu.com/dora/6499/tasks-and-workflows notify_url: 持久化处理结果通知URL """ diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 35b6d677..b41bcdb3 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -9,7 +9,7 @@ class BucketManager(object): """空间管理类 主要涉及了空间资源管理及批量操作接口的实现，具体的接口规格可以参考： - http://developer.qiniu.com/docs/v6/api/reference/rs/ + https://developer.qiniu.com/kodo/1274/rs Attributes: auth: 账号管理密钥对，Auth对象 @@ -33,7 +33,7 @@ def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): 2. 无论 err 值如何，均应该先看 ret.get('items') 是否有内容 3. 如果后续没有更多数据，err 返回 EOF，marker 返回 None（但不通过该特征来判断是否结束） 具体规格参考: - http://developer.qiniu.com/docs/v6/api/reference/rs/list.html + https://developer.qiniu.com/kodo/api/list Args: bucket: 空间名 @@ -111,7 +111,7 @@ def delete(self, bucket, key): """删除文件: 删除指定资源，具体规格参考： - http://developer.qiniu.com/docs/v6/api/reference/rs/delete.html + https://developer.qiniu.com/kodo/api/delete Args: bucket: 待获取信息资源所在的空间 @@ -144,7 +144,7 @@ def move(self, bucket, key, bucket_to, key_to, force='false'): """移动文件: 将资源从一个空间到另一个空间，具体规格参考： - http://developer.qiniu.com/docs/v6/api/reference/rs/move.html + https://developer.qiniu.com/kodo/api/move Args: bucket: 待操作资源所在空间 @@ -164,7 +164,7 @@ def copy(self, bucket, key, bucket_to, key_to, force='false'): """复制文件: 将指定资源复制为新命名资源，具体规格参考： - http://developer.qiniu.com/docs/v6/api/reference/rs/copy.html + https://developer.qiniu.com/kodo/api/copy Args: bucket: 待操作资源所在空间 @@ -183,7 +183,7 @@ def copy(self, bucket, key, bucket_to, key_to, force='false'): def fetch(self, url, bucket, key=None, hostscache_dir=None): """抓取文件: 从指定URL抓取资源，并将该资源存储到指定空间中，具体规格参考： - http://developer.qiniu.com/docs/v6/api/reference/rs/fetch.html + https://developer.qiniu.com/kodo/api/fetch Args: url: 指定的URL @@ -205,7 +205,7 @@ def prefetch(self, bucket, key, hostscache_dir=None): """镜像回源预取文件: 从镜像源站抓取资源到空间中，如果空间中已经存在，则覆盖该资源，具体规格参考 - http://developer.qiniu.com/docs/v6/api/reference/rs/prefetch.html + https://developer.qiniu.com/kodo/api/prefetch Args: bucket: 待获取资源所在的空间 @@ -223,7 +223,7 @@ def change_mime(self, bucket, key, mime): """修改文件mimeType: 主动修改指定资源的文件类型，具体规格参考： - http://developer.qiniu.com/docs/v6/api/reference/rs/chgm.html + https://developer.qiniu.com/kodo/api/chgm Args: bucket: 待操作资源所在空间 @@ -238,7 +238,7 @@ def change_type(self, bucket, key, storage_type): """修改文件的存储类型 修改文件的存储类型，参考文档： - https://developer.qiniu.com/kodo/api/3710/modify-the-file-type + https://developer.qiniu.com/kodo/3710/chtype Args: bucket: 待操作资源所在空间 @@ -252,7 +252,7 @@ def restoreAr(self, bucket, key, freezeAfter_days): """解冻归档存储、深度归档存储文件 对归档存储、深度归档存储文件，进行解冻操作参考文档： - https://developer.qiniu.com/kodo/api/6380/restore-archive + https://developer.qiniu.com/kodo/6380/restore-archive Args: bucket: 待操作资源所在空间 @@ -324,7 +324,7 @@ def batch(self, operations): """批量操作: 在单次请求中进行多个资源管理操作，具体规格参考： - http://developer.qiniu.com/docs/v6/api/reference/rs/batch.html + https://developer.qiniu.com/kodo/api/batch Args: operations: 资源管理操作数组，可通过 diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 81e23d29..965875ba 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -19,7 +19,7 @@ def put_data( up_token: 上传凭证 key: 上传文件名 data: 上传二进制流 - params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + params: 自定义变量，规格参考 https://developer.qiniu.com/kodo/manual/vars#xvar mime_type: 上传数据的mimeType check_crc: 是否校验crc32 progress_handler: 上传进度 @@ -56,7 +56,7 @@ def put_file(up_token, key, file_path, params=None, up_token: 上传凭证 key: 上传文件名 file_path: 上传文件的路径 - params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + params: 自定义变量，规格参考 https://developer.qiniu.com/kodo/manual/vars#xvar mime_type: 上传数据的mimeType check_crc: 是否校验crc32 progress_handler: 上传进度 @@ -153,15 +153,15 @@ class _Resume(object): """断点续上传类 该类主要实现了分块上传，断点续上，以及相应地创建块和创建文件过程，详细规格参考： - http://developer.qiniu.com/docs/v6/api/reference/up/mkblk.html - http://developer.qiniu.com/docs/v6/api/reference/up/mkfile.html + https://developer.qiniu.com/kodo/api/mkblk + https://developer.qiniu.com/kodo/api/mkfile Attributes: up_token: 上传凭证 key: 上传文件名 input_stream: 上传二进制流 data_size: 上传流大小 - params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + params: 自定义变量，规格参考 https://developer.qiniu.com/kodo/manual/vars#xvar mime_type: 上传数据的mimeType progress_handler: 上传进度 upload_progress_recorder: 记录上传进度，用于断点续传 diff --git a/test_qiniu.py b/test_qiniu.py index 80b99304..ed533576 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -322,14 +322,14 @@ def test_prefetch(self): assert ret['key'] == 'python-sdk.html' def test_fetch(self): - ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, + ret, info = self.bucket.fetch('https://developer.qiniu.com/kodo/sdk/python', bucket_name, 'fetch.html', hostscache_dir=hostscache_dir) print(info) assert ret['key'] == 'fetch.html' assert 'hash' in ret def test_fetch_without_key(self): - ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, + ret, info = self.bucket.fetch('https://developer.qiniu.com/kodo/sdk/python', bucket_name, hostscache_dir=hostscache_dir) print(info) assert ret['key'] == ret['hash'] From 0ade9aeb53c4ab10c332e53191a39aeab1ae4c94 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Tue, 15 Nov 2022 14:08:50 +0800 Subject: [PATCH 433/478] Update CHANGELOG.md and __init__.py --- CHANGELOG.md | 6 ++++++ qiniu/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d8c8781..06f3dff8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 7.10.0（2022-11-15） +* 对象存储，修复通过 set_default 设置 rs, rsf 不生效，而 SDK 自动获取的问题（v7.9.0） +* 对象存储，支持直接从 qiniu 导入 UploadProgressRecorder +* 对象存储，优化分片上传 ctx 超时检测 +* 文档，更新注释中文档链接 + ## 7.9.0（2022-07-20） * 对象存储，支持使用时不配置区域信息，SDK 自动获取； * 对象存储，新增 list_domains API 用于查询空间绑定的域名 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 851d7729..e19d7a3e 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.9.0' +__version__ = '7.10.0' from .auth import Auth, QiniuMacAuth From 2e8bedeca0ecd7fdcb9805a9d7fbf465ce60c413 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Thu, 16 Mar 2023 23:27:35 +0800 Subject: [PATCH 434/478] update query backend domains and query bucket domains --- qiniu/region.py | 45 +++++++++++++++----------------- qiniu/services/storage/bucket.py | 13 +++++---- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/qiniu/region.py b/qiniu/region.py index 1342d7de..fd4d33dc 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -23,13 +23,15 @@ def __init__( home_dir=None, scheme="http", rs_host=None, - rsf_host=None): + rsf_host=None, + api_host=None): """初始化Zone类""" self.up_host = up_host self.up_host_backup = up_host_backup self.io_host = io_host self.rs_host = rs_host self.rsf_host = rsf_host + self.api_host = api_host self.home_dir = home_dir self.host_cache = host_cache self.scheme = scheme @@ -91,6 +93,20 @@ def get_rsf_host(self, ak, bucket, home_dir=None): rsf_hosts = bucket_hosts['rsfHosts'] return rsf_hosts[0] + def get_api_host(self, ak, bucket, home_dir=None): + from .config import get_default, is_customized_default + if self.api_host: + return self.api_host + if is_customized_default('default_api_host'): + return get_default('default_api_host') + if home_dir is None: + home_dir = os.getcwd() + bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) + if 'apiHosts' not in bucket_hosts: + bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir, force=True) + api_hosts = bucket_hosts['apiHosts'] + return api_hosts[0] + def get_up_host(self, ak, bucket, home_dir): bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) if 'upHosts' not in bucket_hosts: @@ -123,27 +139,10 @@ def get_bucket_hosts(self, ak, bucket, home_dir, force=False): if not force and len(bucket_hosts) > 0: return bucket_hosts - hosts = {} - hosts.update({self.scheme: {}}) - - hosts[self.scheme].update({'up': []}) - hosts[self.scheme].update({'io': []}) + hosts = compat.json.loads(self.bucket_hosts(ak, bucket)) + default_ttl = 24 * 3600 # 1 day + hosts['ttl'] = hosts['ttl'] if 'ttl' in hosts else default_ttl - if self.up_host is not None: - hosts[self.scheme]['up'].append(self.scheme + "://" + self.up_host) - - if self.up_host_backup is not None: - hosts[self.scheme]['up'].append( - self.scheme + "://" + self.up_host_backup) - - if self.io_host is not None: - hosts[self.scheme]['io'].append(self.scheme + "://" + self.io_host) - - if len(hosts[self.scheme]) == 0 or self.io_host is None: - hosts = compat.json.loads(self.bucket_hosts(ak, bucket)) - else: - # 1 year - hosts['ttl'] = int(time.time()) + 31536000 try: scheme_hosts = hosts[self.scheme] except KeyError: @@ -155,6 +154,7 @@ def get_bucket_hosts(self, ak, bucket, home_dir, force=False): 'ioHosts': scheme_hosts['io'], 'rsHosts': scheme_hosts['rs'], 'rsfHosts': scheme_hosts['rsf'], + 'apiHosts': scheme_hosts['api'], 'deadline': int(time.time()) + hosts['ttl'] } home_dir = "" @@ -166,9 +166,6 @@ def get_bucket_hosts_to_cache(self, key, home_dir): if len(self.host_cache) == 0: self.host_cache_from_file(home_dir) - if self.host_cache == {}: - return ret - if key not in self.host_cache: return ret diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index b41bcdb3..c8e4d080 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -411,11 +411,8 @@ def bucket_domain(self, bucket_name): Args: bucket_name: 存储空间名 """ - options = { - 'tbl': bucket_name, - } - url = "{0}/v6/domain/list?tbl={1}".format(config.get_default("default_api_host"), bucket_name) - return self.__get(url, options) + data = 'tbl={0}'.format(bucket_name) + return self.__api_do(bucket_name, 'v6/domain/list', data) def change_bucket_permission(self, bucket_name, private): """ @@ -428,6 +425,12 @@ def change_bucket_permission(self, bucket_name, private): url = "{0}/private?bucket={1}&private={2}".format(config.get_default("default_uc_host"), bucket_name, private) return self.__post(url) + def __api_do(self, bucket, operation, data=None): + ak = self.auth.get_access_key() + api_host = self.zone.get_api_host(ak, bucket) + url = '{0}/{1}'.format(api_host, operation) + return self.__post(url, data) + def __uc_do(self, operation, *args): return self.__server_do(config.get_default('default_uc_host'), operation, *args) From 0ce8bf320437e326c6adfc2561e145eafc537c13 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Mon, 20 Mar 2023 15:08:30 +0800 Subject: [PATCH 435/478] alias bucket_domain to list_domains --- qiniu/services/storage/bucket.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index c8e4d080..efb2c9b4 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -411,8 +411,7 @@ def bucket_domain(self, bucket_name): Args: bucket_name: 存储空间名 """ - data = 'tbl={0}'.format(bucket_name) - return self.__api_do(bucket_name, 'v6/domain/list', data) + return self.list_domains(bucket_name) def change_bucket_permission(self, bucket_name, private): """ From 1055d14f7aad399f45f793aab30239087b7deff9 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Tue, 21 Mar 2023 23:42:46 +0800 Subject: [PATCH 436/478] update v1/query to v4/query --- qiniu/region.py | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/qiniu/region.py b/qiniu/region.py index fd4d33dc..654229a9 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -139,23 +139,38 @@ def get_bucket_hosts(self, ak, bucket, home_dir, force=False): if not force and len(bucket_hosts) > 0: return bucket_hosts - hosts = compat.json.loads(self.bucket_hosts(ak, bucket)) + hosts = compat.json.loads(self.bucket_hosts(ak, bucket)).get('hosts', []) + + if type(hosts) is not list or len(hosts) == 0: + raise KeyError("Please check your BUCKET_NAME! Server hosts not correct! The hosts is %s" % hosts) + + region = hosts[0] + default_ttl = 24 * 3600 # 1 day - hosts['ttl'] = hosts['ttl'] if 'ttl' in hosts else default_ttl - - try: - scheme_hosts = hosts[self.scheme] - except KeyError: - raise KeyError( - "Please check your BUCKET_NAME! The UpHosts is %s" % - hosts) + region['ttl'] = region.get('ttl', default_ttl) + bucket_hosts = { - 'upHosts': scheme_hosts['up'], - 'ioHosts': scheme_hosts['io'], - 'rsHosts': scheme_hosts['rs'], - 'rsfHosts': scheme_hosts['rsf'], - 'apiHosts': scheme_hosts['api'], - 'deadline': int(time.time()) + hosts['ttl'] + 'upHosts': [ + '{0}://{1}'.format(self.scheme, domain) + for domain in region.get('up', {}).get('domains', []) + ], + 'ioHosts': [ + '{0}://{1}'.format(self.scheme, domain) + for domain in region.get('io', {}).get('domains', []) + ], + 'rsHosts': [ + '{0}://{1}'.format(self.scheme, domain) + for domain in region.get('rs', {}).get('domains', []) + ], + 'rsfHosts': [ + '{0}://{1}'.format(self.scheme, domain) + for domain in region.get('rsf', {}).get('domains', []) + ], + 'apiHosts': [ + '{0}://{1}'.format(self.scheme, domain) + for domain in region.get('api', {}).get('domains', []) + ], + 'deadline': int(time.time()) + region['ttl'] } home_dir = "" self.set_bucket_hosts_to_cache(key, bucket_hosts, home_dir) @@ -208,7 +223,7 @@ def bucket_hosts(self, ak, bucket): uc_host = UC_HOST if is_customized_default('default_uc_host'): uc_host = get_default('default_uc_host') - url = "{0}/v1/query?ak={1}&bucket={2}".format(uc_host, ak, bucket) + url = "{0}/v4/query?ak={1}&bucket={2}".format(uc_host, ak, bucket) ret = requests.get(url) data = compat.json.dumps(ret.json(), separators=(',', ':')) return data From 0d48e4eecd2bf95f61c5b888ab4c902e62ecd484 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Wed, 22 Mar 2023 15:42:36 +0800 Subject: [PATCH 437/478] fix ci --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 2614bb3e..b31c3303 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -21,7 +21,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest pytest-cov freezegun requests scrutinizer-ocular codecov + pip install "coverage<7.2" flake8 pytest pytest-cov freezegun requests scrutinizer-ocular codecov - name: Run cases env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} From 27113574a2602010dd7120fcad9a4d7869b81b71 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Wed, 22 Mar 2023 18:34:42 +0800 Subject: [PATCH 438/478] update hard code api host --- qiniu/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/config.py b/qiniu/config.py index 19fc305c..58ef3650 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -4,7 +4,7 @@ RS_HOST = 'http://rs.qiniu.com' # 管理操作Host RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host -API_HOST = 'http://api.qiniu.com' # 数据处理操作Host +API_HOST = 'http://api.qiniuapi.com' # 数据处理操作Host UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续传分块大小，该参数为接口规格，暂不支持修改 From d19ecd8ed12b88f6299ddeb90677112f854a527b Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Wed, 22 Mar 2023 21:48:22 +0800 Subject: [PATCH 439/478] update CHANGELOG.md --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06f3dff8..43393965 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Next Version +* 对象存储，更新 api 默认域名 +* 对象存储，新增 api 域名的配置与获取 +* 对象存储，修复获取区域域名后无法按照预期进行过期处理 +* 对象存储，更新获取区域域名的接口 +* 对象存储，bucket_domains 修改为 list_domains 的别名 + ## 7.10.0（2022-11-15） * 对象存储，修复通过 set_default 设置 rs, rsf 不生效，而 SDK 自动获取的问题（v7.9.0） * 对象存储，支持直接从 qiniu 导入 UploadProgressRecorder From d90a2ab8fc8e0082d1ac92964427f5ff9d8f2c1a Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Tue, 28 Mar 2023 18:01:45 +0800 Subject: [PATCH 440/478] bump to v7.11.0 --- CHANGELOG.md | 2 +- qiniu/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43393965..202de3cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Next Version +## 7.11.0（2023-03-28） * 对象存储，更新 api 默认域名 * 对象存储，新增 api 域名的配置与获取 * 对象存储，修复获取区域域名后无法按照预期进行过期处理 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index e19d7a3e..778a0454 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.10.0' +__version__ = '7.11.0' from .auth import Auth, QiniuMacAuth From 99081ce08f4070091465d81decb01fe4ebb39651 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Tue, 28 Mar 2023 18:21:19 +0800 Subject: [PATCH 441/478] fix ci by upgrading ubuntu and adding setup python manually --- .github/workflows/ci-test.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b31c3303..e4fce82b 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -6,18 +6,31 @@ jobs: fail-fast: false max-parallel: 1 matrix: - python_version: ['2.7', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9'] - runs-on: ubuntu-18.04 + python_version: ['2.7', '3.4.10', '3.5', '3.6', '3.7', '3.8', '3.9'] + runs-on: ubuntu-20.04 steps: - name: Checkout repo uses: actions/checkout@v2 with: ref: ${{ github.ref }} - name: Setup python + if: ${{ matrix.python_version != '3.4.10' }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python_version }} architecture: x64 + - name: Setup python manually + if: ${{ matrix.python_version == '3.4.10' }} + env: + PYTHON_SOURCE_URL: https://www.python.org/ftp/python/${{ matrix.python_version }}/Python-${{ matrix.python_version }}.tgz + run: | + cd /tmp + curl -s -L "$PYTHON_SOURCE_URL" | tar -zxf - -C ./ + cd Python-${{ matrix.python_version }} + ./configure --enable-optimizations + make + sudo make install + echo 'export PATH="/opt/python/bin:$PATH"' >> $HOME/.bashrc - name: Install dependencies run: | python -m pip install --upgrade pip From 10d19c8912ea05043281ecf8eacf31cb69a95443 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Fri, 12 May 2023 15:14:56 +0800 Subject: [PATCH 442/478] move http to a package --- qiniu/{http.py => http/__init__.py} | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename qiniu/{http.py => http/__init__.py} (99%) diff --git a/qiniu/http.py b/qiniu/http/__init__.py similarity index 99% rename from qiniu/http.py rename to qiniu/http/__init__.py index 91b01605..5381ead1 100644 --- a/qiniu/http.py +++ b/qiniu/http/__init__.py @@ -7,10 +7,9 @@ import requests from requests.auth import AuthBase +from qiniu import config, __version__ from qiniu.compat import is_py2, is_py3 -from qiniu import config import qiniu.auth -from . import __version__ _sys_info = '{0}; {1}'.format(platform.system(), platform.machine()) _python_ver = platform.python_version() From b7fe6bc31fb5e8965206b007241f3b67c14bd195 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Sun, 21 May 2023 14:35:25 +0800 Subject: [PATCH 443/478] add http client and middlewares --- qiniu/http/__init__.py | 96 +++----------- qiniu/http/client.py | 177 +++++++++++++++++++++++++ qiniu/http/middleware/__init__.py | 3 + qiniu/http/middleware/base.py | 36 +++++ qiniu/http/middleware/retry_domains.py | 68 ++++++++++ qiniu/http/middleware/ua.py | 20 +++ qiniu/http/response.py | 75 +++++++++++ test_qiniu.py | 63 ++++++++- 8 files changed, 458 insertions(+), 80 deletions(-) create mode 100644 qiniu/http/client.py create mode 100644 qiniu/http/middleware/__init__.py create mode 100644 qiniu/http/middleware/base.py create mode 100644 qiniu/http/middleware/retry_domains.py create mode 100644 qiniu/http/middleware/ua.py create mode 100644 qiniu/http/response.py diff --git a/qiniu/http/__init__.py b/qiniu/http/__init__.py index 5381ead1..fbad3b61 100644 --- a/qiniu/http/__init__.py +++ b/qiniu/http/__init__.py @@ -5,12 +5,24 @@ from datetime import datetime import requests +from requests.adapters import HTTPAdapter from requests.auth import AuthBase from qiniu import config, __version__ -from qiniu.compat import is_py2, is_py3 import qiniu.auth +from .client import HTTPClient +from .response import ResponseInfo +from .middleware import UserAgentMiddleware + + +qn_http_client = HTTPClient( + middlewares=[ + UserAgentMiddleware(__version__) + ] +) + + _sys_info = '{0}; {1}'.format(platform.system(), platform.machine()) _python_ver = platform.python_version() @@ -47,14 +59,15 @@ def __return_wrapper(resp): def _init(): - session = requests.Session() + global _session + if _session is None: + _session = qn_http_client.session + adapter = requests.adapters.HTTPAdapter( pool_connections=config.get_default('connection_pool'), pool_maxsize=config.get_default('connection_pool'), max_retries=config.get_default('connection_retries')) - session.mount('http://', adapter) - global _session - _session = session + _session.mount('http://', adapter) def _post(url, data, files, auth, headers=None): @@ -228,76 +241,3 @@ def _delete_with_qiniu_mac_and_headers(url, params, auth, headers): except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) - - -class ResponseInfo(object): - """七牛HTTP请求返回信息类 - - 该类主要是用于获取和解析对七牛发起各种请求后的响应包的header和body。 - - Attributes: - status_code: 整数变量，响应状态码 - text_body: 字符串变量，响应的body - req_id: 字符串变量，七牛HTTP扩展字段，参考 https://developer.qiniu.com/kodo/3924/common-request-headers - x_log: 字符串变量，七牛HTTP扩展字段，参考 https://developer.qiniu.com/kodo/3924/common-request-headers - error: 字符串变量，响应的错误内容 - """ - - def __init__(self, response, exception=None): - """用响应包和异常信息初始化ResponseInfo类""" - self.__response = response - self.exception = exception - if response is None: - self.status_code = -1 - self.text_body = None - self.req_id = None - self.x_log = None - self.error = str(exception) - else: - self.status_code = response.status_code - self.text_body = response.text - self.req_id = response.headers.get('X-Reqid') - self.x_log = response.headers.get('X-Log') - if self.status_code >= 400: - if self.__check_json(response): - ret = response.json() if response.text != '' else None - if ret is None: - self.error = 'unknown' - else: - self.error = response.text - else: - self.error = response.text - if self.req_id is None and self.status_code == 200: - self.error = 'server is not qiniu' - - def ok(self): - return self.status_code == 200 and self.req_id is not None - - def need_retry(self): - if self.__response is None or self.req_id is None: - return True - code = self.status_code - if (code // 100 == 5 and code != 579) or code == 996: - return True - return False - - def connect_failed(self): - return self.__response is None or self.req_id is None - - def __str__(self): - if is_py2: - return ', '.join( - ['%s:%s' % item for item in self.__dict__.items()]).encode('utf-8') - elif is_py3: - return ', '.join(['%s:%s' % - item for item in self.__dict__.items()]) - - def __repr__(self): - return self.__str__() - - def __check_json(self, reponse): - try: - reponse.json() - return True - except Exception: - return False diff --git a/qiniu/http/client.py b/qiniu/http/client.py new file mode 100644 index 00000000..cf0aeace --- /dev/null +++ b/qiniu/http/client.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +import logging + +import requests + +from .response import ResponseInfo +from .middleware import Middleware, compose_middleware + + +class HTTPClient: + def __init__(self, middlewares=None, send_opts=None): + self.session = requests.Session() + self.middlewares = [] if middlewares is None else middlewares + self.send_opts = {} if send_opts is None else send_opts + + def send_request(self, request, middlewares=None, **kwargs): + """ + + Args: + request (requests.Request): + requests.Request 对象 + + middlewares (list[Middleware] or (list[Middleware]) -> list[Middleware]): + 仅对本次请求生效的中间件。 + + 如果传入的是列表，那么会作为追加的中间件拼接到 Client 中间件的后面。 + + 也可传入函数，获得 Client 中间件的一个副本来做更细的控制。 + 例如拼接到 Client 中间件的前面，可以这样使用： + + c.send_request(my_req, middlewares=lambda mws: my_mws + mws) + + kwargs: + 将作为其他参数直接透传给 session.send 方法 + + + Returns: + (dict, ResponseInfo): 可拆包的一个元组。 + 第一个元素为响应体的 dict，若响应体为 json 的话。 + 第二个元素为包装过的响应内容，包括了更多的响应内容。 + + """ + + # set default values + middlewares = [] if middlewares is None else middlewares + + # join middlewares and client middlewares + mw_ls = [] + if callable(middlewares): + mw_ls = middlewares(self.middlewares.copy()) + elif isinstance(middlewares, list): + mw_ls = self.middlewares + middlewares + + # send request + try: + handle = compose_middleware( + mw_ls, + lambda req: self.session.send(req.prepare(), **kwargs) + ) + resp = handle(request) + except Exception as e: + return None, ResponseInfo(None, e) + + # wrap resp + resp_info = ResponseInfo(resp) + if not resp_info.ok: + return None, resp_info + + # try dump response info to dict from json + resp.encoding = "utf-8" + try: + ret = resp.json() + except ValueError: + logging.debug("response body decode error: %s" % resp.text) + ret = {} + return ret, resp_info + + def get( + self, + url, + params=None, + auth=None, + headers=None, + middlewares=None, + **kwargs + ): + req = requests.Request( + method='get', + url=url, + params=params, + auth=auth, + headers=headers + ) + send_opts = self.send_opts.copy() + send_opts.update(kwargs) + send_opts.setdefault("allow_redirects", True) + return self.send_request( + req, + middlewares=middlewares, + **send_opts + ) + + def post( + self, + url, + data, + files, + auth=None, + headers=None, + middlewares=None, + **kwargs + ): + req = requests.Request( + method='post', + url=url, + data=data, + files=files, + auth=auth, + headers=headers + ) + send_opts = self.send_opts.copy() + send_opts.update(kwargs) + return self.send_request( + req, + middlewares=middlewares, + **send_opts + ) + + def put( + self, + url, + data, + files, + auth=None, + headers=None, + middlewares=None, + **kwargs + ): + req = requests.Request( + method='put', + url=url, + data=data, + files=files, + auth=auth, + headers=headers + ) + send_opts = self.send_opts.copy() + send_opts.update(kwargs) + return self.send_request( + req, + middlewares=middlewares, + **send_opts + ) + + def delete( + self, + url, + params, + auth=None, + headers=None, + middlewares=None, + **kwargs, + ): + req = requests.Request( + method='delete', + url=url, + params=params, + auth=auth, + headers=headers + ) + send_opts = self.send_opts.copy() + send_opts.update(kwargs) + return self.send_request( + req, + middlewares=middlewares, + **send_opts + ) diff --git a/qiniu/http/middleware/__init__.py b/qiniu/http/middleware/__init__.py new file mode 100644 index 00000000..799253aa --- /dev/null +++ b/qiniu/http/middleware/__init__.py @@ -0,0 +1,3 @@ +from .base import * +from .ua import * +from .retry_domains import * diff --git a/qiniu/http/middleware/base.py b/qiniu/http/middleware/base.py new file mode 100644 index 00000000..f3b1f27b --- /dev/null +++ b/qiniu/http/middleware/base.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +from functools import reduce + + +def compose_middleware(middlewares, handle): + """ + Args: + middlewares (list[Middleware]): Middlewares + handle ((requests.Request) -> requests.Response): The send request handle + + Returns: + (requests.Request) -> requests.Response: Composed handle + + """ + middlewares.reverse() + + return reduce( + lambda h, mw: + lambda req: mw(req, h), + middlewares, + handle + ) + + +class Middleware: + def __call__(self, request, nxt): + """ + Args: + request (requests.Request): + nxt ((requests.Request) -> requests.Response): + + Returns: + requests.Response: + + """ + raise NotImplementedError('{0}.__call__ method is not implemented yet'.format(type(self))) diff --git a/qiniu/http/middleware/retry_domains.py b/qiniu/http/middleware/retry_domains.py new file mode 100644 index 00000000..c3fe49dc --- /dev/null +++ b/qiniu/http/middleware/retry_domains.py @@ -0,0 +1,68 @@ +from qiniu.compat import urlparse + +from .base import Middleware + + +class RetryDomainsMiddleware(Middleware): + def __init__(self, backup_domains, max_retry_times=2): + """ + Args: + backup_domains (list[str]): + max_retry_times (int): + """ + self.backup_domains = backup_domains + self.max_retry_times = max_retry_times + + self.retried_times = 0 + + @staticmethod + def _get_changed_url(url, domain): + url_parse_result = urlparse(url) + + backup_netloc = '' + has_user = False + if url_parse_result.username is not None: + backup_netloc += url_parse_result.username + has_user = True + if url_parse_result.password is not None: + backup_netloc += url_parse_result.password + has_user = True + if has_user: + backup_netloc += '@' + backup_netloc += domain + if url_parse_result.port is not None: + backup_netloc += ':' + str(url_parse_result.port) + url_parse_result = url_parse_result._replace( + netloc=backup_netloc + ) + + return url_parse_result.geturl() + + @staticmethod + def _try_nxt(request, nxt): + resp = None + err = None + try: + resp = nxt(request) + except Exception as e: + err = e + return resp, err + + def __call__(self, request, nxt): + resp, err = None, None + url_parse_result = urlparse(request.url) + + for backup_domain in [str(url_parse_result.hostname)] + self.backup_domains: + request.url = RetryDomainsMiddleware._get_changed_url(request.url, backup_domain) + self.retried_times = 0 + + while (resp is None or not resp.ok) and self.retried_times < self.max_retry_times: + resp, err = RetryDomainsMiddleware._try_nxt(request, nxt) + self.retried_times += 1 + if err is None and (resp is not None and resp.ok): + return resp + + if err is not None: + raise err + + return resp diff --git a/qiniu/http/middleware/ua.py b/qiniu/http/middleware/ua.py new file mode 100644 index 00000000..3661d5cf --- /dev/null +++ b/qiniu/http/middleware/ua.py @@ -0,0 +1,20 @@ +import platform as _platform + +from .base import Middleware + + +class UserAgentMiddleware(Middleware): + def __init__(self, sdk_version): + sys_info = '{0}; {1}'.format(_platform.system(), _platform.machine()) + python_ver = _platform.python_version() + + user_agent = 'QiniuPython/{0} ({1}; ) Python/{2}'.format( + sdk_version, sys_info, python_ver) + + self.user_agent = user_agent + + def __call__(self, request, nxt): + if not request.headers: + request.headers = {} + request.headers['User-Agent'] = self.user_agent + return nxt(request) diff --git a/qiniu/http/response.py b/qiniu/http/response.py new file mode 100644 index 00000000..da5ec96f --- /dev/null +++ b/qiniu/http/response.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +from qiniu.compat import is_py2, is_py3 + + +class ResponseInfo(object): + """七牛HTTP请求返回信息类 + + 该类主要是用于获取和解析对七牛发起各种请求后的响应包的header和body。 + + Attributes: + status_code (int): 整数变量，响应状态码 + text_body (str): 字符串变量，响应的body + req_id (str): 字符串变量，七牛HTTP扩展字段，参考 https://developer.qiniu.com/kodo/3924/common-request-headers + x_log (str): 字符串变量，七牛HTTP扩展字段，参考 https://developer.qiniu.com/kodo/3924/common-request-headers + error (str): 字符串变量，响应的错误内容 + """ + + def __init__(self, response, exception=None): + """用响应包和异常信息初始化ResponseInfo类""" + self.__response = response + self.exception = exception + if response is None: + self.status_code = -1 + self.text_body = None + self.req_id = None + self.x_log = None + self.error = str(exception) + else: + self.status_code = response.status_code + self.text_body = response.text + self.req_id = response.headers.get('X-Reqid') + self.x_log = response.headers.get('X-Log') + if self.status_code >= 400: + if self.__check_json(response): + ret = response.json() if response.text != '' else None + if ret is None: + self.error = 'unknown' + else: + self.error = response.text + else: + self.error = response.text + if self.req_id is None and self.status_code == 200: + self.error = 'server is not qiniu' + + def ok(self): + return self.status_code == 200 and self.req_id is not None + + def need_retry(self): + if self.__response is None or self.req_id is None: + return True + code = self.status_code + if (code // 100 == 5 and code != 579) or code == 996: + return True + return False + + def connect_failed(self): + return self.__response is None or self.req_id is None + + def __str__(self): + if is_py2: + return ', '.join( + ['%s:%s' % item for item in self.__dict__.items()]).encode('utf-8') + elif is_py3: + return ', '.join( + ['%s:%s' % item for item in self.__dict__.items()]) + + def __repr__(self): + return self.__str__() + + def __check_json(self, response): + try: + response.json() + return True + except Exception: + return False diff --git a/test_qiniu.py b/test_qiniu.py index ed533576..6f378557 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # flake8: noqa -import os, time +import os import string import random import tempfile @@ -22,7 +22,8 @@ from qiniu.services.storage.uploader import _form_put -from qiniu.http import __return_wrapper as return_wrapper +from qiniu.http import __return_wrapper as return_wrapper, qn_http_client +from qiniu.http.middleware import Middleware, RetryDomainsMiddleware import qiniu.config @@ -78,7 +79,65 @@ def is_travis(): return os.environ['QINIU_TEST_ENV'] == 'travis' +class MiddlewareRecorder(Middleware): + def __init__(self, rec, label): + self.rec = rec + self.label = label + + def __call__(self, request, nxt): + self.rec.append( + 'bef_{0}{1}'.format(self.label, len(self.rec)) + ) + resp = nxt(request) + self.rec.append( + 'aft_{0}{1}'.format(self.label, len(self.rec)) + ) + return resp + + class HttpTest(unittest.TestCase): + def test_middlewares(self): + rec_ls = [] + mw_a = MiddlewareRecorder(rec_ls, 'A') + mw_b = MiddlewareRecorder(rec_ls, 'B') + qn_http_client.get( + "https://qiniu.com/index.html", + middlewares=[ + mw_a, + mw_b + ] + ) + assert rec_ls == ['bef_A0', 'bef_B1', 'aft_B2', 'aft_A3'] + + + def test_retry_domains(self): + rec_ls = [] + mw_rec = MiddlewareRecorder(rec_ls, 'rec') + ret, resp = qn_http_client.get( + 'https://fake.pysdk.qiniu.com/index.html', + middlewares=[ + RetryDomainsMiddleware( + backup_domains=[ + 'unavailable.pysdk.qiniu.com', + 'qiniu.com' + ], + max_retry_times=3 + ), + mw_rec + ] + ) + # ['bef_rec0', 'bef_rec1', 'bef_rec2'] are first request + # ['bef_rec3', 'bef_rec4', 'bef_rec5'] are 'unavailable.pysdk.qiniu.com' with retried 3 times + # ['bef_rec6', 'aft_rec7'] are 'qiniu.com' and it's success + assert rec_ls == [ + 'bef_rec0', 'bef_rec1', 'bef_rec2', + 'bef_rec3', 'bef_rec4', 'bef_rec5', + 'bef_rec6', 'aft_rec7' + ] + assert ret == {} + assert resp.status_code == 200 + + def test_json_decode_error(self): def mock_res(): r = requests.Response() From 6762bc7d7cf3c383f5eec9526123660a45a20a19 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Sun, 21 May 2023 14:36:15 +0800 Subject: [PATCH 444/478] improve code for x-qiniu-date --- qiniu/auth.py | 41 +++++++++++++++++++++++++++++++++-------- qiniu/http/__init__.py | 21 ++------------------- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index a1466564..d7bba44c 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import base64 +from datetime import datetime import hmac +import os import time from hashlib import sha1 from requests.auth import AuthBase @@ -242,6 +244,14 @@ def __token(self, data): hashed = hmac.new(self.__secret_key, data, sha1) return urlsafe_base64_encode(hashed.digest()) + @property + def should_sign_with_timestamp(self): + if self.disable_qiniu_timestamp_signature is not None: + return not self.disable_qiniu_timestamp_signature + if os.getenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE', '').lower() == 'true': + return False + return True + def token_of_request( self, method, @@ -294,12 +304,12 @@ def token_of_request( return '{0}:{1}'.format(self.__access_key, self.__token(data)) def qiniu_headers(self, headers): - qiniu_fields = list(filter( - lambda k: k.startswith(self.qiniu_header_prefix) and len(k) > len(self.qiniu_header_prefix), - headers, - )) - return "\n".join([ - "%s: %s" % (canonical_mime_header_key(key), headers.get(key)) for key in sorted(qiniu_fields) + qiniu_fields = [ + key for key in headers + if key.startswith(self.qiniu_header_prefix) and len(key) > len(self.qiniu_header_prefix) + ] + return '\n'.join([ + '%s: %s' % (canonical_mime_header_key(key), headers.get(key)) for key in sorted(qiniu_fields) ]) @staticmethod @@ -309,15 +319,30 @@ def __checkKey(access_key, secret_key): class QiniuMacRequestsAuth(AuthBase): + """ + Attributes: + auth (QiniuMacAuth): + """ def __init__(self, auth): + """ + Args: + auth (QiniuMacAuth): + """ self.auth = auth def __call__(self, r): if r.headers.get('Content-Type', None) is None: r.headers['Content-Type'] = 'application/x-www-form-urlencoded' + + if self.auth.should_sign_with_timestamp: + x_qiniu_date = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') + r.headers['X-Qiniu-Date'] = x_qiniu_date + token = self.auth.token_of_request( - r.method, r.headers.get('Host', None), - r.url, self.auth.qiniu_headers(r.headers), + r.method, + r.headers.get('Host', None), + r.url, + self.auth.qiniu_headers(r.headers), r.headers.get('Content-Type', None), r.body ) diff --git a/qiniu/http/__init__.py b/qiniu/http/__init__.py index fbad3b61..a7fed76e 100644 --- a/qiniu/http/__init__.py +++ b/qiniu/http/__init__.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- import logging -import os import platform -from datetime import datetime import requests from requests.adapters import HTTPAdapter @@ -33,19 +31,6 @@ _headers = {'User-Agent': USER_AGENT} -def __add_auth_headers(headers, auth): - x_qiniu_date = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') - if auth.disable_qiniu_timestamp_signature is not None: - if not auth.disable_qiniu_timestamp_signature: - headers['X-Qiniu-Date'] = x_qiniu_date - elif os.getenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE'): - if os.getenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE').lower() != 'true': - headers['X-Qiniu-Date'] = x_qiniu_date - else: - headers['X-Qiniu-Date'] = x_qiniu_date - return headers - - def __return_wrapper(resp): if resp.status_code != 200 or resp.headers.get('X-Reqid') is None: return None, ResponseInfo(resp) @@ -182,18 +167,16 @@ def _post_with_qiniu_mac(url, data, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth( auth ) if auth is not None else None - headers = __add_auth_headers({}, auth) - return _post(url, data, None, qn_auth, headers=headers) + return _post(url, data, None, qn_auth) def _get_with_qiniu_mac(url, params, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth( auth ) if auth is not None else None - headers = __add_auth_headers({}, auth) - return _get(url, params, qn_auth, headers=headers) + return _get(url, params, qn_auth) def _get_with_qiniu_mac_and_headers(url, params, auth, headers): From 76a71e261f6c50ee65189fd6285d11a699b18292 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Mon, 22 May 2023 11:50:25 +0800 Subject: [PATCH 445/478] add get region hosts with backup domains --- qiniu/config.py | 18 +++++++++++++++++- qiniu/http/legacy.py | 0 qiniu/region.py | 22 ++++++++++++++++++---- test_qiniu.py | 21 +++++++++++++++++++++ 4 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 qiniu/http/legacy.py diff --git a/qiniu/config.py b/qiniu/config.py index 58ef3650..9679f181 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -15,6 +15,11 @@ 'default_rsf_host': RSF_HOST, 'default_api_host': API_HOST, 'default_uc_host': UC_HOST, + 'default_uc_backup_hosts': [ + 'kodo-config.qiniuapi.com', + 'api.qiniu.com' + ], + 'default_uc_backup_retry_times': 2, 'connection_timeout': 30, # 链接超时为时间为30s 'connection_retries': 3, # 链接重试次数为3次 'connection_pool': 10, # 链接池个数为10 @@ -27,6 +32,8 @@ 'default_rsf_host': False, 'default_api_host': False, 'default_uc_host': False, + 'default_uc_backup_hosts': False, + 'default_uc_backup_retry_times': False, 'connection_timeout': False, 'connection_retries': False, 'connection_pool': False, @@ -45,7 +52,8 @@ def get_default(key): def set_default( default_zone=None, connection_retries=None, connection_pool=None, connection_timeout=None, default_rs_host=None, default_uc_host=None, - default_rsf_host=None, default_api_host=None, default_upload_threshold=None): + default_rsf_host=None, default_api_host=None, default_upload_threshold=None, + default_uc_backup_hosts=None, default_uc_backup_retry_times=None): if default_zone: _config['default_zone'] = default_zone _is_customized_default['default_zone'] = True @@ -61,6 +69,14 @@ def set_default( if default_uc_host: _config['default_uc_host'] = default_uc_host _is_customized_default['default_uc_host'] = True + _config['default_uc_backup_hosts'] = [] + _is_customized_default['default_uc_backup_hosts'] = True + if default_uc_backup_hosts: + _config['default_uc_backup_hosts'] = default_uc_backup_hosts + _is_customized_default['default_uc_backup_hosts'] = True + if default_uc_backup_retry_times: + _config['default_uc_backup_retry_times'] = default_uc_backup_retry_times + _is_customized_default['default_uc_backup_retry_times'] = True if connection_retries: _config['connection_retries'] = connection_retries _is_customized_default['connection_retries'] = True diff --git a/qiniu/http/legacy.py b/qiniu/http/legacy.py new file mode 100644 index 00000000..e69de29b diff --git a/qiniu/region.py b/qiniu/region.py index 654229a9..711eaea9 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -2,7 +2,7 @@ import logging import os import time -import requests + from qiniu import compat from qiniu import utils @@ -19,13 +19,15 @@ def __init__( up_host=None, up_host_backup=None, io_host=None, - host_cache={}, + host_cache=None, home_dir=None, scheme="http", rs_host=None, rsf_host=None, api_host=None): """初始化Zone类""" + if host_cache is None: + host_cache = {} self.up_host = up_host self.up_host_backup = up_host_backup self.io_host = io_host @@ -220,10 +222,22 @@ def host_cache_to_file(self, home_dir): def bucket_hosts(self, ak, bucket): from .config import get_default, is_customized_default + from .http import qn_http_client + from .http.middleware import RetryDomainsMiddleware uc_host = UC_HOST if is_customized_default('default_uc_host'): uc_host = get_default('default_uc_host') + uc_backup_hosts = get_default('default_uc_backup_hosts') + uc_backup_retry_times = get_default('default_uc_backup_retry_times') url = "{0}/v4/query?ak={1}&bucket={2}".format(uc_host, ak, bucket) - ret = requests.get(url) - data = compat.json.dumps(ret.json(), separators=(',', ':')) + ret, _resp = qn_http_client.get( + url, + middlewares=[ + RetryDomainsMiddleware( + backup_domains=uc_backup_hosts, + max_retry_times=uc_backup_retry_times + ) + ] + ) + data = compat.json.dumps(ret, separators=(',', ':')) return data diff --git a/test_qiniu.py b/test_qiniu.py index 6f378557..6cf35b3c 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -967,6 +967,27 @@ def test_config_compatible(self): qiniu.config._is_customized_default['default_rs_host'] = False qiniu.config._is_customized_default['default_rsf_host'] = False + def test_query_region_with_backup_domains(self): + try: + set_default( + default_uc_host='https://fake-uc.phpsdk.qiniu.com', + default_uc_backup_hosts=[ + 'unavailable-uc.phpsdk.qiniu.com', + 'uc.qbox.me' + ] + ) + zone = Zone() + data = zone.bucket_hosts(access_key, bucket_name) + assert data != "null" + finally: + set_default( + default_uc_host=qiniu.config.UC_HOST, + default_uc_backup_hosts=[ + 'kodo-config.qiniuapi.com', + 'api.qiniu.com' + ] + ) + class ReadWithoutSeek(object): def __init__(self, str): From fe933c5004e548937606e9da7d644a1ecbfc85bd Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Mon, 22 May 2023 12:09:13 +0800 Subject: [PATCH 446/478] format code --- qiniu/http/__init__.py | 2 +- qiniu/http/client.py | 6 +++--- qiniu/http/middleware/__init__.py | 12 +++++++++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/qiniu/http/__init__.py b/qiniu/http/__init__.py index a7fed76e..416d354d 100644 --- a/qiniu/http/__init__.py +++ b/qiniu/http/__init__.py @@ -48,7 +48,7 @@ def _init(): if _session is None: _session = qn_http_client.session - adapter = requests.adapters.HTTPAdapter( + adapter = HTTPAdapter( pool_connections=config.get_default('connection_pool'), pool_maxsize=config.get_default('connection_pool'), max_retries=config.get_default('connection_retries')) diff --git a/qiniu/http/client.py b/qiniu/http/client.py index cf0aeace..afdca951 100644 --- a/qiniu/http/client.py +++ b/qiniu/http/client.py @@ -4,7 +4,7 @@ import requests from .response import ResponseInfo -from .middleware import Middleware, compose_middleware +from .middleware import compose_middleware class HTTPClient: @@ -20,7 +20,7 @@ def send_request(self, request, middlewares=None, **kwargs): request (requests.Request): requests.Request 对象 - middlewares (list[Middleware] or (list[Middleware]) -> list[Middleware]): + middlewares (list[qiniu.http.middleware.Middleware] or (list[qiniu.http.middleware.Middleware]) -> list[qiniu.http.middleware.Middleware]): 仅对本次请求生效的中间件。 如果传入的是列表，那么会作为追加的中间件拼接到 Client 中间件的后面。 @@ -159,7 +159,7 @@ def delete( auth=None, headers=None, middlewares=None, - **kwargs, + **kwargs ): req = requests.Request( method='delete', diff --git a/qiniu/http/middleware/__init__.py b/qiniu/http/middleware/__init__.py index 799253aa..1177dcb4 100644 --- a/qiniu/http/middleware/__init__.py +++ b/qiniu/http/middleware/__init__.py @@ -1,3 +1,9 @@ -from .base import * -from .ua import * -from .retry_domains import * +from .base import Middleware, compose_middleware +from .ua import UserAgentMiddleware +from .retry_domains import RetryDomainsMiddleware + +__all__ = [ + 'Middleware', 'compose_middleware', + 'UserAgentMiddleware', + 'RetryDomainsMiddleware' +] From be912e622ed5f2d5ef8f6d1d6f439405b1f67c39 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Mon, 22 May 2023 12:13:19 +0800 Subject: [PATCH 447/478] move imp to local import in test_qiniu.py by python 3.12 --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 6cf35b3c..ec05ede9 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -4,7 +4,6 @@ import string import random import tempfile -from imp import reload import requests @@ -31,6 +30,7 @@ import sys import StringIO import urllib + from imp import reload reload(sys) sys.setdefaultencoding('utf-8') From 7355e735a65b25576dcdac31fa3972b23490af62 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Mon, 22 May 2023 16:52:44 +0800 Subject: [PATCH 448/478] alias config.UC_HOST to region.UC_HOST --- qiniu/config.py | 7 +++---- qiniu/http/legacy.py | 0 2 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 qiniu/http/legacy.py diff --git a/qiniu/config.py b/qiniu/config.py index 9679f181..6d72702f 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -1,16 +1,15 @@ # -*- coding: utf-8 -*- - -from qiniu import zone +from qiniu import region RS_HOST = 'http://rs.qiniu.com' # 管理操作Host RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host API_HOST = 'http://api.qiniuapi.com' # 数据处理操作Host -UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host +UC_HOST = region.UC_HOST # 获取空间信息Host _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续传分块大小，该参数为接口规格，暂不支持修改 _config = { - 'default_zone': zone.Zone(), + 'default_zone': region.Region(), 'default_rs_host': RS_HOST, 'default_rsf_host': RSF_HOST, 'default_api_host': API_HOST, diff --git a/qiniu/http/legacy.py b/qiniu/http/legacy.py deleted file mode 100644 index e69de29b..00000000 From 0961add2ea77427d2474ee2156d6067120d4cdd0 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Tue, 23 May 2023 15:17:32 +0800 Subject: [PATCH 449/478] add retry_condition to RetryDomainsMiddleware --- qiniu/http/middleware/retry_domains.py | 16 +++++++++++++--- qiniu/region.py | 13 ++++++++++++- test_qiniu.py | 26 ++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/qiniu/http/middleware/retry_domains.py b/qiniu/http/middleware/retry_domains.py index c3fe49dc..e4b93041 100644 --- a/qiniu/http/middleware/retry_domains.py +++ b/qiniu/http/middleware/retry_domains.py @@ -4,14 +4,16 @@ class RetryDomainsMiddleware(Middleware): - def __init__(self, backup_domains, max_retry_times=2): + def __init__(self, backup_domains, max_retry_times=2, retry_condition=None): """ Args: backup_domains (list[str]): max_retry_times (int): + retry_condition ((requests.Response or None, requests.Request)->bool): """ self.backup_domains = backup_domains self.max_retry_times = max_retry_times + self.retry_condition = retry_condition self.retried_times = 0 @@ -48,6 +50,12 @@ def _try_nxt(request, nxt): err = e return resp, err + def _should_retry(self, resp, req): + if callable(self.retry_condition): + return self.retry_condition(resp, req) + + return resp is None or not resp.ok + def __call__(self, request, nxt): resp, err = None, None url_parse_result = urlparse(request.url) @@ -56,10 +64,12 @@ def __call__(self, request, nxt): request.url = RetryDomainsMiddleware._get_changed_url(request.url, backup_domain) self.retried_times = 0 - while (resp is None or not resp.ok) and self.retried_times < self.max_retry_times: + while self.retried_times < self.max_retry_times: resp, err = RetryDomainsMiddleware._try_nxt(request, nxt) self.retried_times += 1 - if err is None and (resp is not None and resp.ok): + if not self._should_retry(resp, request): + if err is not None: + raise err return resp if err is not None: diff --git a/qiniu/region.py b/qiniu/region.py index 711eaea9..a17077ab 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -230,12 +230,23 @@ def bucket_hosts(self, ak, bucket): uc_backup_hosts = get_default('default_uc_backup_hosts') uc_backup_retry_times = get_default('default_uc_backup_retry_times') url = "{0}/v4/query?ak={1}&bucket={2}".format(uc_host, ak, bucket) + + def retry_condition(resp, _): + if resp is None: + return True + if resp.status_code in [612, 631]: + # 612 is app / accesskey is not found + # 631 is no such bucket + return False + return not resp.ok + ret, _resp = qn_http_client.get( url, middlewares=[ RetryDomainsMiddleware( backup_domains=uc_backup_hosts, - max_retry_times=uc_backup_retry_times + max_retry_times=uc_backup_retry_times, + retry_condition=retry_condition ) ] ) diff --git a/test_qiniu.py b/test_qiniu.py index ec05ede9..0f461ac8 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -126,7 +126,7 @@ def test_retry_domains(self): mw_rec ] ) - # ['bef_rec0', 'bef_rec1', 'bef_rec2'] are first request + # ['bef_rec0', 'bef_rec1', 'bef_rec2'] are 'fake.pysdk.qiniu.com' with retried 3 times # ['bef_rec3', 'bef_rec4', 'bef_rec5'] are 'unavailable.pysdk.qiniu.com' with retried 3 times # ['bef_rec6', 'aft_rec7'] are 'qiniu.com' and it's success assert rec_ls == [ @@ -137,6 +137,28 @@ def test_retry_domains(self): assert ret == {} assert resp.status_code == 200 + def test_retry_domains_fail_fast(self): + rec_ls = [] + mw_rec = MiddlewareRecorder(rec_ls, 'rec') + ret, resp = qn_http_client.get( + 'https://fake.pysdk.qiniu.com/index.html', + middlewares=[ + RetryDomainsMiddleware( + backup_domains=[ + 'unavailable.pysdk.qiniu.com', + 'qiniu.com' + ], + retry_condition=lambda _resp, _req: False + ), + mw_rec + ] + ) + # ['bef_rec0'] are 'fake.pysdk.qiniu.com' with fail fast + assert rec_ls == ['bef_rec0'] + assert ret is None + assert resp.status_code == -1 + + def test_json_decode_error(self): def mock_res(): @@ -978,7 +1000,7 @@ def test_query_region_with_backup_domains(self): ) zone = Zone() data = zone.bucket_hosts(access_key, bucket_name) - assert data != "null" + assert data != 'null' finally: set_default( default_uc_host=qiniu.config.UC_HOST, From da7e06ce211bcc3409ed4a8f42a7581f8673e59e Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Tue, 23 May 2023 15:18:47 +0800 Subject: [PATCH 450/478] change ResponseInfo ok to return True when 2xx --- qiniu/http/response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/http/response.py b/qiniu/http/response.py index da5ec96f..af79deeb 100644 --- a/qiniu/http/response.py +++ b/qiniu/http/response.py @@ -43,7 +43,7 @@ def __init__(self, response, exception=None): self.error = 'server is not qiniu' def ok(self): - return self.status_code == 200 and self.req_id is not None + return self.status_code // 100 == 2 def need_retry(self): if self.__response is None or self.req_id is None: From 972bed14b2fd822e8a7d5040c5f1e7cf4ffa1e78 Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Fri, 2 Jun 2023 11:57:08 +0800 Subject: [PATCH 451/478] wrap requests.Response to ResponseInfo with middleware for reusing need_retry --- qiniu/http/client.py | 17 +++++++++-------- qiniu/http/middleware/base.py | 6 +++--- qiniu/http/middleware/retry_domains.py | 15 +++++++++------ qiniu/http/response.py | 21 +++++++++++++++------ qiniu/region.py | 10 ---------- test_qiniu.py | 26 +++++++++++++++++++++++++- 6 files changed, 61 insertions(+), 34 deletions(-) diff --git a/qiniu/http/client.py b/qiniu/http/client.py index afdca951..8dfa2369 100644 --- a/qiniu/http/client.py +++ b/qiniu/http/client.py @@ -13,6 +13,10 @@ def __init__(self, middlewares=None, send_opts=None): self.middlewares = [] if middlewares is None else middlewares self.send_opts = {} if send_opts is None else send_opts + def _wrap_send(self, req, **kwargs): + resp = self.session.send(req.prepare(), **kwargs) + return ResponseInfo(resp, None) + def send_request(self, request, middlewares=None, **kwargs): """ @@ -55,23 +59,20 @@ def send_request(self, request, middlewares=None, **kwargs): try: handle = compose_middleware( mw_ls, - lambda req: self.session.send(req.prepare(), **kwargs) + lambda req: self._wrap_send(req, **kwargs) ) - resp = handle(request) + resp_info = handle(request) except Exception as e: return None, ResponseInfo(None, e) - # wrap resp - resp_info = ResponseInfo(resp) + # if ok try dump response info to dict from json if not resp_info.ok: return None, resp_info - # try dump response info to dict from json - resp.encoding = "utf-8" try: - ret = resp.json() + ret = resp_info.json() except ValueError: - logging.debug("response body decode error: %s" % resp.text) + logging.debug("response body decode error: %s" % resp_info.text_body) ret = {} return ret, resp_info diff --git a/qiniu/http/middleware/base.py b/qiniu/http/middleware/base.py index f3b1f27b..aaaee330 100644 --- a/qiniu/http/middleware/base.py +++ b/qiniu/http/middleware/base.py @@ -6,10 +6,10 @@ def compose_middleware(middlewares, handle): """ Args: middlewares (list[Middleware]): Middlewares - handle ((requests.Request) -> requests.Response): The send request handle + handle ((requests.Request) -> qiniu.http.response.ResponseInfo): The send request handle Returns: - (requests.Request) -> requests.Response: Composed handle + (requests.Request) -> qiniu.http.response.ResponseInfo: Composed handle """ middlewares.reverse() @@ -27,7 +27,7 @@ def __call__(self, request, nxt): """ Args: request (requests.Request): - nxt ((requests.Request) -> requests.Response): + nxt ((requests.Request) -> qiniu.http.response.ResponseInfo): Returns: requests.Response: diff --git a/qiniu/http/middleware/retry_domains.py b/qiniu/http/middleware/retry_domains.py index e4b93041..4f6845a0 100644 --- a/qiniu/http/middleware/retry_domains.py +++ b/qiniu/http/middleware/retry_domains.py @@ -34,6 +34,9 @@ def _get_changed_url(url, domain): backup_netloc += domain if url_parse_result.port is not None: backup_netloc += ':' + str(url_parse_result.port) + + # the _replace is a public method. start with `_` just to prevent conflicts with field names + # see namedtuple docs url_parse_result = url_parse_result._replace( netloc=backup_netloc ) @@ -54,10 +57,10 @@ def _should_retry(self, resp, req): if callable(self.retry_condition): return self.retry_condition(resp, req) - return resp is None or not resp.ok + return resp is None or resp.need_retry() def __call__(self, request, nxt): - resp, err = None, None + resp_info, err = None, None url_parse_result = urlparse(request.url) for backup_domain in [str(url_parse_result.hostname)] + self.backup_domains: @@ -65,14 +68,14 @@ def __call__(self, request, nxt): self.retried_times = 0 while self.retried_times < self.max_retry_times: - resp, err = RetryDomainsMiddleware._try_nxt(request, nxt) + resp_info, err = RetryDomainsMiddleware._try_nxt(request, nxt) self.retried_times += 1 - if not self._should_retry(resp, request): + if not self._should_retry(resp_info, request): if err is not None: raise err - return resp + return resp_info if err is not None: raise err - return resp + return resp_info diff --git a/qiniu/http/response.py b/qiniu/http/response.py index af79deeb..7aa1bba4 100644 --- a/qiniu/http/response.py +++ b/qiniu/http/response.py @@ -46,16 +46,25 @@ def ok(self): return self.status_code // 100 == 2 def need_retry(self): - if self.__response is None or self.req_id is None: - return True - code = self.status_code - if (code // 100 == 5 and code != 579) or code == 996: - return True - return False + if 0 < self.status_code < 500: + return False + # https://developer.qiniu.com/fusion/kb/1352/the-http-request-return-a-status-code + if self.status_code in [ + 501, 509, 573, 579, 608, 612, 614, 616, 618, 630, 631, 632, 640, 701 + ]: + return False + return True def connect_failed(self): return self.__response is None or self.req_id is None + def json(self): + try: + self.__response.encoding = "utf-8" + return self.__response.json() + except Exception: + return {} + def __str__(self): if is_py2: return ', '.join( diff --git a/qiniu/region.py b/qiniu/region.py index a17077ab..1ffc5f46 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -231,22 +231,12 @@ def bucket_hosts(self, ak, bucket): uc_backup_retry_times = get_default('default_uc_backup_retry_times') url = "{0}/v4/query?ak={1}&bucket={2}".format(uc_host, ak, bucket) - def retry_condition(resp, _): - if resp is None: - return True - if resp.status_code in [612, 631]: - # 612 is app / accesskey is not found - # 631 is no such bucket - return False - return not resp.ok - ret, _resp = qn_http_client.get( url, middlewares=[ RetryDomainsMiddleware( backup_domains=uc_backup_hosts, max_retry_times=uc_backup_retry_times, - retry_condition=retry_condition ) ] ) diff --git a/test_qiniu.py b/test_qiniu.py index 0f461ac8..20962a94 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -96,12 +96,36 @@ def __call__(self, request, nxt): class HttpTest(unittest.TestCase): + def test_response_need_retry(self): + _ret, resp_info = qn_http_client.get('https://qiniu.com/index.html') + + resp_info.req_id = 'mocked-req-id' + resp_info.error = None + + def gen_case(code): + if 0 < code < 500: + return code, False + if code in [ + 501, 509, 573, 579, 608, 612, 614, 616, 618, 630, 631, 632, 640, 701 + ]: + return code, False + return code, True + + cases = [ + gen_case(i) for i in range(-1, 800) + ] + + for test_code, should_retry in cases: + resp_info.status_code = test_code + assert_msg = '{0} should{1} retry'.format(test_code, '' if should_retry else ' NOT') + assert resp_info.need_retry() == should_retry, assert_msg + def test_middlewares(self): rec_ls = [] mw_a = MiddlewareRecorder(rec_ls, 'A') mw_b = MiddlewareRecorder(rec_ls, 'B') qn_http_client.get( - "https://qiniu.com/index.html", + 'https://qiniu.com/index.html', middlewares=[ mw_a, mw_b From 9728d3b66cfab5f34c60a8c7d8316d404565a6af Mon Sep 17 00:00:00 2001 From: LiHS <lihsai0@gmail.com> Date: Thu, 8 Jun 2023 15:02:10 +0800 Subject: [PATCH 452/478] update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 202de3cf..2651d981 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ * 对象存储，修复获取区域域名后无法按照预期进行过期处理 * 对象存储，更新获取区域域名的接口 * 对象存储，bucket_domains 修改为 list_domains 的别名 +* 对象存储，新增请求中间件逻辑，方便拓展请求逻辑 +* 对象存储，新增备用 UC 域名用于查询区域域名 ## 7.10.0（2022-11-15） * 对象存储，修复通过 set_default 设置 rs, rsf 不生效，而 SDK 自动获取的问题（v7.9.0） From 4505df77376c19e1d95fee26bab01d4cd5a41ab9 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Fri, 28 Jul 2023 09:37:50 +0800 Subject: [PATCH 453/478] reorder query region hosts & fix ok call & fix ci (#437) --- .github/workflows/ci-test.yml | 38 ++++++++++++++---------- qiniu/config.py | 39 ++++++++++++++++--------- qiniu/http/client.py | 2 +- qiniu/region.py | 4 +-- test_qiniu.py | 55 ++++++++++++++++++++++++++++------- 5 files changed, 95 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index e4fce82b..5d4ba447 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -6,36 +6,42 @@ jobs: fail-fast: false max-parallel: 1 matrix: - python_version: ['2.7', '3.4.10', '3.5', '3.6', '3.7', '3.8', '3.9'] + python_version: ['2.7', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9'] runs-on: ubuntu-20.04 steps: - name: Checkout repo uses: actions/checkout@v2 with: ref: ${{ github.ref }} - - name: Setup python - if: ${{ matrix.python_version != '3.4.10' }} - uses: actions/setup-python@v2 + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@v2 with: + auto-update-conda: true + channels: conda-forge python-version: ${{ matrix.python_version }} - architecture: x64 - - name: Setup python manually - if: ${{ matrix.python_version == '3.4.10' }} + activate-environment: qiniu-sdk + auto-activate-base: false + - name: Setup pip + shell: bash -l {0} env: - PYTHON_SOURCE_URL: https://www.python.org/ftp/python/${{ matrix.python_version }}/Python-${{ matrix.python_version }}.tgz + PYTHON_VERSION: ${{ matrix.python_version }} + PIP_BOOTSTRAP_SCRIPT_PREFIX: https://bootstrap.pypa.io/pip run: | - cd /tmp - curl -s -L "$PYTHON_SOURCE_URL" | tar -zxf - -C ./ - cd Python-${{ matrix.python_version }} - ./configure --enable-optimizations - make - sudo make install - echo 'export PATH="/opt/python/bin:$PATH"' >> $HOME/.bashrc + MAJOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f1) + MINOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f2) + # reinstall pip by some python(<3.7) not compatible + if ! [[ $MAJOR -ge 3 && $MINOR -ge 7 ]]; then + cd /tmp + wget -qLO get-pip.py "$PIP_BOOTSTRAP_SCRIPT_PREFIX/$MAJOR.$MINOR/get-pip.py" + python get-pip.py --user + fi - name: Install dependencies + shell: bash -l {0} run: | python -m pip install --upgrade pip - pip install "coverage<7.2" flake8 pytest pytest-cov freezegun requests scrutinizer-ocular codecov + python -m pip install --ignore-installed "coverage<7.2" flake8 pytest pytest-cov freezegun requests scrutinizer-ocular codecov - name: Run cases + shell: bash -l {0} env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} diff --git a/qiniu/config.py b/qiniu/config.py index 6d72702f..cb2ac57c 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -5,6 +5,7 @@ RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host API_HOST = 'http://api.qiniuapi.com' # 数据处理操作Host UC_HOST = region.UC_HOST # 获取空间信息Host +QUERY_REGION_HOST = 'https://kodo-config.qiniuapi.com' _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续传分块大小，该参数为接口规格，暂不支持修改 @@ -14,11 +15,12 @@ 'default_rsf_host': RSF_HOST, 'default_api_host': API_HOST, 'default_uc_host': UC_HOST, - 'default_uc_backup_hosts': [ - 'kodo-config.qiniuapi.com', + 'default_query_region_host': QUERY_REGION_HOST, + 'default_query_region_backup_hosts': [ + 'uc.qbox.me', 'api.qiniu.com' ], - 'default_uc_backup_retry_times': 2, + 'default_backup_hosts_retry_times': 2, 'connection_timeout': 30, # 链接超时为时间为30s 'connection_retries': 3, # 链接重试次数为3次 'connection_pool': 10, # 链接池个数为10 @@ -31,8 +33,9 @@ 'default_rsf_host': False, 'default_api_host': False, 'default_uc_host': False, - 'default_uc_backup_hosts': False, - 'default_uc_backup_retry_times': False, + 'default_query_region_host': False, + 'default_query_region_backup_hosts': False, + 'default_backup_hosts_retry_times': False, 'connection_timeout': False, 'connection_retries': False, 'connection_pool': False, @@ -52,7 +55,8 @@ def set_default( default_zone=None, connection_retries=None, connection_pool=None, connection_timeout=None, default_rs_host=None, default_uc_host=None, default_rsf_host=None, default_api_host=None, default_upload_threshold=None, - default_uc_backup_hosts=None, default_uc_backup_retry_times=None): + default_query_region_host=None, default_query_region_backup_hosts=None, + default_backup_hosts_retry_times=None): if default_zone: _config['default_zone'] = default_zone _is_customized_default['default_zone'] = True @@ -68,14 +72,21 @@ def set_default( if default_uc_host: _config['default_uc_host'] = default_uc_host _is_customized_default['default_uc_host'] = True - _config['default_uc_backup_hosts'] = [] - _is_customized_default['default_uc_backup_hosts'] = True - if default_uc_backup_hosts: - _config['default_uc_backup_hosts'] = default_uc_backup_hosts - _is_customized_default['default_uc_backup_hosts'] = True - if default_uc_backup_retry_times: - _config['default_uc_backup_retry_times'] = default_uc_backup_retry_times - _is_customized_default['default_uc_backup_retry_times'] = True + _config['default_query_region_host'] = default_uc_host + _is_customized_default['default_query_region_host'] = True + _config['default_query_region_backup_hosts'] = [] + _is_customized_default['default_query_region_backup_hosts'] = True + if default_query_region_host: + _config['default_query_region_host'] = default_query_region_host + _is_customized_default['default_query_region_host'] = True + _config['default_query_region_backup_hosts'] = [] + _is_customized_default['default_query_region_backup_hosts'] = True + if default_query_region_backup_hosts: + _config['default_query_region_backup_hosts'] = default_query_region_backup_hosts + _is_customized_default['default_query_region_backup_hosts'] = True + if default_backup_hosts_retry_times: + _config['default_backup_hosts_retry_times'] = default_backup_hosts_retry_times + _is_customized_default['default_backup_hosts_retry_times'] = True if connection_retries: _config['connection_retries'] = connection_retries _is_customized_default['connection_retries'] = True diff --git a/qiniu/http/client.py b/qiniu/http/client.py index 8dfa2369..97c41980 100644 --- a/qiniu/http/client.py +++ b/qiniu/http/client.py @@ -66,7 +66,7 @@ def send_request(self, request, middlewares=None, **kwargs): return None, ResponseInfo(None, e) # if ok try dump response info to dict from json - if not resp_info.ok: + if not resp_info.ok(): return None, resp_info try: diff --git a/qiniu/region.py b/qiniu/region.py index 1ffc5f46..acab5c4e 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -227,8 +227,8 @@ def bucket_hosts(self, ak, bucket): uc_host = UC_HOST if is_customized_default('default_uc_host'): uc_host = get_default('default_uc_host') - uc_backup_hosts = get_default('default_uc_backup_hosts') - uc_backup_retry_times = get_default('default_uc_backup_retry_times') + uc_backup_hosts = get_default('default_query_region_backup_hosts') + uc_backup_retry_times = get_default('default_backup_hosts_retry_times') url = "{0}/v4/query?ak={1}&bucket={2}".format(uc_host, ak, bucket) ret, _resp = qn_http_client.get( diff --git a/test_qiniu.py b/test_qiniu.py index 20962a94..151b861a 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -1000,6 +1000,24 @@ class RegionTestCase(unittest.TestCase): test_rs_host = 'test.region.compatible.config.rs' test_rsf_host = 'test.region.compatible.config.rsf' + @staticmethod + def restore_hosts(): + set_default( + default_rs_host=qiniu.config.RS_HOST, + default_rsf_host=qiniu.config.RSF_HOST, + default_uc_host=qiniu.config.UC_HOST, + default_query_region_host=qiniu.config.QUERY_REGION_HOST, + default_query_region_backup_hosts=[ + 'uc.qbox.me', + 'api.qiniu.com' + ] + ) + qiniu.config._is_customized_default['default_rs_host'] = False + qiniu.config._is_customized_default['default_rsf_host'] = False + qiniu.config._is_customized_default['default_uc_host'] = False + qiniu.config._is_customized_default['default_query_region_host'] = False + qiniu.config._is_customized_default['default_query_region_backup_hosts'] = False + def test_config_compatible(self): try: set_default(default_rs_host=self.test_rs_host) @@ -1008,16 +1026,24 @@ def test_config_compatible(self): assert zone.get_rs_host("mock_ak", "mock_bucket") == self.test_rs_host assert zone.get_rsf_host("mock_ak", "mock_bucket") == self.test_rsf_host finally: - set_default(default_rs_host=qiniu.config.RS_HOST) - set_default(default_rsf_host=qiniu.config.RSF_HOST) - qiniu.config._is_customized_default['default_rs_host'] = False - qiniu.config._is_customized_default['default_rsf_host'] = False + RegionTestCase.restore_hosts() + + def test_query_region_with_custom_domain(self): + try: + set_default( + default_query_region_host='https://fake-uc.phpsdk.qiniu.com' + ) + zone = Zone() + data = zone.bucket_hosts(access_key, bucket_name) + assert data != 'null' + finally: + RegionTestCase.restore_hosts() def test_query_region_with_backup_domains(self): try: set_default( - default_uc_host='https://fake-uc.phpsdk.qiniu.com', - default_uc_backup_hosts=[ + default_query_region_host='https://fake-uc.phpsdk.qiniu.com', + default_query_region_backup_hosts=[ 'unavailable-uc.phpsdk.qiniu.com', 'uc.qbox.me' ] @@ -1026,13 +1052,22 @@ def test_query_region_with_backup_domains(self): data = zone.bucket_hosts(access_key, bucket_name) assert data != 'null' finally: + RegionTestCase.restore_hosts() + + def test_query_region_with_uc_and_backup_domains(self): + try: set_default( - default_uc_host=qiniu.config.UC_HOST, - default_uc_backup_hosts=[ - 'kodo-config.qiniuapi.com', - 'api.qiniu.com' + default_uc_host='https://fake-uc.phpsdk.qiniu.com', + default_query_region_backup_hosts=[ + 'unavailable-uc.phpsdk.qiniu.com', + 'uc.qbox.me' ] ) + zone = Zone() + data = zone.bucket_hosts(access_key, bucket_name) + assert data != 'null' + finally: + RegionTestCase.restore_hosts() class ReadWithoutSeek(object): From 7d6eb2dde37a53b421cda71709422574eadc8874 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Mon, 21 Aug 2023 15:26:51 +0800 Subject: [PATCH 454/478] fix setup lost some packages by find_packages (#438) * fix setup lost some packages by find_packages * tweak test cases to test with other buckets --- CHANGELOG.md | 3 +++ qiniu/__init__.py | 2 +- setup.py | 21 ++------------------- test_qiniu.py | 44 ++++++++++++++++++++++---------------------- 4 files changed, 28 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2651d981..53ed46a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 7.11.1（2023-08-16） +* 修复 setup.py 打包丢失部分包（v7.11.0 引入） + ## 7.11.0（2023-03-28） * 对象存储，更新 api 默认域名 * 对象存储，新增 api 域名的配置与获取 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 778a0454..a0a26b77 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.11.0' +__version__ = '7.11.1' from .auth import Auth, QiniuMacAuth diff --git a/setup.py b/setup.py index 34badc36..e7e37cce 100644 --- a/setup.py +++ b/setup.py @@ -5,24 +5,7 @@ import os import re -try: - import setuptools - - setup = setuptools.setup -except ImportError: - setuptools = None - from distutils.core import setup - -packages = [ - 'qiniu', - 'qiniu.services', - 'qiniu.services.storage', - 'qiniu.services.processing', - 'qiniu.services.compute', - 'qiniu.services.cdn', - 'qiniu.services.sms', - 'qiniu.services.pili', -] +from setuptools import setup, find_packages def read(*names, **kwargs): @@ -52,7 +35,7 @@ def find_version(*file_paths): license='MIT', url='https://github.com/qiniu/python-sdk', platforms='any', - packages=packages, + packages=find_packages(), classifiers=[ 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', diff --git a/test_qiniu.py b/test_qiniu.py index 151b861a..25e5f8bb 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -17,7 +17,7 @@ build_batch_delete, DomainManager from qiniu import urlsafe_base64_encode, urlsafe_base64_decode, canonical_mime_header_key -from qiniu.compat import is_py2, is_py3, b +from qiniu.compat import is_py2, is_py3, b, json from qiniu.services.storage.uploader import _form_put @@ -414,7 +414,7 @@ def test_list(self): assert len(ret.get('items')) == 4 ret, eof, info = self.bucket.list(bucket_name, limit=1000) print(ret, eof, info) - assert eof is False + assert info.status_code == 200 def test_buckets(self): ret, info = self.bucket.buckets() @@ -699,14 +699,26 @@ def test_putWithoutKey(self): assert info.status_code == 403 # key not match def test_withoutRead_withoutSeek_retry(self): - key = 'retry' - data = 'hello retry!' - set_default(default_zone=Zone('http://a', 'https://upload.qiniup.com')) - token = self.q.upload_token(bucket_name) - ret, info = put_data(token, key, data) - print(info) - assert ret['key'] == key - assert ret['hash'] == 'FlYu0iBR1WpvYi4whKXiBuQpyLLk' + try: + key = 'retry' + data = 'hello retry!' + zone = Zone() + try: + hosts = json.loads( + zone.bucket_hosts(access_key, bucket_name) + ).get('hosts') + up_host_backup = 'https://' + hosts[0].get('up', {}).get('domains')[0] + except IndexError: + up_host_backup = 'https://upload.qiniup.com' + set_default(default_zone=Zone('http://a', up_host_backup)) + token = self.q.upload_token(bucket_name) + ret, info = put_data(token, key, data) + print(info) + assert ret['key'] == key + assert ret['hash'] == 'FlYu0iBR1WpvYi4whKXiBuQpyLLk' + finally: + set_default(default_zone=Zone()) + qiniu.config._is_customized_default['default_zone'] = False def test_putData_without_fname(self): if is_travis(): @@ -780,7 +792,6 @@ def test_put_stream(self): localfile = __file__ key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, @@ -792,7 +803,6 @@ def test_put_stream_v2_without_bucket_name(self): localfile = __file__ key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, @@ -804,7 +814,6 @@ def test_put_2m_stream_v2(self): localfile = create_temp_file(2 * 1024 * 1024 + 1) key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -817,7 +826,6 @@ def test_put_4m_stream_v2(self): localfile = create_temp_file(4 * 1024 * 1024) key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -830,7 +838,6 @@ def test_put_10m_stream_v2(self): localfile = create_temp_file(10 * 1024 * 1024 + 1) key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -844,7 +851,6 @@ def test_put_stream_v2_without_key(self): localfile = create_temp_file(part_size + 1) key = None size = os.stat(localfile).st_size - set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -858,7 +864,6 @@ def test_put_stream_v2_with_empty_return_body(self): localfile = create_temp_file(part_size + 1) key = 'test_file_empty_return_body' size = os.stat(localfile).st_size - set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key, policy={'returnBody': ' '}) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -873,7 +878,6 @@ def test_big_file(self): token = self.q.upload_token(bucket_name, key) localfile = create_temp_file(4 * 1024 * 1024 + 1) progress_handler = lambda progress, total: progress - qiniu.set_default(default_zone=Zone('http://a', 'https://upload.qiniup.com')) ret, info = put_file(token, key, localfile, self.params, self.mime_type, progress_handler=progress_handler) print(info) assert ret['key'] == key @@ -882,7 +886,6 @@ def test_big_file(self): def test_retry(self): localfile = __file__ key = 'test_file_r_retry' - qiniu.set_default(default_zone=Zone('http://a', 'https://upload.qiniup.com')) token = self.q.upload_token(bucket_name, key) ret, info = put_file(token, key, localfile, self.params, self.mime_type) print(info) @@ -893,7 +896,6 @@ def test_put_stream_with_key_limits(self): localfile = __file__ key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key, policy={'keylimit': ['test_file_d']}) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, @@ -910,7 +912,6 @@ def test_put_stream_with_metadata(self): localfile = __file__ key = 'test_put_stream_with_metadata' size = os.stat(localfile).st_size - set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, @@ -927,7 +928,6 @@ def test_put_stream_v2_with_metadata(self): localfile = create_temp_file(part_size + 1) key = 'test_put_stream_v2_with_metadata' size = os.stat(localfile).st_size - set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, From f04c99577cd66f4545555ad36f72a071afa3b25e Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Sun, 8 Oct 2023 10:38:55 +0800 Subject: [PATCH 455/478] resume upload part concurrently (#440) --- .github/workflows/ci-test.yml | 2 +- qiniu/auth.py | 8 + qiniu/compat.py | 11 + qiniu/region.py | 2 + qiniu/services/storage/legacy.py | 354 +++++++++ qiniu/services/storage/uploader.py | 557 ++++---------- qiniu/services/storage/uploaders/__init__.py | 9 + .../storage/uploaders/abc/__init__.py | 7 + .../uploaders/abc/resume_uploader_base.py | 212 +++++ .../storage/uploaders/abc/uploader_base.py | 164 ++++ .../storage/uploaders/form_uploader.py | 240 ++++++ .../services/storage/uploaders/io_chunked.py | 90 +++ .../storage/uploaders/resume_uploader_v1.py | 678 ++++++++++++++++ .../storage/uploaders/resume_uploader_v2.py | 727 ++++++++++++++++++ qiniu/utils.py | 17 +- setup.py | 16 +- test_qiniu.py | 60 +- 17 files changed, 2710 insertions(+), 444 deletions(-) create mode 100644 qiniu/services/storage/legacy.py create mode 100644 qiniu/services/storage/uploaders/__init__.py create mode 100644 qiniu/services/storage/uploaders/abc/__init__.py create mode 100644 qiniu/services/storage/uploaders/abc/resume_uploader_base.py create mode 100644 qiniu/services/storage/uploaders/abc/uploader_base.py create mode 100644 qiniu/services/storage/uploaders/form_uploader.py create mode 100644 qiniu/services/storage/uploaders/io_chunked.py create mode 100644 qiniu/services/storage/uploaders/resume_uploader_v1.py create mode 100644 qiniu/services/storage/uploaders/resume_uploader_v2.py diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 5d4ba447..84c20fd1 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -39,7 +39,7 @@ jobs: shell: bash -l {0} run: | python -m pip install --upgrade pip - python -m pip install --ignore-installed "coverage<7.2" flake8 pytest pytest-cov freezegun requests scrutinizer-ocular codecov + python -m pip install -I -e ".[dev]" - name: Run cases shell: bash -l {0} env: diff --git a/qiniu/auth.py b/qiniu/auth.py index d7bba44c..ff1be077 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -144,6 +144,7 @@ def upload_token( key: 上传的文件名，默认为空 expires: 上传凭证的过期时间，默认为3600s policy: 上传策略，默认为空 + strict_policy: 严格模式，将校验 policy 字段，默认为 True Returns: 上传凭证 @@ -175,6 +176,13 @@ def up_token_decode(up_token): dict_policy = json.loads(decode_policy) return ak, sign, dict_policy + @staticmethod + def get_bucket_name(up_token): + _, _, policy = Auth.up_token_decode(up_token) + if not policy or not policy['scope']: + return None + return policy['scope'].split(':', 1)[0] + def __upload_token(self, policy): data = json.dumps(policy, separators=(',', ':')) return self.token_with_data(data) diff --git a/qiniu/compat.py b/qiniu/compat.py index 3333a58c..4ab4ce78 100644 --- a/qiniu/compat.py +++ b/qiniu/compat.py @@ -4,6 +4,7 @@ pythoncompat """ +import os import sys try: @@ -51,6 +52,13 @@ def s(data): def u(data): return unicode(data, 'unicode_escape') # noqa + def is_seekable(data): + try: + data.seek(0, os.SEEK_CUR) + return True + except (AttributeError, IOError): + return False + elif is_py3: from urllib.parse import urlparse # noqa import io @@ -75,3 +83,6 @@ def s(data): def u(data): return data + + def is_seekable(data): + return data.seekable() diff --git a/qiniu/region.py b/qiniu/region.py index acab5c4e..dcef4398 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -110,6 +110,8 @@ def get_api_host(self, ak, bucket, home_dir=None): return api_hosts[0] def get_up_host(self, ak, bucket, home_dir): + if home_dir is None: + home_dir = os.getcwd() bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) if 'upHosts' not in bucket_hosts: bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir, force=True) diff --git a/qiniu/services/storage/legacy.py b/qiniu/services/storage/legacy.py new file mode 100644 index 00000000..db6f0979 --- /dev/null +++ b/qiniu/services/storage/legacy.py @@ -0,0 +1,354 @@ +# -*- coding: utf-8 -*- +import hashlib +import os +import time + +from qiniu import config, http +from qiniu.auth import Auth +from qiniu.compat import json +from qiniu.utils import _file_iter, crc32, rfc_from_timestamp, urlsafe_base64_encode + +from qiniu.services.storage.upload_progress_recorder import UploadProgressRecorder + + +class _Resume(object): + """deprecated 断点续上传类 + + 该类主要实现了分块上传，断点续上，以及相应地创建块和创建文件过程，详细规格参考： + https://developer.qiniu.com/kodo/api/mkblk + https://developer.qiniu.com/kodo/api/mkfile + + Attributes: + up_token: 上传凭证 + key: 上传文件名 + input_stream: 上传二进制流 + data_size: 上传流大小 + params: 自定义变量，规格参考 https://developer.qiniu.com/kodo/manual/vars#xvar + mime_type: 上传数据的mimeType + progress_handler: 上传进度 + upload_progress_recorder: 记录上传进度，用于断点续传 + modify_time: 上传文件修改日期 + hostscache_dir： host请求 缓存文件保存位置 + version 分片上传版本 目前支持v1/v2版本 默认v1 + part_size 分片上传v2必传字段 分片大小范围为1 MB - 1 GB + bucket_name 分片上传v2字段 空间名称 + """ + + def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, + progress_handler, upload_progress_recorder, modify_time, keep_last_modified, part_size=None, + version=None, bucket_name=None, metadata=None): + """初始化断点续上传""" + self.up_token = up_token + self.key = key + self.input_stream = input_stream + self.file_name = file_name + self.size = data_size + self.hostscache_dir = hostscache_dir + self.blockStatus = [] + self.params = params + self.mime_type = mime_type + self.progress_handler = progress_handler + self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() + self.modify_time = modify_time or time.time() + self.keep_last_modified = keep_last_modified + self.version = version or 'v1' + self.part_size = part_size or config._BLOCK_SIZE + self.bucket_name = bucket_name + self.metadata = metadata + + def record_upload_progress(self, offset): + record_data = { + 'size': self.size, + 'offset': offset, + } + if self.version == 'v1': + record_data['contexts'] = [ + { + 'ctx': block['ctx'], + 'expired_at': block['expired_at'] if 'expired_at' in block else 0 + } for block in self.blockStatus + ] + elif self.version == 'v2': + record_data['etags'] = self.blockStatus + record_data['expired_at'] = self.expiredAt + record_data['upload_id'] = self.uploadId + if self.modify_time: + record_data['modify_time'] = self.modify_time + self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) + + def recovery_from_record(self): + record = self.upload_progress_recorder.get_upload_record(self.file_name, self.key) + if not record: + if self.version == 'v1': + return 0 + elif self.version == 'v2': + return 0, None, None + try: + if not record['modify_time'] or record['size'] != self.size or \ + record['modify_time'] != self.modify_time: + if self.version == 'v1': + return 0 + elif self.version == 'v2': + return 0, None, None + except KeyError: + if self.version == 'v1': + return 0 + elif self.version == 'v2': + return 0, None, None + if self.version == 'v1': + if not record.__contains__('contexts') or len(record['contexts']) == 0: + return 0 + self.blockStatus = [ + # 兼容旧版本的 ctx 持久化 ≤v7.10.0 + ctx if type(ctx) is dict else {'ctx': ctx, 'expired_at': 0} + for ctx in record['contexts'] + ] + return record['offset'] + elif self.version == 'v2': + if not record.__contains__('etags') or len(record['etags']) == 0 or \ + not record.__contains__('expired_at') or float(record['expired_at']) < time.time() or \ + not record.__contains__('upload_id'): + return 0, None, None + self.blockStatus = record['etags'] + return record['offset'], record['upload_id'], record['expired_at'] + + def upload(self): + """上传操作""" + if self.version == 'v1': + return self._upload_v1() + elif self.version == 'v2': + return self._upload_v2() + else: + raise ValueError("version must choose v1 or v2 !") + + def _upload_v1(self): + self.blockStatus = [] + self.recovery_index = 1 + self.expiredAt = None + self.uploadId = None + self.get_bucket() + self.part_size = config._BLOCK_SIZE + + host = self.get_up_host() + offset = self.recovery_from_record() + is_resumed = offset > 0 + + # 检查原来的分片是否过期，如有则重传该分片 + for index, block_status in enumerate(self.blockStatus): + if block_status.get('expired_at', 0) > time.time(): + self.input_stream.seek(self.part_size, os.SEEK_CUR) + else: + block = self.input_stream.read(self.part_size) + response, ok = self._make_block_with_retry(block, host) + ret, info = response + if not ok: + return ret, info + self.blockStatus[index] = ret + self.record_upload_progress(offset) + + # 从断点位置上传 + for block in _file_iter(self.input_stream, self.part_size, offset): + length = len(block) + response, ok = self._make_block_with_retry(block, host) + ret, info = response + if not ok: + return ret, info + + self.blockStatus.append(ret) + offset += length + self.record_upload_progress(offset) + if callable(self.progress_handler): + self.progress_handler(((len(self.blockStatus) - 1) * self.part_size) + len(block), self.size) + + ret, info = self.make_file(host) + if info.status_code == 200 or info.status_code == 701: + self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) + if info.status_code == 701 and is_resumed: + return self.upload() + return ret, info + + def _upload_v2(self): + self.blockStatus = [] + self.recovery_index = 1 + self.expiredAt = None + self.uploadId = None + self.get_bucket() + host = self.get_up_host() + + offset, self.uploadId, self.expiredAt = self.recovery_from_record() + is_resumed = False + if offset > 0 and self.blockStatus != [] and self.uploadId is not None \ + and self.expiredAt is not None: + self.recovery_index = self.blockStatus[-1]['partNumber'] + 1 + is_resumed = True + else: + self.recovery_index = 1 + init_url = self.block_url_v2(host, self.bucket_name) + self.uploadId, self.expiredAt = self.init_upload_task(init_url) + + for index, block in enumerate(_file_iter(self.input_stream, self.part_size, offset)): + length = len(block) + index_ = index + self.recovery_index + url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (self.uploadId, index_) + ret, info = self.make_block_v2(block, url) + if info.status_code == 612: + self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) + if info.status_code == 612 and is_resumed: + return self.upload() + if ret is None and not info.need_retry(): + return ret, info + if info.connect_failed(): + if config.get_default('default_zone').up_host_backup: + host = config.get_default('default_zone').up_host_backup + else: + host = config.get_default('default_zone')\ + .get_up_host_backup_by_token(self.up_token, self.hostscache_dir) + + if info.need_retry(): + url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (self.uploadId, index + 1) + ret, info = self.make_block_v2(block, url) + if info.status_code == 612: + self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) + if info.status_code == 612 and is_resumed: + return self.upload() + if ret is None: + return ret, info + del ret['md5'] + ret['partNumber'] = index_ + self.blockStatus.append(ret) + offset += length + self.record_upload_progress(offset) + if callable(self.progress_handler): + self.progress_handler(((len(self.blockStatus) - 1) * self.part_size) + len(block), self.size) + + make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % self.uploadId + ret, info = self.make_file_v2( + self.blockStatus, + make_file_url, + self.file_name, + self.mime_type, + self.params, + self.metadata) + if info.status_code == 200 or info.status_code == 612: + self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) + if info.status_code == 612 and is_resumed: + return self.upload() + return ret, info + + def make_file_v2(self, block_status, url, file_name=None, mime_type=None, customVars=None, metadata=None): + """completeMultipartUpload""" + parts = self.get_parts(block_status) + headers = { + 'Content-Type': 'application/json', + } + data = { + 'parts': parts, + 'fname': file_name, + 'mimeType': mime_type, + 'customVars': customVars, + 'metadata': metadata + } + return self.post_with_headers(url, json.dumps(data), headers=headers) + + def get_up_host(self): + if config.get_default('default_zone').up_host: + host = config.get_default('default_zone').up_host + else: + host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.hostscache_dir) + return host + + def _make_block_with_retry(self, block_data, up_host): + length = len(block_data) + crc = crc32(block_data) + ret, info = self.make_block(block_data, length, up_host) + if ret is None and not info.need_retry(): + return (ret, info), False + if info.connect_failed(): + if config.get_default('default_zone').up_host_backup: + up_host = config.get_default('default_zone').up_host_backup + else: + up_host = config.get_default('default_zone') \ + .get_up_host_backup_by_token(self.up_token, self.hostscache_dir) + if info.need_retry() or crc != ret['crc32']: + ret, info = self.make_block(block_data, length, up_host) + if ret is None or crc != ret['crc32']: + return (ret, info), False + return (ret, info), True + + def make_block(self, block, block_size, host): + """创建块""" + url = self.block_url(host, block_size) + return self.post(url, block) + + def make_block_v2(self, block, url): + headers = { + 'Content-Type': 'application/octet-stream', + 'Content-MD5': hashlib.md5(block).hexdigest(), + } + return self.put(url, block, headers) + + def block_url(self, host, size): + return '{0}/mkblk/{1}'.format(host, size) + + def block_url_v2(self, host, bucket_name): + encoded_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '~' + return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, encoded_object_name) + + def file_url(self, host): + url = ['{0}/mkfile/{1}'.format(host, self.size)] + if self.mime_type: + url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type))) + + if self.key is not None: + url.append('key/{0}'.format(urlsafe_base64_encode(self.key))) + + if self.file_name is not None: + url.append('fname/{0}'.format(urlsafe_base64_encode(self.file_name))) + + if self.params: + for k, v in self.params.items(): + url.append('{0}/{1}'.format(k, urlsafe_base64_encode(v))) + + if self.modify_time and self.keep_last_modified: + url.append( + "x-qn-meta-!Last-Modified/{0}".format(urlsafe_base64_encode(rfc_from_timestamp(self.modify_time)))) + + if self.metadata: + for k, v in self.metadata.items(): + if k.startswith('x-qn-meta-'): + url.append( + "{0}/{1}".format(k, urlsafe_base64_encode(v))) + + url = '/'.join(url) + return url + + def make_file(self, host): + """创建文件""" + url = self.file_url(host) + body = ','.join([status['ctx'] for status in self.blockStatus]) + return self.post(url, body) + + def init_upload_task(self, url): + body, resp = self.post(url, '') + if body is not None: + return body['uploadId'], body['expireAt'] + else: + return None, None + + def post(self, url, data): + return http._post_with_token(url, data, self.up_token) + + def post_with_headers(self, url, data, headers): + return http._post_with_token_and_headers(url, data, self.up_token, headers) + + def put(self, url, data, headers): + return http._put_with_token_and_headers(url, data, self.up_token, headers) + + def get_parts(self, block_status): + return sorted(block_status, key=lambda i: i['partNumber']) + + def get_bucket(self): + if not self.bucket_name: + bucket_name = Auth.get_bucket_name(self.up_token) + if bucket_name: + self.bucket_name = bucket_name diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 965875ba..a66fae17 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -1,18 +1,22 @@ # -*- coding: utf-8 -*- -import hashlib -import json import os -import time -from qiniu import config, Auth -from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter, rfc_from_timestamp -from qiniu import http -from .upload_progress_recorder import UploadProgressRecorder +from qiniu.config import _BLOCK_SIZE, get_default + +from qiniu.auth import Auth +from qiniu.utils import crc32, file_crc32, rfc_from_timestamp + +from qiniu.services.storage.uploaders import FormUploader, ResumeUploaderV1, ResumeUploaderV2 +from qiniu.services.storage.upload_progress_recorder import UploadProgressRecorder + +# for compact to old sdk +from qiniu.services.storage.legacy import _Resume # noqa def put_data( - up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, - fname=None, hostscache_dir=None, metadata=None): + up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, + fname=None, hostscache_dir=None, metadata=None +): """上传二进制流到七牛 Args: @@ -33,7 +37,7 @@ def put_data( final_data = b'' if hasattr(data, 'read'): while True: - tmp_data = data.read(config._BLOCK_SIZE) + tmp_data = data.read(_BLOCK_SIZE) if len(tmp_data) == 0: break else: @@ -42,14 +46,18 @@ def put_data( final_data = data crc = crc32(final_data) - return _form_put(up_token, key, final_data, params, mime_type, - crc, hostscache_dir, progress_handler, fname, metadata=metadata) - - -def put_file(up_token, key, file_path, params=None, - mime_type='application/octet-stream', check_crc=False, - progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None, - part_size=None, version=None, bucket_name=None, metadata=None): + return _form_put( + up_token, key, final_data, params, mime_type, + crc, hostscache_dir, progress_handler, fname, metadata=metadata + ) + + +def put_file( + up_token, key, file_path, params=None, + mime_type='application/octet-stream', check_crc=False, + progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None, + part_size=None, version=None, bucket_name=None, metadata=None +): """上传文件到七牛 Args: @@ -76,417 +84,116 @@ def put_file(up_token, key, file_path, params=None, with open(file_path, 'rb') as input_stream: file_name = os.path.basename(file_path) modify_time = int(os.path.getmtime(file_path)) - if size > config.get_default('default_upload_threshold'): - ret, info = put_stream(up_token, key, input_stream, file_name, size, hostscache_dir, params, - mime_type, progress_handler, - upload_progress_recorder=upload_progress_recorder, - modify_time=modify_time, keep_last_modified=keep_last_modified, - part_size=part_size, version=version, bucket_name=bucket_name, metadata=metadata) + if size > get_default('default_upload_threshold'): + ret, info = put_stream( + up_token, key, input_stream, file_name, size, hostscache_dir, params, + mime_type, progress_handler, + upload_progress_recorder=upload_progress_recorder, + modify_time=modify_time, keep_last_modified=keep_last_modified, + part_size=part_size, version=version, bucket_name=bucket_name, metadata=metadata + ) else: crc = file_crc32(file_path) - ret, info = _form_put(up_token, key, input_stream, params, mime_type, - crc, hostscache_dir, progress_handler, file_name, - modify_time=modify_time, keep_last_modified=keep_last_modified, metadata=metadata) + ret, info = _form_put( + up_token, key, input_stream, params, mime_type, + crc, hostscache_dir, progress_handler, file_name, + modify_time=modify_time, keep_last_modified=keep_last_modified, metadata=metadata + ) return ret, info -def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, progress_handler=None, file_name=None, - modify_time=None, keep_last_modified=False, metadata=None): - fields = {} - if params: - for k, v in params.items(): - fields[k] = str(v) - if crc: - fields['crc32'] = crc - if key is not None: - fields['key'] = key - - fields['token'] = up_token - if config.get_default('default_zone').up_host: - url = config.get_default('default_zone').up_host - else: - url = config.get_default('default_zone').get_up_host_by_token(up_token, hostscache_dir) - # name = key if key else file_name - - fname = file_name - if not fname or not fname.strip(): - fname = 'file_name' +def _form_put( + up_token, + key, + data, + params, + mime_type, + crc, + hostscache_dir=None, + progress_handler=None, + file_name=None, + modify_time=None, + keep_last_modified=False, + metadata=None +): + bucket_name = Auth.get_bucket_name(up_token) + uploader = FormUploader( + bucket_name, + progress_handler=progress_handler, + hosts_cache_dir=hostscache_dir + ) - # last modify time if modify_time and keep_last_modified: - fields['x-qn-meta-!Last-Modified'] = rfc_from_timestamp(modify_time) - - if metadata: - for k, v in metadata.items(): - if k.startswith('x-qn-meta-'): - fields[k] = str(v) - - r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) - if r is None and info.need_retry(): - if info.connect_failed: - if config.get_default('default_zone').up_host_backup: - url = config.get_default('default_zone').up_host_backup - else: - url = config.get_default('default_zone').get_up_host_backup_by_token(up_token, hostscache_dir) - if hasattr(data, 'read') is False: - pass - elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()): - data.seek(0) - else: - return r, info - r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) - - return r, info - - -def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir=None, params=None, - mime_type=None, progress_handler=None, - upload_progress_recorder=None, modify_time=None, keep_last_modified=False, - part_size=None, version=None, bucket_name=None, metadata=None): - task = _Resume(up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, keep_last_modified, - part_size, version, bucket_name, metadata) - return task.upload() - - -class _Resume(object): - """断点续上传类 - - 该类主要实现了分块上传，断点续上，以及相应地创建块和创建文件过程，详细规格参考： - https://developer.qiniu.com/kodo/api/mkblk - https://developer.qiniu.com/kodo/api/mkfile - - Attributes: - up_token: 上传凭证 - key: 上传文件名 - input_stream: 上传二进制流 - data_size: 上传流大小 - params: 自定义变量，规格参考 https://developer.qiniu.com/kodo/manual/vars#xvar - mime_type: 上传数据的mimeType - progress_handler: 上传进度 - upload_progress_recorder: 记录上传进度，用于断点续传 - modify_time: 上传文件修改日期 - hostscache_dir： host请求 缓存文件保存位置 - version 分片上传版本 目前支持v1/v2版本 默认v1 - part_size 分片上传v2必传字段 分片大小范围为1 MB - 1 GB - bucket_name 分片上传v2字段 空间名称 - """ - - def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, keep_last_modified, part_size=None, - version=None, bucket_name=None, metadata=None): - """初始化断点续上传""" - self.up_token = up_token - self.key = key - self.input_stream = input_stream - self.file_name = file_name - self.size = data_size - self.hostscache_dir = hostscache_dir - self.blockStatus = [] - self.params = params - self.mime_type = mime_type - self.progress_handler = progress_handler - self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() - self.modify_time = modify_time or time.time() - self.keep_last_modified = keep_last_modified - self.version = version or 'v1' - self.part_size = part_size or config._BLOCK_SIZE - self.bucket_name = bucket_name - self.metadata = metadata - - def record_upload_progress(self, offset): - record_data = { - 'size': self.size, - 'offset': offset, - } - if self.version == 'v1': - record_data['contexts'] = [ - { - 'ctx': block['ctx'], - 'expired_at': block['expired_at'] if 'expired_at' in block else 0 - } for block in self.blockStatus - ] - elif self.version == 'v2': - record_data['etags'] = self.blockStatus - record_data['expired_at'] = self.expiredAt - record_data['upload_id'] = self.uploadId - if self.modify_time: - record_data['modify_time'] = self.modify_time - self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) - - def recovery_from_record(self): - record = self.upload_progress_recorder.get_upload_record(self.file_name, self.key) - if not record: - if self.version == 'v1': - return 0 - elif self.version == 'v2': - return 0, None, None - try: - if not record['modify_time'] or record['size'] != self.size or \ - record['modify_time'] != self.modify_time: - if self.version == 'v1': - return 0 - elif self.version == 'v2': - return 0, None, None - except KeyError: - if self.version == 'v1': - return 0 - elif self.version == 'v2': - return 0, None, None - if self.version == 'v1': - if not record.__contains__('contexts') or len(record['contexts']) == 0: - return 0 - self.blockStatus = [ - # 兼容旧版本的 ctx 持久化 - ctx if type(ctx) is dict else {'ctx': ctx, 'expired_at': 0} - for ctx in record['contexts'] - ] - return record['offset'] - elif self.version == 'v2': - if not record.__contains__('etags') or len(record['etags']) == 0 or \ - not record.__contains__('expired_at') or float(record['expired_at']) < time.time() or \ - not record.__contains__('upload_id'): - return 0, None, None - self.blockStatus = record['etags'] - return record['offset'], record['upload_id'], record['expired_at'] - - def upload(self): - """上传操作""" - if self.version == 'v1': - return self._upload_v1() - elif self.version == 'v2': - return self._upload_v2() - else: - raise ValueError("version must choose v1 or v2 !") - - def _upload_v1(self): - self.blockStatus = [] - self.recovery_index = 1 - self.expiredAt = None - self.uploadId = None - self.get_bucket() - self.part_size = config._BLOCK_SIZE - - host = self.get_up_host() - offset = self.recovery_from_record() - is_resumed = offset > 0 - - # 检查原来的分片是否过期，如有则重传该分片 - for index, block_status in enumerate(self.blockStatus): - if block_status.get('expired_at', 0) > time.time(): - self.input_stream.seek(self.part_size, os.SEEK_CUR) - else: - block = self.input_stream.read(self.part_size) - response, ok = self._make_block_with_retry(block, host) - ret, info = response - if not ok: - return ret, info - self.blockStatus[index] = ret - self.record_upload_progress(offset) - - # 从断点位置上传 - for block in _file_iter(self.input_stream, self.part_size, offset): - length = len(block) - response, ok = self._make_block_with_retry(block, host) - ret, info = response - if not ok: - return ret, info - - self.blockStatus.append(ret) - offset += length - self.record_upload_progress(offset) - if callable(self.progress_handler): - self.progress_handler(((len(self.blockStatus) - 1) * self.part_size) + len(block), self.size) - - ret, info = self.make_file(host) - if info.status_code == 200 or info.status_code == 701: - self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) - if info.status_code == 701 and is_resumed: - return self.upload() - return ret, info - - def _upload_v2(self): - self.blockStatus = [] - self.recovery_index = 1 - self.expiredAt = None - self.uploadId = None - self.get_bucket() - host = self.get_up_host() - - offset, self.uploadId, self.expiredAt = self.recovery_from_record() - is_resumed = False - if offset > 0 and self.blockStatus != [] and self.uploadId is not None \ - and self.expiredAt is not None: - self.recovery_index = self.blockStatus[-1]['partNumber'] + 1 - is_resumed = True - else: - self.recovery_index = 1 - init_url = self.block_url_v2(host, self.bucket_name) - self.uploadId, self.expiredAt = self.init_upload_task(init_url) - - for index, block in enumerate(_file_iter(self.input_stream, self.part_size, offset)): - length = len(block) - index_ = index + self.recovery_index - url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (self.uploadId, index_) - ret, info = self.make_block_v2(block, url) - if info.status_code == 612: - self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) - if info.status_code == 612 and is_resumed: - return self.upload() - if ret is None and not info.need_retry(): - return ret, info - if info.connect_failed(): - if config.get_default('default_zone').up_host_backup: - host = config.get_default('default_zone').up_host_backup - else: - host = config.get_default('default_zone')\ - .get_up_host_backup_by_token(self.up_token, self.hostscache_dir) - - if info.need_retry(): - url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (self.uploadId, index + 1) - ret, info = self.make_block_v2(block, url) - if info.status_code == 612: - self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) - if info.status_code == 612 and is_resumed: - return self.upload() - if ret is None: - return ret, info - del ret['md5'] - ret['partNumber'] = index_ - self.blockStatus.append(ret) - offset += length - self.record_upload_progress(offset) - if callable(self.progress_handler): - self.progress_handler(((len(self.blockStatus) - 1) * self.part_size) + len(block), self.size) - - make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % self.uploadId - ret, info = self.make_file_v2( - self.blockStatus, - make_file_url, - self.file_name, - self.mime_type, - self.params, - self.metadata) - if info.status_code == 200 or info.status_code == 612: - self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) - if info.status_code == 612 and is_resumed: - return self.upload() - return ret, info - - def make_file_v2(self, block_status, url, file_name=None, mime_type=None, customVars=None, metadata=None): - """completeMultipartUpload""" - parts = self.get_parts(block_status) - headers = { - 'Content-Type': 'application/json', - } - data = { - 'parts': parts, - 'fname': file_name, - 'mimeType': mime_type, - 'customVars': customVars, - 'metadata': metadata - } - return self.post_with_headers(url, json.dumps(data), headers=headers) - - def get_up_host(self): - if config.get_default('default_zone').up_host: - host = config.get_default('default_zone').up_host - else: - host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.hostscache_dir) - return host - - def _make_block_with_retry(self, block_data, up_host): - length = len(block_data) - crc = crc32(block_data) - ret, info = self.make_block(block_data, length, up_host) - if ret is None and not info.need_retry(): - return (ret, info), False - if info.connect_failed(): - if config.get_default('default_zone').up_host_backup: - up_host = config.get_default('default_zone').up_host_backup - else: - up_host = config.get_default('default_zone') \ - .get_up_host_backup_by_token(self.up_token, self.hostscache_dir) - if info.need_retry() or crc != ret['crc32']: - ret, info = self.make_block(block_data, length, up_host) - if ret is None or crc != ret['crc32']: - return (ret, info), False - return (ret, info), True - - def make_block(self, block, block_size, host): - """创建块""" - url = self.block_url(host, block_size) - return self.post(url, block) - - def make_block_v2(self, block, url): - headers = { - 'Content-Type': 'application/octet-stream', - 'Content-MD5': hashlib.md5(block).hexdigest(), - } - return self.put(url, block, headers) - - def block_url(self, host, size): - return '{0}/mkblk/{1}'.format(host, size) - - def block_url_v2(self, host, bucket_name): - encoded_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '~' - return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, encoded_object_name) - - def file_url(self, host): - url = ['{0}/mkfile/{1}'.format(host, self.size)] - if self.mime_type: - url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type))) - - if self.key is not None: - url.append('key/{0}'.format(urlsafe_base64_encode(self.key))) - - if self.file_name is not None: - url.append('fname/{0}'.format(urlsafe_base64_encode(self.file_name))) - - if self.params: - for k, v in self.params.items(): - url.append('{0}/{1}'.format(k, urlsafe_base64_encode(v))) - - if self.modify_time and self.keep_last_modified: - url.append( - "x-qn-meta-!Last-Modified/{0}".format(urlsafe_base64_encode(rfc_from_timestamp(self.modify_time)))) - - if self.metadata: - for k, v in self.metadata.items(): - if k.startswith('x-qn-meta-'): - url.append( - "{0}/{1}".format(k, urlsafe_base64_encode(v))) - - url = '/'.join(url) - return url - - def make_file(self, host): - """创建文件""" - url = self.file_url(host) - body = ','.join([status['ctx'] for status in self.blockStatus]) - return self.post(url, body) - - def init_upload_task(self, url): - body, resp = self.post(url, '') - if body is not None: - return body['uploadId'], body['expireAt'] - else: - return None, None - - def post(self, url, data): - return http._post_with_token(url, data, self.up_token) - - def post_with_headers(self, url, data, headers): - return http._post_with_token_and_headers(url, data, self.up_token, headers) - - def put(self, url, data, headers): - return http._put_with_token_and_headers(url, data, self.up_token, headers) - - def get_parts(self, block_status): - return sorted(block_status, key=lambda i: i['partNumber']) - - def get_bucket(self): - if self.bucket_name is None or self.bucket_name == '': - _, _, policy = Auth.up_token_decode(self.up_token) - if policy != {}: - self.bucket_name = policy['scope'].split(':')[0] + metadata['x-qn-meta-!Last-Modified'] = rfc_from_timestamp(modify_time) + + return uploader.upload( + key=key, + data=data, + data_size=None, + file_name=file_name, + modify_time=modify_time, + mime_type=mime_type, + metadata=metadata, + params=params, + crc32_int=crc, + up_token=up_token + ) + + +def put_stream( + up_token, + key, + input_stream, + file_name, + data_size, + hostscache_dir=None, + params=None, + mime_type=None, + progress_handler=None, + upload_progress_recorder=None, + modify_time=None, + keep_last_modified=False, + part_size=None, + version='v1', + bucket_name=None, + metadata=None +): + if not bucket_name: + bucket_name = Auth.get_bucket_name(up_token) + if not upload_progress_recorder: + upload_progress_recorder = UploadProgressRecorder() + if not version: + version = 'v1' + if not part_size: + part_size = 4 * (1024 * 1024) + + if version == 'v1': + uploader = ResumeUploaderV1( + bucket_name, + progress_handler=progress_handler, + upload_progress_recorder=upload_progress_recorder, + hosts_cache_dir=hostscache_dir + ) + if modify_time and keep_last_modified: + metadata['x-qn-meta-!Last-Modified'] = rfc_from_timestamp(modify_time) + elif version == 'v2': + uploader = ResumeUploaderV2( + bucket_name, + progress_handler=progress_handler, + upload_progress_recorder=upload_progress_recorder, + part_size=part_size, + hosts_cache_dir=hostscache_dir + ) + else: + raise ValueError('version only could be v1 or v2') + return uploader.upload( + key=key, + data=input_stream, + data_size=data_size, + file_name=file_name, + modify_time=modify_time, + mime_type=mime_type, + metadata=metadata, + params=params, + up_token=up_token + ) diff --git a/qiniu/services/storage/uploaders/__init__.py b/qiniu/services/storage/uploaders/__init__.py new file mode 100644 index 00000000..0e2ea39d --- /dev/null +++ b/qiniu/services/storage/uploaders/__init__.py @@ -0,0 +1,9 @@ +from .form_uploader import FormUploader +from .resume_uploader_v1 import ResumeUploaderV1 +from .resume_uploader_v2 import ResumeUploaderV2 + +__all__ = [ + 'FormUploader', + 'ResumeUploaderV1', + 'ResumeUploaderV2' +] diff --git a/qiniu/services/storage/uploaders/abc/__init__.py b/qiniu/services/storage/uploaders/abc/__init__.py new file mode 100644 index 00000000..2e528ca4 --- /dev/null +++ b/qiniu/services/storage/uploaders/abc/__init__.py @@ -0,0 +1,7 @@ +from .uploader_base import UploaderBase +from .resume_uploader_base import ResumeUploaderBase + +__all__ = [ + 'UploaderBase', + 'ResumeUploaderBase' +] diff --git a/qiniu/services/storage/uploaders/abc/resume_uploader_base.py b/qiniu/services/storage/uploaders/abc/resume_uploader_base.py new file mode 100644 index 00000000..2f6a3c50 --- /dev/null +++ b/qiniu/services/storage/uploaders/abc/resume_uploader_base.py @@ -0,0 +1,212 @@ +import abc +from concurrent import futures + +from qiniu.services.storage.uploaders.io_chunked import ChunkInfo +from qiniu.services.storage.uploaders.abc import UploaderBase + + +class ResumeUploaderBase(UploaderBase): + """ + Attributes + ---------- + part_size: int, optional + progress_handler: function, optional + upload_progress_recorder: UploadProgressRecorder, optional + concurrent_executor: futures.Executor, optional + """ + __metaclass__ = abc.ABCMeta + + def __init__( + self, + bucket_name, + **kwargs + ): + """ + Parameters + ---------- + bucket_name + part_size: int + progress_handler: function + upload_progress_recorder: UploadProgressRecorder + max_concurrent_workers: int + concurrent_executor: futures.Executor + kwargs + """ + super(ResumeUploaderBase, self).__init__(bucket_name, **kwargs) + + self.part_size = kwargs.get('part_size', 4 * (1024 ** 2)) + + self.progress_handler = kwargs.get( + 'progress_handler', + None + ) + + self.upload_progress_recorder = kwargs.get( + 'upload_progress_recorder', + None + ) + + max_workers = kwargs.get('max_concurrent_workers', 3) + self.concurrent_executor = kwargs.get( + 'concurrent_executor', + futures.ThreadPoolExecutor(max_workers=max_workers) + ) + + def gen_chunk_list(self, size, chunk_size=None, uploaded_chunk_no_list=None): + """ + Parameters + ---------- + size: int + chunk_size: int + uploaded_chunk_no_list: list[int] + + Yields + ------- + ChunkInfo + """ + if not chunk_size: + chunk_size = self.part_size + if not uploaded_chunk_no_list: + uploaded_chunk_no_list = [] + + for i, chunk_offset in enumerate(range(0, size, chunk_size)): + chunk_no = i + 1 + if chunk_no in uploaded_chunk_no_list: + continue + yield ChunkInfo( + chunk_no=chunk_no, + chunk_offset=chunk_offset, + chunk_size=min( + chunk_size, + size - chunk_offset + ) + ) + + @abc.abstractmethod + def _recover_from_record( + self, + file_name, + key, + context + ): + """ + Parameters + ---------- + file_name: str + key: str + context: any + + Returns + ------- + any + """ + + @abc.abstractmethod + def _set_to_record( + self, + file_name, + key, + context + ): + """ + Parameters + ---------- + file_name: str + key: str + context: any + """ + + @abc.abstractmethod + def _progress_handler( + self, + file_name, + key, + context, + uploaded_size, + total_size + ): + """ + Parameters + ---------- + file_name: str + key: str + context: any + uploaded_size: int + total_size: int + """ + + @abc.abstractmethod + def initial_parts( + self, + up_token, + key, + file_path, + data, + data_size, + modify_time, + part_size, + **kwargs + ): + """ + Parameters + ---------- + up_token: str + key: str + file_path: str + data: IOBase + data_size: int + modify_time: int + part_size: int + kwargs: dict + + Returns + ------- + ret: dict + resp: ResponseInfo + """ + + @abc.abstractmethod + def upload_parts( + self, + up_token, + data, + data_size, + context, + **kwargs + ): + """ + Parameters + ---------- + up_token: str + data: IOBase + data_size: int + context: any + kwargs: dict + + Returns + ------- + ret: dict + resp: ResponseInfo + """ + + @abc.abstractmethod + def complete_parts( + self, + up_token, + data_size, + context, + **kwargs + ): + """ + Parameters + ---------- + up_token: str + data_size: int + context: any + kwargs: dictr + + Returns + ------- + ret: dict + resp: ResponseInfo + """ diff --git a/qiniu/services/storage/uploaders/abc/uploader_base.py b/qiniu/services/storage/uploaders/abc/uploader_base.py new file mode 100644 index 00000000..e1096b0d --- /dev/null +++ b/qiniu/services/storage/uploaders/abc/uploader_base.py @@ -0,0 +1,164 @@ +import abc + +import qiniu.config as config + +# type import +from qiniu.auth import Auth # noqa + + +class UploaderBase(object): + """ + Attributes + ---------- + bucket_name: str + auth: Auth + regions: list[Region] + """ + __metaclass__ = abc.ABCMeta + + def __init__( + self, + bucket_name, + **kwargs + ): + """ + Parameters + ---------- + bucket_name: str + The name of bucket which you want to upload to. + auth: Auth + The instance of Auth to sign requests. + regions: list[Region], default=[] + The regions of bucket. It will be queried if not specified. + kwargs + The others arguments may be used by subclass. + """ + self.bucket_name = bucket_name + + # change the default when implements AuthProvider + self.auth = kwargs.get('auth', None) + + regions = kwargs.get('regions', []) + # remove the check when implement RegionsProvider + # if not regions: + # raise TypeError('You must provide the regions') + self.regions = regions + + hosts_cache_dir = kwargs.get('hosts_cache_dir', None) + self.hosts_cache_dir = hosts_cache_dir + + def get_up_token(self, **kwargs): + """ + Generate up token + + Parameters + ---------- + bucket_name: str + key: str + expired: int + policy: dict + strict_policy: bool + + Returns + ------- + str + """ + if not self.auth: + raise ValueError('can not get up_token by auth not provided') + + bucket_name = kwargs.get('bucket_name', self.bucket_name) + + kwargs_for_up_token = { + k: kwargs[k] + for k in [ + 'key', 'expires', 'policy', 'strict_policy' + ] if k in kwargs + } + up_token = self.auth.upload_token( + bucket=bucket_name, + **kwargs_for_up_token + ) + return up_token + + def _get_regions(self): + if self.regions: + return self.regions + + default_region = config.get_default('default_zone') + if default_region: + self.regions = [default_region] + + return self.regions + + def _get_up_hosts(self, access_key=None): + """ + This will be deprecated when implement regions and endpoints + + Returns + ------- + list[str] + """ + if not self.auth and not access_key: + raise ValueError('Must provide access_key if auth is unavailable.') + if not access_key: + access_key = self.auth.get_access_key() + + regions = self._get_regions() + + if not regions: + raise ValueError('No region available.') + + if regions[0].up_host and regions[0].up_host_backup: + return [ + regions[0].up_host, + regions[0].up_host_backup + ] + + # this is correct, it does return hosts. bad function name by legacy + return regions[0].get_up_host( + ak=access_key, + bucket=self.bucket_name, + home_dir=self.hosts_cache_dir + ) + + @abc.abstractmethod + def upload( + self, + key, + file_path, + data, + data_size, + modify_time, + + part_size, + mime_type, + metadata, + file_name, + custom_vars, + **kwargs + ): + """ + Upload method + + Parameters + ---------- + key: str + file_path: str + data: IOBase + data_size: int + modify_time: int + + part_size: int + mime_type: str + metadata: dict + file_name: str + custom_vars: dict + kwargs: dict + + Returns + ------- + ret: dict + The parsed response body + info + The response + """ diff --git a/qiniu/services/storage/uploaders/form_uploader.py b/qiniu/services/storage/uploaders/form_uploader.py new file mode 100644 index 00000000..ea096f59 --- /dev/null +++ b/qiniu/services/storage/uploaders/form_uploader.py @@ -0,0 +1,240 @@ +from io import BytesIO +from os import path +from time import time + +from qiniu.compat import is_seekable +from qiniu.utils import b, io_crc32 +from qiniu.auth import Auth +from qiniu.http import qn_http_client + +from qiniu.services.storage.uploaders.abc import UploaderBase + + +class FormUploader(UploaderBase): + def __init__(self, bucket_name, **kwargs): + """ + Parameters + ---------- + bucket_name: str + kwargs + auth, regions + """ + super(FormUploader, self).__init__(bucket_name, **kwargs) + + self.progress_handler = kwargs.get( + 'progress_handler', + None + ) + + def upload( + self, + key, + file_path=None, + data=None, + data_size=None, + modify_time=None, + part_size=None, + mime_type=None, + metadata=None, + file_name=None, + custom_vars=None, + **kwargs + ): + """ + Parameters + ---------- + key: str + file_path: str + data: IOBase + data_size: int + modify_time: int + part_size: int + mime_type: str + metadata: dict + file_name: str + custom_vars: dict + kwargs + up_token, crc32_int + bucket_name, key, expired, policy, strict_policy for get up_token + + Returns + ------- + ret: dict + resp: ResponseInfo + """ + # check and initial arguments + # up_token and up_hosts + up_token = kwargs.get('up_token', None) + if not up_token: + up_token = self.get_up_token(**kwargs) + up_hosts = self._get_up_hosts() + else: + access_key, _, _ = Auth.up_token_decode(up_token) + up_hosts = self._get_up_hosts(access_key) + + # crc32 from outside + crc32_int = kwargs.get('crc32_int', None) + # try to get file_name + if not file_name and file_path: + file_name = path.basename(file_path) + + # must provide file_path or data + if not file_path and not data: + raise TypeError('Must provide one of file_path or data.') + if file_path and data: + raise TypeError('Must provide only one of file_path or data.') + + if not modify_time: + if file_path: + modify_time = int(path.getmtime(file_path)) + else: + modify_time = int(time()) + + # upload + try: + if file_path: + data_size = path.getsize(file_path) + data = open(file_path, 'rb') + elif isinstance(data, bytes): + data_size = len(data) + data = BytesIO(data) + elif isinstance(data, str): + data_size = len(data) + data = BytesIO(b(data)) + if not crc32_int: + crc32_int = self.__get_crc32_int(data) + fields = self.__get_form_fields( + up_hosts=up_hosts, + up_token=up_token, + key=key, + crc32_int=crc32_int, + custom_vars=custom_vars, + metadata=metadata + ) + ret, resp = self.__upload_data( + up_hosts=up_hosts, + fields=fields, + file_name=file_name, + data=data, + data_size=data_size, + mime_type=mime_type + ) + finally: + if file_path: + data.close() + + return ret, resp + + def __upload_data( + self, + up_hosts, + fields, + file_name, + data, + data_size=None, + mime_type='application/octet-stream' + ): + """ + Parameters + ---------- + up_hosts: list[str] + fields: dict + file_name: str + data: IOBase + data_size: int + mime_type: str + + Returns + ------- + ret: dict + resp: ResponseInfo + """ + if not file_name or not file_name.strip(): + file_name = 'file_name' + + ret, resp = None, None + for up_host in up_hosts: + ret, resp = qn_http_client.post( + url=up_host, + data=fields, + files={ + 'file': (file_name, data, mime_type) + } + ) + if resp.ok() and ret: + return ret, resp + if ( + not is_seekable(data) or + not resp.need_retry() + ): + return ret, resp + data.seek(0) + return ret, resp + + def __get_form_fields( + self, + up_token, + **kwargs + ): + """ + Parameters + ---------- + up_token: str + kwargs + key, crc32_int, custom_vars, metadata + + Returns + ------- + dict + """ + key = kwargs.get('key', None) + crc32_int = kwargs.get('crc32_int', None) + custom_vars = kwargs.get('custom_vars', None) + metadata = kwargs.get('metadata', None) + + result = { + 'token': up_token, + } + + if key is not None: + result['key'] = key + + if crc32_int: + result['crc32'] = crc32_int + + if custom_vars: + result.update( + { + k: str(v) + for k, v in custom_vars.items() + if k.startswith('x:') + } + ) + + if metadata: + result.update( + { + k: str(v) + for k, v in metadata.items() + if k.startswith('x-qn-meta-') + } + ) + + return result + + def __get_crc32_int(self, data): + """ + Parameters + ---------- + data: BytesIO + + Returns + ------- + str + """ + result = None + if not is_seekable(data): + return result + result = io_crc32(data) + data.seek(0) + return result diff --git a/qiniu/services/storage/uploaders/io_chunked.py b/qiniu/services/storage/uploaders/io_chunked.py new file mode 100644 index 00000000..79657157 --- /dev/null +++ b/qiniu/services/storage/uploaders/io_chunked.py @@ -0,0 +1,90 @@ +import os +import io +from collections import namedtuple + +from qiniu.compat import is_seekable + + +ChunkInfo = namedtuple( + 'ChunkInfo', + [ + 'chunk_no', + 'chunk_offset', + 'chunk_size' + ] +) + + +class IOChunked(io.IOBase): + def __init__( + self, + base_io, + chunk_offset, + chunk_size, + lock, + buffer_size=4 * (1024 ** 2) # 4MB just for demo + ): + if not is_seekable(base_io): + raise TypeError('"base_io" must be seekable') + self.__base_io = base_io + self.__chunk_start = chunk_offset + self.__chunk_size = chunk_size + self.__chunk_end = chunk_offset + chunk_size + self.__lock = lock + self.__chunk_pos = 0 + + self.buffer_size = min(buffer_size, chunk_size) + + def readable(self): + return self.__base_io.readable() + + def seekable(self): + return True + + def seek(self, offset, whence=0): + if not self.seekable(): + raise io.UnsupportedOperation('does not support seek') + if whence == os.SEEK_SET: + if offset < 0: + raise ValueError('offset should be zero or positive if whence is 0') + self.__chunk_pos = offset + elif whence == os.SEEK_CUR: + self.__chunk_pos += offset + elif whence == os.SEEK_END: + if offset > 0: + raise ValueError('offset should be zero or negative if whence is 2') + self.__chunk_pos = self.__chunk_size + offset + else: + raise ValueError('whence should be 0, 1 or 2') + self.__chunk_pos = max( + 0, + min(self.__chunk_size, self.__chunk_pos) + ) + + def tell(self): + return self.__chunk_pos + + def read(self, size): + if self.__curr_base_pos >= self.__chunk_end: + return b'' + read_size = max(self.buffer_size, size) + read_size = min(self.__rest_chunk_size, read_size) + + # -- ignore size argument -- + with self.__lock: + self.__base_io.seek(self.__curr_base_pos) + data = self.__base_io.read(read_size) + + self.__chunk_pos += len(data) + return data + + def __len__(self): + return self.__chunk_size + + @property + def __curr_base_pos(self): + return self.__chunk_start + self.__chunk_pos + + @property + def __rest_chunk_size(self): + return self.__chunk_end - self.__curr_base_pos diff --git a/qiniu/services/storage/uploaders/resume_uploader_v1.py b/qiniu/services/storage/uploaders/resume_uploader_v1.py new file mode 100644 index 00000000..136ef7ab --- /dev/null +++ b/qiniu/services/storage/uploaders/resume_uploader_v1.py @@ -0,0 +1,678 @@ +import logging +from collections import namedtuple +from concurrent import futures +from io import BytesIO +from itertools import chain +from os import path +from threading import Lock +from time import time + +from qiniu.compat import is_seekable +from qiniu.auth import Auth +from qiniu.http import qn_http_client, ResponseInfo +from qiniu.utils import b, io_crc32, urlsafe_base64_encode + +from qiniu.services.storage.uploaders.abc import ResumeUploaderBase +from qiniu.services.storage.uploaders.io_chunked import IOChunked + + +class ResumeUploaderV1(ResumeUploaderBase): + def _recover_from_record( + self, + file_name, + key, + context + ): + """ + Parameters + ---------- + file_name: str + key: str + context: _ResumeUploadV1Context + + Returns + ------- + _ResumeUploadV1Context + """ + if not isinstance(context, _ResumeUploadV1Context): + raise TypeError('context must be an instance of _ResumeUploadV1Context') + + if not self.upload_progress_recorder or not any([file_name, key]): + return context + + record = self.upload_progress_recorder.get_upload_record( + file_name, + key + ) + + if not record: + return context + + record_up_hosts = record.get('up_hosts', []) + record_part_size = record.get('part_size', None) + record_modify_time = record.get('modify_time', 0) + record_context = record.get('contexts', []) + + # compact with old sdk(<= v7.11.1) + if not record_up_hosts or not record_part_size: + return context + + # filter expired parts + if record_modify_time != context.modify_time: + record_context = [] + else: + now = time() + record_context = [ + ctx + for ctx in record_context + if ctx.get('expired_at', 0) > now + ] + + # assign to context + return context._replace( + up_hosts=record_up_hosts, + part_size=record_part_size, + parts=[ + _ResumeUploadV1Part( + part_no=p['part_no'], + ctx=p['ctx'], + expired_at=p['expired_at'], + ) + for p in record_context + ], + resumed=True + ) + + def _set_to_record( + self, + file_name, + key, + context + ): + """ + Parameters + ---------- + file_name: str + key: str + context: _ResumeUploadV1Context + + """ + if not self.upload_progress_recorder or not any([file_name, key]): + return + + record_data = { + 'up_hosts': context.up_hosts, + 'part_size': context.part_size, + 'modify_time': context.modify_time, + 'contexts': [ + { + 'ctx': part.ctx, + 'expired_at': part.expired_at, + 'part_no': part.part_no + } + for part in context.parts + ] + } + self.upload_progress_recorder.set_upload_record( + file_name, + key, + data=record_data + ) + + def _try_delete_record( + self, + file_name, + key, + context, + resp + ): + """ + Parameters + ---------- + file_name: str + key: str + context: _ResumeUploadV1Context + resp: ResponseInfo + """ + if not self.upload_progress_recorder or not any([file_name, key]): + return + if resp and context and not any([ + resp.ok(), + resp.status_code == 701 and context.resumed + ]): + return + self.upload_progress_recorder.delete_upload_record(file_name, key) + + def _progress_handler( + self, + file_name, + key, + context, + uploaded_size, + total_size + ): + """ + Parameters + ---------- + file_name: str + key: str + context: _ResumeUploadV1Context + uploaded_size: int + total_size: int + + """ + self._set_to_record(file_name, key, context) + if callable(self.progress_handler): + self.progress_handler(uploaded_size, total_size) + + def initial_parts( + self, + up_token, + key, + file_path=None, + data=None, + modify_time=None, + data_size=None, + **kwargs + ): + """ + Parameters + ---------- + up_token + key + file_path + data + modify_time + data_size + + kwargs + + Returns + ------- + context: _ResumeUploadV1Context + resp: None + + """ + # -- check and initial arguments + # must provide file_path or data + if not file_path and not data: + raise TypeError('Must provide one of file_path or data.') + if file_path and data: + raise TypeError('Must provide only one of file_path or data.') + + # data must has length + if not file_path and not data_size: + raise TypeError('Must provide size if use data.') + + if not modify_time: + if file_path: + modify_time = int(path.getmtime(file_path)) + else: + modify_time = int(time()) + + part_size = 4 * (1024 ** 2) + + # -- initial context + context = _ResumeUploadV1Context( + up_hosts=[], + part_size=part_size, + parts=[], + modify_time=modify_time, + resumed=False + ) + + # try to recover from record + file_name = path.basename(file_path) if file_path else None + context = self._recover_from_record( + file_name, + key, + context + ) + + access_key, _, _ = Auth.up_token_decode(up_token) + if not context.up_hosts: + context.up_hosts.extend(self._get_up_hosts(access_key)) + + return context, None + + def upload_parts( + self, + up_token, + data, + data_size, + context, + **kwargs + ): + """ + + Parameters + ---------- + up_token: str + context: _ResumeUploadV1Context + data + data_size: int + + kwargs + key, file_name + + Returns + ------- + part: _ResumeUploadV1Part + resp: ResponseInfo + + """ + # initial arguments + chunk_list = self.gen_chunk_list( + size=data_size, + chunk_size=context.part_size, + uploaded_chunk_no_list=[ + p.part_no for p in context.parts + ] + ) + up_hosts = list(context.up_hosts) + file_name = kwargs.get('file_name', None) + key = kwargs.get('key', None) + + # initial upload state + part, resp = None, None + uploaded_size = 0 + lock = Lock() + + if not self.concurrent_executor: + # upload sequentially + for chunk in chunk_list: + part, resp = self.__upload_part( + data=data, + chunk_info=chunk, + up_hosts=up_hosts, + up_token=up_token, + lock=lock + ) + if not resp.ok(): + return None, resp + elif not part: + return resp.json(), resp + context.parts.append(part) + uploaded_size += chunk.chunk_size + self._progress_handler( + file_name=file_name, + key=key, + context=context, + uploaded_size=uploaded_size, + total_size=data_size + ) + else: + # upload concurrently + future_chunk_dict = {} + for chunk in chunk_list: + ftr = self.concurrent_executor.submit( + self.__upload_part, + data=data, + chunk_info=chunk, + up_hosts=up_hosts, + up_token=up_token, + lock=lock + ) + future_chunk_dict[ftr] = chunk + + first_failed_resp = None + for ftr in futures.as_completed(future_chunk_dict): + if ftr.cancelled(): + continue + elif ftr.exception(): + # only keep first failed future, + # continue instead return to wait running future done. + if first_failed_resp: + continue + first_failed_resp = ResponseInfo(None, ftr.exception()) + for not_done in filter(lambda f: not f.done(), future_chunk_dict): + not_done.cancel() + else: + part, resp = ftr.result() + if not part: + if not first_failed_resp: + first_failed_resp = resp + for not_done in filter(lambda f: not f.done(), future_chunk_dict): + not_done.cancel() + else: + context.parts.append(part) + uploaded_size += future_chunk_dict[ftr].chunk_size + self._progress_handler( + file_name=file_name, + key=key, + context=context, + uploaded_size=uploaded_size, + total_size=data_size + ) + if first_failed_resp: + if first_failed_resp.ok(): + # just compat with old sdk. it's ok when crc32 check failed + return first_failed_resp.json(), first_failed_resp + return None, first_failed_resp + + return part, resp + + def complete_parts( + self, + up_token, + data_size, + context, + **kwargs + ): + """ + + Parameters + ---------- + up_token: str + data_size: int + context: _ResumeUploadV1Context + kwargs: + key, file_name, params, metadata + + Returns + ------- + ret: dict + resp: ResponseInfo + """ + key = kwargs.get('key', None) + file_name = kwargs.get('file_name', None) + params = kwargs.get('params', None) + metadata = kwargs.get('metadata', None) + mime_type = kwargs.get('mime_type', None) + + # sort contexts + sorted_parts = sorted(context.parts, key=lambda part: part.part_no) + body = ','.join((part.ctx for part in sorted_parts)) + + ret, resp = None, None + for up_host in context.up_hosts: + url = self.__get_mkfile_url( + up_host=up_host, + data_size=data_size, + mime_type=mime_type, + key=key, + file_name=file_name, + params=params, + metadata=metadata + ) + ret, resp = qn_http_client.post( + url=url, + data=body, + files=None, + headers={ + 'Authorization': 'UpToken {}'.format(up_token) + } + ) + if resp.ok() or not resp.need_retry(): + break + self._try_delete_record( + file_name, + key, + context, + resp + ) + return ret, resp + + def upload( + self, + key, + file_path=None, + data=None, + data_size=None, + modify_time=None, + + part_size=None, + mime_type=None, + metadata=None, + file_name=None, + custom_vars=None, + **kwargs + ): + """ + + Parameters + ---------- + key + file_path + data + data_size + modify_time + + part_size + mime_type + metadata + file_name + custom_vars + + kwargs: + up_token + bucket_name, expires, policy, strict_policy for generate `up_token` + + Returns + ------- + ret: dict + resp: ResponseInfo + """ + # part_size + if part_size: + logging.warning('ResumeUploader not support part_size. It is fixed to 4MB.') + + # up_token + up_token = kwargs.get('up_token', None) + if not up_token: + up_token = self.get_up_token(**kwargs) + if not file_name and file_path: + file_name = path.basename(file_path) + + # initial_parts + context, resp = self.initial_parts( + up_token, + key, + file_path=file_path, + data=data, + data_size=data_size, + modify_time=modify_time, + ) + + # upload_parts + try: + if file_path: + data_size = path.getsize(file_path) + data = open(file_path, 'rb') + elif isinstance(data, bytes): + data_size = len(data) + data = BytesIO(data) + elif isinstance(data, str): + data_size = len(data) + data = BytesIO(b(data)) + ret, resp = self.upload_parts( + up_token=up_token, + context=context, + data=data, + data_size=data_size, + + key=key, + file_name=file_name + ) + finally: + if file_path: + data.close() + + if resp and not resp.ok(): + return ret, resp + + # complete_parts + ret, resp = self.complete_parts( + up_token=up_token, + data_size=data_size, + context=context, + + key=key, + mime_type=mime_type, + file_name=file_name, + params=custom_vars, + metadata=metadata + ) + + # retry if expired. the record file will be deleted by complete_parts + if resp.status_code == 701 and context.resumed: + return self.upload( + key, + file_path=file_path, + data=data, + data_size=data_size, + modify_time=modify_time, + + mime_type=mime_type, + metadata=metadata, + file_name=file_name, + custom_vars=custom_vars, + **kwargs + ) + + return ret, resp + + def __upload_part( + self, + data, + chunk_info, + up_hosts, + up_token, + lock + ): + """ + Parameters + ---------- + data: IOBase + chunk_info: IOChunked + up_hosts: list[str] + up_token: str + lock: Lock + + Returns + ------- + part: _ResumeUploadV2Part + resp: ResponseInfo + """ + if not up_hosts: + raise ValueError('Must provide one up host at least') + + chunked_data = IOChunked( + base_io=data, + chunk_offset=chunk_info.chunk_offset, + chunk_size=chunk_info.chunk_size, + lock=lock + ) + chunk_crc32 = io_crc32(chunked_data) + chunked_data.seek(0) + part, resp = None, None + for up_host in up_hosts: + url = '/'.join([ + up_host, + 'mkblk', str(chunk_info.chunk_size) + ]) + ret, resp = qn_http_client.post( + url=url, + data=chunked_data, + files=None, + headers={ + 'Authorization': 'UpToken {}'.format(up_token) + } + ) + if resp.ok() and ret: + if ret.get('crc32', 0) != chunk_crc32: + return None, resp + part = _ResumeUploadV1Part( + part_no=chunk_info.chunk_no, + ctx=ret.get('ctx', ''), + expired_at=ret.get('expired_at', 0), + ) + return part, resp + if ( + not is_seekable(chunked_data) or + not resp.need_retry() + ): + return part, resp + chunked_data.seek(0) + return part, resp + + def __get_mkfile_url( + self, + up_host, + data_size, + mime_type=None, + key=None, + file_name=None, + params=None, + metadata=None + ): + """ + Parameters + ---------- + up_host: str + data_size: int + mime_type: str + key: str + file_name: str + params: dict + metadata: dict + + Returns + ------- + str + """ + url_base = [up_host, 'mkfile', str(data_size)] + url_params = [] + + if mime_type: + url_params.append(('mimeType', mime_type)) + + if key: + url_params.append(('key', key)) + + if file_name: + url_params.append(('fname', file_name)) + + if params: + url_params.extend(params.items()) + + if metadata: + url_params.extend( + (k, v) + for k, v in metadata.items() + if k.startswith('x-qn-meta-') + ) + + url_params_iter = chain.from_iterable( + (str(k), urlsafe_base64_encode(str(v))) + for k, v in url_params + ) + + return '/'.join( + chain( + url_base, + url_params_iter + ) + ) + + +# use dataclass instead namedtuple if min version of python update to 3.7 +_ResumeUploadV1Part = namedtuple( + 'ResumeUploadV1Part', + [ + 'part_no', + 'ctx', + 'expired_at', + ] +) + +_ResumeUploadV1Context = namedtuple( + 'ResumeUploadV1Context', + [ + 'up_hosts', + 'part_size', + 'parts', + 'modify_time', # the file last modify time + 'resumed' + ] +) diff --git a/qiniu/services/storage/uploaders/resume_uploader_v2.py b/qiniu/services/storage/uploaders/resume_uploader_v2.py new file mode 100644 index 00000000..82299d0d --- /dev/null +++ b/qiniu/services/storage/uploaders/resume_uploader_v2.py @@ -0,0 +1,727 @@ +from collections import namedtuple +from concurrent import futures +from io import BytesIO +from os import path +from threading import Lock +from time import time + +from qiniu.compat import is_seekable +from qiniu.auth import Auth +from qiniu.http import qn_http_client, ResponseInfo +from qiniu.utils import b, io_md5, urlsafe_base64_encode +from qiniu.compat import json + +from qiniu.services.storage.uploaders.abc import ResumeUploaderBase +from qiniu.services.storage.uploaders.io_chunked import IOChunked + + +class ResumeUploaderV2(ResumeUploaderBase): + def _recover_from_record( + self, + file_name, + key, + context + ): + """ + + Parameters + ---------- + file_name: str + key: str + context: _ResumeUploadV2Context + + Returns + ------- + _ResumeUploadV2Context + """ + if not isinstance(context, _ResumeUploadV2Context): + raise TypeError('context must be an instance of _ResumeUploadV2Context') + + if ( + not self.upload_progress_recorder or + not (file_name or key) + ): + return context + + record = self.upload_progress_recorder.get_upload_record( + file_name, + key + ) + + if not record: + return context + + record_up_hosts = record.get('up_hosts', []) + record_upload_id = record.get('upload_id', '') + record_expired_at = record.get('expired_at', 0) + record_part_size = record.get('part_size', None) + record_modify_time = record.get('modify_time', 0) + record_etags = record.get('etags', []) + + # compact with old sdk(<= v7.11.1) + if not record_up_hosts or not record_part_size: + return context + + if ( + not record_upload_id or + record_modify_time != context.modify_time or + record_expired_at > time() + ): + return context + + return context._replace( + up_hosts=record_up_hosts, + upload_id=record_upload_id, + expired_at=record_expired_at, + part_size=record_part_size, + parts=[ + _ResumeUploadV2Part( + part_no=p['partNumber'], + etag=p['etag'] + ) + for p in record_etags + ], + resumed=True + ) + + def _set_to_record(self, file_name, key, context): + """ + Parameters + ---------- + file_name: str + key: str + context: _ResumeUploadV2Context + + """ + if not self.upload_progress_recorder or not any([file_name, key]): + return + + record_data = { + 'up_hosts': context.up_hosts, + 'upload_id': context.upload_id, + 'expired_at': context.expired_at, + 'part_size': context.part_size, + 'modify_time': context.modify_time, + 'etags': [ + { + 'etag': part.etag, + 'part_no': part.part_no + } + for part in context.parts + ] + } + self.upload_progress_recorder.set_upload_record( + file_name, + key, + data=record_data + ) + + def _try_delete_record( + self, + file_name, + key, + context, + resp + ): + """ + Parameters + ---------- + file_name: str + key: str + context: _ResumeUploadV2Context + resp: ResponseInfo + """ + if not self.upload_progress_recorder or not any([file_name, key]): + return + if resp and context and not any([ + resp.ok(), + resp.status_code == 612 and context.resumed + ]): + return + self.upload_progress_recorder.delete_upload_record(file_name, key) + + def _progress_handler( + self, + file_name, + key, + context, + uploaded_size, + total_size + ): + """ + Parameters + ---------- + file_name: str + key: str + context: _ResumeUploadV2Context + uploaded_size: int + total_size: int + """ + self._set_to_record(file_name, key, context) + if callable(self.progress_handler): + self.progress_handler(uploaded_size, total_size) + + def initial_parts( + self, + up_token, + key, + file_path=None, + data=None, + data_size=None, + modify_time=None, + part_size=None, + **kwargs + ): + """ + + Parameters + ---------- + up_token: str + key: str + file_path: str + data: IOBase + data_size: int + modify_time: int + part_size: int + kwargs + + Returns + ------- + ret: _ResumeUploadV2Context + resp: ResponseInfo + """ + # -- check and initial arguments + # must provide file_path or data + if not file_path and not data: + raise TypeError('Must provide one of file_path or data.') + if file_path and data: + raise TypeError('Must provide only one of file_path or data.') + + # data must has length + if not file_path and not data_size: + raise TypeError('Must provide size if use data.') + + if not modify_time: + if file_path: + modify_time = int(path.getmtime(file_path)) + else: + modify_time = int(time()) + + if not part_size: + part_size = self.part_size + + # -- initial context + context = _ResumeUploadV2Context( + up_hosts=[], + upload_id='', + expired_at=0, + part_size=part_size, + parts=[], + modify_time=modify_time, + resumed=False + ) + + # try to recover from record + file_name = path.basename(file_path) if file_path else None + context = self._recover_from_record( + file_name, + key, + context + ) + + if ( + context.up_hosts and + context.upload_id and + context.expired_at + ): + return context, None + + # -- get a new upload id + access_key, _, _ = Auth.up_token_decode(up_token) + if not context.up_hosts: + context.up_hosts.extend(self._get_up_hosts(access_key)) + + bucket_name = Auth.get_bucket_name(up_token) + + resp = None + for up_host in context.up_hosts: + url = self.__get_url_for_upload( + up_host, + bucket_name, + key + ) + ret, resp = qn_http_client.post( + url=url, + data='', + files=None, + headers={ + 'Authorization': 'UpToken {}'.format(up_token) + } + ) + if not resp.ok() and not resp.need_retry(): + break + if resp.ok() and ret: + context = context._replace( + upload_id=ret.get('uploadId', ''), + expired_at=ret.get('expireAt', 0) + ) + break + + return context, resp + + def upload_parts( + self, + up_token, + data, + data_size, + context, + **kwargs + ): + """ + Parameters + ---------- + up_token: str + data + data_size: int + context: _ResumeUploadV2Context + kwargs + key, file_name + + Returns + ------- + part: _ResumeUploadV2Part + resp: ResponseInfo + + """ + # initial arguments + chunk_list = self.gen_chunk_list( + size=data_size, + chunk_size=context.part_size, + uploaded_chunk_no_list=[ + p.part_no for p in context.parts + ] + ) + up_hosts = list(context.up_hosts) + file_name = kwargs.get('file_name', None) + key = kwargs.get('key', None) + + # initial upload state + part, resp = None, None + uploaded_size = 0 + lock = Lock() + + if not self.concurrent_executor: + # upload sequentially + for chunk in chunk_list: + part, resp = self.__upload_part( + data=data, + chunk_info=chunk, + up_hosts=up_hosts, + up_token=up_token, + upload_id=context.upload_id, + key=key, + lock=lock + ) + if not resp.ok(): + return None, resp + context.parts.append(part) + uploaded_size += chunk.chunk_size + self._progress_handler( + file_name=file_name, + key=key, + context=context, + uploaded_size=uploaded_size, + total_size=data_size + ) + else: + # upload concurrently + future_chunk_dict = {} + for chunk in chunk_list: + ftr = self.concurrent_executor.submit( + self.__upload_part, + data=data, + chunk_info=chunk, + up_hosts=up_hosts, + up_token=up_token, + upload_id=context.upload_id, + key=key, + lock=lock + ) + future_chunk_dict[ftr] = chunk + + first_failed_resp = None + for ftr in futures.as_completed(future_chunk_dict): + if ftr.cancelled(): + continue + elif ftr.exception(): + if first_failed_resp: + continue + first_failed_resp = ResponseInfo(None, ftr.exception()) + for not_done in filter(lambda f: not f.done(), future_chunk_dict): + not_done.cancel() + else: + part, resp = ftr.result() + if not part: + if not first_failed_resp: + first_failed_resp = resp + for not_done in filter(lambda f: not f.done(), future_chunk_dict): + not_done.cancel() + else: + context.parts.append(part) + uploaded_size += future_chunk_dict[ftr].chunk_size + self._progress_handler( + file_name=file_name, + key=key, + context=context, + uploaded_size=uploaded_size, + total_size=data_size + ) + if first_failed_resp: + return None, first_failed_resp + + return part, resp + + def complete_parts( + self, + up_token, + data_size, + context, + **kwargs + ): + """ + Parameters + ---------- + up_token: str + data_size: int + context: _ResumeUploadV2Context + kwargs + key, file_name, params, metadata + Returns + ------- + ret: dict + resp: ResponseInfo + """ + key = kwargs.get('key', None) + file_name = kwargs.get('file_name', None) + mime_type = kwargs.get('mime_type', None) + params = kwargs.get('params', None) + metadata = kwargs.get('metadata', None) + + # sort contexts + sorted_parts = sorted(context.parts, key=lambda part: part.part_no) + + bucket_name = Auth.get_bucket_name(up_token) + + ret, resp = None, None + for up_host in context.up_hosts: + url = self.__get_url_for_upload( + up_host, + bucket_name, + key, + upload_id=context.upload_id + ) + data = { + 'parts': [ + { + 'etag': p.etag, + 'partNumber': p.part_no + } + for p in sorted_parts + ], + 'fname': file_name, + 'mimeType': mime_type, + 'customVars': params, + 'metadata': metadata + } + ret, resp = qn_http_client.post( + url=url, + data=json.dumps(data), + files=None, + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'UpToken {}'.format(up_token) + } + ) + if resp.ok() or not resp.need_retry(): + break + self._try_delete_record( + file_name, + key, + context, + resp + ) + return ret, resp + + def upload( + self, + key, + file_path=None, + data=None, + data_size=None, + + part_size=None, + modify_time=None, + mime_type=None, + metadata=None, + file_name=None, + custom_vars=None, + **kwargs + ): + """ + Parameters + ---------- + key: str + file_path: str + data: IOBase + data_size: int + part_size: int + modify_time: int + mime_type: str + metadata: dict + file_name: str + custom_vars: dict + kwargs + up_token + bucket_name, expires, policy, strict_policy for generate `up_token` + + Returns + ------- + + """ + # up_token + up_token = kwargs.get('up_token', None) + if not up_token: + up_token = self.get_up_token(**kwargs) + if not file_name and file_path: + file_name = path.basename(file_path) + + # initial_parts + context, resp = self.initial_parts( + up_token, + key, + file_path=file_path, + data=data, + data_size=data_size, + modify_time=modify_time, + part_size=part_size + ) + + if ( + not context.up_hosts or + not context.upload_id or + not context.expired_at + ): + return None, resp + + # upload parts + try: + if file_path: + data_size = path.getsize(file_path) + data = open(file_path, 'rb') + elif isinstance(data, bytes): + data_size = len(data) + data = BytesIO(data) + elif isinstance(data, str): + data_size = len(data) + data = BytesIO(b(data)) + ret, resp = self.upload_parts( + up_token=up_token, + data=data, + data_size=data_size, + context=context, + + key=key, + file_name=file_name + ) + finally: + if file_path: + data.close() + + if resp and not resp.ok(): + if resp.status_code == 612 and context.resumed: + return self.upload( + key, + file_path=file_path, + data=data, + data_size=data_size, + modify_time=modify_time, + + mime_type=mime_type, + metadata=metadata, + file_name=file_name, + custom_vars=custom_vars, + **kwargs + ) + return ret, resp + + # complete parts + ret, resp = self.complete_parts( + up_token=up_token, + data_size=data_size, + context=context, + + key=key, + mime_type=mime_type, + file_name=file_name, + params=custom_vars, + metadata=metadata + ) + + # retry if expired. the record file will be deleted by complete_parts + if resp.status_code == 612 and context.resumed: + return self.upload( + key, + file_path=file_path, + data=data, + data_size=data_size, + modify_time=modify_time, + + mime_type=mime_type, + metadata=metadata, + file_name=file_name, + custom_vars=custom_vars, + **kwargs + ) + + return ret, resp + + def __get_url_for_upload( + self, + up_host, + bucket_name, + key, + upload_id=None, + part_no=None, + ): + """ + Parameters + ---------- + up_host: str + bucket_name: str + key: str + upload_id: str + part_no: int + + Returns + ------- + str + """ + if not bucket_name: + bucket_name = self.bucket_name + + object_entry = '~' + if key: + object_entry = urlsafe_base64_encode(key) + + url_segs = [ + up_host, + 'buckets', bucket_name, + 'objects', object_entry, + 'uploads', + ] + + if upload_id: + url_segs.append(upload_id) + + if part_no: + url_segs.append(str(part_no)) + + return '/'.join(url_segs) + + def __upload_part( + # resort arguments + self, + data, + chunk_info, + up_hosts, + up_token, + upload_id, + key, + lock + ): + """ + Parameters + ---------- + data: IOBase + chunk_info: ChunkInfo + up_hosts: list[str] + up_token: str + upload_id: str + key: str + lock: Lock + + Returns + ------- + part: _ResumeUploadV2Part + resp: ResponseInfo + """ + if not up_hosts: + raise ValueError('Must provide on up host at least') + + bucket_name = Auth.get_bucket_name(up_token) + if not bucket_name: + bucket_name = self.bucket_name + + chunked_data = IOChunked( + base_io=data, + chunk_offset=chunk_info.chunk_offset, + chunk_size=chunk_info.chunk_size, + lock=lock + ) + chunk_md5 = io_md5(chunked_data) + chunked_data.seek(0) + part, resp = None, None + for up_host in up_hosts: + url = self.__get_url_for_upload( + up_host, + bucket_name, + key, + upload_id=upload_id, + part_no=chunk_info.chunk_no + ) + ret, resp = qn_http_client.put( + url=url, + data=chunked_data, + files=None, + headers={ + 'Content-Type': 'application/octet-stream', + 'Content-MD5': chunk_md5, + 'Authorization': 'UpToken {}'.format(up_token) + } + ) + if resp.ok() and ret: + part = _ResumeUploadV2Part( + part_no=chunk_info.chunk_no, + etag=ret.get('etag', '') + ) + return part, resp + if ( + not is_seekable(chunked_data) or + not resp.need_retry() + ): + return part, resp + chunked_data.seek(0) + return part, resp + + +# use dataclass instead namedtuple if min version of python update to 3.7 +_ResumeUploadV2Part = namedtuple( + 'ResumeUploadV2Part', + [ + 'part_no', + 'etag' + ] +) + +_ResumeUploadV2Context = namedtuple( + 'ResumeUploadV2Context', + [ + 'up_hosts', + 'upload_id', + 'expired_at', + 'part_size', + 'parts', + 'modify_time', + 'resumed' + ] +) diff --git a/qiniu/utils.py b/qiniu/utils.py index 92a81449..033f003f 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- - -from hashlib import sha1 +from hashlib import sha1, new as hashlib_new from base64 import urlsafe_b64encode, urlsafe_b64decode from datetime import datetime from .compat import b, s @@ -63,6 +62,20 @@ def file_crc32(filePath): return crc +def io_crc32(io_data): + result = 0 + for d in io_data: + result = binascii.crc32(d, result) & 0xFFFFFFFF + return result + + +def io_md5(io_data): + h = hashlib_new('md5') + for d in io_data: + h.update(d) + return h.hexdigest() + + def crc32(data): """计算输入流的crc32检验码: diff --git a/setup.py b/setup.py index e7e37cce..e9809d2b 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,21 @@ def find_version(*file_paths): 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 'Topic :: Software Development :: Libraries :: Python Modules' ], - install_requires=['requests'], + install_requires=[ + 'requests', + 'futures; python_version == "2.7"' + ], + extras_require={ + 'dev': [ + 'coverage<7.2', + 'flake8', + 'pytest', + 'pytest-cov', + 'freezegun', + 'scrutinizer-ocular', + 'codecov' + ] + }, entry_points={ 'console_scripts': [ diff --git a/test_qiniu.py b/test_qiniu.py index 25e5f8bb..86072de5 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -4,6 +4,7 @@ import string import random import tempfile +import functools import requests @@ -43,6 +44,23 @@ StringIO = io.StringIO urlopen = urllib.request.urlopen +if hasattr(functools, 'cache'): + cache_decorator = functools.cache +else: + def cache_decorator(func): + cache = {} + + @functools.wraps(func) + def wrapper(*args, **kwargs): + key = (args, frozenset(kwargs.items())) + if key in cache: + return cache[key] + result = func(*args, **kwargs) + cache[key] = result + return result + + return wrapper + access_key = os.getenv('QINIU_ACCESS_KEY') secret_key = os.getenv('QINIU_SECRET_KEY') bucket_name = os.getenv('QINIU_TEST_BUCKET') @@ -637,6 +655,19 @@ def test_invalid_x_qiniu_date_env_be_ignored(self): assert info.status_code == 403 +@cache_decorator +def get_valid_up_host(): + zone = Zone() + try: + hosts = json.loads( + zone.bucket_hosts(access_key, bucket_name) + ).get('hosts') + up_host = 'https://' + hosts[0].get('up', {}).get('domains')[0] + except IndexError: + up_host = 'https://upload.qiniup.com' + return up_host + + class UploaderTestCase(unittest.TestCase): mime_type = "text/plain" params = {'x:a': 'a'} @@ -702,14 +733,7 @@ def test_withoutRead_withoutSeek_retry(self): try: key = 'retry' data = 'hello retry!' - zone = Zone() - try: - hosts = json.loads( - zone.bucket_hosts(access_key, bucket_name) - ).get('hosts') - up_host_backup = 'https://' + hosts[0].get('up', {}).get('domains')[0] - except IndexError: - up_host_backup = 'https://upload.qiniup.com' + up_host_backup = get_valid_up_host() set_default(default_zone=Zone('http://a', up_host_backup)) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) @@ -884,13 +908,19 @@ def test_big_file(self): remove_temp_file(localfile) def test_retry(self): - localfile = __file__ - key = 'test_file_r_retry' - token = self.q.upload_token(bucket_name, key) - ret, info = put_file(token, key, localfile, self.params, self.mime_type) - print(info) - assert ret['key'] == key - assert ret['hash'] == etag(localfile) + try: + localfile = __file__ + key = 'test_file_r_retry' + token = self.q.upload_token(bucket_name, key) + up_host_backup = get_valid_up_host() + set_default(default_zone=Zone('http://a', up_host_backup)) + ret, info = put_file(token, key, localfile, self.params, self.mime_type) + print(info) + assert ret['key'] == key + assert ret['hash'] == etag(localfile) + finally: + set_default(default_zone=Zone()) + qiniu.config._is_customized_default['default_zone'] = False def test_put_stream_with_key_limits(self): localfile = __file__ From 02872abae9dda5efadc598da4d6f5de0c7054a02 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Sun, 8 Oct 2023 15:48:04 +0800 Subject: [PATCH 456/478] Bump to v7.12.0 (#441) --- CHANGELOG.md | 2 ++ qiniu/__init__.py | 2 +- setup.py | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53ed46a3..7fb6c694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ # Changelog +## 7.12.0（2023-10-08） +* 对象存储，分片上传支持并发上传 ## 7.11.1（2023-08-16） * 修复 setup.py 打包丢失部分包（v7.11.0 引入） diff --git a/qiniu/__init__.py b/qiniu/__init__.py index a0a26b77..c7409e69 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.11.1' +__version__ = '7.12.0' from .auth import Auth, QiniuMacAuth diff --git a/setup.py b/setup.py index e9809d2b..5ddc9c4d 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,8 @@ def find_version(*file_paths): 'Topic :: Software Development :: Libraries :: Python Modules' ], install_requires=[ - 'requests', + 'requests; python_version >= "3.7"', + 'requests<2.28; python_version < "3.7"', 'futures; python_version == "2.7"' ], extras_require={ From 16e8262defdaf9fce6251ebbf12378af1d8f4040 Mon Sep 17 00:00:00 2001 From: Wintercom <89456349+Wintercom@users.noreply.github.com> Date: Mon, 20 Nov 2023 18:46:39 +0800 Subject: [PATCH 457/478] =?UTF-8?q?FUSION-19874=20python-sdk=E5=88=A0?= =?UTF-8?q?=E9=99=A4cdn=E5=9F=9F=E5=90=8D=E4=BB=A3=E7=A0=81=E9=97=AE?= =?UTF-8?q?=E9=A2=98=E4=BF=AE=E5=A4=8D=20(#443)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 3 +++ qiniu/__init__.py | 2 +- qiniu/services/cdn/manager.py | 6 +++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fb6c694..32da40d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 7.12.1（2023-11-20） +* 修复 CDN 删除域名代码问题 + ## 7.12.0（2023-10-08） * 对象存储，分片上传支持并发上传 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index c7409e69..7d9ec548 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.12.0' +__version__ = '7.12.1' from .auth import Auth, QiniuMacAuth diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 788e0ac3..bf107926 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -238,7 +238,7 @@ def delete_domain(self, name): - ResponseInfo 请求的Response信息 """ url = '{0}/domain/{1}'.format(self.server, name) - return self.__get(url) + return self.__del(url) def get_domain(self, name): """ @@ -312,6 +312,10 @@ def __get(self, url, data=None): headers = {'Content-Type': 'application/json'} return http._get_with_auth_and_headers(url, data, self.auth, headers) + def __del(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._delete_with_qiniu_mac_and_headers(url, data, self.auth, headers) + def create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline): """ From 4d54f2127d78a904a0c6e805e0cbfa72ee7a1fb5 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Wed, 6 Dec 2023 17:30:24 +0800 Subject: [PATCH 458/478] object lifecycle archive ir (#444) --- qiniu/auth.py | 2 +- qiniu/services/storage/bucket.py | 11 +++++++---- .../services/storage/uploaders/resume_uploader_v1.py | 2 +- test_qiniu.py | 2 ++ 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index ff1be077..5e49693d 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -36,7 +36,7 @@ 'persistentNotifyUrl', # 持久化处理结果通知URL 'persistentPipeline', # 持久化处理独享队列 'deleteAfterDays', # 文件多少天后自动删除 - 'fileType', # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储 + 'fileType', # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储，4为归档直读存储 'isPrefixalScope' # 指定上传文件必须使用的前缀 ]) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index efb2c9b4..0f1106f5 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -60,8 +60,8 @@ def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): options['delimiter'] = delimiter ak = self.auth.get_access_key() - rs_host = self.zone.get_rsf_host(ak, bucket) - url = '{0}/list'.format(rs_host) + rsf_host = self.zone.get_rsf_host(ak, bucket) + url = '{0}/list'.format(rsf_host) ret, info = self.__get(url, options) eof = False @@ -243,7 +243,7 @@ def change_type(self, bucket, key, storage_type): Args: bucket: 待操作资源所在空间 key: 待操作资源文件名 - storage_type: 待操作资源存储类型，0为普通存储，1为低频存储，2 为归档存储，3 为深度归档 + storage_type: 待操作资源存储类型，0为普通存储，1为低频存储，2 为归档存储，3 为深度归档，4 为归档直读存储 """ resource = entry(bucket, key) return self.__rs_do(bucket, 'chtype', resource, 'type/{0}'.format(storage_type)) @@ -289,7 +289,8 @@ def set_object_lifecycle( to_archive_after_days=0, to_deep_archive_after_days=0, delete_after_days=0, - cond=None + cond=None, + to_archive_ir_after_days=0 ): """ @@ -303,6 +304,7 @@ def set_object_lifecycle( to_deep_archive_after_days: 多少天后将文件转为深度归档存储，设置为 -1 表示取消已设置的转深度归档存储的生命周期规则， 0 表示不修改转深度归档生命周期规则 delete_after_days: 多少天后将文件删除，设置为 -1 表示取消已设置的删除存储的生命周期规则， 0 表示不修改删除存储的生命周期规则。 cond: 匹配条件，只有条件匹配才会设置成功，当前支持设置 hash、mime、fsize、putTime。 + to_archive_ir_after_days: 多少天后将文件转为归档直读存储，设置为 -1 表示取消已设置的转归档只读存储的生命周期规则， 0 表示不修改转归档只读存储生命周期规则。 Returns: resBody, respInfo @@ -310,6 +312,7 @@ def set_object_lifecycle( """ options = [ 'toIAAfterDays', str(to_line_after_days), + 'toArchiveIRAfterDays', str(to_archive_ir_after_days), 'toArchiveAfterDays', str(to_archive_after_days), 'toDeepArchiveAfterDays', str(to_deep_archive_after_days), 'deleteAfterDays', str(delete_after_days) diff --git a/qiniu/services/storage/uploaders/resume_uploader_v1.py b/qiniu/services/storage/uploaders/resume_uploader_v1.py index 136ef7ab..af184780 100644 --- a/qiniu/services/storage/uploaders/resume_uploader_v1.py +++ b/qiniu/services/storage/uploaders/resume_uploader_v1.py @@ -544,7 +544,7 @@ def __upload_part( Parameters ---------- data: IOBase - chunk_info: IOChunked + chunk_info: ChunkInfo up_hosts: list[str] up_token: str lock: Lock diff --git a/test_qiniu.py b/test_qiniu.py index 86072de5..0b07c9e3 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -592,6 +592,7 @@ def test_set_object_lifecycle(self): bucket=bucket_name, key=key, to_line_after_days=10, + to_archive_ir_after_days=15, to_archive_after_days=20, to_deep_archive_after_days=30, delete_after_days=40 @@ -609,6 +610,7 @@ def test_set_object_lifecycle_with_cond(self): bucket=bucket_name, key=key, to_line_after_days=10, + to_archive_ir_after_days=15, to_archive_after_days=20, to_deep_archive_after_days=30, delete_after_days=40, From cb0d415c04858733b15769aaef9b9d76c100fe60 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Mon, 11 Dec 2023 09:35:02 +0800 Subject: [PATCH 459/478] fix batch rs host (#445) --- qiniu/__init__.py | 2 +- qiniu/services/storage/bucket.py | 17 ++++++- qiniu/utils.py | 4 ++ test_qiniu.py | 86 +++++++++++++++++++++++++++++++- 4 files changed, 105 insertions(+), 4 deletions(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 7d9ec548..a09123d4 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -28,4 +28,4 @@ from .services.compute.qcos_api import QcosClient from .services.sms.sms import Sms from .services.pili.rtc_server_manager import RtcServer, get_room_token -from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry, canonical_mime_header_key +from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry, decode_entry, canonical_mime_header_key diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 0f1106f5..8edb00a3 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -2,7 +2,7 @@ from qiniu import config, QiniuMacAuth from qiniu import http -from qiniu.utils import urlsafe_base64_encode, entry +from qiniu.utils import urlsafe_base64_encode, entry, decode_entry class BucketManager(object): @@ -344,7 +344,20 @@ def batch(self, operations): ] 一个ResponseInfo对象 """ - url = '{0}/batch'.format(config.get_default('default_rs_host')) + if not operations: + raise Exception('operations is empty') + bucket = '' + for op in operations: + segments = op.split('/') + e = segments[1] if len(segments) >= 2 else '' + bucket, _ = decode_entry(e) + if bucket: + break + if not bucket: + raise Exception('bucket is empty') + ak = self.auth.get_access_key() + rs_host = self.zone.get_rs_host(ak, bucket) + url = '{0}/batch'.format(rs_host) return self.__post(url, dict(op=operations)) def buckets(self): diff --git a/qiniu/utils.py b/qiniu/utils.py index 033f003f..fa750707 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -175,6 +175,10 @@ def entry(bucket, key): return urlsafe_base64_encode('{0}:{1}'.format(bucket, key)) +def decode_entry(e): + return (s(urlsafe_base64_decode(e)).split(':') + [None] * 2)[:2] + + def rfc_from_timestamp(timestamp): """将时间戳转换为HTTP RFC格式 diff --git a/test_qiniu.py b/test_qiniu.py index 0b07c9e3..d52caeea 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -16,7 +16,7 @@ from qiniu import put_data, put_file, put_stream from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, \ build_batch_delete, DomainManager -from qiniu import urlsafe_base64_encode, urlsafe_base64_decode, canonical_mime_header_key +from qiniu import urlsafe_base64_encode, urlsafe_base64_decode, canonical_mime_header_key, entry, decode_entry from qiniu.compat import is_py2, is_py3, b, json @@ -255,6 +255,90 @@ def test_canonical_mime_header_key(self): for i in range(len(field_names)): assert canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i] + def test_entry(self): + case_list = [ + { + 'msg': 'normal', + 'bucket': 'qiniuphotos', + 'key': 'gogopher.jpg', + 'expect': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' + }, + { + 'msg': 'key empty', + 'bucket': 'qiniuphotos', + 'key': '', + 'expect': 'cWluaXVwaG90b3M6' + }, + { + 'msg': 'key undefined', + 'bucket': 'qiniuphotos', + 'key': None, + 'expect': 'cWluaXVwaG90b3M=' + }, + { + 'msg': 'key need replace plus symbol', + 'bucket': 'qiniuphotos', + 'key': '012ts>a', + 'expect': 'cWluaXVwaG90b3M6MDEydHM-YQ==' + }, + { + 'msg': 'key need replace slash symbol', + 'bucket': 'qiniuphotos', + 'key': '012ts?a', + 'expect': 'cWluaXVwaG90b3M6MDEydHM_YQ==' + } + ] + for c in case_list: + assert c.get('expect') == entry(c.get('bucket'), c.get('key')), c.get('msg') + + def test_decode_entry(self): + case_list = [ + { + 'msg': 'normal', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': 'gogopher.jpg' + }, + 'entry': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' + }, + { + 'msg': 'key empty', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': '' + }, + 'entry': 'cWluaXVwaG90b3M6' + }, + { + 'msg': 'key undefined', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': None + }, + 'entry': 'cWluaXVwaG90b3M=' + }, + { + 'msg': 'key need replace plus symbol', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': '012ts>a' + }, + 'entry': 'cWluaXVwaG90b3M6MDEydHM-YQ==' + }, + { + 'msg': 'key need replace slash symbol', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': '012ts?a' + }, + 'entry': 'cWluaXVwaG90b3M6MDEydHM_YQ==' + } + ] + for c in case_list: + bucket, key = decode_entry(c.get('entry')) + assert bucket == c.get('expect').get('bucket'), c.get('msg') + assert key == c.get('expect').get('key'), c.get('msg') + class AuthTestCase(unittest.TestCase): def test_token(self): From 9c087badd9c9dc0e892840551dec083af91a14b8 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Mon, 25 Dec 2023 16:26:51 +0800 Subject: [PATCH 460/478] bump version to v7.13.0 and update CHANGELOG.md (#446) --- CHANGELOG.md | 4 ++++ qiniu/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32da40d7..a4de016d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 7.13.0（2023-12-11） +* 对象存储，新增支持归档直读存储 +* 对象存储，批量操作支持自动查询 rs 服务域名 + ## 7.12.1（2023-11-20） * 修复 CDN 删除域名代码问题 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index a09123d4..63cb78db 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.12.1' +__version__ = '7.13.0' from .auth import Auth, QiniuMacAuth From 7c25d034d81edbf4c8a3a746a9543915ba0dc2cc Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Wed, 7 Feb 2024 11:20:20 +0800 Subject: [PATCH 461/478] Fix upload config compatibility and improve test cases (#447) --- .github/workflows/ci-test.yml | 23 +- CHANGELOG.md | 3 + qiniu/__init__.py | 2 +- qiniu/compat.py | 3 +- qiniu/http/__init__.py | 14 + qiniu/http/client.py | 4 + qiniu/http/response.py | 8 +- .../uploaders/abc/resume_uploader_base.py | 2 + .../storage/uploaders/abc/uploader_base.py | 14 +- .../storage/uploaders/resume_uploader_v1.py | 18 +- .../storage/uploaders/resume_uploader_v2.py | 20 +- test_qiniu.py | 466 +------------- tests/cases/__init__.py | 0 tests/cases/conftest.py | 76 +++ tests/cases/test_http/__init__.py | 0 tests/cases/test_http/conftest.py | 11 + tests/cases/test_http/test_middleware.py | 88 +++ tests/cases/test_http/test_qiniu_conf.py | 117 ++++ tests/cases/test_http/test_resp.py | 64 ++ tests/cases/test_services/__init__.py | 0 .../test_services/test_storage/__init__.py | 0 .../test_services/test_storage/conftest.py | 8 + .../test_storage/test_uploader.py | 594 ++++++++++++++++++ tests/mock_server/main.py | 65 ++ tests/mock_server/routes/__init__.py | 10 + tests/mock_server/routes/echo.py | 34 + tests/mock_server/routes/retry_me.py | 145 +++++ tests/mock_server/routes/timeout.py | 36 ++ 28 files changed, 1340 insertions(+), 485 deletions(-) create mode 100644 tests/cases/__init__.py create mode 100644 tests/cases/conftest.py create mode 100644 tests/cases/test_http/__init__.py create mode 100644 tests/cases/test_http/conftest.py create mode 100644 tests/cases/test_http/test_middleware.py create mode 100644 tests/cases/test_http/test_qiniu_conf.py create mode 100644 tests/cases/test_http/test_resp.py create mode 100644 tests/cases/test_services/__init__.py create mode 100644 tests/cases/test_services/test_storage/__init__.py create mode 100644 tests/cases/test_services/test_storage/conftest.py create mode 100644 tests/cases/test_services/test_storage/test_uploader.py create mode 100644 tests/mock_server/main.py create mode 100644 tests/mock_server/routes/__init__.py create mode 100644 tests/mock_server/routes/echo.py create mode 100644 tests/mock_server/routes/retry_me.py create mode 100644 tests/mock_server/routes/timeout.py diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 84c20fd1..af9670fd 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -35,24 +35,37 @@ jobs: wget -qLO get-pip.py "$PIP_BOOTSTRAP_SCRIPT_PREFIX/$MAJOR.$MINOR/get-pip.py" python get-pip.py --user fi + - name: Setup mock server + shell: bash -el {0} + run: | + conda create -y -n mock-server python=3.10 + conda activate mock-server + python3 --version + nohup python3 tests/mock_server/main.py --port 9000 > py-mock-server.log & + echo $! > mock-server.pid + conda deactivate - name: Install dependencies shell: bash -l {0} run: | python -m pip install --upgrade pip python -m pip install -I -e ".[dev]" - name: Run cases - shell: bash -l {0} + shell: bash -el {0} env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }} QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }} QINIU_TEST_ENV: "travis" + MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" PYTHONPATH: "$PYTHONPATH:." run: | - set -e - flake8 --show-source --max-line-length=160 . - py.test --cov qiniu + flake8 --show-source --max-line-length=160 ./qiniu + coverage run -m pytest ./test_qiniu.py ./tests/cases ocular --data-file .coverage - coverage run test_qiniu.py codecov + cat mock-server.pid | xargs kill + - name: Print mock server log + if: ${{ failure() }} + run: | + cat py-mock-server.log diff --git a/CHANGELOG.md b/CHANGELOG.md index a4de016d..c5192eb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 7.13.1（2024-02-04） +* 对象存储，修复上传部分配置项的兼容 + ## 7.13.0（2023-12-11） * 对象存储，新增支持归档直读存储 * 对象存储，批量操作支持自动查询 rs 服务域名 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 63cb78db..5edeb952 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.13.0' +__version__ = '7.13.1' from .auth import Auth, QiniuMacAuth diff --git a/qiniu/compat.py b/qiniu/compat.py index 4ab4ce78..6a418c99 100644 --- a/qiniu/compat.py +++ b/qiniu/compat.py @@ -33,6 +33,7 @@ # --------- if is_py2: + from urllib import urlencode # noqa from urlparse import urlparse # noqa import StringIO StringIO = BytesIO = StringIO.StringIO @@ -60,7 +61,7 @@ def is_seekable(data): return False elif is_py3: - from urllib.parse import urlparse # noqa + from urllib.parse import urlparse, urlencode # noqa import io StringIO = io.StringIO BytesIO = io.BytesIO diff --git a/qiniu/http/__init__.py b/qiniu/http/__init__.py index 416d354d..2b61e0fe 100644 --- a/qiniu/http/__init__.py +++ b/qiniu/http/__init__.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import logging import platform +import functools import requests from requests.adapters import HTTPAdapter @@ -21,6 +22,19 @@ ) +# compatibility with some config from qiniu.config +def _before_send(func): + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + if _session is None: + _init() + return func(self, *args, **kwargs) + + return wrapper + + +qn_http_client.send_request = _before_send(qn_http_client.send_request) + _sys_info = '{0}; {1}'.format(platform.system(), platform.machine()) _python_ver = platform.python_version() diff --git a/qiniu/http/client.py b/qiniu/http/client.py index 97c41980..6b850a38 100644 --- a/qiniu/http/client.py +++ b/qiniu/http/client.py @@ -3,6 +3,7 @@ import requests +from qiniu.config import get_default from .response import ResponseInfo from .middleware import compose_middleware @@ -14,6 +15,9 @@ def __init__(self, middlewares=None, send_opts=None): self.send_opts = {} if send_opts is None else send_opts def _wrap_send(self, req, **kwargs): + # compatibility with setting timeout by qiniu.config.set_default + kwargs.setdefault('timeout', get_default('connection_timeout')) + resp = self.session.send(req.prepare(), **kwargs) return ResponseInfo(resp, None) diff --git a/qiniu/http/response.py b/qiniu/http/response.py index 7aa1bba4..8e9fd84d 100644 --- a/qiniu/http/response.py +++ b/qiniu/http/response.py @@ -46,7 +46,13 @@ def ok(self): return self.status_code // 100 == 2 def need_retry(self): - if 0 < self.status_code < 500: + if 100 <= self.status_code < 500: + return False + if all([ + self.status_code < 0, + self.exception is not None, + 'BadStatusLine' in str(self.exception) + ]): return False # https://developer.qiniu.com/fusion/kb/1352/the-http-request-return-a-status-code if self.status_code in [ diff --git a/qiniu/services/storage/uploaders/abc/resume_uploader_base.py b/qiniu/services/storage/uploaders/abc/resume_uploader_base.py index 2f6a3c50..06d827df 100644 --- a/qiniu/services/storage/uploaders/abc/resume_uploader_base.py +++ b/qiniu/services/storage/uploaders/abc/resume_uploader_base.py @@ -145,6 +145,7 @@ def initial_parts( data_size, modify_time, part_size, + file_name, **kwargs ): """ @@ -157,6 +158,7 @@ def initial_parts( data_size: int modify_time: int part_size: int + file_name: str kwargs: dict Returns diff --git a/qiniu/services/storage/uploaders/abc/uploader_base.py b/qiniu/services/storage/uploaders/abc/uploader_base.py index e1096b0d..bb39fbfc 100644 --- a/qiniu/services/storage/uploaders/abc/uploader_base.py +++ b/qiniu/services/storage/uploaders/abc/uploader_base.py @@ -84,6 +84,7 @@ def _get_regions(self): if self.regions: return self.regions + # handle compatibility for default_zone default_region = config.get_default('default_zone') if default_region: self.regions = [default_region] @@ -108,11 +109,14 @@ def _get_up_hosts(self, access_key=None): if not regions: raise ValueError('No region available.') - if regions[0].up_host and regions[0].up_host_backup: - return [ - regions[0].up_host, - regions[0].up_host_backup - ] + # get up hosts in region + up_hosts = [ + regions[0].up_host, + regions[0].up_host_backup + ] + up_hosts = [h for h in up_hosts if h] + if up_hosts: + return up_hosts # this is correct, it does return hosts. bad function name by legacy return regions[0].get_up_host( diff --git a/qiniu/services/storage/uploaders/resume_uploader_v1.py b/qiniu/services/storage/uploaders/resume_uploader_v1.py index af184780..7f1ae89f 100644 --- a/qiniu/services/storage/uploaders/resume_uploader_v1.py +++ b/qiniu/services/storage/uploaders/resume_uploader_v1.py @@ -1,4 +1,5 @@ import logging +import math from collections import namedtuple from concurrent import futures from io import BytesIO @@ -65,7 +66,11 @@ def _recover_from_record( record_context = [ ctx for ctx in record_context - if ctx.get('expired_at', 0) > now + if ( + ctx.get('expired_at', 0) > now and + ctx.get('part_no', None) and + ctx.get('ctx', None) + ) ] # assign to context @@ -173,6 +178,7 @@ def initial_parts( data=None, modify_time=None, data_size=None, + file_name=None, **kwargs ): """ @@ -184,6 +190,7 @@ def initial_parts( data modify_time data_size + file_name kwargs @@ -222,7 +229,8 @@ def initial_parts( ) # try to recover from record - file_name = path.basename(file_path) if file_path else None + if not file_name and file_path: + file_name = path.basename(file_path) context = self._recover_from_record( file_name, key, @@ -275,7 +283,10 @@ def upload_parts( # initial upload state part, resp = None, None - uploaded_size = 0 + uploaded_size = context.part_size * len(context.parts) + if math.ceil(data_size / context.part_size) in [p.part_no for p in context.parts]: + # if last part uploaded, should correct the uploaded size + uploaded_size += (data_size % context.part_size) - context.part_size lock = Lock() if not self.concurrent_executor: @@ -469,6 +480,7 @@ def upload( up_token, key, file_path=file_path, + file_name=file_name, data=data, data_size=data_size, modify_time=modify_time, diff --git a/qiniu/services/storage/uploaders/resume_uploader_v2.py b/qiniu/services/storage/uploaders/resume_uploader_v2.py index 82299d0d..db73b182 100644 --- a/qiniu/services/storage/uploaders/resume_uploader_v2.py +++ b/qiniu/services/storage/uploaders/resume_uploader_v2.py @@ -1,3 +1,4 @@ +import math from collections import namedtuple from concurrent import futures from io import BytesIO @@ -65,7 +66,7 @@ def _recover_from_record( if ( not record_upload_id or record_modify_time != context.modify_time or - record_expired_at > time() + record_expired_at < time() ): return context @@ -80,6 +81,10 @@ def _recover_from_record( etag=p['etag'] ) for p in record_etags + if ( + p.get('partNumber', None) and + p.get('etag', None) + ) ], resumed=True ) @@ -105,7 +110,7 @@ def _set_to_record(self, file_name, key, context): 'etags': [ { 'etag': part.etag, - 'part_no': part.part_no + 'partNumber': part.part_no } for part in context.parts ] @@ -170,6 +175,7 @@ def initial_parts( data_size=None, modify_time=None, part_size=None, + file_name=None, **kwargs ): """ @@ -183,6 +189,7 @@ def initial_parts( data_size: int modify_time: int part_size: int + file_name: str kwargs Returns @@ -222,7 +229,8 @@ def initial_parts( ) # try to recover from record - file_name = path.basename(file_path) if file_path else None + if not file_name and file_path: + file_name = path.basename(file_path) context = self._recover_from_record( file_name, key, @@ -307,7 +315,10 @@ def upload_parts( # initial upload state part, resp = None, None - uploaded_size = 0 + uploaded_size = context.part_size * len(context.parts) + if math.ceil(data_size / context.part_size) in [p.part_no for p in context.parts]: + # if last part uploaded, should correct the uploaded size + uploaded_size += (data_size % context.part_size) - context.part_size lock = Lock() if not self.concurrent_executor: @@ -500,6 +511,7 @@ def upload( up_token, key, file_path=file_path, + file_name=file_name, data=data, data_size=data_size, modify_time=modify_time, diff --git a/test_qiniu.py b/test_qiniu.py index d52caeea..c048d4c3 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -13,20 +13,15 @@ from freezegun import freeze_time from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone, QiniuMacAuth -from qiniu import put_data, put_file, put_stream from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, \ build_batch_delete, DomainManager from qiniu import urlsafe_base64_encode, urlsafe_base64_decode, canonical_mime_header_key, entry, decode_entry from qiniu.compat import is_py2, is_py3, b, json -from qiniu.services.storage.uploader import _form_put - -from qiniu.http import __return_wrapper as return_wrapper, qn_http_client -from qiniu.http.middleware import Middleware, RetryDomainsMiddleware - import qiniu.config + if is_py2: import sys import StringIO @@ -44,23 +39,6 @@ StringIO = io.StringIO urlopen = urllib.request.urlopen -if hasattr(functools, 'cache'): - cache_decorator = functools.cache -else: - def cache_decorator(func): - cache = {} - - @functools.wraps(func) - def wrapper(*args, **kwargs): - key = (args, frozenset(kwargs.items())) - if key in cache: - return cache[key] - result = func(*args, **kwargs) - cache[key] = result - return result - - return wrapper - access_key = os.getenv('QINIU_ACCESS_KEY') secret_key = os.getenv('QINIU_SECRET_KEY') bucket_name = os.getenv('QINIU_TEST_BUCKET') @@ -93,131 +71,6 @@ def remove_temp_file(file): pass -def is_travis(): - return os.environ['QINIU_TEST_ENV'] == 'travis' - - -class MiddlewareRecorder(Middleware): - def __init__(self, rec, label): - self.rec = rec - self.label = label - - def __call__(self, request, nxt): - self.rec.append( - 'bef_{0}{1}'.format(self.label, len(self.rec)) - ) - resp = nxt(request) - self.rec.append( - 'aft_{0}{1}'.format(self.label, len(self.rec)) - ) - return resp - - -class HttpTest(unittest.TestCase): - def test_response_need_retry(self): - _ret, resp_info = qn_http_client.get('https://qiniu.com/index.html') - - resp_info.req_id = 'mocked-req-id' - resp_info.error = None - - def gen_case(code): - if 0 < code < 500: - return code, False - if code in [ - 501, 509, 573, 579, 608, 612, 614, 616, 618, 630, 631, 632, 640, 701 - ]: - return code, False - return code, True - - cases = [ - gen_case(i) for i in range(-1, 800) - ] - - for test_code, should_retry in cases: - resp_info.status_code = test_code - assert_msg = '{0} should{1} retry'.format(test_code, '' if should_retry else ' NOT') - assert resp_info.need_retry() == should_retry, assert_msg - - def test_middlewares(self): - rec_ls = [] - mw_a = MiddlewareRecorder(rec_ls, 'A') - mw_b = MiddlewareRecorder(rec_ls, 'B') - qn_http_client.get( - 'https://qiniu.com/index.html', - middlewares=[ - mw_a, - mw_b - ] - ) - assert rec_ls == ['bef_A0', 'bef_B1', 'aft_B2', 'aft_A3'] - - - def test_retry_domains(self): - rec_ls = [] - mw_rec = MiddlewareRecorder(rec_ls, 'rec') - ret, resp = qn_http_client.get( - 'https://fake.pysdk.qiniu.com/index.html', - middlewares=[ - RetryDomainsMiddleware( - backup_domains=[ - 'unavailable.pysdk.qiniu.com', - 'qiniu.com' - ], - max_retry_times=3 - ), - mw_rec - ] - ) - # ['bef_rec0', 'bef_rec1', 'bef_rec2'] are 'fake.pysdk.qiniu.com' with retried 3 times - # ['bef_rec3', 'bef_rec4', 'bef_rec5'] are 'unavailable.pysdk.qiniu.com' with retried 3 times - # ['bef_rec6', 'aft_rec7'] are 'qiniu.com' and it's success - assert rec_ls == [ - 'bef_rec0', 'bef_rec1', 'bef_rec2', - 'bef_rec3', 'bef_rec4', 'bef_rec5', - 'bef_rec6', 'aft_rec7' - ] - assert ret == {} - assert resp.status_code == 200 - - def test_retry_domains_fail_fast(self): - rec_ls = [] - mw_rec = MiddlewareRecorder(rec_ls, 'rec') - ret, resp = qn_http_client.get( - 'https://fake.pysdk.qiniu.com/index.html', - middlewares=[ - RetryDomainsMiddleware( - backup_domains=[ - 'unavailable.pysdk.qiniu.com', - 'qiniu.com' - ], - retry_condition=lambda _resp, _req: False - ), - mw_rec - ] - ) - # ['bef_rec0'] are 'fake.pysdk.qiniu.com' with fail fast - assert rec_ls == ['bef_rec0'] - assert ret is None - assert resp.status_code == -1 - - - - def test_json_decode_error(self): - def mock_res(): - r = requests.Response() - r.status_code = 200 - r.headers.__setitem__('X-Reqid', 'mockedReqid') - - def json_func(): - raise ValueError('%s: line %d column %d (char %d)' % ('Expecting value', 0, 0, 0)) - r.json = json_func - - return r - mocked_res = mock_res() - ret, _ = return_wrapper(mocked_res) - assert ret == {} - - class UtilsTest(unittest.TestCase): def test_urlsafe(self): a = 'hello\x96' @@ -740,323 +593,6 @@ def test_invalid_x_qiniu_date_env_be_ignored(self): assert ret is None assert info.status_code == 403 - -@cache_decorator -def get_valid_up_host(): - zone = Zone() - try: - hosts = json.loads( - zone.bucket_hosts(access_key, bucket_name) - ).get('hosts') - up_host = 'https://' + hosts[0].get('up', {}).get('domains')[0] - except IndexError: - up_host = 'https://upload.qiniup.com' - return up_host - - -class UploaderTestCase(unittest.TestCase): - mime_type = "text/plain" - params = {'x:a': 'a'} - metadata = { - 'x-qn-meta-name': 'qiniu', - 'x-qn-meta-age': '18' - } - q = Auth(access_key, secret_key) - bucket = BucketManager(q) - - def test_put(self): - key = 'a\\b\\c"hello' - data = 'hello bubby!' - token = self.q.upload_token(bucket_name) - ret, info = put_data(token, key, data) - print(info) - assert ret['key'] == key - - def test_put_crc(self): - key = '' - data = 'hello bubby!' - token = self.q.upload_token(bucket_name, key) - ret, info = put_data(token, key, data, check_crc=True) - print(info) - assert ret['key'] == key - - def test_putfile(self): - localfile = __file__ - key = 'test_file' - - token = self.q.upload_token(bucket_name, key) - ret, info = put_file(token, key, localfile, mime_type=self.mime_type, check_crc=True) - print(info) - assert ret['key'] == key - assert ret['hash'] == etag(localfile) - - def test_putInvalidCrc(self): - key = 'test_invalid' - data = 'hello bubby!' - crc32 = 'wrong crc32' - token = self.q.upload_token(bucket_name) - ret, info = _form_put(token, key, data, None, None, crc=crc32) - print(info) - assert ret is None - assert info.status_code == 400 - - def test_putWithoutKey(self): - key = None - data = 'hello bubby!' - token = self.q.upload_token(bucket_name) - ret, info = put_data(token, key, data) - print(info) - assert ret['hash'] == ret['key'] - - data = 'hello bubby!' - token = self.q.upload_token(bucket_name, 'nokey2') - ret, info = put_data(token, None, data) - print(info) - assert ret is None - assert info.status_code == 403 # key not match - - def test_withoutRead_withoutSeek_retry(self): - try: - key = 'retry' - data = 'hello retry!' - up_host_backup = get_valid_up_host() - set_default(default_zone=Zone('http://a', up_host_backup)) - token = self.q.upload_token(bucket_name) - ret, info = put_data(token, key, data) - print(info) - assert ret['key'] == key - assert ret['hash'] == 'FlYu0iBR1WpvYi4whKXiBuQpyLLk' - finally: - set_default(default_zone=Zone()) - qiniu.config._is_customized_default['default_zone'] = False - - def test_putData_without_fname(self): - if is_travis(): - return - localfile = create_temp_file(30 * 1024 * 1024) - key = 'test_putData_without_fname' - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name) - ret, info = put_data(token, key, input_stream) - print(info) - assert ret is not None - - def test_putData_without_fname1(self): - if is_travis(): - return - localfile = create_temp_file(30 * 1024 * 1024) - key = 'test_putData_without_fname1' - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name) - ret, info = put_data(token, key, input_stream, self.params, self.mime_type, False, None, "") - print(info) - assert ret is not None - - def test_putData_without_fname2(self): - if is_travis(): - return - localfile = create_temp_file(30 * 1024 * 1024) - key = 'test_putData_without_fname2' - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name) - ret, info = put_data(token, key, input_stream, self.params, self.mime_type, False, None, " ") - print(info) - assert ret is not None - - def test_put_file_with_metadata(self): - localfile = __file__ - key = 'test_file_with_metadata' - - token = self.q.upload_token(bucket_name, key) - ret, info = put_file(token, key, localfile, metadata=self.metadata) - assert ret['key'] == key - assert ret['hash'] == etag(localfile) - ret, info = self.bucket.stat(bucket_name, key) - assert 'x-qn-meta' in ret - assert ret['x-qn-meta']['name'] == 'qiniu' - assert ret['x-qn-meta']['age'] == '18' - - def test_put_data_with_metadata(self): - key = 'put_data_with_metadata' - data = 'hello metadata!' - token = self.q.upload_token(bucket_name, key) - ret, info = put_data(token, key, data, metadata=self.metadata) - assert ret['key'] == key - ret, info = self.bucket.stat(bucket_name, key) - assert 'x-qn-meta' in ret - assert ret['x-qn-meta']['name'] == 'qiniu' - assert ret['x-qn-meta']['age'] == '18' - - -class ResumableUploaderTestCase(unittest.TestCase): - mime_type = "text/plain" - params = {'x:a': 'a'} - metadata = { - 'x-qn-meta-name': 'qiniu', - 'x-qn-meta-age': '18' - } - q = Auth(access_key, secret_key) - bucket = BucketManager(q) - - def test_put_stream(self): - localfile = __file__ - key = 'test_file_r' - size = os.stat(localfile).st_size - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, - self.params, - self.mime_type, part_size=None, version=None, bucket_name=None) - assert ret['key'] == key - - def test_put_stream_v2_without_bucket_name(self): - localfile = __file__ - key = 'test_file_r' - size = os.stat(localfile).st_size - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, - self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2') - assert ret['key'] == key - - def test_put_2m_stream_v2(self): - localfile = create_temp_file(2 * 1024 * 1024 + 1) - key = 'test_file_r' - size = os.stat(localfile).st_size - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, - self.params, - self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) - assert ret['key'] == key - remove_temp_file(localfile) - - def test_put_4m_stream_v2(self): - localfile = create_temp_file(4 * 1024 * 1024) - key = 'test_file_r' - size = os.stat(localfile).st_size - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, - self.params, - self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) - assert ret['key'] == key - remove_temp_file(localfile) - - def test_put_10m_stream_v2(self): - localfile = create_temp_file(10 * 1024 * 1024 + 1) - key = 'test_file_r' - size = os.stat(localfile).st_size - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, - self.params, - self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) - assert ret['key'] == key - remove_temp_file(localfile) - - def test_put_stream_v2_without_key(self): - part_size = 1024 * 1024 * 4 - localfile = create_temp_file(part_size + 1) - key = None - size = os.stat(localfile).st_size - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, - self.params, - self.mime_type, part_size=part_size, version='v2', bucket_name=bucket_name) - assert ret['key'] == ret['hash'] - remove_temp_file(localfile) - - def test_put_stream_v2_with_empty_return_body(self): - part_size = 1024 * 1024 * 4 - localfile = create_temp_file(part_size + 1) - key = 'test_file_empty_return_body' - size = os.stat(localfile).st_size - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name, key, policy={'returnBody': ' '}) - ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, - self.params, - self.mime_type, part_size=part_size, version='v2', bucket_name=bucket_name) - assert info.status_code == 200 - assert ret == {} - remove_temp_file(localfile) - - def test_big_file(self): - key = 'big' - token = self.q.upload_token(bucket_name, key) - localfile = create_temp_file(4 * 1024 * 1024 + 1) - progress_handler = lambda progress, total: progress - ret, info = put_file(token, key, localfile, self.params, self.mime_type, progress_handler=progress_handler) - print(info) - assert ret['key'] == key - remove_temp_file(localfile) - - def test_retry(self): - try: - localfile = __file__ - key = 'test_file_r_retry' - token = self.q.upload_token(bucket_name, key) - up_host_backup = get_valid_up_host() - set_default(default_zone=Zone('http://a', up_host_backup)) - ret, info = put_file(token, key, localfile, self.params, self.mime_type) - print(info) - assert ret['key'] == key - assert ret['hash'] == etag(localfile) - finally: - set_default(default_zone=Zone()) - qiniu.config._is_customized_default['default_zone'] = False - - def test_put_stream_with_key_limits(self): - localfile = __file__ - key = 'test_file_r' - size = os.stat(localfile).st_size - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name, key, policy={'keylimit': ['test_file_d']}) - ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, - self.params, - self.mime_type) - assert info.status_code == 403 - token = self.q.upload_token(bucket_name, key, policy={'keylimit': ['test_file_d', 'test_file_r']}) - ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, - self.params, - self.mime_type) - assert info.status_code == 200 - - def test_put_stream_with_metadata(self): - localfile = __file__ - key = 'test_put_stream_with_metadata' - size = os.stat(localfile).st_size - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, - self.params, self.mime_type, - part_size=None, version=None, bucket_name=None, metadata=self.metadata) - assert ret['key'] == key - ret, info = self.bucket.stat(bucket_name, key) - assert 'x-qn-meta' in ret - assert ret['x-qn-meta']['name'] == 'qiniu' - assert ret['x-qn-meta']['age'] == '18' - - def test_put_stream_v2_with_metadata(self): - part_size = 1024 * 1024 * 4 - localfile = create_temp_file(part_size + 1) - key = 'test_put_stream_v2_with_metadata' - size = os.stat(localfile).st_size - with open(localfile, 'rb') as input_stream: - token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, - self.params, self.mime_type, - part_size=part_size, version='v2', bucket_name=bucket_name, metadata=self.metadata) - assert ret['key'] == key - remove_temp_file(localfile) - ret, info = self.bucket.stat(bucket_name, key) - assert 'x-qn-meta' in ret - assert ret['x-qn-meta']['name'] == 'qiniu' - assert ret['x-qn-meta']['age'] == '18' - - class DownloadTestCase(unittest.TestCase): q = Auth(access_key, secret_key) diff --git a/tests/cases/__init__.py b/tests/cases/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/conftest.py b/tests/cases/conftest.py new file mode 100644 index 00000000..56792f20 --- /dev/null +++ b/tests/cases/conftest.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +import os + +import pytest + +from qiniu import config as qn_config +from qiniu import region +from qiniu import Auth + + +@pytest.fixture(scope='session') +def access_key(): + yield os.getenv('QINIU_ACCESS_KEY') + + +@pytest.fixture(scope='session') +def secret_key(): + yield os.getenv('QINIU_SECRET_KEY') + + +@pytest.fixture(scope='session') +def bucket_name(): + yield os.getenv('QINIU_TEST_BUCKET') + + +@pytest.fixture(scope='session') +def qn_auth(access_key, secret_key): + yield Auth(access_key, secret_key) + + +@pytest.fixture(scope='session') +def is_travis(): + """ + migrate from old test cases. + seems useless. + """ + yield os.getenv('QINIU_TEST_ENV') == 'travis' + + +@pytest.fixture(scope='function') +def set_conf_default(request): + if hasattr(request, 'param'): + qn_config.set_default(**request.param) + yield + qn_config._config = { + 'default_zone': region.Region(), + 'default_rs_host': qn_config.RS_HOST, + 'default_rsf_host': qn_config.RSF_HOST, + 'default_api_host': qn_config.API_HOST, + 'default_uc_host': qn_config.UC_HOST, + 'default_query_region_host': qn_config.QUERY_REGION_HOST, + 'default_query_region_backup_hosts': [ + 'uc.qbox.me', + 'api.qiniu.com' + ], + 'default_backup_hosts_retry_times': 2, + 'connection_timeout': 30, # 链接超时为时间为30s + 'connection_retries': 3, # 链接重试次数为3次 + 'connection_pool': 10, # 链接池个数为10 + 'default_upload_threshold': 2 * qn_config._BLOCK_SIZE # put_file上传方式的临界默认值 + } + + _is_customized_default = { + 'default_zone': False, + 'default_rs_host': False, + 'default_rsf_host': False, + 'default_api_host': False, + 'default_uc_host': False, + 'default_query_region_host': False, + 'default_query_region_backup_hosts': False, + 'default_backup_hosts_retry_times': False, + 'connection_timeout': False, + 'connection_retries': False, + 'connection_pool': False, + 'default_upload_threshold': False + } diff --git a/tests/cases/test_http/__init__.py b/tests/cases/test_http/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/test_http/conftest.py b/tests/cases/test_http/conftest.py new file mode 100644 index 00000000..9f2c23a8 --- /dev/null +++ b/tests/cases/test_http/conftest.py @@ -0,0 +1,11 @@ +import os + +import pytest + +from qiniu.compat import urlparse + + +@pytest.fixture(scope='session') +def mock_server_addr(): + addr = os.getenv('MOCK_SERVER_ADDRESS', 'http://localhost:9000') + yield urlparse(addr) diff --git a/tests/cases/test_http/test_middleware.py b/tests/cases/test_http/test_middleware.py new file mode 100644 index 00000000..c2292afa --- /dev/null +++ b/tests/cases/test_http/test_middleware.py @@ -0,0 +1,88 @@ +from qiniu.http.middleware import Middleware, RetryDomainsMiddleware +from qiniu.http import qn_http_client + + +class MiddlewareRecorder(Middleware): + def __init__(self, rec, label): + self.rec = rec + self.label = label + + def __call__(self, request, nxt): + self.rec.append( + 'bef_{0}{1}'.format(self.label, len(self.rec)) + ) + resp = nxt(request) + self.rec.append( + 'aft_{0}{1}'.format(self.label, len(self.rec)) + ) + return resp + + +class TestMiddleware: + def test_middlewares(self, mock_server_addr): + rec_ls = [] + mw_a = MiddlewareRecorder(rec_ls, 'A') + mw_b = MiddlewareRecorder(rec_ls, 'B') + qn_http_client.get( + '{scheme}://{host}/echo?status=200'.format( + scheme=mock_server_addr.scheme, + host=mock_server_addr.netloc + ), + middlewares=[ + mw_a, + mw_b + ] + ) + assert rec_ls == ['bef_A0', 'bef_B1', 'aft_B2', 'aft_A3'] + + def test_retry_domains(self, mock_server_addr): + rec_ls = [] + mw_rec = MiddlewareRecorder(rec_ls, 'rec') + ret, resp = qn_http_client.get( + '{scheme}://fake.pysdk.qiniu.com/echo?status=200'.format( + scheme=mock_server_addr.scheme + ), + middlewares=[ + RetryDomainsMiddleware( + backup_domains=[ + 'unavailable.pysdk.qiniu.com', + mock_server_addr.netloc + ], + max_retry_times=3 + ), + mw_rec + ] + ) + # ['bef_rec0', 'bef_rec1', 'bef_rec2'] are 'fake.pysdk.qiniu.com' with retried 3 times + # ['bef_rec3', 'bef_rec4', 'bef_rec5'] are 'unavailable.pysdk.qiniu.com' with retried 3 times + # ['bef_rec6', 'aft_rec7'] are mock_server and it's success + assert rec_ls == [ + 'bef_rec0', 'bef_rec1', 'bef_rec2', + 'bef_rec3', 'bef_rec4', 'bef_rec5', + 'bef_rec6', 'aft_rec7' + ] + assert ret == {} + assert resp.status_code == 200 + + def test_retry_domains_fail_fast(self, mock_server_addr): + rec_ls = [] + mw_rec = MiddlewareRecorder(rec_ls, 'rec') + ret, resp = qn_http_client.get( + '{scheme}://fake.pysdk.qiniu.com/echo?status=200'.format( + scheme=mock_server_addr.scheme + ), + middlewares=[ + RetryDomainsMiddleware( + backup_domains=[ + 'unavailable.pysdk.qiniu.com', + mock_server_addr.netloc + ], + retry_condition=lambda _resp, _req: False + ), + mw_rec + ] + ) + # ['bef_rec0'] are 'fake.pysdk.qiniu.com' with fail fast + assert rec_ls == ['bef_rec0'] + assert ret is None + assert resp.status_code == -1 diff --git a/tests/cases/test_http/test_qiniu_conf.py b/tests/cases/test_http/test_qiniu_conf.py new file mode 100644 index 00000000..3ce4c5a0 --- /dev/null +++ b/tests/cases/test_http/test_qiniu_conf.py @@ -0,0 +1,117 @@ +import pytest +import requests + +from qiniu.compat import urlencode +import qiniu.http as qiniu_http + + +@pytest.fixture(scope='function') +def retry_id(request, mock_server_addr): + success_times = [] + failure_times = [] + if hasattr(request, 'param'): + success_times = request.param.get('success_times', success_times) + failure_times = request.param.get('failure_times', failure_times) + query_dict = { + 's': success_times, + 'f': failure_times, + } + query_params = urlencode( + query_dict, + doseq=True + ) + request_url = '{scheme}://{host}/retry_me/__mgr__?{query_params}'.format( + scheme=mock_server_addr.scheme, + host=mock_server_addr.netloc, + query_params=query_params + ) + resp = requests.put(request_url) + resp.raise_for_status() + record_id = resp.text + yield record_id + request_url = '{scheme}://{host}/retry_me/__mgr__?id={id}'.format( + scheme=mock_server_addr.scheme, + host=mock_server_addr.netloc, + id=record_id + ) + resp = requests.delete(request_url) + resp.raise_for_status() + + +@pytest.fixture(scope='function') +def reset_session(): + qiniu_http._session = None + yield + + +class TestQiniuConf: + @pytest.mark.usefixtures('reset_session') + @pytest.mark.parametrize( + 'set_conf_default', + [ + { + 'connection_timeout': 0.3, + 'connection_retries': 0 + } + ], + indirect=True + ) + @pytest.mark.parametrize( + 'method,opts', + [ + ('get', {}), + ('put', {'data': None, 'files': None}), + ('post', {'data': None, 'files': None}), + ('delete', {'params': None}) + ], + ids=lambda v: v if type(v) is str else 'opts' + ) + def test_timeout_conf(self, mock_server_addr, method, opts, set_conf_default): + request_url = '{scheme}://{host}/timeout?delay=0.5'.format( + scheme=mock_server_addr.scheme, + host=mock_server_addr.netloc + ) + send = getattr(qiniu_http.qn_http_client, method) + _ret, resp = send(request_url, **opts) + assert 'Read timed out' in str(resp.exception) + + @pytest.mark.usefixtures('reset_session') + @pytest.mark.parametrize( + 'retry_id', + [ + { + 'success_times': [0, 1], + 'failure_times': [5, 0], + }, + ], + indirect=True + ) + @pytest.mark.parametrize( + 'set_conf_default', + [ + { + 'connection_retries': 5 + } + ], + indirect=True + ) + @pytest.mark.parametrize( + 'method,opts', + [ + # post not retry default, see + # https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry.DEFAULT_ALLOWED_METHODS + ('get', {}), + ('put', {'data': None, 'files': None}), + ('delete', {'params': None}) + ], + ids=lambda v: v if type(v) is str else 'opts' + ) + def test_retry_times(self, retry_id, mock_server_addr, method, opts, set_conf_default): + request_url = '{scheme}://{host}/retry_me?id={id}'.format( + scheme=mock_server_addr.scheme, + host=mock_server_addr.netloc, + id=retry_id + ) + send = getattr(qiniu_http.qn_http_client, method) + _ret, resp = send(request_url, **opts) + assert resp.status_code == 200 diff --git a/tests/cases/test_http/test_resp.py b/tests/cases/test_http/test_resp.py new file mode 100644 index 00000000..ddfeadcf --- /dev/null +++ b/tests/cases/test_http/test_resp.py @@ -0,0 +1,64 @@ +import requests + +from qiniu.http import qn_http_client, __return_wrapper as return_wrapper + + +class TestResponse: + def test_response_need_retry(self, mock_server_addr): + def gen_case(code): + if 0 <= code < 500: + return code, False + if code in [ + 501, 509, 573, 579, 608, 612, 614, 616, 618, 630, 631, 632, 640, 701 + ]: + return code, False + return code, True + + cases = [ + gen_case(i) for i in range(-1, 800) + ] + + for test_code, should_retry in cases: + req_url = '{scheme}://{host}/echo?status={status}'.format( + scheme=mock_server_addr.scheme, + host=mock_server_addr.netloc, + status=test_code + ) + if test_code < 0: + req_url = 'http://fake.python-sdk.qiniu.com/' + _ret, resp_info = qn_http_client.get(req_url) + assert_msg = '{code} should{adv} retry'.format( + code=test_code, + adv='' if should_retry else ' NOT' + ) + assert resp_info.need_retry() == should_retry, assert_msg + + def test_json_decode_error(self, mock_server_addr): + req_url = '{scheme}://{host}/echo?status=200'.format( + scheme=mock_server_addr.scheme, + host=mock_server_addr.netloc + ) + ret, resp = qn_http_client.get(req_url) + assert resp.text_body is not None + assert ret == {} + + def test_old_json_decode_error(self): + """ + test old return_wrapper + """ + + def mock_res(): + r = requests.Response() + r.status_code = 200 + r.headers.__setitem__('X-Reqid', 'mockedReqid') + + def json_func(): + raise ValueError('%s: line %d column %d (char %d)' % ('Expecting value', 0, 0, 0)) + + r.json = json_func + + return r + + mocked_res = mock_res() + ret, _ = return_wrapper(mocked_res) + assert ret == {} diff --git a/tests/cases/test_services/__init__.py b/tests/cases/test_services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/test_services/test_storage/__init__.py b/tests/cases/test_services/test_storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/test_services/test_storage/conftest.py b/tests/cases/test_services/test_storage/conftest.py new file mode 100644 index 00000000..a791092c --- /dev/null +++ b/tests/cases/test_services/test_storage/conftest.py @@ -0,0 +1,8 @@ +import pytest + +from qiniu import BucketManager + + +@pytest.fixture() +def bucket_manager(qn_auth): + yield BucketManager(qn_auth) diff --git a/tests/cases/test_services/test_storage/test_uploader.py b/tests/cases/test_services/test_storage/test_uploader.py new file mode 100644 index 00000000..8cd397a5 --- /dev/null +++ b/tests/cases/test_services/test_storage/test_uploader.py @@ -0,0 +1,594 @@ +import os +from collections import namedtuple + +import tempfile +import pytest + +from qiniu.compat import json, is_py2 +from qiniu import ( + Zone, + etag, + set_default, + put_file, + put_data, + put_stream +) +from qiniu import config as qn_config +from qiniu.services.storage.uploader import _form_put + +KB = 1024 +MB = 1024 * KB +GB = 1024 * MB + + +@pytest.fixture(scope='session') +def valid_up_host(access_key, bucket_name): + zone = Zone() + try: + hosts = json.loads( + zone.bucket_hosts(access_key, bucket_name) + ).get('hosts') + up_host = 'https://' + hosts[0].get('up', {}).get('domains')[0] + except IndexError: + up_host = 'https://upload.qiniup.com' + return up_host + + +CommonlyOptions = namedtuple( + 'CommonlyOptions', + [ + 'mime_type', + 'params', + 'metadata' + ] +) + + +@pytest.fixture() +def commonly_options(request): + res = CommonlyOptions( + mime_type='text/plain', + params={'x:a': 'a'}, + metadata={ + 'x-qn-meta-name': 'qiniu', + 'x-qn-meta-age': '18' + } + ) + if hasattr(request, 'params'): + res = res._replace(**request.params) + yield res + + +@pytest.fixture(scope='function') +def set_default_up_host_zone(request, valid_up_host): + zone_args = { + 'up_host': valid_up_host, + } + if hasattr(request, 'param') and request.param is not None: + zone_args = { + 'up_host': request.param, + 'up_host_backup': valid_up_host + } + set_default( + default_zone=Zone(**zone_args) + ) + yield + set_default(default_zone=Zone()) + qn_config._is_customized_default['default_zone'] = False + + +@pytest.fixture(scope='function') +def temp_file(request): + size = 4 * KB + if hasattr(request, 'param'): + size = request.param + + tmp_file_path = tempfile.mktemp() + chunk_size = 4 * KB + + with open(tmp_file_path, 'wb') as f: + remaining_bytes = size + while remaining_bytes > 0: + chunk = os.urandom(min(chunk_size, remaining_bytes)) + f.write(chunk) + remaining_bytes -= len(chunk) + + yield tmp_file_path + + try: + os.remove(tmp_file_path) + except Exception: + pass + + +class TestUploadFuncs: + def test_put(self, qn_auth, bucket_name): + key = 'a\\b\\c"hello' + data = 'hello bubby!' + token = qn_auth.upload_token(bucket_name) + ret, info = put_data(token, key, data) + print(info) + assert ret['key'] == key + + def test_put_crc(self, qn_auth, bucket_name): + key = '' + data = 'hello bubby!' + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_data(token, key, data, check_crc=True) + print(info) + assert ret['key'] == key + + @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) + def test_put_file(self, qn_auth, bucket_name, temp_file, commonly_options): + key = 'test_file' + + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_file( + token, + key, + temp_file, + mime_type=commonly_options.mime_type, + check_crc=True + ) + print(info) + assert ret['key'] == key + assert ret['hash'] == etag(temp_file) + + def test_put_with_invalid_crc(self, qn_auth, bucket_name): + key = 'test_invalid' + data = 'hello bubby!' + crc32 = 'wrong crc32' + token = qn_auth.upload_token(bucket_name) + ret, info = _form_put(token, key, data, None, None, crc=crc32) + assert ret is None, info + assert info.status_code == 400, info + + def test_put_without_key(self, qn_auth, bucket_name): + key = None + data = 'hello bubby!' + token = qn_auth.upload_token(bucket_name) + ret, info = put_data(token, key, data) + print(info) + assert ret['hash'] == ret['key'] + + data = 'hello bubby!' + token = qn_auth.upload_token(bucket_name, 'nokey2') + ret, info = put_data(token, None, data) + print(info) + assert ret is None + assert info.status_code == 403 # key not match + + @pytest.mark.parametrize( + 'set_default_up_host_zone', + [ + 'http://fake.qiniu.com', + None + ], + indirect=True + ) + def test_without_read_without_seek_retry(self, set_default_up_host_zone, qn_auth, bucket_name): + key = 'retry' + data = 'hello retry!' + token = qn_auth.upload_token(bucket_name) + ret, info = put_data(token, key, data) + print(info) + assert ret['key'] == key + assert ret['hash'] == 'FlYu0iBR1WpvYi4whKXiBuQpyLLk' + + @pytest.mark.parametrize('temp_file', [30 * MB], indirect=True) + def test_put_data_without_fname( + self, + qn_auth, + bucket_name, + is_travis, + temp_file + ): + if is_travis: + return + key = 'test_putData_without_fname' + with open(temp_file, 'rb') as input_stream: + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_data(token, key, input_stream) + print(info) + assert ret is not None + + @pytest.mark.parametrize('temp_file', [30 * MB], indirect=True) + def test_put_data_with_empty_fname( + self, + qn_auth, + bucket_name, + is_travis, + temp_file, + commonly_options + ): + if is_travis: + return + key = 'test_putData_without_fname1' + with open(temp_file, 'rb') as input_stream: + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_data( + token, + key, + input_stream, + commonly_options.params, + commonly_options.mime_type, + False, + None, + '' + ) + print(info) + assert ret is not None + + @pytest.mark.parametrize('temp_file', [30 * MB], indirect=True) + def test_put_data_with_space_only_fname( + self, + qn_auth, + bucket_name, + is_travis, + temp_file, + commonly_options + ): + if is_travis: + return + key = 'test_putData_without_fname2' + with open(temp_file, 'rb') as input_stream: + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_data( + token, + key, + input_stream, + commonly_options.params, + commonly_options.mime_type, + False, + None, + ' ' + ) + print(info) + assert ret is not None + + @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) + def test_put_file_with_metadata( + self, + qn_auth, + bucket_name, + temp_file, + commonly_options, + bucket_manager + ): + key = 'test_file_with_metadata' + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_file(token, key, temp_file, metadata=commonly_options.metadata) + assert ret['key'] == key + assert ret['hash'] == etag(temp_file) + ret, info = bucket_manager.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + + def test_put_data_with_metadata( + self, + qn_auth, + bucket_name, + commonly_options, + bucket_manager + ): + key = 'put_data_with_metadata' + data = 'hello metadata!' + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_data(token, key, data, metadata=commonly_options.metadata) + assert ret['key'] == key + ret, info = bucket_manager.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + + +class TestResumableUploader: + @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) + def test_put_stream(self, qn_auth, bucket_name, temp_file, commonly_options): + key = 'test_file_r' + size = os.stat(temp_file).st_size + with open(temp_file, 'rb') as input_stream: + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_stream( + token, + key, + input_stream, + os.path.basename(temp_file), + size, + None, + commonly_options.params, + commonly_options.mime_type, + part_size=None, + version=None, + bucket_name=None + ) + assert ret['key'] == key + + @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) + def test_put_stream_v2_without_bucket_name(self, qn_auth, bucket_name, temp_file, commonly_options): + key = 'test_file_r' + size = os.stat(temp_file).st_size + with open(temp_file, 'rb') as input_stream: + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_stream( + token, + key, + input_stream, + os.path.basename(temp_file), + size, + None, + commonly_options.params, + commonly_options.mime_type, + part_size=1024 * 1024 * 10, + version='v2' + ) + assert ret['key'] == key + + @pytest.mark.parametrize( + 'temp_file', + [ + 2 * MB + 1, + 4 * MB, + 10 * MB + 1 + ], + ids=[ + '2MB+', + '4MB', + '10MB+' + ], + indirect=True + ) + def test_put_stream_v2(self, qn_auth, bucket_name, temp_file, commonly_options): + key = 'test_file_r' + size = os.stat(temp_file).st_size + with open(temp_file, 'rb') as input_stream: + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_stream( + token, + key, + input_stream, + os.path.basename(temp_file), + size, + None, + commonly_options.params, + commonly_options.mime_type, + part_size=1024 * 1024 * 4, + version='v2', + bucket_name=bucket_name + ) + assert ret['key'] == key + + @pytest.mark.parametrize('temp_file', [4 * MB + 1], indirect=True) + def test_put_stream_v2_without_key(self, qn_auth, bucket_name, temp_file, commonly_options): + part_size = 4 * MB + key = None + size = os.stat(temp_file).st_size + with open(temp_file, 'rb') as input_stream: + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_stream( + token, + key, + input_stream, + os.path.basename(temp_file), + size, + None, + commonly_options.params, + commonly_options.mime_type, + part_size=part_size, + version='v2', + bucket_name=bucket_name + ) + assert ret['key'] == ret['hash'] + + @pytest.mark.parametrize('temp_file', [4 * MB + 1], indirect=True) + def test_put_stream_v2_with_empty_return_body(self, qn_auth, bucket_name, temp_file, commonly_options): + part_size = 4 * MB + key = 'test_file_empty_return_body' + size = os.stat(temp_file).st_size + with open(temp_file, 'rb') as input_stream: + token = qn_auth.upload_token(bucket_name, key, policy={'returnBody': ' '}) + ret, info = put_stream( + token, + key, + input_stream, + os.path.basename(temp_file), + size, + None, + commonly_options.params, + commonly_options.mime_type, + part_size=part_size, + version='v2', + bucket_name=bucket_name + ) + assert info.status_code == 200 + assert ret == {} + + @pytest.mark.parametrize('temp_file', [4 * MB + 1], indirect=True) + def test_big_file(self, qn_auth, bucket_name, temp_file, commonly_options): + key = 'big' + token = qn_auth.upload_token(bucket_name, key) + + ret, info = put_file( + token, + key, + temp_file, + commonly_options.params, + commonly_options.mime_type, + progress_handler=lambda progress, total: progress + ) + print(info) + assert ret['key'] == key + + @pytest.mark.parametrize( + 'set_default_up_host_zone', + [ + 'http://fake.qiniu.com', + None + ], + indirect=True + ) + @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) + def test_retry(self, set_default_up_host_zone, qn_auth, bucket_name, temp_file, commonly_options): + key = 'test_file_r_retry' + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_file( + token, + key, + temp_file, + commonly_options.params, + commonly_options.mime_type + ) + print(info) + assert ret['key'] == key + assert ret['hash'] == etag(temp_file) + + @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) + def test_put_stream_with_key_limits(self, qn_auth, bucket_name, temp_file, commonly_options): + key = 'test_file_r' + size = os.stat(temp_file).st_size + with open(temp_file, 'rb') as input_stream: + token = qn_auth.upload_token(bucket_name, key, policy={'keylimit': ['test_file_d']}) + ret, info = put_stream( + token, + key, + input_stream, + os.path.basename(temp_file), + size, + None, + commonly_options.params, + commonly_options.mime_type + ) + assert info.status_code == 403 + token = qn_auth.upload_token( + bucket_name, + key, + policy={'keylimit': ['test_file_d', 'test_file_r']} + ) + ret, info = put_stream( + token, + key, + input_stream, + os.path.basename(temp_file), + size, + None, + commonly_options.params, + commonly_options.mime_type + ) + assert info.status_code == 200 + + @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) + def test_put_stream_with_metadata( + self, + qn_auth, + bucket_name, + temp_file, + commonly_options, + bucket_manager + ): + key = 'test_put_stream_with_metadata' + size = os.stat(temp_file).st_size + with open(temp_file, 'rb') as input_stream: + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_stream( + token, + key, + input_stream, + os.path.basename(temp_file), + size, + None, + commonly_options.params, + commonly_options.mime_type, + part_size=None, + version=None, + bucket_name=None, + metadata=commonly_options.metadata + ) + assert ret['key'] == key + ret, info = bucket_manager.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + + @pytest.mark.parametrize('temp_file', [4 * MB + 1], indirect=True) + def test_put_stream_v2_with_metadata( + self, + qn_auth, + bucket_name, + temp_file, + commonly_options, + bucket_manager + ): + part_size = 4 * MB + key = 'test_put_stream_v2_with_metadata' + size = os.stat(temp_file).st_size + with open(temp_file, 'rb') as input_stream: + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_stream( + token, + key, + input_stream, + os.path.basename(temp_file), + size, + None, + commonly_options.params, + commonly_options.mime_type, + part_size=part_size, + version='v2', + bucket_name=bucket_name, + metadata=commonly_options.metadata + ) + assert ret['key'] == key + ret, info = bucket_manager.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + + @pytest.mark.parametrize('temp_file', [30 * MB], indirect=True) + @pytest.mark.parametrize('version', ['v1', 'v2']) + def test_resume_upload(self, bucket_name, qn_auth, temp_file, version): + key = 'test_resume_upload_{}'.format(version) + size = os.stat(temp_file).st_size + part_size = 4 * MB + + def mock_fail(uploaded_size, _total_size): + if uploaded_size > 10 * MB: + raise Exception('Mock Fail') + + try: + token = qn_auth.upload_token(bucket_name, key) + try: + _ret, _into = put_file( + up_token=token, + key=key, + file_path=temp_file, + hostscache_dir=None, + part_size=part_size, + version=version, + bucket_name=bucket_name, + progress_handler=mock_fail + ) + except Exception as e: + if 'Mock Fail' not in str(e): + raise e + except IOError: + if is_py2: + # https://github.com/pytest-dev/pytest/issues/2370 + # https://github.com/pytest-dev/pytest/pull/3305 + pass + + def should_start_from_resume(uploaded_size, _total_size): + assert uploaded_size // part_size >= 3 + + token = qn_auth.upload_token(bucket_name, key) + ret, into = put_file( + up_token=token, + key=key, + file_path=temp_file, + hostscache_dir=None, + part_size=part_size, + version=version, + bucket_name=bucket_name, + progress_handler=should_start_from_resume + ) + assert ret['key'] == key diff --git a/tests/mock_server/main.py b/tests/mock_server/main.py new file mode 100644 index 00000000..d85129ba --- /dev/null +++ b/tests/mock_server/main.py @@ -0,0 +1,65 @@ +import argparse +import http.server +import http.client +import logging +import sys +from urllib.parse import urlparse + +from routes import routes + + +class MockHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.handle_request('GET') + + def do_POST(self): + self.handle_request('POST') + + def do_PUT(self): + self.handle_request('PUT') + + def do_DELETE(self): + self.handle_request('DELETE') + + def do_OPTIONS(self): + self.handle_request('OPTIONS') + + def do_HEAD(self): + self.handle_request('HEAD') + + def handle_request(self, method): + parsed_uri = urlparse(self.path) + handle = routes.get(parsed_uri.path) + if callable(handle): + try: + handle(method=method, parsed_uri=parsed_uri, request_handler=self) + except Exception: + logging.exception('Exception while handling.') + else: + self.send_response(404) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(b'404 Not Found') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '--port', + type=int, + default=8000, + ) + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + datefmt='%Y-%m-%d %H:%M:%S', + format='[%(asctime)s %(levelname)s] %(message)s', + handlers=[logging.StreamHandler(sys.stdout)], + ) + + server_address = ('', args.port) + httpd = http.server.HTTPServer(server_address, MockHandler) + logging.info('Mock Server running on port {}...'.format(args.port)) + + httpd.serve_forever() diff --git a/tests/mock_server/routes/__init__.py b/tests/mock_server/routes/__init__.py new file mode 100644 index 00000000..7eba32f1 --- /dev/null +++ b/tests/mock_server/routes/__init__.py @@ -0,0 +1,10 @@ +from .timeout import * +from .echo import * +from .retry_me import * + +routes = { + '/timeout': handle_timeout, + '/echo': handle_echo, + '/retry_me': handle_retry_me, + '/retry_me/__mgr__': handle_mgr_retry_me, +} diff --git a/tests/mock_server/routes/echo.py b/tests/mock_server/routes/echo.py new file mode 100644 index 00000000..30174c77 --- /dev/null +++ b/tests/mock_server/routes/echo.py @@ -0,0 +1,34 @@ +import http +import logging +from urllib.parse import parse_qs + + +def handle_echo(method, parsed_uri, request_handler): + """ + Parameters + ---------- + method: str + HTTP method + parsed_uri: urllib.parse.ParseResult + parsed URI + request_handler: http.server.BaseHTTPRequestHandler + request handler + """ + if method not in []: + # all method allowed + pass + echo_status = parse_qs(parsed_uri.query).get('status') + if not echo_status: + echo_status = http.HTTPStatus.BAD_REQUEST + logging.error('No echo status specified') + echo_body = f'param status is required' + else: + echo_status = int(echo_status[0]) + echo_body = f'Response echo status is {echo_status}' + + request_handler.send_response(echo_status) + request_handler.send_header('Content-Type', 'text/plain') + request_handler.send_header('X-Reqid', 'mocked-req-id') + request_handler.end_headers() + + request_handler.wfile.write(echo_body.encode('utf-8')) diff --git a/tests/mock_server/routes/retry_me.py b/tests/mock_server/routes/retry_me.py new file mode 100644 index 00000000..af4458f7 --- /dev/null +++ b/tests/mock_server/routes/retry_me.py @@ -0,0 +1,145 @@ +import http +import random +import string + +from urllib.parse import parse_qs + +__failure_record = {} + + +def should_fail_by_times(success_times=None, failure_times=None): + """ + Parameters + ---------- + success_times: list[int], default=[1] + failure_times: list[int], default=[0] + + Returns + ------- + Generator[bool, None, None] + + Examples + -------- + + should_fail_by_times([2], [3]) + will succeed 2 times and failed 3 times, and loop + + should_fail_by_times([2, 4], [3]) + will succeed 2 times and failed 3 times, + then succeeded 4 times and failed 3 time, and loop + """ + if not success_times: + success_times = [1] + if not failure_times: + failure_times = [0] + + def success_times_gen(): + while True: + for i in success_times: + yield i + + def failure_times_gen(): + while True: + for i in failure_times: + yield i + + success_times_iter = success_times_gen() + fail_times_iter = failure_times_gen() + + while True: + success = next(success_times_iter) + fail = next(fail_times_iter) + for _ in range(success): + yield False + for _ in range(fail): + yield True + + +def handle_mgr_retry_me(method, parsed_uri, request_handler): + """ + Parameters + ---------- + method: str + HTTP method + parsed_uri: urllib.parse.ParseResult + parsed URI + request_handler: http.server.BaseHTTPRequestHandler + request handler + """ + if method not in ['PUT', 'DELETE']: + request_handler.send_response(http.HTTPStatus.METHOD_NOT_ALLOWED) + return + match method: + case 'PUT': + # s for success + success_times = parse_qs(parsed_uri.query).get('s', []) + # f for failure + failure_times = parse_qs(parsed_uri.query).get('f', []) + + record_id = ''.join(random.choices(string.ascii_letters, k=16)) + + __failure_record[record_id] = should_fail_by_times( + success_times=[int(n) for n in success_times], + failure_times=[int(n) for n in failure_times] + ) + + request_handler.send_response(http.HTTPStatus.OK) + request_handler.send_header('Content-Type', 'text/plain') + request_handler.send_header('X-Reqid', record_id) + request_handler.end_headers() + + request_handler.wfile.write(record_id.encode('utf-8')) + case 'DELETE': + record_id = parse_qs(parsed_uri.query).get('id') + if not record_id or not record_id[0]: + request_handler.send_response(http.HTTPStatus.BAD_REQUEST) + return + record_id = record_id[0] + + if record_id in __failure_record: + del __failure_record[record_id] + + request_handler.send_response(http.HTTPStatus.NO_CONTENT) + request_handler.send_header('X-Reqid', record_id) + request_handler.end_headers() + + +def handle_retry_me(method, parsed_uri, request_handler): + """ + Parameters + ---------- + method: str + HTTP method + parsed_uri: urllib.parse.ParseResult + parsed URI + request_handler: http.server.BaseHTTPRequestHandler + request handler + """ + if method not in []: + # all method allowed + pass + record_id = parse_qs(parsed_uri.query).get('id') + if not record_id or not record_id[0]: + request_handler.send_response(http.HTTPStatus.BAD_REQUEST) + return + record_id = record_id[0] + + should_fail = next(__failure_record[record_id]) + + if should_fail: + request_handler.send_response(-1) + request_handler.send_header('Content-Type', 'text/plain') + request_handler.send_header('X-Reqid', record_id) + request_handler.end_headers() + + resp_body = 'service unavailable' + request_handler.wfile.write(resp_body.encode('utf-8')) + return + + request_handler.send_response(http.HTTPStatus.OK) + request_handler.send_header('Content-Type', 'text/plain') + request_handler.send_header('X-Reqid', record_id) + request_handler.end_headers() + + resp_body = 'ok' + request_handler.wfile.write(resp_body.encode('utf-8')) diff --git a/tests/mock_server/routes/timeout.py b/tests/mock_server/routes/timeout.py new file mode 100644 index 00000000..1cdaf70a --- /dev/null +++ b/tests/mock_server/routes/timeout.py @@ -0,0 +1,36 @@ +import http +import logging +import time + +from urllib.parse import parse_qs + + +def handle_timeout(method, parsed_uri, request_handler): + """ + Parameters + ---------- + method: str + HTTP method + parsed_uri: urllib.parse.ParseResult + parsed URI + request_handler: http.server.BaseHTTPRequestHandler + request handler + """ + if method not in []: + # all method allowed + pass + delay = parse_qs(parsed_uri.query).get('delay') + if not delay: + delay = 3 + logging.info('No delay specified. Fallback to %s seconds.', delay) + else: + delay = float(delay[0]) + + time.sleep(delay) + request_handler.send_response(http.HTTPStatus.OK) + request_handler.send_header('Content-Type', 'text/plain') + request_handler.send_header('X-Reqid', 'mocked-req-id') + request_handler.end_headers() + + resp_body = f'Response after {delay} seconds' + request_handler.wfile.write(resp_body.encode('utf-8')) From 7db3a45d7c820df8a3709157d7ebd22a588fe2c3 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Tue, 12 Mar 2024 15:00:41 +0800 Subject: [PATCH 462/478] feat adding put policy fields (#448) --- CHANGELOG.md | 3 ++- qiniu/auth.py | 62 ++++++++++++++++++++++++++++----------------------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5192eb8..4734b0e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog -## 7.13.1（2024-02-04） +## 7.13.1（2024-02-21） * 对象存储，修复上传部分配置项的兼容 +* 对象存储，添加上传策略部分字段 ## 7.13.0（2023-12-11） * 对象存储，新增支持归档直读存储 diff --git a/qiniu/auth.py b/qiniu/auth.py index 5e49693d..b86d8bf5 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -11,34 +11,40 @@ # 上传策略，参数规格详见 # https://developer.qiniu.com/kodo/manual/1206/put-policy -_policy_fields = set([ - 'callbackUrl', # 回调URL - 'callbackBody', # 回调Body - 'callbackHost', # 回调URL指定的Host - 'callbackBodyType', # 回调Body的Content-Type - 'callbackFetchKey', # 回调FetchKey模式开关 - - 'returnUrl', # 上传端的303跳转URL - 'returnBody', # 上传端简单反馈获取的Body - - 'endUser', # 回调时上传端标识 - 'saveKey', # 自定义资源名 - 'forceSaveKey', # saveKey的优先级设置。为 true 时，saveKey不能为空，会忽略客户端指定的key，强制使用saveKey进行文件命名。参数不设置时，默认值为false - 'insertOnly', # 插入模式开关 - - 'detectMime', # MimeType侦测开关 - 'mimeLimit', # MimeType限制 - 'fsizeLimit', # 上传文件大小限制 - 'fsizeMin', # 上传文件最少字节数 - 'keylimit', # 设置允许上传的key列表，字符串数组类型，数组长度不可超过20个，如果设置了这个字段，上传时必须提供key - - 'persistentOps', # 持久化处理操作 - 'persistentNotifyUrl', # 持久化处理结果通知URL - 'persistentPipeline', # 持久化处理独享队列 - 'deleteAfterDays', # 文件多少天后自动删除 - 'fileType', # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储，4为归档直读存储 - 'isPrefixalScope' # 指定上传文件必须使用的前缀 -]) +# the `str()` prevent implicit concatenation of string. DON'T remove it. +# for example, avoid you lost comma at the end of line in middle. +_policy_fields = { + str('callbackUrl'), # 回调URL + str('callbackBody'), # 回调Body + str('callbackHost'), # 回调URL指定的Host + str('callbackBodyType'), # 回调Body的Content-Type + str('callbackFetchKey'), # 回调FetchKey模式开关 + + str('returnUrl'), # 上传端的303跳转URL + str('returnBody'), # 上传端简单反馈获取的Body + + str('endUser'), # 回调时上传端标识 + str('saveKey'), # 自定义资源名 + str('forceSaveKey'), # saveKey的优先级设置。为 true 时，saveKey不能为空，会忽略客户端指定的key，强制使用saveKey进行文件命名。参数不设置时，默认值为false + str('insertOnly'), # 插入模式开关 + + str('detectMime'), # MimeType侦测开关 + str('mimeLimit'), # MimeType限制 + str('fsizeLimit'), # 上传文件大小限制 + str('fsizeMin'), # 上传文件最少字节数 + str('keylimit'), # 设置允许上传的key列表，字符串数组类型，数组长度不可超过20个，如果设置了这个字段，上传时必须提供key + + str('persistentOps'), # 持久化处理操作 + str('persistentNotifyUrl'), # 持久化处理结果通知URL + str('persistentPipeline'), # 持久化处理独享队列 + str('deleteAfterDays'), # 文件多少天后自动删除 + str('fileType'), # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储，4为归档直读存储 + str('isPrefixalScope'), # 指定上传文件必须使用的前缀 + + str('transform'), # deprecated + str('transformFallbackKey'), # deprecated + str('transformFallbackMode'), # deprecated +} class Auth(object): From 26ec27880fecf9d951a0ce369cd9b3e4e3fd0291 Mon Sep 17 00:00:00 2001 From: Rong Zhou <zhourong@qiniu.com> Date: Mon, 13 May 2024 12:21:37 +0800 Subject: [PATCH 463/478] add version check --- .github/workflows/version-check.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/version-check.yml diff --git a/.github/workflows/version-check.yml b/.github/workflows/version-check.yml new file mode 100644 index 00000000..a951ed06 --- /dev/null +++ b/.github/workflows/version-check.yml @@ -0,0 +1,19 @@ +name: Python SDK Version Check +on: + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+" +jobs: + linux: + name: Version Check + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set env + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/v}" >> $GITHUB_ENV + - name: Check + run: | + set -e + grep -qF "## ${RELEASE_VERSION}" CHANGELOG.md + grep -qF "__version__ = '${RELEASE_VERSION}'" qiniu/__init__.py From 3ac1cf8645d3548b6c3d5f40d06fdfc0707b8a01 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Fri, 31 May 2024 11:35:14 +0800 Subject: [PATCH 464/478] fix: upload custom vars not work (#449) --- .github/workflows/ci-test.yml | 1 + CHANGELOG.md | 3 + qiniu/__init__.py | 2 +- qiniu/services/storage/uploader.py | 4 +- tests/cases/conftest.py | 5 + .../test_storage/test_uploader.py | 143 ++++++++++++++++++ 6 files changed, 155 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index af9670fd..cea3133a 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -56,6 +56,7 @@ jobs: QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }} QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }} + QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}} QINIU_TEST_ENV: "travis" MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" PYTHONPATH: "$PYTHONPATH:." diff --git a/CHANGELOG.md b/CHANGELOG.md index 4734b0e9..63f6ff70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 7.13.2（2024-05-28） +* 对象存储，修复上传回调设置自定义变量失效（v7.12.0 引入） + ## 7.13.1（2024-02-21） * 对象存储，修复上传部分配置项的兼容 * 对象存储，添加上传策略部分字段 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 5edeb952..4378ee9d 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.13.1' +__version__ = '7.13.2' from .auth import Auth, QiniuMacAuth diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index a66fae17..049cc876 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -134,7 +134,7 @@ def _form_put( modify_time=modify_time, mime_type=mime_type, metadata=metadata, - params=params, + custom_vars=params, crc32_int=crc, up_token=up_token ) @@ -194,6 +194,6 @@ def put_stream( modify_time=modify_time, mime_type=mime_type, metadata=metadata, - params=params, + custom_vars=params, up_token=up_token ) diff --git a/tests/cases/conftest.py b/tests/cases/conftest.py index 56792f20..dcf802f7 100644 --- a/tests/cases/conftest.py +++ b/tests/cases/conftest.py @@ -23,6 +23,11 @@ def bucket_name(): yield os.getenv('QINIU_TEST_BUCKET') +@pytest.fixture(scope='session') +def upload_callback_url(): + yield os.getenv('QINIU_UPLOAD_CALLBACK_URL') + + @pytest.fixture(scope='session') def qn_auth(access_key, secret_key): yield Auth(access_key, secret_key) diff --git a/tests/cases/test_services/test_storage/test_uploader.py b/tests/cases/test_services/test_storage/test_uploader.py index 8cd397a5..44b153e6 100644 --- a/tests/cases/test_services/test_storage/test_uploader.py +++ b/tests/cases/test_services/test_storage/test_uploader.py @@ -282,6 +282,68 @@ def test_put_data_with_metadata( assert ret['x-qn-meta']['name'] == 'qiniu' assert ret['x-qn-meta']['age'] == '18' + @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) + def test_put_file_with_callback( + self, + qn_auth, + bucket_name, + temp_file, + commonly_options, + bucket_manager, + upload_callback_url + ): + key = 'test_file_with_callback' + policy = { + 'callbackUrl': upload_callback_url, + 'callbackBody': '{"custom_vars":{"a":$(x:a)},"key":$(key),"hash":$(etag)}', + 'callbackBodyType': 'application/json', + } + token = qn_auth.upload_token(bucket_name, key, policy=policy) + ret, info = put_file( + token, + key, + temp_file, + metadata=commonly_options.metadata, + params=commonly_options.params, + ) + assert ret['key'] == key + assert ret['hash'] == etag(temp_file) + assert ret['custom_vars']['a'] == 'a' + ret, info = bucket_manager.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + + def test_put_data_with_callback( + self, + qn_auth, + bucket_name, + commonly_options, + bucket_manager, + upload_callback_url + ): + key = 'put_data_with_metadata' + data = 'hello metadata!' + policy = { + 'callbackUrl': upload_callback_url, + 'callbackBody': '{"custom_vars":{"a":$(x:a)},"key":$(key),"hash":$(etag)}', + 'callbackBodyType': 'application/json', + } + token = qn_auth.upload_token(bucket_name, key, policy=policy) + ret, info = put_data( + token, + key, + data, + metadata=commonly_options.metadata, + params=commonly_options.params + ) + assert ret['key'] == key + assert ret['custom_vars']['a'] == 'a' + ret, info = bucket_manager.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + class TestResumableUploader: @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) @@ -544,6 +606,87 @@ def test_put_stream_v2_with_metadata( assert ret['x-qn-meta']['name'] == 'qiniu' assert ret['x-qn-meta']['age'] == '18' + @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) + def test_put_stream_with_callback( + self, + qn_auth, + bucket_name, + temp_file, + commonly_options, + bucket_manager, + upload_callback_url + ): + key = 'test_put_stream_with_callback' + size = os.stat(temp_file).st_size + with open(temp_file, 'rb') as input_stream: + policy = { + 'callbackUrl': upload_callback_url, + 'callbackBody': '{"custom_vars":{"a":$(x:a)},"key":$(key),"hash":$(etag)}', + 'callbackBodyType': 'application/json', + } + token = qn_auth.upload_token(bucket_name, key, policy=policy) + ret, info = put_stream( + token, + key, + input_stream, + os.path.basename(temp_file), + size, + None, + commonly_options.params, + commonly_options.mime_type, + part_size=None, + version=None, + bucket_name=None, + metadata=commonly_options.metadata + ) + assert ret['key'] == key + assert ret['custom_vars']['a'] == 'a' + ret, info = bucket_manager.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + + @pytest.mark.parametrize('temp_file', [4 * MB + 1], indirect=True) + def test_put_stream_v2_with_callback( + self, + qn_auth, + bucket_name, + temp_file, + commonly_options, + bucket_manager, + upload_callback_url + ): + part_size = 4 * MB + key = 'test_put_stream_v2_with_metadata' + size = os.stat(temp_file).st_size + with open(temp_file, 'rb') as input_stream: + policy = { + 'callbackUrl': upload_callback_url, + 'callbackBody': '{"custom_vars":{"a":$(x:a)},"key":$(key),"hash":$(etag)}', + 'callbackBodyType': 'application/json', + } + token = qn_auth.upload_token(bucket_name, key, policy=policy) + ret, info = put_stream( + token, + key, + input_stream, + os.path.basename(temp_file), + size, + None, + commonly_options.params, + commonly_options.mime_type, + part_size=part_size, + version='v2', + bucket_name=bucket_name, + metadata=commonly_options.metadata + ) + assert ret['key'] == key + assert ret['custom_vars']['a'] == 'a' + ret, info = bucket_manager.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + @pytest.mark.parametrize('temp_file', [30 * MB], indirect=True) @pytest.mark.parametrize('version', ['v1', 'v2']) def test_resume_upload(self, bucket_name, qn_auth, temp_file, version): From d8ed8780d9c494415737609f57a5f4c73f5f2d2e Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Fri, 20 Sep 2024 21:55:08 +0800 Subject: [PATCH 465/478] bump to 7.14.0 (#450) * feat: add retrier * chore(utils): deprecate etag * refactor: update circle import solution between region and conf Make `qiniu.config` more independent and move test cases to `tests/cases/test_zone` from `test_qiniu.py` BREAK CHANGE: default value of protected filed `qiniu.config._config['default_zone']` changes to `None` from `qiniu.region.Region()`, which should be never used directly in user code. * feat: add new region and endpoint for source/accelerate uploading with retrying * fix: default_client will not work with config.set_default by call once * chore: remove iter self for endpoint * chore: improve error text * fix: some features are not compatible with old version and improve error text - change preferred scheme to http from https - change UploaderBase._get_regions return type to list[LegacyRegion] from list[LegacyRegion or Region] - change all inner legacy region type name from Region to LegacyRegion - LegacyRegion.get_bucket_hosts homecache argument not work * fix: TokenExpiredRetryPolicy not work * fix: AccUnavailableRetryPolicy modify origin region service * fix: argument name typo on put_data * feat: bucket support regions and endpoints retry * test: add and improve test cases * feat: add field persistentType to strict policy fields * style: fix flake8 code styles * fix: enum on python2 * test: fix compatibility of test cases on python 2.7 * test: change test region to na0 from z0 * chore: ci add no accelerate bucket * test: fix test error with python2 * fix: LegacyRegion.get_bucket_host not working in python2 * doc: add more type info to functions * chore: change default hosts for querying regions * fix: CachedRegionsProvider shrink not working * feat: add uc backup hosts * feat: update version and changelog * add Qiniu auth verify callback * chore: remove `Region.from_region_id` backup domain qbox and s3 * feat: add idle-time fop support and get fop status * docs: fix authorization token link * fix: form retry not working by no resume recorder * chore: fix flake8 lint on python >= 3.8 * Update CHANGELOG.md * fix: legacy region get_xxx_host and add more cases --- .github/workflows/ci-test.yml | 1 + CHANGELOG.md | 7 + examples/upload.py | 24 +- examples/upload_callback.py | 3 +- examples/upload_pfops.py | 3 +- examples/upload_with_qvmzone.py | 3 +- qiniu/__init__.py | 6 +- qiniu/auth.py | 112 ++- qiniu/config.py | 50 +- qiniu/http/__init__.py | 33 +- qiniu/http/default_client.py | 37 + qiniu/http/endpoint.py | 68 ++ qiniu/http/endpoints_provider.py | 13 + qiniu/http/endpoints_retry_policy.py | 56 ++ qiniu/http/region.py | 184 +++++ qiniu/http/regions_provider.py | 743 ++++++++++++++++++ qiniu/http/regions_retry_policy.py | 162 ++++ qiniu/http/response.py | 2 + qiniu/region.py | 283 ++++--- qiniu/retry/__init__.py | 7 + qiniu/retry/abc/__init__.py | 5 + qiniu/retry/abc/policy.py | 61 ++ qiniu/retry/attempt.py | 18 + qiniu/retry/retrier.py | 183 +++++ qiniu/services/processing/pfop.py | 54 +- .../storage/_bucket_default_retrier.py | 25 + qiniu/services/storage/bucket.py | 543 +++++++++++-- .../storage/upload_progress_recorder.py | 47 +- qiniu/services/storage/uploader.py | 54 +- .../storage/uploaders/_default_retrier.py | 216 +++++ .../uploaders/abc/resume_uploader_base.py | 2 +- .../storage/uploaders/abc/uploader_base.py | 181 ++++- .../storage/uploaders/form_uploader.py | 97 ++- .../storage/uploaders/resume_uploader_v1.py | 346 +++++--- .../storage/uploaders/resume_uploader_v2.py | 308 +++++--- qiniu/utils.py | 35 +- qiniu/zone.py | 4 +- setup.py | 3 +- test_qiniu.py | 260 +----- tests/cases/conftest.py | 43 +- tests/cases/test_auth.py | 212 +++++ tests/cases/test_http/test_endpoint.py | 27 + .../test_http/test_endpoints_retry_policy.py | 75 ++ tests/cases/test_http/test_qiniu_conf.py | 2 +- tests/cases/test_http/test_region.py | 186 +++++ .../cases/test_http/test_regions_provider.py | 267 +++++++ .../test_http/test_regions_retry_policy.py | 263 +++++++ tests/cases/test_retry/__init__.py | 0 tests/cases/test_retry/test_retrier.py | 142 ++++ .../test_services/test_processing/__init__.py | 0 .../test_processing/test_pfop.py | 49 ++ .../test_services/test_storage/conftest.py | 134 +++- .../test_storage/test_bucket_manager.py | 205 +++++ .../test_storage/test_upload_pfop.py | 66 ++ .../test_storage/test_uploader.py | 464 +++++++---- .../test_uploaders_default_retrier.py | 235 ++++++ tests/cases/test_zone/__init__.py | 0 tests/cases/test_zone/test_lagacy_region.py | 111 +++ tests/cases/test_zone/test_qiniu_conf.py | 99 +++ 59 files changed, 5828 insertions(+), 991 deletions(-) create mode 100644 qiniu/http/default_client.py create mode 100644 qiniu/http/endpoint.py create mode 100644 qiniu/http/endpoints_provider.py create mode 100644 qiniu/http/endpoints_retry_policy.py create mode 100644 qiniu/http/region.py create mode 100644 qiniu/http/regions_provider.py create mode 100644 qiniu/http/regions_retry_policy.py create mode 100644 qiniu/retry/__init__.py create mode 100644 qiniu/retry/abc/__init__.py create mode 100644 qiniu/retry/abc/policy.py create mode 100644 qiniu/retry/attempt.py create mode 100644 qiniu/retry/retrier.py create mode 100644 qiniu/services/storage/_bucket_default_retrier.py create mode 100644 qiniu/services/storage/uploaders/_default_retrier.py create mode 100644 tests/cases/test_auth.py create mode 100644 tests/cases/test_http/test_endpoint.py create mode 100644 tests/cases/test_http/test_endpoints_retry_policy.py create mode 100644 tests/cases/test_http/test_region.py create mode 100644 tests/cases/test_http/test_regions_provider.py create mode 100644 tests/cases/test_http/test_regions_retry_policy.py create mode 100644 tests/cases/test_retry/__init__.py create mode 100644 tests/cases/test_retry/test_retrier.py create mode 100644 tests/cases/test_services/test_processing/__init__.py create mode 100644 tests/cases/test_services/test_processing/test_pfop.py create mode 100644 tests/cases/test_services/test_storage/test_bucket_manager.py create mode 100644 tests/cases/test_services/test_storage/test_upload_pfop.py create mode 100644 tests/cases/test_services/test_storage/test_uploaders_default_retrier.py create mode 100644 tests/cases/test_zone/__init__.py create mode 100644 tests/cases/test_zone/test_lagacy_region.py create mode 100644 tests/cases/test_zone/test_qiniu_conf.py diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index cea3133a..c705019b 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -55,6 +55,7 @@ jobs: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }} + QINIU_TEST_NO_ACC_BUCKET: ${{ secrets.QINIU_TEST_NO_ACC_BUCKET }} QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }} QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}} QINIU_TEST_ENV: "travis" diff --git a/CHANGELOG.md b/CHANGELOG.md index 63f6ff70..6bf26f46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ # Changelog +## 7.14.0 +* 对象存储，空间管理、上传文件新增备用域名重试逻辑 +* 对象存储，调整查询区域主备域名 +* 对象存储，支持空间级别加速域名开关 +* 对象存储，回调签名验证函数新增兼容 Qiniu 签名 +* 对象存储，持久化处理支持闲时任务 + ## 7.13.2（2024-05-28） * 对象存储，修复上传回调设置自定义变量失效（v7.12.0 引入） diff --git a/examples/upload.py b/examples/upload.py index 690ea046..25aef7cc 100755 --- a/examples/upload.py +++ b/examples/upload.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # flake8: noqa +# import hashlib -from qiniu import Auth, put_file, etag, urlsafe_base64_encode +from qiniu import Auth, put_file, urlsafe_base64_encode import qiniu.config from qiniu.compat import is_py2, is_py3 @@ -24,7 +25,24 @@ # 要上传文件的本地路径 localfile = '/Users/jemy/Documents/qiniu.png' -ret, info = put_file(token, key, localfile) +# 上传时，sdk 会自动计算文件 hash 作为参数传递给服务端确保上传完整性 +# （若不一致，服务端会拒绝完成上传） +# 但在访问文件时，服务端可能不会提供 MD5 或者编码格式不是期望的 +# 因此若有需有，请通过元数据功能自定义 MD5 或其他 hash 字段 +# hasher = hashlib.md5() +# with open(localfile, 'rb') as f: +# for d in f: +# hasher.update(d) +# object_metadata = { +# 'x-qn-meta-md5': hasher.hexdigest() +# } + +ret, info = put_file( + token, + key, + localfile + # metadata=object_metadata +) print(ret) print(info) @@ -32,5 +50,3 @@ assert ret['key'].encode('utf-8') == key elif is_py3: assert ret['key'] == key - -assert ret['hash'] == etag(localfile) diff --git a/examples/upload_callback.py b/examples/upload_callback.py index d8a0a788..468120a5 100755 --- a/examples/upload_callback.py +++ b/examples/upload_callback.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # flake8: noqa -from qiniu import Auth, put_file, etag +from qiniu import Auth, put_file access_key = '...' secret_key = '...' @@ -25,4 +25,3 @@ ret, info = put_file(token, key, localfile) print(info) assert ret['key'] == key -assert ret['hash'] == etag(localfile) diff --git a/examples/upload_pfops.py b/examples/upload_pfops.py index 7cc2b9e1..d8546c3f 100755 --- a/examples/upload_pfops.py +++ b/examples/upload_pfops.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # flake8: noqa -from qiniu import Auth, put_file, etag, urlsafe_base64_encode +from qiniu import Auth, put_file, urlsafe_base64_encode access_key = '...' secret_key = '...' @@ -36,4 +36,3 @@ ret, info = put_file(token, key, localfile) print(info) assert ret['key'] == key -assert ret['hash'] == etag(localfile) diff --git a/examples/upload_with_qvmzone.py b/examples/upload_with_qvmzone.py index 54f8b603..4d298f59 100644 --- a/examples/upload_with_qvmzone.py +++ b/examples/upload_with_qvmzone.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # flake8: noqa -from qiniu import Auth, put_file, etag, urlsafe_base64_encode +from qiniu import Auth, put_file, urlsafe_base64_encode import qiniu.config from qiniu import Zone, set_default @@ -37,4 +37,3 @@ ret, info = put_file(token, key, localfile) print(info) assert ret['key'] == key -assert ret['hash'] == etag(localfile) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 4378ee9d..55acfb25 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,16 +9,16 @@ # flake8: noqa -__version__ = '7.13.2' +__version__ = '7.14.0' from .auth import Auth, QiniuMacAuth from .config import set_default from .zone import Zone -from .region import Region +from .region import LegacyRegion as Region from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ - build_batch_stat, build_batch_delete, build_batch_restoreAr + build_batch_stat, build_batch_delete, build_batch_restoreAr, build_batch_restore_ar from .services.storage.uploader import put_data, put_file, put_stream from .services.storage.upload_progress_recorder import UploadProgressRecorder from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url, DomainManager diff --git a/qiniu/auth.py b/qiniu/auth.py index b86d8bf5..d3e0a055 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -37,6 +37,8 @@ str('persistentOps'), # 持久化处理操作 str('persistentNotifyUrl'), # 持久化处理结果通知URL str('persistentPipeline'), # 持久化处理独享队列 + str('persistentType'), # 为 `1` 时，开启闲时任务，必须是 int 类型 + str('deleteAfterDays'), # 文件多少天后自动删除 str('fileType'), # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储，4为归档直读存储 str('isPrefixalScope'), # 指定上传文件必须使用的前缀 @@ -194,22 +196,56 @@ def __upload_token(self, policy): return self.token_with_data(data) def verify_callback( - self, - origin_authorization, - url, - body, - content_type='application/x-www-form-urlencoded'): - """回调验证 - - Args: - origin_authorization: 回调时请求Header中的Authorization字段 - url: 回调请求的url - body: 回调请求的body - content_type: 回调请求body的Content-Type - - Returns: - 返回true表示验证成功，返回false表示验证失败 + self, + origin_authorization, + url, + body, + content_type='application/x-www-form-urlencoded', + method='GET', + headers=None + ): + """ + Qbox 回调验证 + + Parameters + ---------- + origin_authorization: str + 回调时请求 Header 中的 Authorization 字段 + url: str + 回调请求的 url + body: str + 回调请求的 body + content_type: str + 回调请求的 Content-Type + method: str + 回调请求的 method，Qiniu 签名必须传入，默认 GET + headers: dict + 回调请求的 headers，Qiniu 签名必须传入，默认为空字典 + + Returns + ------- + bool + 返回 True 表示验证成功，返回 False 表示验证失败 """ + if headers is None: + headers = {} + + # 兼容 Qiniu 签名 + if origin_authorization.startswith("Qiniu"): + qn_auth = QiniuMacAuth( + access_key=self.__access_key, + secret_key=self.__secret_key, + disable_qiniu_timestamp_signature=True + ) + return qn_auth.verify_callback( + origin_authorization, + url=url, + body=body, + content_type=content_type, + method=method, + headers=headers + ) + token = self.token_of_request(url, body, content_type) authorization = 'QBox {0}'.format(token) return origin_authorization == authorization @@ -243,7 +279,7 @@ class QiniuMacAuth(object): __access_key __secret_key - http://kirk-docs.qiniu.com/apidocs/#TOC_325b437b89e8465e62e958cccc25c63f + https://developer.qiniu.com/kodo/1201/access-token """ def __init__(self, access_key, secret_key, disable_qiniu_timestamp_signature=None): @@ -326,6 +362,50 @@ def qiniu_headers(self, headers): '%s: %s' % (canonical_mime_header_key(key), headers.get(key)) for key in sorted(qiniu_fields) ]) + def verify_callback( + self, + origin_authorization, + url, + body, + content_type='application/x-www-form-urlencoded', + method='GET', + headers=None + ): + """ + Qiniu 回调验证 + + Parameters + ---------- + origin_authorization: str + 回调时请求 Header 中的 Authorization 字段 + url: str + 回调请求的 url + body: str + 回调请求的 body + content_type: str + 回调请求的 Content-Type + method: str + 回调请求的 Method + headers: dict + 回调请求的 headers + + Returns + ------- + + """ + if headers is None: + headers = {} + token = self.token_of_request( + method=method, + host=headers.get('Host', None), + url=url, + qheaders=self.qiniu_headers(headers), + content_type=content_type, + body=body + ) + authorization = 'Qiniu {0}'.format(token) + return origin_authorization == authorization + @staticmethod def __checkKey(access_key, secret_key): if not (access_key and secret_key): diff --git a/qiniu/config.py b/qiniu/config.py index cb2ac57c..338c1399 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -1,26 +1,27 @@ # -*- coding: utf-8 -*- -from qiniu import region - RS_HOST = 'http://rs.qiniu.com' # 管理操作Host RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host API_HOST = 'http://api.qiniuapi.com' # 数据处理操作Host -UC_HOST = region.UC_HOST # 获取空间信息Host -QUERY_REGION_HOST = 'https://kodo-config.qiniuapi.com' +QUERY_REGION_HOST = 'https://uc.qiniuapi.com' +QUERY_REGION_BACKUP_HOSTS = [ + 'kodo-config.qiniuapi.com', + 'uc.qbox.me' +] +UC_HOST = QUERY_REGION_HOST # 获取空间信息Host +UC_BACKUP_HOSTS = QUERY_REGION_BACKUP_HOSTS _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续传分块大小，该参数为接口规格，暂不支持修改 _config = { - 'default_zone': region.Region(), + 'default_zone': None, 'default_rs_host': RS_HOST, 'default_rsf_host': RSF_HOST, 'default_api_host': API_HOST, 'default_uc_host': UC_HOST, + 'default_uc_backup_hosts': UC_BACKUP_HOSTS, 'default_query_region_host': QUERY_REGION_HOST, - 'default_query_region_backup_hosts': [ - 'uc.qbox.me', - 'api.qiniu.com' - ], - 'default_backup_hosts_retry_times': 2, + 'default_query_region_backup_hosts': QUERY_REGION_BACKUP_HOSTS, + 'default_backup_hosts_retry_times': 3, # 仅控制旧区域 LegacyRegion 查询 Hosts 的重试次数 'connection_timeout': 30, # 链接超时为时间为30s 'connection_retries': 3, # 链接重试次数为3次 'connection_pool': 10, # 链接池个数为10 @@ -28,18 +29,8 @@ } _is_customized_default = { - 'default_zone': False, - 'default_rs_host': False, - 'default_rsf_host': False, - 'default_api_host': False, - 'default_uc_host': False, - 'default_query_region_host': False, - 'default_query_region_backup_hosts': False, - 'default_backup_hosts_retry_times': False, - 'connection_timeout': False, - 'connection_retries': False, - 'connection_pool': False, - 'default_upload_threshold': False + k: False + for k in _config.keys() } @@ -48,6 +39,10 @@ def is_customized_default(key): def get_default(key): + if key == 'default_zone' and not _is_customized_default[key]: + # prevent circle import + from .region import LegacyRegion + return LegacyRegion() return _config[key] @@ -56,7 +51,7 @@ def set_default( connection_timeout=None, default_rs_host=None, default_uc_host=None, default_rsf_host=None, default_api_host=None, default_upload_threshold=None, default_query_region_host=None, default_query_region_backup_hosts=None, - default_backup_hosts_retry_times=None): + default_backup_hosts_retry_times=None, default_uc_backup_hosts=None): if default_zone: _config['default_zone'] = default_zone _is_customized_default['default_zone'] = True @@ -72,16 +67,23 @@ def set_default( if default_uc_host: _config['default_uc_host'] = default_uc_host _is_customized_default['default_uc_host'] = True + _config['default_uc_backup_hosts'] = [] + _is_customized_default['default_uc_backup_hosts'] = True _config['default_query_region_host'] = default_uc_host _is_customized_default['default_query_region_host'] = True _config['default_query_region_backup_hosts'] = [] _is_customized_default['default_query_region_backup_hosts'] = True + if default_uc_backup_hosts is not None: + _config['default_uc_backup_hosts'] = default_uc_backup_hosts + _is_customized_default['default_uc_backup_hosts'] = True + _config['default_query_region_backup_hosts'] = default_uc_backup_hosts + _is_customized_default['default_query_region_backup_hosts'] = True if default_query_region_host: _config['default_query_region_host'] = default_query_region_host _is_customized_default['default_query_region_host'] = True _config['default_query_region_backup_hosts'] = [] _is_customized_default['default_query_region_backup_hosts'] = True - if default_query_region_backup_hosts: + if default_query_region_backup_hosts is not None: _config['default_query_region_backup_hosts'] = default_query_region_backup_hosts _is_customized_default['default_query_region_backup_hosts'] = True if default_backup_hosts_retry_times: diff --git a/qiniu/http/__init__.py b/qiniu/http/__init__.py index 2b61e0fe..83a837a4 100644 --- a/qiniu/http/__init__.py +++ b/qiniu/http/__init__.py @@ -1,39 +1,15 @@ # -*- coding: utf-8 -*- import logging import platform -import functools import requests -from requests.adapters import HTTPAdapter from requests.auth import AuthBase from qiniu import config, __version__ import qiniu.auth -from .client import HTTPClient from .response import ResponseInfo -from .middleware import UserAgentMiddleware - - -qn_http_client = HTTPClient( - middlewares=[ - UserAgentMiddleware(__version__) - ] -) - - -# compatibility with some config from qiniu.config -def _before_send(func): - @functools.wraps(func) - def wrapper(self, *args, **kwargs): - if _session is None: - _init() - return func(self, *args, **kwargs) - - return wrapper - - -qn_http_client.send_request = _before_send(qn_http_client.send_request) +from .default_client import qn_http_client, _init_http_adapter _sys_info = '{0}; {1}'.format(platform.system(), platform.machine()) _python_ver = platform.python_version() @@ -61,12 +37,7 @@ def _init(): global _session if _session is None: _session = qn_http_client.session - - adapter = HTTPAdapter( - pool_connections=config.get_default('connection_pool'), - pool_maxsize=config.get_default('connection_pool'), - max_retries=config.get_default('connection_retries')) - _session.mount('http://', adapter) + _init_http_adapter() def _post(url, data, files, auth, headers=None): diff --git a/qiniu/http/default_client.py b/qiniu/http/default_client.py new file mode 100644 index 00000000..7d7ccc60 --- /dev/null +++ b/qiniu/http/default_client.py @@ -0,0 +1,37 @@ +import functools + +from requests.adapters import HTTPAdapter + +from qiniu import config, __version__ + +from .client import HTTPClient +from .middleware import UserAgentMiddleware + +qn_http_client = HTTPClient( + middlewares=[ + UserAgentMiddleware(__version__) + ] +) + + +# compatibility with some config from qiniu.config +def _before_send(func): + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + _init_http_adapter() + return func(self, *args, **kwargs) + + return wrapper + + +qn_http_client.send_request = _before_send(qn_http_client.send_request) + + +def _init_http_adapter(): + # may be optimized: + # only called when config changed, not every time before send request + adapter = HTTPAdapter( + pool_connections=config.get_default('connection_pool'), + pool_maxsize=config.get_default('connection_pool'), + max_retries=config.get_default('connection_retries')) + qn_http_client.session.mount('http://', adapter) diff --git a/qiniu/http/endpoint.py b/qiniu/http/endpoint.py new file mode 100644 index 00000000..307542b9 --- /dev/null +++ b/qiniu/http/endpoint.py @@ -0,0 +1,68 @@ +class Endpoint: + @staticmethod + def from_host(host): + """ + Autodetect scheme from host string + + Parameters + ---------- + host: str + + Returns + ------- + Endpoint + """ + if '://' in host: + scheme, host = host.split('://') + return Endpoint(host=host, default_scheme=scheme) + else: + return Endpoint(host=host) + + def __init__(self, host, default_scheme='https'): + """ + Parameters + ---------- + host: str + default_scheme: str + """ + self.host = host + self.default_scheme = default_scheme + + def __str__(self): + return 'Endpoint(host:\'{0}\',default_scheme:\'{1}\')'.format( + self.host, + self.default_scheme + ) + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + if not isinstance(other, Endpoint): + raise TypeError('Cannot compare Endpoint with {0}'.format(type(other))) + + return self.host == other.host and self.default_scheme == other.default_scheme + + def get_value(self, scheme=None): + """ + Parameters + ---------- + scheme: str + + Returns + ------- + str + """ + scheme = scheme if scheme is not None else self.default_scheme + return ''.join([scheme, '://', self.host]) + + def clone(self): + """ + Returns + ------- + Endpoint + """ + return Endpoint( + host=self.host, + default_scheme=self.default_scheme + ) diff --git a/qiniu/http/endpoints_provider.py b/qiniu/http/endpoints_provider.py new file mode 100644 index 00000000..ccfb3b43 --- /dev/null +++ b/qiniu/http/endpoints_provider.py @@ -0,0 +1,13 @@ +import abc + + +class EndpointsProvider: + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def __iter__(self): + """ + Returns + ------- + list[Endpoint] + """ diff --git a/qiniu/http/endpoints_retry_policy.py b/qiniu/http/endpoints_retry_policy.py new file mode 100644 index 00000000..f648a29e --- /dev/null +++ b/qiniu/http/endpoints_retry_policy.py @@ -0,0 +1,56 @@ +from qiniu.retry.abc import RetryPolicy + + +class EndpointsRetryPolicy(RetryPolicy): + def __init__(self, endpoints_provider=None, skip_init_context=False): + """ + Parameters + ---------- + endpoints_provider: Iterable[Endpoint] + skip_init_context: bool + """ + self.endpoints_provider = endpoints_provider if endpoints_provider else [] + self.skip_init_context = skip_init_context + + def init_context(self, context): + """ + Parameters + ---------- + context: dict + + Returns + ------- + None + """ + if self.skip_init_context: + return + context['alternative_endpoints'] = list(self.endpoints_provider) + if not context['alternative_endpoints']: + raise ValueError('There isn\'t available endpoint') + context['endpoint'] = context['alternative_endpoints'].pop(0) + + def should_retry(self, attempt): + """ + Parameters + ---------- + attempt: qiniu.retry.Attempt + + Returns + ------- + bool + """ + return len(attempt.context['alternative_endpoints']) > 0 + + def prepare_retry(self, attempt): + """ + Parameters + ---------- + attempt: qiniu.retry.Attempt + + Returns + ------- + None + """ + if not attempt.context['alternative_endpoints']: + raise Exception('There isn\'t available endpoint for next try') + attempt.context['endpoint'] = attempt.context['alternative_endpoints'].pop(0) diff --git a/qiniu/http/region.py b/qiniu/http/region.py new file mode 100644 index 00000000..09a9917c --- /dev/null +++ b/qiniu/http/region.py @@ -0,0 +1,184 @@ +from datetime import datetime, timedelta + +from enum import Enum + +from .endpoint import Endpoint + + +# Use StrEnum when min version of python update to >= 3.11 +# to make the json stringify more readable, +# or find another way to simple the json stringify +class ServiceName(Enum): + UC = 'uc' + UP = 'up' + UP_ACC = 'up_acc' + IO = 'io' + # IO_SRC = 'io_src' + RS = 'rs' + RSF = 'rsf' + API = 'api' + + +class Region: + @staticmethod + def merge(*args): + """ + Parameters + ---------- + args: list[list[Region]] + + Returns + ------- + + """ + if not args: + raise TypeError('There aren\'ta any regions to merge') + source, rest = args[0], args[1:] + target = source.clone() + for r in rest: + for sn, el in r.services.items(): + if sn not in target.services: + target.services[sn] = [e.clone() for e in el] + else: + target_values = [e.get_value() for e in target.services[sn]] + target.services[sn] += [ + e.clone() + for e in el + if e.get_value() not in target_values + ] + + return target + + @staticmethod + def from_region_id(region_id, **kwargs): + """ + Parameters + ---------- + region_id: str + kwargs: dict + s3_region_id: str + ttl: int + create_time: datetime + extended_services: dict[str, list[Region]] + preferred_scheme: str + + Returns + ------- + Region + """ + # create services endpoints + endpoint_kwargs = { + } + if 'preferred_scheme' in kwargs: + endpoint_kwargs['default_scheme'] = kwargs.get('preferred_scheme') + + is_z0 = region_id == 'z0' + services_hosts = { + ServiceName.UC: ['uc.qiniuapi.com'], + ServiceName.UP: [ + 'upload-{0}.qiniup.com'.format(region_id), + 'up-{0}.qiniup.com'.format(region_id) + ] if not is_z0 else [ + 'upload.qiniup.com', + 'up.qiniup.com' + ], + ServiceName.IO: [ + 'iovip-{0}.qiniuio.com'.format(region_id), + ] if not is_z0 else [ + 'iovip.qiniuio.com', + ], + ServiceName.RS: [ + 'rs-{0}.qiniuapi.com'.format(region_id), + ], + ServiceName.RSF: [ + 'rsf-{0}.qiniuapi.com'.format(region_id), + ], + ServiceName.API: [ + 'api-{0}.qiniuapi.com'.format(region_id), + ] + } + services = { + k: [ + Endpoint(h, **endpoint_kwargs) for h in v + ] + for k, v in services_hosts.items() + } + services.update(kwargs.get('extended_services', {})) + + # create region + region_kwargs = { + k: kwargs.get(k) + for k in [ + 's3_region_id', + 'ttl', + 'create_time' + ] if k in kwargs + } + region_kwargs['region_id'] = region_id + region_kwargs.setdefault('s3_region_id', region_id) + region_kwargs['services'] = services + + return Region(**region_kwargs) + + def __init__( + self, + region_id=None, + s3_region_id=None, + services=None, + ttl=86400, + create_time=None + ): + """ + Parameters + ---------- + region_id: str + s3_region_id: str + services: dict[ServiceName or str, list[Endpoint]] + ttl: int, default 86400 + create_time: datetime, default datetime.now() + """ + self.region_id = region_id + self.s3_region_id = s3_region_id if s3_region_id else region_id + + self.services = services if services else {} + self.services.update( + { + k: [] + for k in ServiceName + if + k not in self.services or + not isinstance(self.services[k], list) + } + ) + + self.ttl = ttl + self.create_time = create_time if create_time else datetime.now() + + @property + def is_live(self): + """ + Returns + ------- + bool + """ + if self.ttl < 0: + return True + live_time = datetime.now() - self.create_time + return live_time < timedelta(seconds=self.ttl) + + def clone(self): + """ + Returns + ------- + Region + """ + return Region( + region_id=self.region_id, + s3_region_id=self.s3_region_id, + services={ + k: [endpoint.clone() for endpoint in self.services[k]] + for k in self.services + }, + ttl=self.ttl, + create_time=self.create_time + ) diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py new file mode 100644 index 00000000..8b52822c --- /dev/null +++ b/qiniu/http/regions_provider.py @@ -0,0 +1,743 @@ +import abc +import datetime +import itertools +from collections import namedtuple +import logging +import tempfile +import os + +from qiniu.compat import json, b as to_bytes +from qiniu.utils import io_md5 + +from .endpoint import Endpoint +from .region import Region, ServiceName +from .default_client import qn_http_client +from .middleware import RetryDomainsMiddleware + + +class RegionsProvider: + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def __iter__(self): + """ + Returns + ------- + list[Region] + """ + + +class MutableRegionsProvider(RegionsProvider): + @abc.abstractmethod + def set_regions(self, regions): + """ + Parameters + ---------- + regions: list[Region] + """ + + +# --- serializers for QueryRegionsProvider --- + +def _get_region_from_query(data, **kwargs): + preferred_scheme = kwargs.get('preferred_scheme') + if not preferred_scheme: + preferred_scheme = 'http' + + domain_path_map = { + k: (k.value, 'domains') + for k in ServiceName + if k not in [ServiceName.UP_ACC] + } + domain_path_map[ServiceName.UP_ACC] = ('up', 'acc_domains') + + services = { + # sn service name, dsn data service name + sn: [ + Endpoint(h, default_scheme=preferred_scheme) + for h in data.get(dsn, {}).get(k, []) + ] + for sn, (dsn, k) in domain_path_map.items() + } + + return Region( + region_id=data.get('region'), + s3_region_id=data.get('s3', {}).get('region_alias', None), + services=services, + ttl=data.get('ttl', None) + ) + + +class QueryRegionsProvider(RegionsProvider): + def __init__( + self, + access_key, + bucket_name, + endpoints_provider, + preferred_scheme='http', + max_retry_times_per_endpoint=1, + ): + """ + Parameters + ---------- + access_key: str + bucket_name: str + endpoints_provider: Iterable[Endpoint] + preferred_scheme: str + max_retry_times_per_endpoint: int + """ + self.access_key = access_key + self.bucket_name = bucket_name + self.endpoints_provider = endpoints_provider + self.preferred_scheme = preferred_scheme + self.max_retry_times_per_endpoint = max_retry_times_per_endpoint + + def __iter__(self): + regions = self.__fetch_regions() + # change to `yield from` when min version of python update to >= 3.3 + for r in regions: + yield r + + def __fetch_regions(self): + endpoints = list(self.endpoints_provider) + if not endpoints: + raise ValueError('There aren\'t any available endpoints to query regions') + endpoint, alternative_endpoints = endpoints[0], endpoints[1:] + + url = '{0}/v4/query?ak={1}&bucket={2}'.format(endpoint.get_value(), self.access_key, self.bucket_name) + ret, resp = qn_http_client.get( + url, + middlewares=[ + RetryDomainsMiddleware( + backup_domains=[e.host for e in alternative_endpoints], + max_retry_times=self.max_retry_times_per_endpoint + ) + ] + ) + + if not resp.ok(): + raise RuntimeError( + ( + 'Query regions failed with ' + 'HTTP Status Code {0}, ' + 'Body {1}' + ).format(resp.status_code, resp.text_body) + ) + + return [ + _get_region_from_query(d, preferred_scheme=self.preferred_scheme) + for d in ret.get('hosts', []) + ] + + +# --- helpers for CachedRegionsProvider --- +class FileAlreadyLocked(RuntimeError): + def __init__(self, message): + super(FileAlreadyLocked, self).__init__(message) + + +class _FileLocker: + def __init__(self, origin_file): + self._origin_file = origin_file + + def __enter__(self): + if os.access(self.lock_file_path, os.R_OK | os.W_OK): + raise FileAlreadyLocked('File {0} already locked'.format(self._origin_file)) + with open(self.lock_file_path, 'w'): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + os.remove(self.lock_file_path) + + @property + def lock_file_path(self): + """ + Returns + ------- + str + """ + return self._origin_file + '.lock' + + +# use dataclass instead namedtuple if min version of python update to 3.7 +CacheScope = namedtuple( + 'CacheScope', + [ + 'memo_cache', + 'persist_path', + 'last_shrink_at', + 'shrink_interval', + 'should_shrink_expired_regions' + ] +) + + +_global_cache_scope = CacheScope( + memo_cache={}, + persist_path=os.path.join( + tempfile.gettempdir(), + 'qn-regions-cache.jsonl' + ), + last_shrink_at=datetime.datetime.fromtimestamp(0), + shrink_interval=datetime.timedelta(-1), # useless for now + should_shrink_expired_regions=False +) + + +# --- serializers for CachedRegionsProvider --- + +_PersistedEndpoint = namedtuple( + 'PersistedEndpoint', + [ + 'host', + 'defaultScheme' + ] +) + + +def _persist_endpoint(endpoint): + """ + Parameters + ---------- + endpoint: Endpoint + + Returns + ------- + dict + """ + return _PersistedEndpoint( + defaultScheme=endpoint.default_scheme, + host=endpoint.host + )._asdict() + + +def _get_endpoint_from_persisted(data): + """ + Parameters + ---------- + data: dict + + Returns + ------- + Endpoint + """ + persisted_endpoint = _PersistedEndpoint(**data) + return Endpoint( + persisted_endpoint.host, + default_scheme=persisted_endpoint.defaultScheme + ) + + +_PersistedRegion = namedtuple( + 'PersistedRegion', + [ + 'regionId', + 's3RegionId', + 'services', + 'ttl', + 'createTime' + ] +) + + +def _persist_region(region): + """ + Parameters + ---------- + region: Region + + Returns + ------- + dict + """ + return _PersistedRegion( + regionId=region.region_id, + s3RegionId=region.s3_region_id, + services={ + # The StrEnum not available in python < 3.11 + # so need stringify the key manually + k.value if isinstance(k, ServiceName) else k: [ + _persist_endpoint(e) + for e in v + ] + for k, v in region.services.items() + }, + ttl=region.ttl, + # use datetime.datetime.timestamp() when min version of python >= 3 + createTime=int(float(region.create_time.strftime('%s.%f')) * 1000) + )._asdict() + + +def _get_region_from_persisted(data): + """ + Parameters + ---------- + data: dict + + Returns + ------- + Region + """ + def _get_service_name(k): + try: + return ServiceName(k) + except ValueError: + return k + + persisted_region = _PersistedRegion(**data) + + return Region( + region_id=persisted_region.regionId, + s3_region_id=persisted_region.s3RegionId, + services={ + # The StrEnum not available in python < 3.11 + # so need parse the key manually + _get_service_name(k): [ + _get_endpoint_from_persisted(d) + for d in v + ] + for k, v in persisted_region.services.items() + }, + ttl=persisted_region.ttl, + create_time=datetime.datetime.fromtimestamp(persisted_region.createTime / 1000) + ) + + +def _parse_persisted_regions(persisted_data): + """ + Parameters + ---------- + persisted_data: str + + Returns + ------- + cache_key: str + regions: list[Region] + """ + parsed_data = json.loads(persisted_data) + regions = [ + _get_region_from_persisted(d) + for d in parsed_data.get('regions', []) + ] + return parsed_data.get('cacheKey'), regions + + +def _walk_persist_cache_file(persist_path, ignore_parse_error=False): + """ + Parameters + ---------- + persist_path: str + ignore_parse_error: bool + + Returns + ------- + Iterable[(str, list[Region])] + """ + if not os.access(persist_path, os.R_OK): + return + + with open(persist_path, 'r') as f: + for line in f: + try: + cache_key, regions = _parse_persisted_regions(line) + yield cache_key, regions + except Exception as err: + if not ignore_parse_error: + raise err + + +def _merge_regions(*args): + """ + merge two regions by region id. + if the same region id, the last create region will be keep. + Parameters + ---------- + args: list[Region] + + Returns + ------- + list[Region] + """ + regions_dict = {} + + for r in itertools.chain(*args): + if r.region_id not in regions_dict: + regions_dict[r.region_id] = r + else: + if r.create_time > regions_dict[r.region_id].create_time: + regions_dict[r.region_id] = r + + return regions_dict.values() + + +class CachedRegionsProvider(MutableRegionsProvider): + def __init__( + self, + cache_key, + base_regions_provider, + **kwargs + ): + """ + Parameters + ---------- + cache_key: str + base_regions_provider: Iterable[Region] + kwargs + persist_path: str + shrink_interval: datetime.timedelta + should_shrink_expired_regions: bool + """ + self.cache_key = cache_key + self.base_regions_provider = base_regions_provider + + persist_path = kwargs.get('persist_path', None) + if persist_path is None: + persist_path = _global_cache_scope.persist_path + + shrink_interval = kwargs.get('shrink_interval', None) + if shrink_interval is None: + shrink_interval = datetime.timedelta(days=1) + + should_shrink_expired_regions = kwargs.get('should_shrink_expired_regions', None) + if should_shrink_expired_regions is None: + should_shrink_expired_regions = False + + self._cache_scope = CacheScope( + memo_cache=_global_cache_scope.memo_cache, + persist_path=persist_path, + last_shrink_at=datetime.datetime.fromtimestamp(0), + shrink_interval=shrink_interval, + should_shrink_expired_regions=should_shrink_expired_regions, + ) + + def __iter__(self): + if self.__should_shrink: + self.__shrink_cache() + + get_regions_fns = [ + self.__get_regions_from_memo, + self.__get_regions_from_file, + self.__get_regions_from_base_provider + ] + + regions = None + for get_regions in get_regions_fns: + regions = get_regions(fallback=regions) + if regions and all(r.is_live for r in regions): + break + + # change to `yield from` when min version of python update to >= 3.3 + for r in regions: + yield r + + def set_regions(self, regions): + """ + Parameters + ---------- + regions: list[Region] + """ + self._cache_scope.memo_cache[self.cache_key] = regions + + if not self._cache_scope.persist_path: + return + + try: + with open(self._cache_scope.persist_path, 'a') as f: + f.write(json.dumps({ + 'cacheKey': self.cache_key, + 'regions': [_persist_region(r) for r in regions] + }) + os.linesep) + except Exception as err: + logging.warning('failed to cache regions result to file', err) + + @property + def persist_path(self): + """ + Returns + ------- + str + """ + return self._cache_scope.persist_path + + @persist_path.setter + def persist_path(self, value): + """ + Parameters + ---------- + value: str + """ + self._cache_scope = self._cache_scope._replace( + persist_path=value + ) + + @property + def last_shrink_at(self): + """ + Returns + ------- + datetime.datetime + """ + # copy the datetime make sure it is read-only + return self._cache_scope.last_shrink_at.replace() + + @property + def shrink_interval(self): + """ + Returns + ------- + datetime.timedelta + """ + return self._cache_scope.shrink_interval + + @shrink_interval.setter + def shrink_interval(self, value): + """ + Parameters + ---------- + value: datetime.timedelta + """ + self._cache_scope = self._cache_scope._replace( + shrink_interval=value + ) + + @property + def should_shrink_expired_regions(self): + """ + Returns + ------- + bool + """ + return self._cache_scope.should_shrink_expired_regions + + @should_shrink_expired_regions.setter + def should_shrink_expired_regions(self, value): + """ + Parameters + ---------- + value: bool + """ + self._cache_scope = self._cache_scope._replace( + should_shrink_expired_regions=value + ) + + def __get_regions_from_memo(self, fallback=None): + """ + Parameters + ---------- + fallback: list[Region] + + Returns + ------- + list[Region] + """ + regions = self._cache_scope.memo_cache.get(self.cache_key) + + if regions: + return regions + + return fallback + + def __get_regions_from_file(self, fallback=None): + """ + Parameters + ---------- + fallback: list[Region] + + Returns + ------- + list[Region] + """ + if not self._cache_scope.persist_path: + return fallback + + try: + self.__flush_file_cache_to_memo() + except Exception as err: + if fallback is not None: + return fallback + else: + raise err + + return self.__get_regions_from_memo(fallback) + + def __get_regions_from_base_provider(self, fallback=None): + """ + Parameters + ---------- + fallback: list[Region] + + Returns + ------- + list[Region] + """ + try: + regions = list(self.base_regions_provider) + except Exception as err: + if fallback is not None: + return fallback + else: + raise err + self.set_regions(regions) + return regions + + def __flush_file_cache_to_memo(self): + for cache_key, regions in _walk_persist_cache_file( + persist_path=self._cache_scope.persist_path + # ignore_parse_error=True + ): + if cache_key not in self._cache_scope.memo_cache: + self._cache_scope.memo_cache[cache_key] = regions + return + memo_regions = self._cache_scope.memo_cache[cache_key] + self._cache_scope.memo_cache[cache_key] = _merge_regions( + memo_regions, + regions + ) + + @property + def __should_shrink(self): + """ + Returns + ------- + bool + """ + return datetime.datetime.now() - self._cache_scope.last_shrink_at > self._cache_scope.shrink_interval + + def __shrink_cache(self): + # shrink memory cache + if self._cache_scope.should_shrink_expired_regions: + kept_memo_cache = {} + for k, regions in self._cache_scope.memo_cache.items(): + live_regions = [r for r in regions if r.is_live] + if live_regions: + kept_memo_cache[k] = live_regions + self._cache_scope = self._cache_scope._replace(memo_cache=kept_memo_cache) + + # shrink file cache + if not self._cache_scope.persist_path: + self._cache_scope = self._cache_scope._replace( + last_shrink_at=datetime.datetime.now() + ) + return + + shrink_file_path = self._cache_scope.persist_path + '.shrink' + try: + with _FileLocker(shrink_file_path): + # filter data + shrunk_cache = {} + for cache_key, regions in _walk_persist_cache_file( + persist_path=self._cache_scope.persist_path + ): + kept_regions = regions + if self._cache_scope.should_shrink_expired_regions: + kept_regions = [ + r for r in kept_regions if r.is_live + ] + + if cache_key not in shrunk_cache: + shrunk_cache[cache_key] = kept_regions + else: + shrunk_cache[cache_key] = _merge_regions( + shrunk_cache[cache_key], + kept_regions + ) + + # write data + with open(shrink_file_path, 'a') as f: + for cache_key, regions in shrunk_cache.items(): + f.write( + json.dumps( + { + 'cacheKey': cache_key, + 'regions': [_persist_region(r) for r in regions] + } + ) + os.linesep + ) + + # rename file + os.rename(shrink_file_path, self._cache_scope.persist_path) + except FileAlreadyLocked: + pass + finally: + self._cache_scope = self._cache_scope._replace( + last_shrink_at=datetime.datetime.now() + ) + + +def get_default_regions_provider( + query_endpoints_provider, + access_key, + bucket_name, + accelerate_uploading=False, + force_query=False, + **kwargs +): + """ + Parameters + ---------- + query_endpoints_provider: Iterable[Endpoint] + access_key: str + bucket_name: str + accelerate_uploading: bool + force_query: bool + kwargs + preferred_scheme: str + option of QueryRegionsProvider + max_retry_times_per_endpoint: int + option of QueryRegionsProvider + persist_path: str + option of CachedRegionsProvider + shrink_interval: datetime.timedelta + option of CachedRegionsProvider + should_shrink_expired_regions: bool + option of CachedRegionsProvider + + Returns + ------- + Iterable[Region] + """ + query_regions_provider_opts = { + 'access_key': access_key, + 'bucket_name': bucket_name, + 'endpoints_provider': query_endpoints_provider, + } + query_regions_provider_opts.update({ + k: v + for k, v in kwargs.items() + if k in ['preferred_scheme', 'max_retry_times_per_endpoint'] + }) + + query_regions_provider = QueryRegionsProvider(**query_regions_provider_opts) + + if force_query: + return query_regions_provider + + query_endpoints = list(query_endpoints_provider) + + endpoints_md5 = io_md5([ + to_bytes(e.host) for e in query_endpoints + ]) + cache_key = ':'.join([ + endpoints_md5, + access_key, + bucket_name, + 'true' if accelerate_uploading else 'false' + ]) + + cached_regions_provider_opts = { + 'cache_key': cache_key, + 'base_regions_provider': query_regions_provider, + } + cached_regions_provider_opts.update({ + k: v + for k, v in kwargs.items() + if k in [ + 'persist_path', + 'shrink_interval', + 'should_shrink_expired_regions' + ] + }) + + return CachedRegionsProvider( + **cached_regions_provider_opts + ) diff --git a/qiniu/http/regions_retry_policy.py b/qiniu/http/regions_retry_policy.py new file mode 100644 index 00000000..29b2f9a9 --- /dev/null +++ b/qiniu/http/regions_retry_policy.py @@ -0,0 +1,162 @@ +from qiniu.retry.abc import RetryPolicy + +from .region import Region, ServiceName + + +class RegionsRetryPolicy(RetryPolicy): + def __init__( + self, + regions_provider, + service_names, + preferred_endpoints_provider=None, + on_change_region=None + ): + """ + Parameters + ---------- + regions_provider: Iterable[Region] + service_names: list[ServiceName or str] + preferred_endpoints_provider: Iterable[Endpoint] + on_change_region: Callable + `(context: dict) -> None` + """ + self.regions_provider = regions_provider + self.service_names = service_names + if not service_names: + raise ValueError('Must provide at least one service name') + if preferred_endpoints_provider is None: + preferred_endpoints_provider = [] + self.preferred_endpoints_provider = preferred_endpoints_provider + self.on_change_region = on_change_region + + def init_context(self, context): + """ + Parameters + ---------- + context: dict + """ + self._init_regions(context) + self._prepare_endpoints(context) + + def should_retry(self, attempt): + """ + Parameters + ---------- + attempt: Attempt + """ + return ( + len(attempt.context.get('alternative_regions', [])) > 0 or + len(attempt.context.get('alternative_service_names', [])) > 0 + ) + + def prepare_retry(self, attempt): + """ + Parameters + ---------- + attempt: Attempt + """ + if attempt.context.get('alternative_service_names'): + # change service for next try + attempt.context['service_name'] = attempt.context.get('alternative_service_names').pop(0) + elif attempt.context.get('alternative_regions'): + # change region for next try + attempt.context['region'] = attempt.context.get('alternative_regions').pop(0) + if callable(self.on_change_region): + self.on_change_region(attempt.context) + else: + raise RuntimeError('There isn\'t available region or service for next try') + self._prepare_endpoints(attempt.context) + + def _init_regions(self, context): + """ + Parameters + ---------- + context: dict + """ + regions = list(self.regions_provider) + preferred_endpoints = list(self.preferred_endpoints_provider) + if not regions and not preferred_endpoints: + raise ValueError('There isn\'t available region or preferred endpoint') + + if not preferred_endpoints: + # regions are not empty implicitly by above if condition + context['alternative_regions'] = regions + context['region'] = context['alternative_regions'].pop(0) + # shallow copy list + # change to `list.copy` for more readable when min version of python update to >= 3 + context['alternative_service_names'] = self.service_names[:] + context['service_name'] = context['alternative_service_names'].pop(0) + return + + # find preferred service name and region by preferred endpoints + preferred_region_index = -1 + preferred_service_index = -1 + for ri, region in enumerate(regions): + for si, service_name in enumerate(self.service_names): + if any( + pe.host in [ + e.host for e in region.services.get(service_name, []) + ] + for pe in preferred_endpoints + ): + preferred_region_index = ri + preferred_service_index = si + break + + # initialize the order of service_names and regions + if preferred_region_index < 0: + # shallow copy list + # change to `list.copy` for more readable when min version of python update to >= 3 + context['alternative_service_names'] = self.service_names[:] + context['service_name'] = context['alternative_service_names'].pop(0) + + context['region'] = Region( + region_id='preferred_region', + services={ + context['service_name']: preferred_endpoints + } + ) + context['alternative_regions'] = regions + else: + # regions are not empty implicitly by above if condition + # preferred endpoints are in a known region, then reorder the regions and services + context['alternative_regions'] = regions + context['region'] = context['alternative_regions'].pop(preferred_region_index) + # shallow copy list + # change to `list.copy` for more readable when min version of python update to >= 3 + context['alternative_service_names'] = self.service_names[:] + context['service_name'] = context['alternative_service_names'].pop(preferred_service_index) + + def _prepare_endpoints(self, context): + """ + Parameters + ---------- + context: dict + """ + # shallow copy list + # change to `list.copy` for more readable when min version of python update to >= 3 + endpoints = context['region'].services.get(context['service_name'], [])[:] + while not endpoints: + if context['alternative_service_names']: + context['service_name'] = context['alternative_service_names'].pop(0) + endpoints = context['region'].services.get(context['service_name'], [])[:] + elif context['alternative_regions']: + context['region'] = context['alternative_regions'].pop(0) + # shallow copy list + # change to `list.copy` for more readable when min version of python update to >= 3 + context['alternative_service_names'] = self.service_names[:] + context['service_name'] = context['alternative_service_names'].pop(0) + endpoints = context['region'].services.get(context['service_name'], [])[:] + if callable(self.on_change_region): + self.on_change_region(context) + else: + raise RuntimeError( + 'There isn\'t available endpoint for {0} service(s) in any available regions'.format( + ', '.join( + sn.value if isinstance(sn, ServiceName) else sn + for sn in self.service_names + ) + ) + ) + context['alternative_endpoints'] = endpoints + context['endpoint'] = context['alternative_endpoints'].pop(0) diff --git a/qiniu/http/response.py b/qiniu/http/response.py index 8e9fd84d..6450438d 100644 --- a/qiniu/http/response.py +++ b/qiniu/http/response.py @@ -20,12 +20,14 @@ def __init__(self, response, exception=None): self.__response = response self.exception = exception if response is None: + self.url = None self.status_code = -1 self.text_body = None self.req_id = None self.x_log = None self.error = str(exception) else: + self.url = response.url self.status_code = response.status_code self.text_body = response.text self.req_id = response.headers.get('X-Reqid') diff --git a/qiniu/region.py b/qiniu/region.py index dcef4398..09ac791d 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -1,31 +1,52 @@ # -*- coding: utf-8 -*- +import functools import logging import os import time -from qiniu import compat -from qiniu import utils -UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host +from .compat import json, s as str_from_bytes +from .utils import urlsafe_base64_decode +from .config import UC_HOST, is_customized_default, get_default +from .http.endpoint import Endpoint as _HTTPEndpoint +from .http.regions_provider import Region as _HTTPRegion, ServiceName, get_default_regions_provider -class Region(object): +def _legacy_default_get(key): + def decorator(func): + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + if hasattr(self, key) and getattr(self, key): + return getattr(self, key) + if is_customized_default('default_' + key): + return get_default('default_' + key) + return func(self, *args, **kwargs) + + return wrapper + + return decorator + + +class LegacyRegion(_HTTPRegion, object): """七牛上传区域类 该类主要内容上传区域地址。 """ def __init__( - self, - up_host=None, - up_host_backup=None, - io_host=None, - host_cache=None, - home_dir=None, - scheme="http", - rs_host=None, - rsf_host=None, - api_host=None): + self, + up_host=None, + up_host_backup=None, + io_host=None, + host_cache=None, + home_dir=None, + scheme="http", + rs_host=None, + rsf_host=None, + api_host=None, + accelerate_uploading=False + ): """初始化Zone类""" + super(LegacyRegion, self).__init__() if host_cache is None: host_cache = {} self.up_host = up_host @@ -37,6 +58,20 @@ def __init__( self.home_dir = home_dir self.host_cache = host_cache self.scheme = scheme + self.services.update({ + k: [ + _HTTPEndpoint.from_host(h) + for h in v if h + ] + for k, v in { + ServiceName.UP: [up_host, up_host_backup], + ServiceName.IO: [io_host], + ServiceName.RS: [rs_host], + ServiceName.RSF: [rsf_host], + ServiceName.API: [api_host] + }.items() + }) + self.accelerate_uploading = accelerate_uploading def get_up_host_by_token(self, up_token, home_dir): ak, bucket = self.unmarshal_up_token(up_token) @@ -67,12 +102,8 @@ def get_io_host(self, ak, bucket, home_dir=None): io_hosts = bucket_hosts['ioHosts'] return io_hosts[0] + @_legacy_default_get('rs_host') def get_rs_host(self, ak, bucket, home_dir=None): - from .config import get_default, is_customized_default - if self.rs_host: - return self.rs_host - if is_customized_default('default_rs_host'): - return get_default('default_rs_host') if home_dir is None: home_dir = os.getcwd() bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) @@ -81,12 +112,8 @@ def get_rs_host(self, ak, bucket, home_dir=None): rs_hosts = bucket_hosts['rsHosts'] return rs_hosts[0] + @_legacy_default_get('rsf_host') def get_rsf_host(self, ak, bucket, home_dir=None): - from .config import get_default, is_customized_default - if self.rsf_host: - return self.rsf_host - if is_customized_default('default_rsf_host'): - return get_default('default_rsf_host') if home_dir is None: home_dir = os.getcwd() bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) @@ -95,12 +122,8 @@ def get_rsf_host(self, ak, bucket, home_dir=None): rsf_hosts = bucket_hosts['rsfHosts'] return rsf_hosts[0] + @_legacy_default_get('api_host') def get_api_host(self, ak, bucket, home_dir=None): - from .config import get_default, is_customized_default - if self.api_host: - return self.api_host - if is_customized_default('default_api_host'): - return get_default('default_api_host') if home_dir is None: home_dir = os.getcwd() bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) @@ -120,67 +143,72 @@ def get_up_host(self, ak, bucket, home_dir): def unmarshal_up_token(self, up_token): token = up_token.split(':') - if (len(token) != 3): + if len(token) != 3: raise ValueError('invalid up_token') ak = token[0] - policy = compat.json.loads( - compat.s( - utils.urlsafe_base64_decode( + policy = json.loads( + str_from_bytes( + urlsafe_base64_decode( token[2]))) scope = policy["scope"] bucket = scope - if (':' in scope): + if ':' in scope: bucket = scope.split(':')[0] return ak, bucket - def get_bucket_hosts(self, ak, bucket, home_dir, force=False): - key = self.scheme + ":" + ak + ":" + bucket - - bucket_hosts = self.get_bucket_hosts_to_cache(key, home_dir) - if not force and len(bucket_hosts) > 0: - return bucket_hosts - - hosts = compat.json.loads(self.bucket_hosts(ak, bucket)).get('hosts', []) - - if type(hosts) is not list or len(hosts) == 0: - raise KeyError("Please check your BUCKET_NAME! Server hosts not correct! The hosts is %s" % hosts) + def get_bucket_hosts(self, ak, bucket, home_dir=None, force=False): + cache_persist_path = os.path.join(home_dir, 'qn-regions-cache.jsonl') if home_dir else None + regions = self.__get_bucket_regions( + ak, + bucket, + force_query=force, + cache_persist_path=cache_persist_path + ) - region = hosts[0] + if not regions: + raise KeyError("Please check your BUCKET_NAME! Server hosts not correct! The hosts is empty") - default_ttl = 24 * 3600 # 1 day - region['ttl'] = region.get('ttl', default_ttl) + region = regions[0] bucket_hosts = { - 'upHosts': [ - '{0}://{1}'.format(self.scheme, domain) - for domain in region.get('up', {}).get('domains', []) - ], - 'ioHosts': [ - '{0}://{1}'.format(self.scheme, domain) - for domain in region.get('io', {}).get('domains', []) - ], - 'rsHosts': [ - '{0}://{1}'.format(self.scheme, domain) - for domain in region.get('rs', {}).get('domains', []) - ], - 'rsfHosts': [ - '{0}://{1}'.format(self.scheme, domain) - for domain in region.get('rsf', {}).get('domains', []) - ], - 'apiHosts': [ - '{0}://{1}'.format(self.scheme, domain) - for domain in region.get('api', {}).get('domains', []) - ], - 'deadline': int(time.time()) + region['ttl'] + k: [ + e.get_value(scheme=self.scheme) + for e in region.services[sn] + if e + ] + for k, sn in { + 'upHosts': ServiceName.UP, + 'ioHosts': ServiceName.IO, + 'rsHosts': ServiceName.RS, + 'rsfHosts': ServiceName.RSF, + 'apiHosts': ServiceName.API + }.items() } - home_dir = "" - self.set_bucket_hosts_to_cache(key, bucket_hosts, home_dir) + + ttl = region.ttl if region.ttl > 0 else 24 * 3600 # 1 day + # use datetime.datetime.timestamp() when min version of python >= 3 + create_time = int(float(region.create_time.strftime('%s.%f')) * 1000) + bucket_hosts['deadline'] = create_time + ttl + return bucket_hosts def get_bucket_hosts_to_cache(self, key, home_dir): + """ + .. deprecated:: + The cache has been replaced by CachedRegionsProvider + + Parameters + ---------- + key: str + home_dir: str + + Returns + ------- + dict + """ ret = {} if len(self.host_cache) == 0: self.host_cache_from_file(home_dir) @@ -194,11 +222,29 @@ def get_bucket_hosts_to_cache(self, key, home_dir): return ret def set_bucket_hosts_to_cache(self, key, val, home_dir): + """ + .. deprecated:: + The cache has been replaced by CachedRegionsProvider + + Parameters + ---------- + key: str + val: dict + home_dir: str + """ self.host_cache[key] = val self.host_cache_to_file(home_dir) return def host_cache_from_file(self, home_dir): + """ + .. deprecated:: + The cache has been replaced by CachedRegionsProvider + + Parameters + ---------- + home_dir: str + """ if home_dir is not None: self.home_dir = home_dir path = self.host_cache_file_path() @@ -206,7 +252,7 @@ def host_cache_from_file(self, home_dir): return None with open(path, 'r') as f: try: - bucket_hosts = compat.json.load(f) + bucket_hosts = json.load(f) self.host_cache = bucket_hosts except Exception as e: logging.error(e) @@ -214,33 +260,86 @@ def host_cache_from_file(self, home_dir): return def host_cache_file_path(self): + """ + .. deprecated:: + The cache has been replaced by CachedRegionsProvider + + Returns + ------- + str + """ return os.path.join(self.home_dir, ".qiniu_pythonsdk_hostscache.json") def host_cache_to_file(self, home_dir): + """ + .. deprecated:: + The cache has been replaced by CachedRegionsProvider + + Parameters + ---------- + home_dir: str + + """ path = self.host_cache_file_path() with open(path, 'w') as f: - compat.json.dump(self.host_cache, f) + json.dump(self.host_cache, f) f.close() def bucket_hosts(self, ak, bucket): - from .config import get_default, is_customized_default - from .http import qn_http_client - from .http.middleware import RetryDomainsMiddleware - uc_host = UC_HOST - if is_customized_default('default_uc_host'): - uc_host = get_default('default_uc_host') - uc_backup_hosts = get_default('default_query_region_backup_hosts') - uc_backup_retry_times = get_default('default_backup_hosts_retry_times') - url = "{0}/v4/query?ak={1}&bucket={2}".format(uc_host, ak, bucket) - - ret, _resp = qn_http_client.get( - url, - middlewares=[ - RetryDomainsMiddleware( - backup_domains=uc_backup_hosts, - max_retry_times=uc_backup_retry_times, - ) + regions = self.__get_bucket_regions(ak, bucket) + + data_dict = { + 'hosts': [ + { + k.value if isinstance(k, ServiceName) else k: { + 'domains': [ + e.host for e in v + ] + } + for k, v in r.services.items() + } + for r in regions ] - ) - data = compat.json.dumps(ret, separators=(',', ':')) + } + for r in data_dict['hosts']: + if 'up_acc' in r: + r.setdefault('up', {}) + r['up'].update(acc_domains=r['up_acc'].get('domains', [])) + del r['up_acc'] + + data = json.dumps(data_dict) + return data + + def __get_bucket_regions( + self, + access_key, + bucket_name, + force_query=False, + cache_persist_path=None + ): + query_region_host = UC_HOST + if is_customized_default('default_query_region_host'): + query_region_host = get_default('default_query_region_host') + query_region_backup_hosts = get_default('default_query_region_backup_hosts') + query_region_backup_retry_times = get_default('default_backup_hosts_retry_times') + + regions_provider = get_default_regions_provider( + query_endpoints_provider=[ + _HTTPEndpoint.from_host(h) + for h in [query_region_host] + query_region_backup_hosts + if h + ], + access_key=access_key, + bucket_name=bucket_name, + accelerate_uploading=self.accelerate_uploading, + force_query=force_query, + preferred_scheme=self.scheme, + persist_path=cache_persist_path, + max_retry_times_per_endpoint=query_region_backup_retry_times + ) + + return list(regions_provider) + + +Region = LegacyRegion diff --git a/qiniu/retry/__init__.py b/qiniu/retry/__init__.py new file mode 100644 index 00000000..e726010f --- /dev/null +++ b/qiniu/retry/__init__.py @@ -0,0 +1,7 @@ +from .attempt import Attempt +from .retrier import Retrier + +__all__ = [ + 'Attempt', + 'Retrier' +] diff --git a/qiniu/retry/abc/__init__.py b/qiniu/retry/abc/__init__.py new file mode 100644 index 00000000..4f458a73 --- /dev/null +++ b/qiniu/retry/abc/__init__.py @@ -0,0 +1,5 @@ +from .policy import RetryPolicy + +__all__ = [ + 'RetryPolicy' +] diff --git a/qiniu/retry/abc/policy.py b/qiniu/retry/abc/policy.py new file mode 100644 index 00000000..b5b792bf --- /dev/null +++ b/qiniu/retry/abc/policy.py @@ -0,0 +1,61 @@ +import abc + + +class RetryPolicy(object): + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def init_context(self, context): + """ + initial context values the policy required + + Parameters + ---------- + context: dict + """ + + @abc.abstractmethod + def should_retry(self, attempt): + """ + if returns True, this policy will be applied + + Parameters + ---------- + attempt: qiniu.retry.attempt.Attempt + + Returns + ------- + bool + """ + + @abc.abstractmethod + def prepare_retry(self, attempt): + """ + apply this policy to change the context values for next attempt + + Parameters + ---------- + attempt: qiniu.retry.attempt.Attempt + """ + + def is_important(self, attempt): + """ + if returns True, this policy will be applied, whether it should retry or not. + this is useful when want to stop retry. + + Parameters + ---------- + attempt: qiniu.retry.attempt.Attempt + + Returns + ------- + bool + """ + + def after_retry(self, attempt, policy): + """ + Parameters + ---------- + attempt: qiniu.retry.attempt.Attempt + policy: RetryPolicy + """ diff --git a/qiniu/retry/attempt.py b/qiniu/retry/attempt.py new file mode 100644 index 00000000..460145c6 --- /dev/null +++ b/qiniu/retry/attempt.py @@ -0,0 +1,18 @@ +class Attempt: + def __init__(self, custom_context=None): + """ + Parameters + ---------- + custom_context: dict or None + """ + self.context = custom_context if custom_context is not None else {} + self.exception = None + self.result = None + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None and exc_val is not None: + self.exception = exc_val + return True # Swallow exception. diff --git a/qiniu/retry/retrier.py b/qiniu/retry/retrier.py new file mode 100644 index 00000000..23ff23b6 --- /dev/null +++ b/qiniu/retry/retrier.py @@ -0,0 +1,183 @@ +import functools + +from .attempt import Attempt + + +def before_retry_nothing(attempt, policy): + return True + + +class Retrier: + def __init__(self, policies=None, before_retry=None): + """ + Parameters + ---------- + policies: list[qiniu.retry.abc.RetryPolicy] + before_retry: callable + `(attempt: Attempt, policy: qiniu.retry.abc.RetryPolicy) -> bool` + """ + self.policies = policies if policies is not None else [] + self.before_retry = before_retry if before_retry is not None else before_retry_nothing + + def __iter__(self): + retrying = Retrying( + # change to `list.copy` for more readable when min version of python update to >= 3 + policies=self.policies[:], + before_retry=self.before_retry + ) + retrying.init_context() + while True: + attempt = Attempt(retrying.context) + yield attempt + if ( + hasattr(attempt.exception, 'no_need_retry') and + attempt.exception.no_need_retry + ): + break + policy = retrying.get_retry_policy(attempt) + if not policy: + break + if not self.before_retry(attempt, policy): + break + policy.prepare_retry(attempt) + retrying.after_retried(attempt, policy) + if attempt.exception: + raise attempt.exception + + def try_do( + self, + func, + *args, + **kwargs + ): + attempt = None + for attempt in self: + with attempt: + if kwargs.get('with_retry_context', False): + # inject retry_context + kwargs['retry_context'] = attempt.context + if 'with_retry_context' in kwargs: + del kwargs['with_retry_context'] + + # store result + attempt.result = func(*args, **kwargs) + + if attempt is None: + raise RuntimeError('attempt is none') + + return attempt.result + + def _wrap(self, with_retry_context=False): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + return self.try_do( + func, + with_retry_context=with_retry_context, + *args, + **kwargs + ) + + return wrapper + + return decorator + + def retry(self, *args, **kwargs): + """ + decorator to retry + """ + if len(args) == 1 and callable(args[0]): + return self.retry()(args[0]) + else: + return self._wrap(**kwargs) + + +class Retrying: + def __init__(self, policies, before_retry): + """ + Parameters + ---------- + policies: list[qiniu.retry.abc.RetryPolicy] + before_retry: callable + `(attempt: Attempt, policy: qiniu.retry.abc.RetryPolicy) -> bool` + """ + self.policies = policies + self.before_retry = before_retry + self.context = {} + + def init_context(self): + for policy in self.policies: + policy.init_context(self.context) + + def get_retry_policy(self, attempt): + """ + + Parameters + ---------- + attempt: Attempt + + Returns + ------- + qiniu.retry.abc.RetryPolicy + + """ + policy = None + + # find important policy + for p in self.policies: + if p.is_important(attempt): + policy = p + break + if policy and policy.should_retry(attempt): + return policy + else: + policy = None + + # find retry policy + for p in self.policies: + if p.should_retry(attempt): + policy = p + break + + return policy + + def after_retried(self, attempt, policy): + for p in self.policies: + p.after_retry(attempt, policy) + + +""" +Examples +-------- +retrier = Retrier() +result = None +for attempt in retrier: + with attempt: + endpoint = attempt.context.get('endpoint') + result = upload(endpoint) + attempt.result = result +return result +""" + +""" +Examples +-------- +def foo(): + print('hi') + +retrier = Retrier() +retrier.try_do(foo) +""" + +""" +Examples +-------- +retrier = Retrier() + + +@retrier.retry +def foo(): + print('hi') + +foo() +""" diff --git a/qiniu/services/processing/pfop.py b/qiniu/services/processing/pfop.py index fa414930..346e6277 100644 --- a/qiniu/services/processing/pfop.py +++ b/qiniu/services/processing/pfop.py @@ -24,17 +24,25 @@ def __init__(self, auth, bucket, pipeline=None, notify_url=None): self.pipeline = pipeline self.notify_url = notify_url - def execute(self, key, fops, force=None): - """执行持久化处理: - - Args: - key: 待处理的源文件 - fops: 处理详细操作，规格详见 https://developer.qiniu.com/dora/manual/1291/persistent-data-processing-pfop - force: 强制执行持久化处理开关 + def execute(self, key, fops, force=None, persistent_type=None): + """ + 执行持久化处理 - Returns: - 一个dict变量，返回持久化处理的persistentId，类似{"persistentId": 5476bedf7823de4068253bae}; - 一个ResponseInfo对象 + Parameters + ---------- + key: str + 待处理的源文件 + fops: list[str] + 处理详细操作，规格详见 https://developer.qiniu.com/dora/manual/1291/persistent-data-processing-pfop + force: int or str, optional + 强制执行持久化处理开关 + persistent_type: int or str, optional + 持久化处理类型，为 '1' 时开启闲时任务 + Returns + ------- + ret: dict + 持久化处理的 persistentId，类似 {"persistentId": 5476bedf7823de4068253bae}; + resp: ResponseInfo """ ops = ';'.join(fops) data = {'bucket': self.bucket, 'key': key, 'fops': ops} @@ -42,8 +50,30 @@ def execute(self, key, fops, force=None): data['pipeline'] = self.pipeline if self.notify_url: data['notifyURL'] = self.notify_url - if force == 1: - data['force'] = 1 + if force == 1 or force == '1': + data['force'] = str(force) + if persistent_type and type(int(persistent_type)) is int: + data['type'] = str(persistent_type) url = '{0}/pfop'.format(config.get_default('default_api_host')) return http._post_with_auth(url, data, self.auth) + + def get_status(self, persistent_id): + """ + 获取持久化处理状态 + + Parameters + ---------- + persistent_id: str + + Returns + ------- + ret: dict + 持久化处理的状态，详见 https://developer.qiniu.com/dora/1294/persistent-processing-status-query-prefop + resp: ResponseInfo + """ + url = '{0}/status/get/prefop'.format(config.get_default('default_api_host')) + data = { + 'id': persistent_id + } + return http._get_with_auth(url, data, self.auth) diff --git a/qiniu/services/storage/_bucket_default_retrier.py b/qiniu/services/storage/_bucket_default_retrier.py new file mode 100644 index 00000000..70758e30 --- /dev/null +++ b/qiniu/services/storage/_bucket_default_retrier.py @@ -0,0 +1,25 @@ +from qiniu.http.endpoints_retry_policy import EndpointsRetryPolicy +from qiniu.http.regions_retry_policy import RegionsRetryPolicy +from qiniu.retry import Retrier + + +def get_default_retrier( + regions_provider, + service_names, + preferred_endpoints_provider=None, +): + if not service_names: + raise ValueError('service_names should not be empty') + + retry_policies = [ + EndpointsRetryPolicy( + skip_init_context=True + ), + RegionsRetryPolicy( + regions_provider=regions_provider, + service_names=service_names, + preferred_endpoints_provider=preferred_endpoints_provider + ) + ] + + return Retrier(retry_policies) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 8edb00a3..5e21b6c4 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -1,8 +1,12 @@ # -*- coding: utf-8 -*- - from qiniu import config, QiniuMacAuth from qiniu import http from qiniu.utils import urlsafe_base64_encode, entry, decode_entry +from qiniu.http.endpoint import Endpoint +from qiniu.http.region import Region, ServiceName +from qiniu.http.regions_provider import get_default_regions_provider + +from ._bucket_default_retrier import get_default_retrier class BucketManager(object): @@ -15,17 +19,38 @@ class BucketManager(object): auth: 账号管理密钥对，Auth对象 """ - def __init__(self, auth, zone=None): + def __init__( + self, + auth, + zone=None, + regions=None, + query_regions_endpoints=None, + preferred_scheme='http' + ): + """ + Parameters + ---------- + auth: Auth + zone: LegacyRegion + regions: list[Region] + query_regions_endpoints: list[Endpoint] + preferred_scheme: str, default='http' + """ self.auth = auth self.mac_auth = QiniuMacAuth( auth.get_access_key(), auth.get_secret_key(), auth.disable_qiniu_timestamp_signature) - if (zone is None): + + if zone is None: self.zone = config.get_default('default_zone') else: self.zone = zone + self.regions = regions + self.query_regions_endpoints = query_regions_endpoints + self.preferred_scheme = preferred_scheme + def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): """前缀查询: @@ -59,10 +84,13 @@ def list(self, bucket, prefix=None, marker=None, limit=None, delimiter=None): if delimiter is not None: options['delimiter'] = delimiter - ak = self.auth.get_access_key() - rsf_host = self.zone.get_rsf_host(ak, bucket) - url = '{0}/list'.format(rsf_host) - ret, info = self.__get(url, options) + ret, info = self.__server_do_with_retrier( + bucket, + [ServiceName.RSF], + '/list', + data=options, + method='GET' + ) eof = False if ret and not ret.get('marker'): @@ -81,7 +109,7 @@ def list_domains(self, bucket): resBody, respInfo resBody 为绑定的域名列表，格式：["example.com"] """ - return self.__uc_do('v2/domains?tbl={0}'.format(bucket)) + return self.__uc_do_with_retrier('/v2/domains?tbl={0}'.format(bucket)) def stat(self, bucket, key): """获取文件信息: @@ -105,7 +133,11 @@ def stat(self, bucket, key): 一个ResponseInfo对象 """ resource = entry(bucket, key) - return self.__rs_do(bucket, 'stat', resource) + return self.__server_do_with_retrier( + bucket, + [ServiceName.RS], + '/stat/{0}'.format(resource) + ) def delete(self, bucket, key): """删除文件: @@ -122,7 +154,11 @@ def delete(self, bucket, key): 一个ResponseInfo对象 """ resource = entry(bucket, key) - return self.__rs_do(bucket, 'delete', resource) + return self.__server_do_with_retrier( + bucket, + [ServiceName.RS], + '/delete/{0}'.format(resource) + ) def rename(self, bucket, key, key_to, force='false'): """重命名文件: @@ -133,6 +169,7 @@ def rename(self, bucket, key, key_to, force='false'): bucket: 待操作资源所在空间 key: 待操作资源文件名 key_to: 目标资源文件名 + force: 是否强制覆盖 Returns: 一个dict变量，成功返回NULL，失败返回{"error": "<errMsg string>"} @@ -151,14 +188,23 @@ def move(self, bucket, key, bucket_to, key_to, force='false'): bucket_to: 目标资源空间名 key: 待操作资源文件名 key_to: 目标资源文件名 + force: 是否强制覆盖 Returns: 一个dict变量，成功返回NULL，失败返回{"error": "<errMsg string>"} 一个ResponseInfo对象 """ - resource = entry(bucket, key) - to = entry(bucket_to, key_to) - return self.__rs_do(bucket, 'move', resource, to, 'force/{0}'.format(force)) + src = entry(bucket, key) + dst = entry(bucket_to, key_to) + return self.__server_do_with_retrier( + bucket, + [ServiceName.RS], + '/move/{src}/{dst}/force/{force}'.format( + src=src, + dst=dst, + force=force + ) + ) def copy(self, bucket, key, bucket_to, key_to, force='false'): """复制文件: @@ -171,14 +217,23 @@ def copy(self, bucket, key, bucket_to, key_to, force='false'): bucket_to: 目标资源空间名 key: 待操作资源文件名 key_to: 目标资源文件名 + force: 是否强制覆盖 Returns: 一个dict变量，成功返回NULL，失败返回{"error": "<errMsg string>"} 一个ResponseInfo对象 """ - resource = entry(bucket, key) - to = entry(bucket_to, key_to) - return self.__rs_do(bucket, 'copy', resource, to, 'force/{0}'.format(force)) + src = entry(bucket, key) + dst = entry(bucket_to, key_to) + return self.__server_do_with_retrier( + bucket, + [ServiceName.RS], + '/copy/{src}/{dst}/force/{force}'.format( + src=src, + dst=dst, + force=force + ) + ) def fetch(self, url, bucket, key=None, hostscache_dir=None): """抓取文件: @@ -189,7 +244,8 @@ def fetch(self, url, bucket, key=None, hostscache_dir=None): url: 指定的URL bucket: 目标资源空间 key: 目标资源文件名 - hostscache_dir： host请求 缓存文件保存位置 + hostscache_dir: deprecated, 此参数不再生效，可修改 get_default_regions_provider 返回对象的属性达成同样功能； + 查询区域缓存文件保存位置 Returns: 一个dict变量： @@ -199,7 +255,11 @@ def fetch(self, url, bucket, key=None, hostscache_dir=None): """ resource = urlsafe_base64_encode(url) to = entry(bucket, key) - return self.__io_do(bucket, 'fetch', hostscache_dir, resource, 'to/{0}'.format(to)) + return self.__server_do_with_retrier( + bucket, + [ServiceName.IO], + '/fetch/{0}/to/{1}'.format(resource, to) + ) def prefetch(self, bucket, key, hostscache_dir=None): """镜像回源预取文件: @@ -210,14 +270,19 @@ def prefetch(self, bucket, key, hostscache_dir=None): Args: bucket: 待获取资源所在的空间 key: 代获取资源文件名 - hostscache_dir： host请求 缓存文件保存位置 + hostscache_dir: deprecated, 此参数不再生效，可修改 get_default_regions_provider 返回对象的属性达成同样功能； + 查询区域缓存文件保存位置 Returns: 一个dict变量，成功返回NULL，失败返回{"error": "<errMsg string>"} 一个ResponseInfo对象 """ resource = entry(bucket, key) - return self.__io_do(bucket, 'prefetch', hostscache_dir, resource) + return self.__server_do_with_retrier( + bucket, + [ServiceName.IO], + '/prefetch/{0}'.format(resource) + ) def change_mime(self, bucket, key, mime): """修改文件mimeType: @@ -232,7 +297,11 @@ def change_mime(self, bucket, key, mime): """ resource = entry(bucket, key) encode_mime = urlsafe_base64_encode(mime) - return self.__rs_do(bucket, 'chgm', resource, 'mime/{0}'.format(encode_mime)) + return self.__server_do_with_retrier( + bucket, + [ServiceName.RS], + '/chgm/{0}/mime/{1}'.format(resource, encode_mime) + ) def change_type(self, bucket, key, storage_type): """修改文件的存储类型 @@ -246,21 +315,52 @@ def change_type(self, bucket, key, storage_type): storage_type: 待操作资源存储类型，0为普通存储，1为低频存储，2 为归档存储，3 为深度归档，4 为归档直读存储 """ resource = entry(bucket, key) - return self.__rs_do(bucket, 'chtype', resource, 'type/{0}'.format(storage_type)) + return self.__server_do_with_retrier( + bucket, + [ServiceName.RS], + '/chtype/{0}/type/{1}'.format(resource, storage_type) + ) def restoreAr(self, bucket, key, freezeAfter_days): - """解冻归档存储、深度归档存储文件 - - 对归档存储、深度归档存储文件，进行解冻操作参考文档： - https://developer.qiniu.com/kodo/6380/restore-archive + """ + restore_ar 的别名，用于兼容旧版本 Args: bucket: 待操作资源所在空间 key: 待操作资源文件名 freezeAfter_days: 解冻有效时长，取值范围 1～7 """ + return self.restore_ar( + bucket, + key, + freezeAfter_days + ) + + def restore_ar(self, bucket, key, freeze_after_days): + """ + 解冻归档存储、深度归档存储文件 + + 对归档存储、深度归档存储文件，进行解冻操作参考文档： + https://developer.qiniu.com/kodo/6380/restore-archive + + Parameters + ---------- + bucket: str + key: str + freeze_after_days: int + + Returns + ------- + ret: dict + resp: ResponseInfo + """ + resource = entry(bucket, key) - return self.__rs_do(bucket, 'restoreAr', resource, 'freezeAfterDays/{0}'.format(freezeAfter_days)) + return self.__server_do_with_retrier( + bucket, + [ServiceName.RS], + '/restoreAr/{0}/freezeAfterDays/{1}'.format(resource, freeze_after_days) + ) def change_status(self, bucket, key, status, cond): """修改文件的状态 @@ -270,16 +370,23 @@ def change_status(self, bucket, key, status, cond): Args: bucket: 待操作资源所在空间 key: 待操作资源文件名 - storage_type: 待操作资源存储类型，0为启用，1为禁用 + status: 待操作资源存储类型，0为启用，1为禁用 """ resource = entry(bucket, key) + url_resource = '/chstatus/{0}/status/{1}'.format(resource, status) if cond and isinstance(cond, dict): - condstr = "" - for k, v in cond.items(): - condstr += "{0}={1}&".format(k, v) - condstr = urlsafe_base64_encode(condstr[:-1]) - return self.__rs_do(bucket, 'chstatus', resource, 'status/{0}'.format(status), 'cond', condstr) - return self.__rs_do(bucket, 'chstatus', resource, 'status/{0}'.format(status)) + condstr = urlsafe_base64_encode( + '&'.join( + '='.join([k, v]) + for k, v in cond.items() + ) + ) + url_resource += '/cond/{0}'.format(condstr) + return self.__server_do_with_retrier( + bucket, + [ServiceName.RS], + url_resource + ) def set_object_lifecycle( self, @@ -318,10 +425,17 @@ def set_object_lifecycle( 'deleteAfterDays', str(delete_after_days) ] if cond and isinstance(cond, dict): - cond_str = '&'.join(["{0}={1}".format(k, v) for k, v in cond.items()]) + cond_str = '&'.join( + '='.join([k, v]) + for k, v in cond.items() + ) options += ['cond', urlsafe_base64_encode(cond_str)] resource = entry(bucket, key) - return self.__rs_do(bucket, 'lifecycle', resource, *options) + return self.__server_do_with_retrier( + bucket, + service_names=[ServiceName.RS], + url_resource='/lifecycle/{0}/{1}'.format(resource, '/'.join(options)), + ) def batch(self, operations): """批量操作: @@ -345,6 +459,7 @@ def batch(self, operations): 一个ResponseInfo对象 """ if not operations: + # change to ValueError when make break changes version raise Exception('operations is empty') bucket = '' for op in operations: @@ -354,11 +469,15 @@ def batch(self, operations): if bucket: break if not bucket: + # change to ValueError when make break changes version raise Exception('bucket is empty') - ak = self.auth.get_access_key() - rs_host = self.zone.get_rs_host(ak, bucket) - url = '{0}/batch'.format(rs_host) - return self.__post(url, dict(op=operations)) + + return self.__server_do_with_retrier( + bucket, + [ServiceName.RS], + '/batch', + {'op': operations} + ) def buckets(self): """获取所有空间名: @@ -370,7 +489,7 @@ def buckets(self): [ <Bucket1>, <Bucket2>, ... ] 一个ResponseInfo对象 """ - return self.__uc_do('buckets') + return self.__uc_do_with_retrier('/buckets') def delete_after_days(self, bucket, key, days): """更新文件生命周期 @@ -392,7 +511,11 @@ def delete_after_days(self, bucket, key, days): days: 指定天数 """ resource = entry(bucket, key) - return self.__rs_do(bucket, 'deleteAfterDays', resource, days) + return self.__server_do_with_retrier( + bucket, + [ServiceName.RS], + '/deleteAfterDays/{0}/{1}'.format(resource, days) + ) def mkbucketv3(self, bucket_name, region): """ @@ -402,7 +525,9 @@ def mkbucketv3(self, bucket_name, region): bucket_name: 存储空间名 region: 存储区域 """ - return self.__uc_do('mkbucketv3', bucket_name, 'region', region) + return self.__uc_do_with_retrier( + '/mkbucketv3/{0}/region/{1}'.format(bucket_name, region) + ) def list_bucket(self, region): """ @@ -410,7 +535,7 @@ def list_bucket(self, region): Args: """ - return self.__uc_do('v3/buckets?region={0}'.format(region)) + return self.__uc_do_with_retrier('/v3/buckets?region={0}'.format(region)) def bucket_info(self, bucket_name): """ @@ -419,7 +544,7 @@ def bucket_info(self, bucket_name): Args: bucket_name: 存储空间名 """ - return self.__uc_do('v2/bucketInfo?bucket={}'.format(bucket_name), ) + return self.__uc_do_with_retrier('/v2/bucketInfo?bucket={0}'.format(bucket_name)) def bucket_domain(self, bucket_name): """ @@ -437,32 +562,146 @@ def change_bucket_permission(self, bucket_name, private): bucket_name: 存储空间名 private: 0 公开；1 私有 ,str类型 """ - url = "{0}/private?bucket={1}&private={2}".format(config.get_default("default_uc_host"), bucket_name, private) - return self.__post(url) + return self.__uc_do_with_retrier( + '/private?bucket={0}&private={1}'.format(bucket_name, private) + ) - def __api_do(self, bucket, operation, data=None): - ak = self.auth.get_access_key() - api_host = self.zone.get_api_host(ak, bucket) - url = '{0}/{1}'.format(api_host, operation) - return self.__post(url, data) + def _get_regions_provider(self, bucket_name): + """ + Parameters + ---------- + bucket_name: str - def __uc_do(self, operation, *args): - return self.__server_do(config.get_default('default_uc_host'), operation, *args) + Returns + ------- + Iterable[Region] + """ + if self.regions: + return self.regions + + # handle compatibility for legacy config + if self.zone and any( + hasattr(self.zone, attr_name) and getattr(self.zone, attr_name) + for attr_name in [ + 'io_host', + 'rs_host', + 'rsf_host', + 'api_host' + ] + ): + return [self.zone] + + # handle compatibility for default_query_region_host + query_regions_endpoints = self.query_regions_endpoints + if not query_regions_endpoints: + query_region_host = config.get_default('default_query_region_host') + query_region_backup_hosts = config.get_default('default_query_region_backup_hosts') + query_regions_endpoints = [ + Endpoint.from_host(h) + for h in [query_region_host] + query_region_backup_hosts + ] + + return get_default_regions_provider( + query_endpoints_provider=query_regions_endpoints, + access_key=self.auth.get_access_key(), + bucket_name=bucket_name, + preferred_scheme=self.preferred_scheme + ) + + def __uc_do_with_retrier(self, url_resource, data=None): + """ + Parameters + ---------- + url_resource: url + data: dict or None + + Returns + ------- + ret: dict or None + resp: ResponseInfo + """ + regions = self.regions + + # ignore self.zone by no uc in it + # handle compatibility for default_uc + if not regions: + uc_host = config.get_default('default_uc_host') + uc_backup_hosts = config.get_default('default_uc_backup_hosts') + uc_endpoints = [ + Endpoint.from_host(h) + for h in [uc_host] + uc_backup_hosts + ] + regions = [Region(services={ServiceName.UC: uc_endpoints})] + + retrier = get_default_retrier( + regions_provider=regions, + service_names=[ServiceName.UC] + ) + + attempt = None + for attempt in retrier: + with attempt: + host = attempt.context.get('endpoint').get_value(scheme=self.preferred_scheme) + url = host + url_resource + attempt.result = self.__post(url, data) + ret, resp = attempt.result + if resp.ok() and ret: + return attempt.result + if not resp.need_retry(): + return attempt.result + + if attempt is None: + raise RuntimeError('Retrier is not working. attempt is None') + + return attempt.result + + def __server_do_with_retrier(self, bucket_name, service_names, url_resource, data=None, method='POST'): + """ + Parameters + ---------- + bucket_name: str + service_names: List[ServiceName] + url_resource: str + data: dict or None + method: str + + Returns + ------- + ret: dict or None + resp: ResponseInfo + """ + if not service_names: + raise ValueError('service_names is empty') + + retrier = get_default_retrier( + regions_provider=self._get_regions_provider(bucket_name=bucket_name), + service_names=service_names + ) + + method = method.upper() + if method == 'POST': + send_request = self.__post + elif method == 'GET': + send_request = self.__get + else: + raise ValueError('"method" must be "POST" or "GET"') - def __rs_do(self, bucket, operation, *args): - ak = self.auth.get_access_key() - rs_host = self.zone.get_rs_host(ak, bucket) - return self.__server_do(rs_host, operation, *args) + attempt = None + for attempt in retrier: + with attempt: + host = attempt.context.get('endpoint').get_value(scheme=self.preferred_scheme) + url = host + url_resource + attempt.result = send_request(url, data) + ret, resp = attempt.result + if resp.ok() and ret: + return attempt.result + if not resp.need_retry(): + return attempt.result - def __io_do(self, bucket, operation, home_dir, *args): - ak = self.auth.get_access_key() - io_host = self.zone.get_io_host(ak, bucket, home_dir) - return self.__server_do(io_host, operation, *args) + if attempt is None: + raise RuntimeError('Retrier is not working. attempt is None') - def __server_do(self, host, operation, *args): - cmd = _build_op(operation, *args) - url = '{0}/{1}'.format(host, cmd) - return self.__post(url) + return attempt.result def __post(self, url, data=None): return http._post_with_qiniu_mac(url, data, self.mac_auth) @@ -472,44 +711,200 @@ def __get(self, url, params=None): def _build_op(*args): - return '/'.join(args) + return '/'.join(map(str, args)) def build_batch_copy(source_bucket, key_pairs, target_bucket, force='false'): + """ + Parameters + ---------- + source_bucket: str + key_pairs: dict + target_bucket: str + force: str + + Returns + ------- + list[str] + """ return _two_key_batch('copy', source_bucket, key_pairs, target_bucket, force) def build_batch_rename(bucket, key_pairs, force='false'): + """ + Parameters + ---------- + bucket: str + key_pairs: dict + force: str + + Returns + ------- + list[str] + """ return build_batch_move(bucket, key_pairs, bucket, force) def build_batch_move(source_bucket, key_pairs, target_bucket, force='false'): + """ + Parameters + ---------- + source_bucket: str + key_pairs: dict + target_bucket: str + force: str + + Returns + ------- + list[str] + """ return _two_key_batch('move', source_bucket, key_pairs, target_bucket, force) def build_batch_restoreAr(bucket, keys): - return _three_key_batch('restoreAr', bucket, keys) + """ + alias for build_batch_restore_ar for compatibility with old version + + Parameters + ---------- + bucket: str + keys: dict + + Returns + ------- + list[str] + """ + return build_batch_restore_ar(bucket, keys) + + +def build_batch_restore_ar(bucket, keys): + """ + Parameters + ---------- + bucket: str + keys: dict + + Returns + ------- + list[str] + """ + keys = { + k: ['freezeAfterDays', v] + for k, v in keys.items() + } + return _one_key_batch('restoreAr', bucket, keys) def build_batch_delete(bucket, keys): + """ + Parameters + ---------- + bucket: str + keys: list[str] + + Returns + ------- + list[str] + """ return _one_key_batch('delete', bucket, keys) def build_batch_stat(bucket, keys): + """ + Parameters + ---------- + bucket: str + keys: list[str] + + Returns + ------- + list[str] + """ return _one_key_batch('stat', bucket, keys) def _one_key_batch(operation, bucket, keys): - return [_build_op(operation, entry(bucket, key)) for key in keys] + """ + Parameters + ---------- + operation: str + bucket: str + keys: list[str] or dict + + Returns + ------- + list[str] + """ + # use functools.singledispatch to refactor when min version of python >= 3.4 + if isinstance(keys, list): + return [ + _build_op( + operation, + entry(bucket, key), + ) + for key in keys + ] + elif isinstance(keys, dict): + return [ + _build_op( + operation, + entry(bucket, key), + *opts + ) + for key, opts in keys.items() + ] + else: + raise TypeError('"keys" only support list or dict') + +def _two_key_batch(operation, source_bucket, key_pairs, target_bucket=None, force='false'): + """ -def _two_key_batch(operation, source_bucket, key_pairs, target_bucket, force='false'): + Parameters + ---------- + operation: str + source_bucket: str + key_pairs: dict + target_bucket: str + force: str + + Returns + ------- + list[str] + """ if target_bucket is None: target_bucket = source_bucket - return [_build_op(operation, entry(source_bucket, k), entry(target_bucket, v), 'force/{0}'.format(force)) for k, v - in key_pairs.items()] + return _one_key_batch( + operation, + source_bucket, + { + src_key: [ + entry(target_bucket, dst_key), + 'force', + force + ] + for src_key, dst_key in key_pairs.items() + } + ) def _three_key_batch(operation, bucket, keys): - return [_build_op(operation, entry(bucket, k), 'freezeAfterDays/{0}'.format(v)) for k, v - in keys.items()] + """ + .. deprecated: Use `_one_key_batch` instead. + `keys` could be `{key: [freezeAfterDays, days]}` + + Parameters + ---------- + operation: str + bucket: str + keys: dict + + Returns + ------- + list[str] + """ + keys = { + k: ['freezeAfterDays', v] + for k, v in keys.items() + } + return _one_key_batch(operation, bucket, keys) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index 5be466d8..8673b198 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import hashlib import json import os @@ -8,16 +7,11 @@ class UploadProgressRecorder(object): - """持久化上传记录类 + """ + 持久化上传记录类 该类默认保存每个文件的上传记录到文件系统中，用于断点续传 - 上传记录为json格式： - { - "size": file_size, - "offset": upload_offset, - "modify_time": file_modify_time, - "contexts": contexts - } + 上传记录为json格式 Attributes: record_folder: 保存上传记录的目录 @@ -26,46 +20,37 @@ class UploadProgressRecorder(object): def __init__(self, record_folder=tempfile.gettempdir()): self.record_folder = record_folder - def get_upload_record(self, file_name, key): + def __get_upload_record_file_path(self, file_name, key): record_key = '{0}/{1}'.format(key, file_name) if is_py2: record_file_name = hashlib.md5(record_key).hexdigest() else: record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() + return os.path.join(self.record_folder, record_file_name) - upload_record_file_path = os.path.join(self.record_folder, record_file_name) - if not os.path.isfile(upload_record_file_path): + def has_upload_record(self, file_name, key): + upload_record_file_path = self.__get_upload_record_file_path(file_name, key) + return os.path.isfile(upload_record_file_path) + + def get_upload_record(self, file_name, key): + upload_record_file_path = self.__get_upload_record_file_path(file_name, key) + if not self.has_upload_record(file_name, key): return None try: with open(upload_record_file_path, 'r') as f: - try: - json_data = json.load(f) - except ValueError: - json_data = None - except IOError: + json_data = json.load(f) + except (IOError, ValueError): json_data = None return json_data def set_upload_record(self, file_name, key, data): - record_key = '{0}/{1}'.format(key, file_name) - if is_py2: - record_file_name = hashlib.md5(record_key).hexdigest() - else: - record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() - - upload_record_file_path = os.path.join(self.record_folder, record_file_name) + upload_record_file_path = self.__get_upload_record_file_path(file_name, key) with open(upload_record_file_path, 'w') as f: json.dump(data, f) def delete_upload_record(self, file_name, key): - record_key = '{0}/{1}'.format(key, file_name) - if is_py2: - record_file_name = hashlib.md5(record_key).hexdigest() - else: - record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() - - upload_record_file_path = os.path.join(self.record_folder, record_file_name) + upload_record_file_path = self.__get_upload_record_file_path(file_name, key) try: os.remove(upload_record_file_path) except OSError: diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 049cc876..5bcdf4e5 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -9,13 +9,23 @@ from qiniu.services.storage.uploaders import FormUploader, ResumeUploaderV1, ResumeUploaderV2 from qiniu.services.storage.upload_progress_recorder import UploadProgressRecorder -# for compact to old sdk +# for compat to old sdk (<= v7.11.1) from qiniu.services.storage.legacy import _Resume # noqa def put_data( - up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, - fname=None, hostscache_dir=None, metadata=None + up_token, + key, + data, + params=None, + mime_type='application/octet-stream', + check_crc=False, + progress_handler=None, + fname=None, + hostscache_dir=None, + metadata=None, + regions=None, + accelerate_uploading=False ): """上传二进制流到七牛 @@ -27,8 +37,11 @@ def put_data( mime_type: 上传数据的mimeType check_crc: 是否校验crc32 progress_handler: 上传进度 + fname: 文件名 hostscache_dir: host请求 缓存文件保存位置 metadata: 元数据 + regions: 区域信息，默认自动查询 + accelerate_uploading: 是否优先使用加速上传 Returns: 一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"} @@ -48,7 +61,8 @@ def put_data( crc = crc32(final_data) return _form_put( up_token, key, final_data, params, mime_type, - crc, hostscache_dir, progress_handler, fname, metadata=metadata + crc, hostscache_dir, progress_handler, fname, metadata=metadata, + regions=regions, accelerate_uploading=accelerate_uploading ) @@ -56,7 +70,8 @@ def put_file( up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None, - part_size=None, version=None, bucket_name=None, metadata=None + part_size=None, version=None, bucket_name=None, metadata=None, + regions=None, accelerate_uploading=False ): """上传文件到七牛 @@ -69,11 +84,14 @@ def put_file( check_crc: 是否校验crc32 progress_handler: 上传进度 upload_progress_recorder: 记录上传进度，用于断点续传 + keep_last_modified: 是否保留文件的最后修改时间 hostscache_dir: host请求 缓存文件保存位置 version: 分片上传版本 目前支持v1/v2版本 默认v1 part_size: 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB bucket_name: 分片上传v2字段 空间名称 metadata: 元数据信息 + regions: region信息 + accelerate_uploading: 是否开启加速上传 Returns: 一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"} @@ -90,14 +108,16 @@ def put_file( mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, modify_time=modify_time, keep_last_modified=keep_last_modified, - part_size=part_size, version=version, bucket_name=bucket_name, metadata=metadata + part_size=part_size, version=version, bucket_name=bucket_name, metadata=metadata, + regions=regions, accelerate_uploading=accelerate_uploading ) else: crc = file_crc32(file_path) ret, info = _form_put( up_token, key, input_stream, params, mime_type, crc, hostscache_dir, progress_handler, file_name, - modify_time=modify_time, keep_last_modified=keep_last_modified, metadata=metadata + modify_time=modify_time, keep_last_modified=keep_last_modified, metadata=metadata, + regions=regions, accelerate_uploading=accelerate_uploading ) return ret, info @@ -114,13 +134,17 @@ def _form_put( file_name=None, modify_time=None, keep_last_modified=False, - metadata=None + metadata=None, + regions=None, + accelerate_uploading=False ): bucket_name = Auth.get_bucket_name(up_token) uploader = FormUploader( bucket_name, progress_handler=progress_handler, - hosts_cache_dir=hostscache_dir + regions=regions, + accelerate_uploading=accelerate_uploading, + preferred_scheme=get_default('default_zone').scheme ) if modify_time and keep_last_modified: @@ -156,7 +180,9 @@ def put_stream( part_size=None, version='v1', bucket_name=None, - metadata=None + metadata=None, + regions=None, + accelerate_uploading=False ): if not bucket_name: bucket_name = Auth.get_bucket_name(up_token) @@ -172,7 +198,9 @@ def put_stream( bucket_name, progress_handler=progress_handler, upload_progress_recorder=upload_progress_recorder, - hosts_cache_dir=hostscache_dir + regions=regions, + accelerate_uploading=accelerate_uploading, + preferred_scheme=get_default('default_zone').scheme ) if modify_time and keep_last_modified: metadata['x-qn-meta-!Last-Modified'] = rfc_from_timestamp(modify_time) @@ -182,7 +210,9 @@ def put_stream( progress_handler=progress_handler, upload_progress_recorder=upload_progress_recorder, part_size=part_size, - hosts_cache_dir=hostscache_dir + regions=regions, + accelerate_uploading=accelerate_uploading, + preferred_scheme=get_default('default_zone').scheme ) else: raise ValueError('version only could be v1 or v2') diff --git a/qiniu/services/storage/uploaders/_default_retrier.py b/qiniu/services/storage/uploaders/_default_retrier.py new file mode 100644 index 00000000..6c15df1a --- /dev/null +++ b/qiniu/services/storage/uploaders/_default_retrier.py @@ -0,0 +1,216 @@ +from collections import namedtuple + +from qiniu.http.endpoints_retry_policy import EndpointsRetryPolicy +from qiniu.http.region import ServiceName +from qiniu.http.regions_retry_policy import RegionsRetryPolicy +from qiniu.retry.abc import RetryPolicy +from qiniu.retry import Retrier + + +_TokenExpiredRetryState = namedtuple( + 'TokenExpiredRetryState', + [ + 'retried_times', + 'upload_api_version' + ] +) + + +class TokenExpiredRetryPolicy(RetryPolicy): + def __init__( + self, + upload_api_version, + record_delete_handler, + record_exists_handler, + max_retry_times=1 + ): + """ + Parameters + ---------- + upload_api_version: str + record_delete_handler: callable + `() -> None` + record_exists_handler: callable + `() -> bool` + max_retry_times: int + """ + self.upload_api_version = upload_api_version + self.record_delete_handler = record_delete_handler + self.record_exists_handler = record_exists_handler + self.max_retry_times = max_retry_times + + def init_context(self, context): + """ + Parameters + ---------- + context: dict + """ + context[self] = _TokenExpiredRetryState( + retried_times=0, + upload_api_version=self.upload_api_version + ) + + def should_retry(self, attempt): + """ + Parameters + ---------- + attempt: qiniu.retry.Attempt + + Returns + ------- + bool + """ + state = attempt.context[self] + + if ( + state.retried_times >= self.max_retry_times or + not self.record_exists_handler() + ): + return False + + if not attempt.result: + return False + + _ret, resp = attempt.result + + if ( + state.upload_api_version == 'v1' and + resp.status_code == 701 + ): + return True + + if ( + state.upload_api_version == 'v2' and + resp.status_code == 612 + ): + return True + + return False + + def prepare_retry(self, attempt): + """ + Parameters + ---------- + attempt: qiniu.retry.Attempt + """ + state = attempt.context[self] + attempt.context[self] = state._replace(retried_times=state.retried_times + 1) + + if not self.record_exists_handler(): + return + + self.record_delete_handler() + + +class AccUnavailableRetryPolicy(RetryPolicy): + def __init__(self): + pass + + def init_context(self, context): + pass + + def should_retry(self, attempt): + """ + Parameters + ---------- + attempt: qiniu.retry.Attempt + + Returns + ------- + bool + """ + if not attempt.result: + return False + + region = attempt.context.get('region') + if not region: + return False + + if all( + not region.services[sn] + for sn in attempt.context.get('alternative_service_names') + ): + return False + + _ret, resp = attempt.result + + return resp.status_code == 400 and \ + 'transfer acceleration is not configured on this bucket' in resp.text_body + + def prepare_retry(self, attempt): + """ + Parameters + ---------- + attempt: qiniu.retry.Attempt + """ + endpoints = [] + while not endpoints: + if not attempt.context.get('alternative_service_names'): + raise RuntimeError('No alternative service available') + attempt.context['service_name'] = attempt.context.get('alternative_service_names').pop(0) + # shallow copy list + # change to `list.copy` for more readable when min version of python update to >= 3 + endpoints = attempt.context['region'].services.get(attempt.context['service_name'], [])[:] + attempt.context['alternative_endpoints'] = endpoints + attempt.context['endpoint'] = attempt.context['alternative_endpoints'].pop(0) + + +ProgressRecord = namedtuple( + 'ProgressRecorder', + [ + 'upload_api_version', + 'exists', + 'delete' + ] +) + + +def get_default_retrier( + regions_provider, + preferred_endpoints_provider=None, + progress_record=None, + accelerate_uploading=False +): + """ + Parameters + ---------- + regions_provider: Iterable[Region] + preferred_endpoints_provider: Iterable[Endpoint] + progress_record: ProgressRecord + accelerate_uploading: bool + + Returns + ------- + Retrier + """ + retry_policies = [] + upload_service_names = [ServiceName.UP] + handle_change_region = None + + if accelerate_uploading: + retry_policies.append(AccUnavailableRetryPolicy()) + upload_service_names.insert(0, ServiceName.UP_ACC) + + if progress_record: + retry_policies.append(TokenExpiredRetryPolicy( + upload_api_version=progress_record.upload_api_version, + record_delete_handler=progress_record.delete, + record_exists_handler=progress_record.exists + )) + + def _handle_change_region(_): + progress_record.delete() + + handle_change_region = _handle_change_region + + retry_policies += [ + EndpointsRetryPolicy(skip_init_context=True), + RegionsRetryPolicy( + regions_provider=regions_provider, + service_names=upload_service_names, + preferred_endpoints_provider=preferred_endpoints_provider, + on_change_region=handle_change_region + ) + ] + + return Retrier(retry_policies) diff --git a/qiniu/services/storage/uploaders/abc/resume_uploader_base.py b/qiniu/services/storage/uploaders/abc/resume_uploader_base.py index 06d827df..2965dee0 100644 --- a/qiniu/services/storage/uploaders/abc/resume_uploader_base.py +++ b/qiniu/services/storage/uploaders/abc/resume_uploader_base.py @@ -205,7 +205,7 @@ def complete_parts( up_token: str data_size: int context: any - kwargs: dictr + kwargs: dict Returns ------- diff --git a/qiniu/services/storage/uploaders/abc/uploader_base.py b/qiniu/services/storage/uploaders/abc/uploader_base.py index bb39fbfc..5907aa1c 100644 --- a/qiniu/services/storage/uploaders/abc/uploader_base.py +++ b/qiniu/services/storage/uploaders/abc/uploader_base.py @@ -1,9 +1,13 @@ import abc import qiniu.config as config +from qiniu.region import LegacyRegion +from qiniu.http.endpoint import Endpoint +from qiniu.http.regions_provider import get_default_regions_provider # type import from qiniu.auth import Auth # noqa +from qiniu.http.region import Region, ServiceName # noqa class UploaderBase(object): @@ -33,21 +37,37 @@ def __init__( kwargs The others arguments may be used by subclass. """ + # default bucket_name self.bucket_name = bucket_name # change the default when implements AuthProvider self.auth = kwargs.get('auth', None) - regions = kwargs.get('regions', []) - # remove the check when implement RegionsProvider - # if not regions: - # raise TypeError('You must provide the regions') + # regions config + regions = kwargs.get('regions', None) + if not regions: + regions = [] self.regions = regions - hosts_cache_dir = kwargs.get('hosts_cache_dir', None) - self.hosts_cache_dir = hosts_cache_dir + query_regions_endpoints = kwargs.get('query_regions_endpoints', None) + if not query_regions_endpoints: + query_regions_endpoints = [] + self.query_regions_endpoints = query_regions_endpoints + + self.preferred_scheme = kwargs.get('preferred_scheme', 'http') + + # change the default value to False when remove config.get_default('default_zone') + self.accelerate_uploading = kwargs.get('accelerate_uploading', None) - def get_up_token(self, **kwargs): + def get_up_token( + self, + bucket_name=None, + key=None, + expired=None, + policy=None, + strict_policy=None, + **_kwargs + ): """ Generate up token @@ -56,8 +76,11 @@ def get_up_token(self, **kwargs): bucket_name: str key: str expired: int + seconds policy: dict strict_policy: bool + _kwargs: dict + useless for now, just for compatibility Returns ------- @@ -66,64 +89,148 @@ def get_up_token(self, **kwargs): if not self.auth: raise ValueError('can not get up_token by auth not provided') - bucket_name = kwargs.get('bucket_name', self.bucket_name) + bucket_name = bucket_name if bucket_name else self.bucket_name kwargs_for_up_token = { - k: kwargs[k] - for k in [ - 'key', 'expires', 'policy', 'strict_policy' - ] if k in kwargs + k: v + for k, v in { + 'bucket': bucket_name, + 'key': key, + 'expired': expired, + 'policy': policy, + 'strict_policy': strict_policy + }.items() + if k } - up_token = self.auth.upload_token( - bucket=bucket_name, - **kwargs_for_up_token - ) + up_token = self.auth.upload_token(**kwargs_for_up_token) return up_token - def _get_regions(self): + def _get_regions_provider(self, access_key=None, bucket_name=None): + """ + Parameters + ---------- + access_key: str + bucket_name: str + + Returns + ------- + Iterable[Region or LegacyRegion] + """ if self.regions: return self.regions # handle compatibility for default_zone - default_region = config.get_default('default_zone') - if default_region: - self.regions = [default_region] + if config.is_customized_default('default_zone'): + return [config.get_default('default_zone')] - return self.regions + # handle compatibility for default_query_region_host + query_regions_endpoints = self.query_regions_endpoints + if not query_regions_endpoints: + query_region_host = config.get_default('default_query_region_host') + query_region_backup_hosts = config.get_default('default_query_region_backup_hosts') + query_regions_endpoints = [ + Endpoint.from_host(h) + for h in [query_region_host] + query_region_backup_hosts + ] + + # get regions from default regions provider + if not self.auth and not access_key: + raise ValueError('Must provide access_key and bucket_name if auth is unavailable.') + if not access_key: + access_key = self.auth.get_access_key() + if not bucket_name: + bucket_name = self.bucket_name - def _get_up_hosts(self, access_key=None): + return get_default_regions_provider( + query_endpoints_provider=query_regions_endpoints, + access_key=access_key, + bucket_name=bucket_name, + accelerate_uploading=self.accelerate_uploading, + preferred_scheme=self.preferred_scheme, + ) + + def _get_regions(self, access_key=None, bucket_name=None): """ - This will be deprecated when implement regions and endpoints + .. deprecated:: + This has been deprecated by implemented regions provider and endpoints + + Parameters + ---------- + access_key: str + bucket_name: str + + Returns + ------- + list[LegacyRegion] + """ + def get_legacy_region(r): + if isinstance(r, LegacyRegion): + return r + opts = { + 'scheme': self.preferred_scheme, + 'accelerate_uploading': self.accelerate_uploading + } + if r.services[ServiceName.UP]: + opts['up_host'] = r.services[ServiceName.UP][0].get_value(self.preferred_scheme) + if len(r.services[ServiceName.UP]) > 1: + opts['up_host_backup'] = [ + e.get_value(self.preferred_scheme) + for e in r.services[ServiceName.UP][1:] + ] + if r.services[ServiceName.IO]: + opts['io_host'] = r.services[ServiceName.IO][0].get_value(self.preferred_scheme) + if r.services[ServiceName.RS]: + opts['rs_host'] = r.services[ServiceName.RS][0].get_value(self.preferred_scheme) + if r.services[ServiceName.RSF]: + opts['rsf_host'] = r.services[ServiceName.RSF][0].get_value(self.preferred_scheme) + if r.services[ServiceName.API]: + opts['api_host'] = r.services[ServiceName.API][0].get_value(self.preferred_scheme) + result = LegacyRegion(**opts) + result.services = r.services + result.region_id = r.region_id + result.s3_region_id = r.s3_region_id + result.ttl = r.ttl + result.create_time = r.create_time + return result + + return [ + get_legacy_region(r) + for r in self._get_regions_provider(access_key, bucket_name) + ] + + def _get_up_hosts(self, access_key=None, bucket_name=None): + """ + get hosts of upload by access key or the first region + + .. deprecated:: + This has been deprecated by implemented regions provider and endpoints Returns ------- list[str] """ + if not bucket_name: + bucket_name = self.bucket_name if not self.auth and not access_key: raise ValueError('Must provide access_key if auth is unavailable.') if not access_key: access_key = self.auth.get_access_key() - regions = self._get_regions() + regions = self._get_regions(access_key, bucket_name) if not regions: raise ValueError('No region available.') # get up hosts in region - up_hosts = [ - regions[0].up_host, - regions[0].up_host_backup + service_names = [ServiceName.UP] + if self.accelerate_uploading: + service_names.insert(0, ServiceName.UP_ACC) + + return [ + e.get_value() + for sn in service_names + for e in regions[0].services[sn] ] - up_hosts = [h for h in up_hosts if h] - if up_hosts: - return up_hosts - - # this is correct, it does return hosts. bad function name by legacy - return regions[0].get_up_host( - ak=access_key, - bucket=self.bucket_name, - home_dir=self.hosts_cache_dir - ) @abc.abstractmethod def upload( diff --git a/qiniu/services/storage/uploaders/form_uploader.py b/qiniu/services/storage/uploaders/form_uploader.py index ea096f59..288a69da 100644 --- a/qiniu/services/storage/uploaders/form_uploader.py +++ b/qiniu/services/storage/uploaders/form_uploader.py @@ -7,7 +7,8 @@ from qiniu.auth import Auth from qiniu.http import qn_http_client -from qiniu.services.storage.uploaders.abc import UploaderBase +from .abc import UploaderBase +from ._default_retrier import get_default_retrier class FormUploader(UploaderBase): @@ -54,8 +55,16 @@ def upload( file_name: str custom_vars: dict kwargs - up_token, crc32_int - bucket_name, key, expired, policy, strict_policy for get up_token + up_token: str + crc32_int: int + bucket_name: str + is required if upload to another bucket + expired: int + option for generate up_token if not provide up_token. seconds + policy: dict + option for generate up_token if not provide up_token. details see `auth.Auth` + strict_policy: bool + option for generate up_token if not provide up_token Returns ------- @@ -63,14 +72,16 @@ def upload( resp: ResponseInfo """ # check and initial arguments - # up_token and up_hosts + # bucket_name + bucket_name = kwargs.get('bucket_name', self.bucket_name) + + # up_token up_token = kwargs.get('up_token', None) if not up_token: up_token = self.get_up_token(**kwargs) - up_hosts = self._get_up_hosts() + access_key = self.auth.get_access_key() else: access_key, _, _ = Auth.up_token_decode(up_token) - up_hosts = self._get_up_hosts(access_key) # crc32 from outside crc32_int = kwargs.get('crc32_int', None) @@ -84,6 +95,7 @@ def upload( if file_path and data: raise TypeError('Must provide only one of file_path or data.') + # useless for form upload if not modify_time: if file_path: modify_time = int(path.getmtime(file_path)) @@ -104,15 +116,17 @@ def upload( if not crc32_int: crc32_int = self.__get_crc32_int(data) fields = self.__get_form_fields( - up_hosts=up_hosts, up_token=up_token, key=key, crc32_int=crc32_int, custom_vars=custom_vars, metadata=metadata ) - ret, resp = self.__upload_data( - up_hosts=up_hosts, + ret, resp = self.__upload_data_with_retrier( + # retrier options + access_key=access_key, + bucket_name=bucket_name, + # upload_data options fields=fields, file_name=file_name, data=data, @@ -125,9 +139,45 @@ def upload( return ret, resp + def __upload_data_with_retrier( + self, + access_key, + bucket_name, + **upload_data_opts + ): + retrier = get_default_retrier( + regions_provider=self._get_regions_provider( + access_key=access_key, + bucket_name=bucket_name + ), + accelerate_uploading=self.accelerate_uploading + ) + data = upload_data_opts.get('data') + attempt = None + for attempt in retrier: + with attempt: + attempt.result = self.__upload_data( + up_endpoint=attempt.context.get('endpoint'), + **upload_data_opts + ) + ret, resp = attempt.result + if resp.ok() and ret: + return attempt.result + if ( + not is_seekable(data) or + not resp.need_retry() + ): + return attempt.result + data.seek(0) + + if attempt is None: + raise RuntimeError('Retrier is not working. attempt is None') + + return attempt.result + def __upload_data( self, - up_hosts, + up_endpoint, fields, file_name, data, @@ -137,7 +187,7 @@ def __upload_data( """ Parameters ---------- - up_hosts: list[str] + up_endpoint: Endpoint fields: dict file_name: str data: IOBase @@ -149,26 +199,17 @@ def __upload_data( ret: dict resp: ResponseInfo """ + req_url = up_endpoint.get_value(scheme=self.preferred_scheme) if not file_name or not file_name.strip(): file_name = 'file_name' - ret, resp = None, None - for up_host in up_hosts: - ret, resp = qn_http_client.post( - url=up_host, - data=fields, - files={ - 'file': (file_name, data, mime_type) - } - ) - if resp.ok() and ret: - return ret, resp - if ( - not is_seekable(data) or - not resp.need_retry() - ): - return ret, resp - data.seek(0) + ret, resp = qn_http_client.post( + url=req_url, + data=fields, + files={ + 'file': (file_name, data, mime_type) + } + ) return ret, resp def __get_form_fields( diff --git a/qiniu/services/storage/uploaders/resume_uploader_v1.py b/qiniu/services/storage/uploaders/resume_uploader_v1.py index 7f1ae89f..8a0e9cfb 100644 --- a/qiniu/services/storage/uploaders/resume_uploader_v1.py +++ b/qiniu/services/storage/uploaders/resume_uploader_v1.py @@ -1,5 +1,6 @@ import logging import math +import functools from collections import namedtuple from concurrent import futures from io import BytesIO @@ -11,10 +12,12 @@ from qiniu.compat import is_seekable from qiniu.auth import Auth from qiniu.http import qn_http_client, ResponseInfo +from qiniu.http.endpoint import Endpoint from qiniu.utils import b, io_crc32, urlsafe_base64_encode -from qiniu.services.storage.uploaders.abc import ResumeUploaderBase -from qiniu.services.storage.uploaders.io_chunked import IOChunked +from ._default_retrier import ProgressRecord, get_default_retrier +from .abc import ResumeUploaderBase +from .io_chunked import IOChunked class ResumeUploaderV1(ResumeUploaderBase): @@ -36,7 +39,7 @@ def _recover_from_record( _ResumeUploadV1Context """ if not isinstance(context, _ResumeUploadV1Context): - raise TypeError('context must be an instance of _ResumeUploadV1Context') + raise TypeError('"context" must be an instance of _ResumeUploadV1Context') if not self.upload_progress_recorder or not any([file_name, key]): return context @@ -54,7 +57,7 @@ def _recover_from_record( record_modify_time = record.get('modify_time', 0) record_context = record.get('contexts', []) - # compact with old sdk(<= v7.11.1) + # compat with old sdk(<= v7.11.1) if not record_up_hosts or not record_part_size: return context @@ -128,23 +131,20 @@ def _try_delete_record( self, file_name, key, - context, - resp + context=None, + resp=None ): """ Parameters ---------- - file_name: str - key: str + file_name: str or None + key: str or None context: _ResumeUploadV1Context resp: ResponseInfo """ if not self.upload_progress_recorder or not any([file_name, key]): return - if resp and context and not any([ - resp.ok(), - resp.status_code == 701 and context.resumed - ]): + if resp and not resp.ok(): return self.upload_progress_recorder.delete_upload_record(file_name, key) @@ -167,8 +167,46 @@ def _progress_handler( """ self._set_to_record(file_name, key, context) - if callable(self.progress_handler): + if not callable(self.progress_handler): + return + try: self.progress_handler(uploaded_size, total_size) + except Exception as err: + err.no_need_retry = True + raise err + + def _initial_context( + self, + key, + file_name, + modify_time + ): + """ + Parameters + ---------- + key: str + file_name: str + modify_time: float + + Returns + ------- + _ResumeUploadV1Context + """ + part_size = 4 * (1024 ** 2) + context = _ResumeUploadV1Context( + up_hosts=[], + part_size=part_size, + parts=[], + modify_time=modify_time, + resumed=False + ) + + # try to recover from record + return self._recover_from_record( + key=key, + file_name=file_name, + context=context + ) def initial_parts( self, @@ -176,23 +214,28 @@ def initial_parts( key, file_path=None, data=None, - modify_time=None, data_size=None, + modify_time=None, + part_size=None, file_name=None, + up_endpoint=None, **kwargs ): """ Parameters ---------- - up_token - key - file_path - data - modify_time - data_size - file_name + up_token: str + key: str + file_path: str or None + data: str or None + modify_time: float + data_size: int + part_size: None + useless for v1 by fixed part size + file_name: str + up_endpoint: Endpoint - kwargs + kwargs: dict Returns ------- @@ -217,28 +260,20 @@ def initial_parts( else: modify_time = int(time()) - part_size = 4 * (1024 ** 2) - - # -- initial context - context = _ResumeUploadV1Context( - up_hosts=[], - part_size=part_size, - parts=[], - modify_time=modify_time, - resumed=False - ) - - # try to recover from record if not file_name and file_path: file_name = path.basename(file_path) - context = self._recover_from_record( - file_name, - key, - context + + context = self._initial_context( + key=key, + file_name=file_name, + modify_time=modify_time ) - access_key, _, _ = Auth.up_token_decode(up_token) + if not context.up_hosts and up_endpoint: + context.up_hosts.extend([up_endpoint.get_value(self.preferred_scheme)]) + if not context.up_hosts: + access_key, _, _ = Auth.up_token_decode(up_token) context.up_hosts.extend(self._get_up_hosts(access_key)) return context, None @@ -285,7 +320,7 @@ def upload_parts( part, resp = None, None uploaded_size = context.part_size * len(context.parts) if math.ceil(data_size / context.part_size) in [p.part_no for p in context.parts]: - # if last part uploaded, should correct the uploaded size + # if last part has been uploaded, should correct the uploaded size uploaded_size += (data_size % context.part_size) - context.part_size lock = Lock() @@ -424,57 +459,95 @@ def complete_parts( ) return ret, resp - def upload( + def __upload_with_retrier( self, - key, - file_path=None, - data=None, - data_size=None, - modify_time=None, - - part_size=None, - mime_type=None, - metadata=None, - file_name=None, - custom_vars=None, - **kwargs + access_key, + bucket_name, + **upload_opts ): - """ + file_name = upload_opts.get('file_name', None) + key = upload_opts.get('key', None) + modify_time = upload_opts.get('modify_time', None) - Parameters - ---------- - key - file_path - data - data_size - modify_time + context = self._initial_context( + key=key, + file_name=file_name, + modify_time=modify_time + ) + preferred_endpoints = None + if context.up_hosts: + preferred_endpoints = [ + Endpoint.from_host(h) + for h in context.up_hosts + ] - part_size - mime_type - metadata - file_name - custom_vars + progress_record = None + if all([ + self.upload_progress_recorder, + file_name, + key + ]): + progress_record = ProgressRecord( + upload_api_version='v1', + exists=functools.partial( + self.upload_progress_recorder.has_upload_record, + file_name=file_name, + key=key + ), + delete=functools.partial( + self.upload_progress_recorder.delete_upload_record, + file_name=file_name, + key=key + ) + ) - kwargs: - up_token - bucket_name, expires, policy, strict_policy for generate `up_token` + retrier = get_default_retrier( + regions_provider=self._get_regions_provider( + access_key=access_key, + bucket_name=bucket_name + ), + preferred_endpoints_provider=preferred_endpoints, + progress_record=progress_record, + accelerate_uploading=self.accelerate_uploading, + ) - Returns - ------- - ret: dict - resp: ResponseInfo - """ - # part_size - if part_size: - logging.warning('ResumeUploader not support part_size. It is fixed to 4MB.') + data = upload_opts.get('data') + attempt = None + for attempt in retrier: + with attempt: + upload_opts['up_endpoint'] = attempt.context.get('endpoint') + attempt.result = self.__upload( + **upload_opts + ) + ret, resp = attempt.result + if resp.ok() and ret: + return attempt.result + if ( + not is_seekable(data) or + not resp.need_retry() + ): + return attempt.result + data.seek(0) - # up_token - up_token = kwargs.get('up_token', None) - if not up_token: - up_token = self.get_up_token(**kwargs) - if not file_name and file_path: - file_name = path.basename(file_path) + if attempt is None: + raise RuntimeError('Retrier is not working. attempt is None') + return attempt.result + + def __upload( + self, + up_token, + key, + file_path, + file_name, + data, + data_size, + modify_time, + mime_type, + custom_vars, + metadata, + up_endpoint + ): # initial_parts context, resp = self.initial_parts( up_token, @@ -484,6 +557,7 @@ def upload( data=data, data_size=data_size, modify_time=modify_time, + up_endpoint=up_endpoint ) # upload_parts @@ -526,23 +600,89 @@ def upload( metadata=metadata ) - # retry if expired. the record file will be deleted by complete_parts - if resp.status_code == 701 and context.resumed: - return self.upload( - key, - file_path=file_path, - data=data, - data_size=data_size, - modify_time=modify_time, + return ret, resp - mime_type=mime_type, - metadata=metadata, - file_name=file_name, - custom_vars=custom_vars, - **kwargs - ) + def upload( + self, + key, + file_path=None, + data=None, + data_size=None, + modify_time=None, - return ret, resp + part_size=None, + mime_type=None, + metadata=None, + file_name=None, + custom_vars=None, + **kwargs + ): + """ + + Parameters + ---------- + key + file_path + data + data_size + modify_time + + part_size + mime_type + metadata + file_name + custom_vars + + kwargs: + up_token: str + crc32_int: int + bucket_name: str + is required if upload to another bucket + expired: int + option for generate up_token if not provide up_token. seconds + policy: dict + option for generate up_token if not provide up_token. details see `auth.Auth` + strict_policy: bool + option for generate up_token if not provide up_token + + Returns + ------- + ret: dict + resp: ResponseInfo + """ + # part_size + if part_size: + logging.warning('ResumeUploader not support part_size. It is fixed to 4MB.') + + # up_token + up_token = kwargs.get('up_token', None) + if not up_token: + kwargs.setdefault('up_token', self.get_up_token(**kwargs)) + access_key = self.auth.get_access_key() + else: + access_key, _, _ = Auth.up_token_decode(up_token) + + # bucket_name + kwargs['bucket_name'] = Auth.get_bucket_name(up_token) + + # file_name + if not file_name and file_path: + file_name = path.basename(file_path) + + # upload + return self.__upload_with_retrier( + access_key=access_key, + key=key, + file_path=file_path, + data=data, + data_size=data_size, + modify_time=modify_time, + mime_type=mime_type, + metadata=metadata, + file_name=file_name, + custom_vars=custom_vars, + **kwargs + ) def __upload_part( self, @@ -623,11 +763,11 @@ def __get_mkfile_url( ---------- up_host: str data_size: int - mime_type: str - key: str - file_name: str - params: dict - metadata: dict + mime_type: str or None + key: str or None + file_name: str or None + params: dict or None + metadata: dict or None Returns ------- diff --git a/qiniu/services/storage/uploaders/resume_uploader_v2.py b/qiniu/services/storage/uploaders/resume_uploader_v2.py index db73b182..3e165e2f 100644 --- a/qiniu/services/storage/uploaders/resume_uploader_v2.py +++ b/qiniu/services/storage/uploaders/resume_uploader_v2.py @@ -1,3 +1,4 @@ +import functools import math from collections import namedtuple from concurrent import futures @@ -9,11 +10,13 @@ from qiniu.compat import is_seekable from qiniu.auth import Auth from qiniu.http import qn_http_client, ResponseInfo +from qiniu.http.endpoint import Endpoint from qiniu.utils import b, io_md5, urlsafe_base64_encode from qiniu.compat import json -from qiniu.services.storage.uploaders.abc import ResumeUploaderBase -from qiniu.services.storage.uploaders.io_chunked import IOChunked +from ._default_retrier import ProgressRecord, get_default_retrier +from .abc import ResumeUploaderBase +from .io_chunked import IOChunked class ResumeUploaderV2(ResumeUploaderBase): @@ -36,7 +39,7 @@ def _recover_from_record( _ResumeUploadV2Context """ if not isinstance(context, _ResumeUploadV2Context): - raise TypeError('context must be an instance of _ResumeUploadV2Context') + raise TypeError('"context" must be an instance of _ResumeUploadV2Context') if ( not self.upload_progress_recorder or @@ -59,7 +62,7 @@ def _recover_from_record( record_modify_time = record.get('modify_time', 0) record_etags = record.get('etags', []) - # compact with old sdk(<= v7.11.1) + # compat with old sdk(<= v7.11.1) if not record_up_hosts or not record_part_size: return context @@ -125,8 +128,8 @@ def _try_delete_record( self, file_name, key, - context, - resp + context=None, + resp=None ): """ Parameters @@ -138,10 +141,7 @@ def _try_delete_record( """ if not self.upload_progress_recorder or not any([file_name, key]): return - if resp and context and not any([ - resp.ok(), - resp.status_code == 612 and context.resumed - ]): + if resp and not resp.ok(): return self.upload_progress_recorder.delete_upload_record(file_name, key) @@ -163,8 +163,38 @@ def _progress_handler( total_size: int """ self._set_to_record(file_name, key, context) - if callable(self.progress_handler): + if not callable(self.progress_handler): + return + try: self.progress_handler(uploaded_size, total_size) + except Exception as err: + err.no_need_retry = True + raise err + + def _initial_context( + self, + key, + file_name, + modify_time, + part_size + ): + context = _ResumeUploadV2Context( + up_hosts=[], + upload_id='', + expired_at=0, + part_size=part_size, + parts=[], + modify_time=modify_time, + resumed=False + ) + + # try to recover from record + + return self._recover_from_record( + file_name, + key, + context + ) def initial_parts( self, @@ -176,6 +206,7 @@ def initial_parts( modify_time=None, part_size=None, file_name=None, + up_endpoint=None, **kwargs ): """ @@ -190,6 +221,7 @@ def initial_parts( modify_time: int part_size: int file_name: str + up_endpoint: Endpoint kwargs Returns @@ -218,23 +250,13 @@ def initial_parts( part_size = self.part_size # -- initial context - context = _ResumeUploadV2Context( - up_hosts=[], - upload_id='', - expired_at=0, - part_size=part_size, - parts=[], - modify_time=modify_time, - resumed=False - ) - - # try to recover from record if not file_name and file_path: file_name = path.basename(file_path) - context = self._recover_from_record( - file_name, - key, - context + context = self._initial_context( + key=key, + file_name=file_name, + modify_time=modify_time, + part_size=part_size ) if ( @@ -245,8 +267,11 @@ def initial_parts( return context, None # -- get a new upload id - access_key, _, _ = Auth.up_token_decode(up_token) + if not context.up_hosts and up_endpoint: + context.up_hosts.extend([up_endpoint.get_value(scheme=self.preferred_scheme)]) + if not context.up_hosts: + access_key, _, _ = Auth.up_token_decode(up_token) context.up_hosts.extend(self._get_up_hosts(access_key)) bucket_name = Auth.get_bucket_name(up_token) @@ -463,49 +488,100 @@ def complete_parts( ) return ret, resp - def upload( + def __upload_with_retrier( self, - key, - file_path=None, - data=None, - data_size=None, - - part_size=None, - modify_time=None, - mime_type=None, - metadata=None, - file_name=None, - custom_vars=None, - **kwargs + access_key, + bucket_name, + **upload_opts ): - """ - Parameters - ---------- - key: str - file_path: str - data: IOBase - data_size: int - part_size: int - modify_time: int - mime_type: str - metadata: dict - file_name: str - custom_vars: dict - kwargs - up_token - bucket_name, expires, policy, strict_policy for generate `up_token` + file_name = upload_opts.get('file_name', None) + key = upload_opts.get('key', None) + modify_time = upload_opts.get('modify_time', None) + part_size = upload_opts.get('part_size', self.part_size) - Returns - ------- + context = self._initial_context( + key=key, + file_name=file_name, + modify_time=modify_time, + part_size=part_size + ) + preferred_endpoints = None + if context.up_hosts: + preferred_endpoints = [ + Endpoint.from_host(h) + for h in context.up_hosts + ] - """ - # up_token - up_token = kwargs.get('up_token', None) - if not up_token: - up_token = self.get_up_token(**kwargs) - if not file_name and file_path: - file_name = path.basename(file_path) + progress_record = None + if all( + [ + self.upload_progress_recorder, + file_name, + key + ] + ): + progress_record = ProgressRecord( + upload_api_version='v1', + exists=functools.partial( + self.upload_progress_recorder.has_upload_record, + file_name=file_name, + key=key + ), + delete=functools.partial( + self.upload_progress_recorder.delete_upload_record, + file_name=file_name, + key=key + ) + ) + + retrier = get_default_retrier( + regions_provider=self._get_regions_provider( + access_key=access_key, + bucket_name=bucket_name + ), + preferred_endpoints_provider=preferred_endpoints, + progress_record=progress_record, + accelerate_uploading=self.accelerate_uploading + ) + + data = upload_opts.get('data') + attempt = None + for attempt in retrier: + with attempt: + upload_opts['up_endpoint'] = attempt.context.get('endpoint') + attempt.result = self.__upload( + **upload_opts + ) + ret, resp = attempt.result + if resp.ok() and ret: + return attempt.result + if ( + not is_seekable(data) or + not resp.need_retry() + ): + return attempt.result + data.seek(0) + + if attempt is None: + raise RuntimeError('Retrier is not working. attempt is None') + return attempt.result + + def __upload( + self, + up_token, + key, + file_path, + file_name, + data, + data_size, + part_size, + modify_time, + mime_type, + custom_vars, + metadata, + up_endpoint + ): # initial_parts context, resp = self.initial_parts( up_token, @@ -515,7 +591,8 @@ def upload( data=data, data_size=data_size, modify_time=modify_time, - part_size=part_size + part_size=part_size, + up_endpoint=up_endpoint ) if ( @@ -550,20 +627,6 @@ def upload( data.close() if resp and not resp.ok(): - if resp.status_code == 612 and context.resumed: - return self.upload( - key, - file_path=file_path, - data=data, - data_size=data_size, - modify_time=modify_time, - - mime_type=mime_type, - metadata=metadata, - file_name=file_name, - custom_vars=custom_vars, - **kwargs - ) return ret, resp # complete parts @@ -579,23 +642,78 @@ def upload( metadata=metadata ) - # retry if expired. the record file will be deleted by complete_parts - if resp.status_code == 612 and context.resumed: - return self.upload( - key, - file_path=file_path, - data=data, - data_size=data_size, - modify_time=modify_time, + return ret, resp - mime_type=mime_type, - metadata=metadata, - file_name=file_name, - custom_vars=custom_vars, - **kwargs - ) + def upload( + self, + key, + file_path=None, + data=None, + data_size=None, - return ret, resp + part_size=None, + modify_time=None, + mime_type=None, + metadata=None, + file_name=None, + custom_vars=None, + **kwargs + ): + """ + Parameters + ---------- + key: str + file_path: str + data: IOBase + data_size: int + part_size: int + modify_time: int + mime_type: str + metadata: dict + file_name: str + custom_vars: dict + kwargs + up_token: str + bucket_name: str, + expired: int, + policy: dict, + strict_policy: bool + + Returns + ------- + ret: dict + resp: ResponseInfo + """ + # up_token + up_token = kwargs.get('up_token', None) + if not up_token: + kwargs.setdefault('up_token', self.get_up_token(**kwargs)) + access_key = self.auth.get_access_key() + else: + access_key, _, _ = Auth.up_token_decode(up_token) + + # bucket_name + kwargs['bucket_name'] = Auth.get_bucket_name(up_token) + + # file_name + if not file_name and file_path: + file_name = path.basename(file_path) + + # upload + return self.__upload_with_retrier( + access_key=access_key, + key=key, + file_path=file_path, + file_name=file_name, + data=data, + data_size=data_size, + part_size=part_size, + modify_time=modify_time, + mime_type=mime_type, + custom_vars=custom_vars, + metadata=metadata, + **kwargs + ) def __get_url_for_upload( self, diff --git a/qiniu/utils.py b/qiniu/utils.py index fa750707..f8517e35 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -121,15 +121,21 @@ def _sha1(data): def etag_stream(input_stream): - """计算输入流的etag: + """ + 计算输入流的etag - etag规格参考 https://developer.qiniu.com/kodo/manual/1231/appendix#3 + .. deprecated:: + 在 v2 分片上传使用 4MB 以外分片大小时无法正常工作 - Args: - input_stream: 待计算etag的二进制流 + Parameters + ---------- + input_stream: io.IOBase + 支持随机访问的文件型对象 + + Returns + ------- + str - Returns: - 输入流的etag值 """ array = [_sha1(block) for block in _file_iter(input_stream, _BLOCK_SIZE)] if len(array) == 0: @@ -145,12 +151,21 @@ def etag_stream(input_stream): def etag(filePath): - """计算文件的etag: + """ + 计算文件的etag: - Args: - filePath: 待计算etag的文件路径 + .. deprecated:: + 在 v2 分片上传使用 4MB 以外分片大小时无法正常工作 - Returns: + + Parameters + ---------- + filePath: str + 待计算 etag 的文件路径 + + Returns + ------- + str 输入文件的etag值 """ with open(filePath, 'rb') as f: diff --git a/qiniu/zone.py b/qiniu/zone.py index acb34d00..0a213eaa 100644 --- a/qiniu/zone.py +++ b/qiniu/zone.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from qiniu.region import Region +from qiniu.region import LegacyRegion -class Zone(Region): +class Zone(LegacyRegion): pass diff --git a/setup.py b/setup.py index 5ddc9c4d..cf97eae2 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,8 @@ def find_version(*file_paths): install_requires=[ 'requests; python_version >= "3.7"', 'requests<2.28; python_version < "3.7"', - 'futures; python_version == "2.7"' + 'futures; python_version == "2.7"', + 'enum34; python_version == "2.7"' ], extras_require={ 'dev': [ diff --git a/test_qiniu.py b/test_qiniu.py index c048d4c3..2b71aa22 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -45,11 +45,6 @@ hostscache_dir = None -dummy_access_key = 'abcdefghklmnopq' -dummy_secret_key = '1234567890' -dummy_auth = Auth(dummy_access_key, dummy_secret_key) - - def rand_string(length): lib = string.ascii_uppercase return ''.join([random.choice(lib) for i in range(0, length)]) @@ -193,172 +188,6 @@ def test_decode_entry(self): assert key == c.get('expect').get('key'), c.get('msg') -class AuthTestCase(unittest.TestCase): - def test_token(self): - token = dummy_auth.token('test') - assert token == 'abcdefghklmnopq:mSNBTR7uS2crJsyFr2Amwv1LaYg=' - - def test_token_with_data(self): - token = dummy_auth.token_with_data('test') - assert token == 'abcdefghklmnopq:-jP8eEV9v48MkYiBGs81aDxl60E=:dGVzdA==' - - def test_noKey(self): - with pytest.raises(ValueError): - Auth(None, None).token('nokey') - with pytest.raises(ValueError): - Auth('', '').token('nokey') - - def test_token_of_request(self): - token = dummy_auth.token_of_request('https://www.qiniu.com?go=1', 'test', '') - assert token == 'abcdefghklmnopq:cFyRVoWrE3IugPIMP5YJFTO-O-Y=' - token = dummy_auth.token_of_request('https://www.qiniu.com?go=1', 'test', 'application/x-www-form-urlencoded') - assert token == 'abcdefghklmnopq:svWRNcacOE-YMsc70nuIYdaa1e4=' - - def test_QiniuMacRequestsAuth(self): - auth = QiniuMacAuth("ak", "sk") - test_cases = [ - { - "method": "GET", - "host": None, - "url": "", - "qheaders": { - "X-Qiniu-": "a", - "X-Qiniu": "b", - "Content-Type": "application/x-www-form-urlencoded", - }, - "content_type": "application/x-www-form-urlencoded", - "body": "{\"name\": \"test\"}", - "except_sign_token": "ak:0i1vKClRDWFyNkcTFzwcE7PzX74=", - }, - { - "method": "GET", - "host": None, - "url": "", - "qheaders": { - "Content-Type": "application/json", - }, - "content_type": "application/json", - "body": "{\"name\": \"test\"}", - "except_sign_token": "ak:K1DI0goT05yhGizDFE5FiPJxAj4=", - }, - { - "method": "POST", - "host": None, - "url": "", - "qheaders": { - "Content-Type": "application/json", - "X-Qiniu": "b", - }, - "content_type": "application/json", - "body": "{\"name\": \"test\"}", - "except_sign_token": "ak:0ujEjW_vLRZxebsveBgqa3JyQ-w=", - }, - { - "method": "GET", - "host": "upload.qiniup.com", - "url": "http://upload.qiniup.com", - "qheaders": { - "X-Qiniu-": "a", - "X-Qiniu": "b", - "Content-Type": "application/x-www-form-urlencoded", - }, - "content_type": "application/x-www-form-urlencoded", - "body": "{\"name\": \"test\"}", - "except_sign_token": "ak:GShw5NitGmd5TLoo38nDkGUofRw=", - }, - { - "method": "GET", - "host": "upload.qiniup.com", - "url": "http://upload.qiniup.com", - "qheaders": { - "Content-Type": "application/json", - "X-Qiniu-Bbb": "BBB", - "X-Qiniu-Aaa": "DDD", - "X-Qiniu-": "a", - "X-Qiniu": "b", - }, - "content_type": "application/json", - "body": "{\"name\": \"test\"}", - "except_sign_token": "ak:DhNA1UCaBqSHCsQjMOLRfVn63GQ=", - }, - { - "method": "GET", - "host": "upload.qiniup.com", - "url": "http://upload.qiniup.com", - "qheaders": { - "Content-Type": "application/x-www-form-urlencoded", - "X-Qiniu-Bbb": "BBB", - "X-Qiniu-Aaa": "DDD", - "X-Qiniu-": "a", - "X-Qiniu": "b", - }, - "content_type": "application/x-www-form-urlencoded", - "body": "name=test&language=go", - "except_sign_token": "ak:KUAhrYh32P9bv0COD8ugZjDCmII=", - }, - { - "method": "GET", - "host": "upload.qiniup.com", - "url": "http://upload.qiniup.com", - "qheaders": { - "Content-Type": "application/x-www-form-urlencoded", - "X-Qiniu-Bbb": "BBB", - "X-Qiniu-Aaa": "DDD", - }, - "content_type": "application/x-www-form-urlencoded", - "body": "name=test&language=go", - "except_sign_token": "ak:KUAhrYh32P9bv0COD8ugZjDCmII=", - }, - { - "method": "GET", - "host": "upload.qiniup.com", - "url": "http://upload.qiniup.com/mkfile/sdf.jpg", - "qheaders": { - "Content-Type": "application/x-www-form-urlencoded", - "X-Qiniu-Bbb": "BBB", - "X-Qiniu-Aaa": "DDD", - "X-Qiniu-": "a", - "X-Qiniu": "b", - }, - "content_type": "application/x-www-form-urlencoded", - "body": "name=test&language=go", - "except_sign_token": "ak:fkRck5_LeyfwdkyyLk-hyNwGKac=", - }, - { - "method": "GET", - "host": "upload.qiniup.com", - "url": "http://upload.qiniup.com/mkfile/sdf.jpg?s=er3&df", - "qheaders": { - "Content-Type": "application/x-www-form-urlencoded", - "X-Qiniu-Bbb": "BBB", - "X-Qiniu-Aaa": "DDD", - "X-Qiniu-": "a", - "X-Qiniu": "b", - }, - "content_type": "application/x-www-form-urlencoded", - "body": "name=test&language=go", - "except_sign_token": "ak:PUFPWsEUIpk_dzUvvxTTmwhp3p4=", - }, - ] - - for test_case in test_cases: - sign_token = auth.token_of_request( - method=test_case["method"], - host=test_case["host"], - url=test_case["url"], - qheaders=auth.qiniu_headers(test_case["qheaders"]), - content_type=test_case["content_type"], - body=test_case["body"], - ) - assert sign_token == test_case["except_sign_token"] - - def test_verify_callback(self): - body = 'name=sunflower.jpg&hash=Fn6qeQi4VDLQ347NiRm-RlQx_4O2&location=Shanghai&price=1500.00&uid=123' - url = 'test.qiniu.com/callback' - ok = dummy_auth.verify_callback('QBox abcdefghklmnopq:ZWyeM5ljWMRFwuPTPOwQ4RwSto4=', url, body) - assert ok - - class BucketTestCase(unittest.TestCase): q = Auth(access_key, secret_key) bucket = BucketManager(q) @@ -368,8 +197,7 @@ def test_list(self): assert eof is False assert len(ret.get('items')) == 4 ret, eof, info = self.bucket.list(bucket_name, limit=1000) - print(ret, eof, info) - assert info.status_code == 200 + assert info.status_code == 200, info def test_buckets(self): ret, info = self.bucket.buckets() @@ -606,18 +434,6 @@ def test_private_url(self): assert r.status_code == 200 -class MediaTestCase(unittest.TestCase): - def test_pfop(self): - q = Auth(access_key, secret_key) - pfop = PersistentFop(q, 'testres', 'sdktest') - op = op_save('avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', 'pythonsdk', 'pfoptest') - ops = [] - ops.append(op) - ret, info = pfop.execute('sintel_trailer.mp4', ops, 1) - print(info) - assert ret['persistentId'] is not None - - class EtagTestCase(unittest.TestCase): def test_zero_size(self): open("x", 'a').close() @@ -648,80 +464,6 @@ def test_get_domain(self): assert info.status_code == 200 -class RegionTestCase(unittest.TestCase): - test_rs_host = 'test.region.compatible.config.rs' - test_rsf_host = 'test.region.compatible.config.rsf' - - @staticmethod - def restore_hosts(): - set_default( - default_rs_host=qiniu.config.RS_HOST, - default_rsf_host=qiniu.config.RSF_HOST, - default_uc_host=qiniu.config.UC_HOST, - default_query_region_host=qiniu.config.QUERY_REGION_HOST, - default_query_region_backup_hosts=[ - 'uc.qbox.me', - 'api.qiniu.com' - ] - ) - qiniu.config._is_customized_default['default_rs_host'] = False - qiniu.config._is_customized_default['default_rsf_host'] = False - qiniu.config._is_customized_default['default_uc_host'] = False - qiniu.config._is_customized_default['default_query_region_host'] = False - qiniu.config._is_customized_default['default_query_region_backup_hosts'] = False - - def test_config_compatible(self): - try: - set_default(default_rs_host=self.test_rs_host) - set_default(default_rsf_host=self.test_rsf_host) - zone = Zone() - assert zone.get_rs_host("mock_ak", "mock_bucket") == self.test_rs_host - assert zone.get_rsf_host("mock_ak", "mock_bucket") == self.test_rsf_host - finally: - RegionTestCase.restore_hosts() - - def test_query_region_with_custom_domain(self): - try: - set_default( - default_query_region_host='https://fake-uc.phpsdk.qiniu.com' - ) - zone = Zone() - data = zone.bucket_hosts(access_key, bucket_name) - assert data != 'null' - finally: - RegionTestCase.restore_hosts() - - def test_query_region_with_backup_domains(self): - try: - set_default( - default_query_region_host='https://fake-uc.phpsdk.qiniu.com', - default_query_region_backup_hosts=[ - 'unavailable-uc.phpsdk.qiniu.com', - 'uc.qbox.me' - ] - ) - zone = Zone() - data = zone.bucket_hosts(access_key, bucket_name) - assert data != 'null' - finally: - RegionTestCase.restore_hosts() - - def test_query_region_with_uc_and_backup_domains(self): - try: - set_default( - default_uc_host='https://fake-uc.phpsdk.qiniu.com', - default_query_region_backup_hosts=[ - 'unavailable-uc.phpsdk.qiniu.com', - 'uc.qbox.me' - ] - ) - zone = Zone() - data = zone.bucket_hosts(access_key, bucket_name) - assert data != 'null' - finally: - RegionTestCase.restore_hosts() - - class ReadWithoutSeek(object): def __init__(self, str): self.str = str diff --git a/tests/cases/conftest.py b/tests/cases/conftest.py index dcf802f7..13f41618 100644 --- a/tests/cases/conftest.py +++ b/tests/cases/conftest.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- import os +import random +import string import pytest from qiniu import config as qn_config -from qiniu import region from qiniu import Auth @@ -23,6 +24,16 @@ def bucket_name(): yield os.getenv('QINIU_TEST_BUCKET') +@pytest.fixture(scope='session') +def no_acc_bucket_name(): + yield os.getenv('QINIU_TEST_NO_ACC_BUCKET') + + +@pytest.fixture(scope='session') +def download_domain(): + yield os.getenv('QINIU_TEST_DOMAIN') + + @pytest.fixture(scope='session') def upload_callback_url(): yield os.getenv('QINIU_UPLOAD_CALLBACK_URL') @@ -48,11 +59,12 @@ def set_conf_default(request): qn_config.set_default(**request.param) yield qn_config._config = { - 'default_zone': region.Region(), + 'default_zone': None, 'default_rs_host': qn_config.RS_HOST, 'default_rsf_host': qn_config.RSF_HOST, 'default_api_host': qn_config.API_HOST, 'default_uc_host': qn_config.UC_HOST, + 'default_uc_backup_hosts': qn_config.UC_BACKUP_HOSTS, 'default_query_region_host': qn_config.QUERY_REGION_HOST, 'default_query_region_backup_hosts': [ 'uc.qbox.me', @@ -79,3 +91,30 @@ def set_conf_default(request): 'connection_pool': False, 'default_upload_threshold': False } + + +@pytest.fixture(scope='session') +def rand_string(): + def _rand_string(length): + # use random.choices when min version of python >= 3.6 + return ''.join( + random.choice(string.ascii_letters + string.digits) + for _ in range(length) + ) + yield _rand_string + + +class Ref: + """ + python2 not support nonlocal keyword + """ + def __init__(self, value=None): + self.value = value + + +@pytest.fixture(scope='session') +def use_ref(): + def _use_ref(value): + return Ref(value) + + yield _use_ref diff --git a/tests/cases/test_auth.py b/tests/cases/test_auth.py new file mode 100644 index 00000000..d294f018 --- /dev/null +++ b/tests/cases/test_auth.py @@ -0,0 +1,212 @@ +import pytest + +from qiniu.auth import Auth, QiniuMacAuth + + +@pytest.fixture(scope="module") +def dummy_auth(): + dummy_access_key = 'abcdefghklmnopq' + dummy_secret_key = '1234567890' + yield Auth(dummy_access_key, dummy_secret_key) + + +class TestAuth: + def test_token(self, dummy_auth): + token = dummy_auth.token('test') + assert token == 'abcdefghklmnopq:mSNBTR7uS2crJsyFr2Amwv1LaYg=' + + def test_token_with_data(self, dummy_auth): + token = dummy_auth.token_with_data('test') + assert token == 'abcdefghklmnopq:-jP8eEV9v48MkYiBGs81aDxl60E=:dGVzdA==' + + def test_nokey(self, dummy_auth): + with pytest.raises(ValueError): + Auth(None, None).token('nokey') + with pytest.raises(ValueError): + Auth('', '').token('nokey') + + def test_token_of_request(self, dummy_auth): + token = dummy_auth.token_of_request('https://www.qiniu.com?go=1', 'test', '') + assert token == 'abcdefghklmnopq:cFyRVoWrE3IugPIMP5YJFTO-O-Y=' + token = dummy_auth.token_of_request('https://www.qiniu.com?go=1', 'test', 'application/x-www-form-urlencoded') + assert token == 'abcdefghklmnopq:svWRNcacOE-YMsc70nuIYdaa1e4=' + + @pytest.mark.parametrize( + 'opts, except_token', + [ + ( + { + "method": "GET", + "host": None, + "url": "", + "qheaders": { + "X-Qiniu-": "a", + "X-Qiniu": "b", + "Content-Type": "application/x-www-form-urlencoded", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "{\"name\": \"test\"}", + }, + "ak:0i1vKClRDWFyNkcTFzwcE7PzX74=", + ), + ( + { + "method": "GET", + "host": None, + "url": "", + "qheaders": { + "Content-Type": "application/json", + }, + "content_type": "application/json", + "body": "{\"name\": \"test\"}", + }, + "ak:K1DI0goT05yhGizDFE5FiPJxAj4=", + ), + ( + { + "method": "POST", + "host": None, + "url": "", + "qheaders": { + "Content-Type": "application/json", + "X-Qiniu": "b", + }, + "content_type": "application/json", + "body": "{\"name\": \"test\"}", + }, + "ak:0ujEjW_vLRZxebsveBgqa3JyQ-w=", + ), + ( + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com", + "qheaders": { + "X-Qiniu-": "a", + "X-Qiniu": "b", + "Content-Type": "application/x-www-form-urlencoded", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "{\"name\": \"test\"}", + }, + "ak:GShw5NitGmd5TLoo38nDkGUofRw=", + ), + ( + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com", + "qheaders": { + "Content-Type": "application/json", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + "X-Qiniu-": "a", + "X-Qiniu": "b", + }, + "content_type": "application/json", + "body": "{\"name\": \"test\"}", + }, + "ak:DhNA1UCaBqSHCsQjMOLRfVn63GQ=", + ), + ( + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com", + "qheaders": { + "Content-Type": "application/x-www-form-urlencoded", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + "X-Qiniu-": "a", + "X-Qiniu": "b", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "name=test&language=go", + }, + "ak:KUAhrYh32P9bv0COD8ugZjDCmII=", + ), + ( + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com", + "qheaders": { + "Content-Type": "application/x-www-form-urlencoded", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "name=test&language=go", + }, + "ak:KUAhrYh32P9bv0COD8ugZjDCmII=", + ), + ( + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com/mkfile/sdf.jpg", + "qheaders": { + "Content-Type": "application/x-www-form-urlencoded", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + "X-Qiniu-": "a", + "X-Qiniu": "b", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "name=test&language=go", + }, + "ak:fkRck5_LeyfwdkyyLk-hyNwGKac=", + ), + ( + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com/mkfile/sdf.jpg?s=er3&df", + "qheaders": { + "Content-Type": "application/x-www-form-urlencoded", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + "X-Qiniu-": "a", + "X-Qiniu": "b", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "name=test&language=go", + }, + "ak:PUFPWsEUIpk_dzUvvxTTmwhp3p4=", + ) + ] + ) + def test_qiniu_mac_requests_auth(self, dummy_auth, opts, except_token): + auth = QiniuMacAuth("ak", "sk") + + sign_token = auth.token_of_request( + method=opts["method"], + host=opts["host"], + url=opts["url"], + qheaders=auth.qiniu_headers(opts["qheaders"]), + content_type=opts["content_type"], + body=opts["body"], + ) + assert sign_token == except_token + + def test_qbox_verify_callback(self, dummy_auth): + ok = dummy_auth.verify_callback( + 'QBox abcdefghklmnopq:T7F-SjxX7X2zI4Fc1vANiNt1AUE=', + url='https://test.qiniu.com/callback', + body='name=sunflower.jpg&hash=Fn6qeQi4VDLQ347NiRm-RlQx_4O2&location=Shanghai&price=1500.00&uid=123' + ) + assert ok + + def test_qiniu_verify_token(self, dummy_auth): + ok = dummy_auth.verify_callback( + 'Qiniu abcdefghklmnopq:ZqS7EZuAKrhZaEIxqNGxDJi41IQ=', + url='https://test.qiniu.com/callback', + body='name=sunflower.jpg&hash=Fn6qeQi4VDLQ347NiRm-RlQx_4O2&location=Shanghai&price=1500.00&uid=123', + content_type='application/x-www-form-urlencoded', + method='GET', + headers={ + 'X-Qiniu-Bbb': 'BBB', + } + ) + assert ok + diff --git a/tests/cases/test_http/test_endpoint.py b/tests/cases/test_http/test_endpoint.py new file mode 100644 index 00000000..9bfbeb66 --- /dev/null +++ b/tests/cases/test_http/test_endpoint.py @@ -0,0 +1,27 @@ +from qiniu.http.endpoint import Endpoint + + +class TestEndpoint: + def test_endpoint_with_default_scheme(self): + endpoint = Endpoint('uc.python-sdk.qiniu.com') + assert endpoint.get_value() == 'https://uc.python-sdk.qiniu.com' + + def test_endpoint_with_custom_scheme(self): + endpoint = Endpoint('uc.python-sdk.qiniu.com', default_scheme='http') + assert endpoint.get_value() == 'http://uc.python-sdk.qiniu.com' + + def test_endpoint_with_get_value_with_custom_scheme(self): + endpoint = Endpoint('uc.python-sdk.qiniu.com', default_scheme='http') + assert endpoint.get_value('https') == 'https://uc.python-sdk.qiniu.com' + + def test_create_endpoint_from_host_with_scheme(self): + endpoint = Endpoint.from_host('http://uc.python-sdk.qiniu.com') + assert endpoint.default_scheme == 'http' + assert endpoint.get_value() == 'http://uc.python-sdk.qiniu.com' + + def test_clone_endpoint(self): + endpoint = Endpoint('uc.python-sdk.qiniu.com') + another_endpoint = endpoint.clone() + another_endpoint.host = 'another-uc.python-sdk.qiniu.com' + assert endpoint.get_value() == 'https://uc.python-sdk.qiniu.com' + assert another_endpoint.get_value() == 'https://another-uc.python-sdk.qiniu.com' diff --git a/tests/cases/test_http/test_endpoints_retry_policy.py b/tests/cases/test_http/test_endpoints_retry_policy.py new file mode 100644 index 00000000..a8135ca2 --- /dev/null +++ b/tests/cases/test_http/test_endpoints_retry_policy.py @@ -0,0 +1,75 @@ +import pytest + +from qiniu.http.endpoint import Endpoint +from qiniu.http.endpoints_retry_policy import EndpointsRetryPolicy +from qiniu.retry.attempt import Attempt + + +@pytest.fixture(scope='function') +def mocked_endpoints_provider(): + yield [ + Endpoint('a'), + Endpoint('b'), + Endpoint('c') + ] + + +class TestEndpointsRetryPolicy: + def test_init_context(self, mocked_endpoints_provider): + endpoints_retry_policy = EndpointsRetryPolicy( + endpoints_provider=mocked_endpoints_provider + ) + + mocked_context = {} + endpoints_retry_policy.init_context(mocked_context) + + assert mocked_context['endpoint'].get_value() == mocked_endpoints_provider[0].get_value() + assert [ + e.get_value() + for e in mocked_context['alternative_endpoints'] + ] == [ + e.get_value() + for e in mocked_endpoints_provider[1:] + ] + + def test_should_retry(self, mocked_endpoints_provider): + mocked_attempt = Attempt() + + endpoints_retry_policy = EndpointsRetryPolicy( + endpoints_provider=mocked_endpoints_provider + ) + endpoints_retry_policy.init_context(mocked_attempt.context) + assert endpoints_retry_policy.should_retry(mocked_attempt) + + def test_prepare_retry(self, mocked_endpoints_provider): + mocked_attempt = Attempt() + + endpoints_retry_policy = EndpointsRetryPolicy( + endpoints_provider=mocked_endpoints_provider + ) + endpoints_retry_policy.init_context(mocked_attempt.context) + + actual_tried_endpoints = [ + mocked_attempt.context.get('endpoint') + ] + while endpoints_retry_policy.should_retry(mocked_attempt): + endpoints_retry_policy.prepare_retry(mocked_attempt) + actual_tried_endpoints.append(mocked_attempt.context.get('endpoint')) + + assert [ + e.get_value() for e in actual_tried_endpoints + ] == [ + e.get_value() for e in mocked_endpoints_provider + ] + + def test_skip_init_context(self, mocked_endpoints_provider): + endpoints_retry_policy = EndpointsRetryPolicy( + endpoints_provider=mocked_endpoints_provider, + skip_init_context=True + ) + + mocked_context = {} + endpoints_retry_policy.init_context(mocked_context) + + assert not mocked_context.get('endpoint') + assert not mocked_context.get('alternative_endpoints') diff --git a/tests/cases/test_http/test_qiniu_conf.py b/tests/cases/test_http/test_qiniu_conf.py index 3ce4c5a0..29c6fd05 100644 --- a/tests/cases/test_http/test_qiniu_conf.py +++ b/tests/cases/test_http/test_qiniu_conf.py @@ -44,7 +44,7 @@ def reset_session(): yield -class TestQiniuConf: +class TestQiniuConfWithHTTP: @pytest.mark.usefixtures('reset_session') @pytest.mark.parametrize( 'set_conf_default', diff --git a/tests/cases/test_http/test_region.py b/tests/cases/test_http/test_region.py new file mode 100644 index 00000000..a66b16c9 --- /dev/null +++ b/tests/cases/test_http/test_region.py @@ -0,0 +1,186 @@ +from datetime import datetime, timedelta +from itertools import chain + +from qiniu.http.endpoint import Endpoint +from qiniu.http.region import Region, ServiceName + + +class TestRegion: + def test_default_options(self): + region = Region('z0') + assert region.region_id == 'z0' + assert region.s3_region_id == 'z0' + assert all(k in region.services for k in ServiceName) + assert datetime.now() - region.create_time < timedelta(seconds=1) + assert region.ttl == 86400 + assert region.is_live + + def test_custom_options(self): + region = Region( + region_id='z0', + s3_region_id='s3-z0', + services={ + ServiceName.UP: [ + Endpoint('uc.python-sdk.qiniu.com') + ], + 'custom-service': [ + Endpoint('custom-service.python-sdk.qiniu.com') + ] + }, + create_time=datetime.now() - timedelta(days=1), + ttl=3600 + ) + assert region.region_id == 'z0' + assert region.s3_region_id == 's3-z0' + assert all( + k in region.services + for k in chain(ServiceName, ['custom-service']) + ) + assert datetime.now() - region.create_time > timedelta(days=1) + assert region.ttl == 3600 + assert not region.is_live + + def test_from_region_id(self): + region = Region.from_region_id('z0') + + expect_services_endpoint_value = { + ServiceName.UC: [ + 'https://uc.qiniuapi.com' + ], + ServiceName.UP: [ + 'https://upload.qiniup.com', + 'https://up.qiniup.com' + ], + ServiceName.UP_ACC: [], + ServiceName.IO: [ + 'https://iovip.qiniuio.com', + ], + ServiceName.RS: [ + 'https://rs-z0.qiniuapi.com', + ], + ServiceName.RSF: [ + 'https://rsf-z0.qiniuapi.com', + ], + ServiceName.API: [ + 'https://api-z0.qiniuapi.com', + ] + } + + assert region.region_id == 'z0' + assert region.s3_region_id == 'z0' + + assert { + k: [ + e.get_value() + for e in v + ] + for k, v in region.services.items() + } == expect_services_endpoint_value + + assert datetime.now() - region.create_time < timedelta(seconds=1) + assert region.ttl == 86400 + assert region.is_live + + def test_from_region_id_with_custom_options(self): + preferred_scheme = 'http' + custom_service_endpoint = Endpoint('custom-service.python-sdk.qiniu.com') + region_z1 = Region.from_region_id( + 'z1', + s3_region_id='s3-z1', + ttl=-1, + create_time=datetime.fromtimestamp(0), + extended_services= { + 'custom-service': [ + custom_service_endpoint + ] + }, + preferred_scheme=preferred_scheme + ) + + expect_services_endpoint_value = { + ServiceName.UC: [ + preferred_scheme + '://uc.qiniuapi.com' + ], + ServiceName.UP: [ + preferred_scheme + '://upload-z1.qiniup.com', + preferred_scheme + '://up-z1.qiniup.com' + ], + ServiceName.UP_ACC: [], + ServiceName.IO: [ + preferred_scheme + '://iovip-z1.qiniuio.com', + ], + ServiceName.RS: [ + preferred_scheme + '://rs-z1.qiniuapi.com', + ], + ServiceName.RSF: [ + preferred_scheme + '://rsf-z1.qiniuapi.com', + ], + ServiceName.API: [ + preferred_scheme + '://api-z1.qiniuapi.com', + ], + 'custom-service': [ + custom_service_endpoint.get_value() + ] + } + + assert region_z1.region_id == 'z1' + assert region_z1.s3_region_id == 's3-z1' + assert { + k: [ + e.get_value() + for e in v + ] + for k, v in region_z1.services.items() + } == expect_services_endpoint_value + assert region_z1.ttl == -1 + assert region_z1.create_time == datetime.fromtimestamp(0) + assert region_z1.is_live + + def test_clone(self): + region = Region.from_region_id('z0') + cloned_region = region.clone() + cloned_region.region_id = 'another' + cloned_region.services[ServiceName.UP][0].host = 'another-uc.qiniuapi.com' + assert region.region_id == 'z0' + assert region.services[ServiceName.UP][0].get_value() == 'https://upload.qiniup.com' + assert cloned_region.services[ServiceName.UP][0].get_value() == 'https://another-uc.qiniuapi.com' + + def test_merge(self): + r1 = Region.from_region_id('z0') + r2 = Region( + region_id='r2', + s3_region_id='s3-r2', + services={ + ServiceName.UP: [ + Endpoint('up-r2.python-sdk.qiniu.com') + ], + 'custom-service': [ + Endpoint('custom-service-r2.python-sdk.qiniu.com') + ] + }, + create_time=datetime.now() - timedelta(days=1), + ttl=3600 + ) + + merged_region = Region.merge(r1, r2) + + assert merged_region.region_id == r1.region_id + assert merged_region.s3_region_id == r1.s3_region_id + assert merged_region.create_time == r1.create_time + assert merged_region.ttl == r1.ttl + + assert all( + k in merged_region.services + for k in [ + ServiceName.UP, + 'custom-service' + ] + ), merged_region.services.keys() + + for k, v in merged_region.services.items(): + if k == ServiceName.UP: + assert v == list(chain(r1.services[k], r2.services[k])) + elif k == 'custom-service': + assert v == r2.services[k] + else: + assert v == r1.services[k] diff --git a/tests/cases/test_http/test_regions_provider.py b/tests/cases/test_http/test_regions_provider.py new file mode 100644 index 00000000..163f19d2 --- /dev/null +++ b/tests/cases/test_http/test_regions_provider.py @@ -0,0 +1,267 @@ +import os +import datetime +import tempfile +import json + +import pytest + +from qiniu.compat import urlparse +from qiniu.config import QUERY_REGION_HOST, QUERY_REGION_BACKUP_HOSTS +from qiniu.http.endpoint import Endpoint +from qiniu.http.region import Region +from qiniu.http.regions_provider import QueryRegionsProvider, CachedRegionsProvider, _global_cache_scope, _persist_region + + +@pytest.fixture(scope='session') +def query_regions_endpoints_provider(): + query_region_host = urlparse(QUERY_REGION_HOST).hostname + endpoints_provider = [ + Endpoint(h) + for h in [query_region_host] + QUERY_REGION_BACKUP_HOSTS + ] + yield endpoints_provider + + +@pytest.fixture(scope='function') +def query_regions_provider(access_key, bucket_name, query_regions_endpoints_provider): + query_regions_provider = QueryRegionsProvider( + access_key=access_key, + bucket_name=bucket_name, + endpoints_provider=query_regions_endpoints_provider + ) + yield query_regions_provider + + +class TestQueryRegionsProvider: + def test_getter(self, query_regions_provider): + ret = list(query_regions_provider) + assert len(ret) > 0 + + def test_error_with_bad_ak(self, query_regions_endpoints_provider): + query_regions_provider = QueryRegionsProvider( + access_key='fake', + bucket_name='fake', + endpoints_provider=query_regions_endpoints_provider + ) + with pytest.raises(Exception) as exc: + list(query_regions_provider) + assert '612' in str(exc) + + def test_error_with_bad_endpoint(self, query_regions_provider): + query_regions_provider.endpoints_provider = [ + Endpoint('fake-uc.python.qiniu.com') + ] + with pytest.raises(Exception) as exc: + list(query_regions_provider) + assert '-1' in str(exc) + + def test_getter_with_retried(self, query_regions_provider, query_regions_endpoints_provider): + query_regions_provider.endpoints_provider = [ + Endpoint('fake-uc.python.qiniu.com'), + ] + list(query_regions_endpoints_provider) + + ret = list(query_regions_provider) + assert len(ret) > 0 + + def test_getter_with_preferred_scheme(self, query_regions_provider): + query_regions_provider.preferred_scheme = 'http' + for region in query_regions_provider: + for endpoints in region.services.values(): + assert all( + e.get_value().startswith('http://') + for e in endpoints + ) + + +@pytest.fixture(scope='function') +def cached_regions_provider(request): + if not hasattr(request, 'param') or not isinstance(request.param, dict): + request.param = {} + request.param.setdefault('cache_key', 'test-cache-key') + request.param.setdefault('base_regions_provider', []) + + cached_regions_provider = CachedRegionsProvider( + **request.param + ) + yield cached_regions_provider + + # clear memo_cache for test cases will affect each other with same cache_key + _global_cache_scope.memo_cache.clear() + persist_path = request.param.get('persist_path') + if persist_path: + try: + os.remove(persist_path) + except OSError: + pass + + +@pytest.fixture(scope='function') +def bad_regions_provider(): + regions_provider = QueryRegionsProvider( + access_key='fake', + bucket_name='fake', + endpoints_provider=[ + Endpoint('fake-uc.python.qiniu.com') + ] + ) + yield regions_provider + + +class TestCachedQueryRegionsProvider: + @pytest.mark.parametrize( + 'cached_regions_provider', + [ + {'base_regions_provider': [Region.from_region_id('z0')]}, + ], + indirect=True + ) + def test_getter_normally(self, cached_regions_provider): + ret = list(cached_regions_provider) + assert len(ret) > 0 + + def test_setter(self, cached_regions_provider): + regions = [Region.from_region_id('z0')] + cached_regions_provider.set_regions(regions) + assert list(cached_regions_provider) == regions + + def test_getter_with_expired_file_cache(self, cached_regions_provider): + expired_region = Region.from_region_id('z0') + expired_region.create_time = datetime.datetime.now() + + r_z0 = Region.from_region_id('z0') + r_z0.ttl = 86400 + + with open(cached_regions_provider.persist_path, 'w') as f: + json.dump({ + 'cacheKey': cached_regions_provider.cache_key, + 'regions': [_persist_region(r) for r in [expired_region]] + }, f) + + cached_regions_provider._cache_scope.memo_cache[cached_regions_provider.cache_key] = [r_z0] + + assert list(cached_regions_provider) == [r_z0] + try: + os.remove(cached_regions_provider.persist_path) + except OSError: + pass + + @pytest.mark.parametrize( + 'cached_regions_provider', + [ + { + 'persist_path': os.path.join(tempfile.gettempdir(), 'test-disable-persist.jsonl'), + }, + { + 'persist_path': None, + } + ], + indirect=True + ) + def test_disable_persist(self, cached_regions_provider): + if cached_regions_provider.persist_path: + old_persist_path = cached_regions_provider.persist_path + cached_regions_provider.persist_path = None + else: + old_persist_path = _global_cache_scope.persist_path + + regions = [Region.from_region_id('z0')] + cached_regions_provider.set_regions(regions) + + assert list(cached_regions_provider) == regions + assert not os.path.exists(old_persist_path) + + @pytest.mark.parametrize( + 'cached_regions_provider', + [ + { + 'persist_path': os.path.join(tempfile.gettempdir(), 'test-base-provider.jsonl'), + 'base_regions_provider': [Region.from_region_id('z0')] + } + ], + indirect=True + ) + def test_getter_with_base_regions_provider(self, cached_regions_provider): + assert not os.path.exists(cached_regions_provider.persist_path) + regions = list(cached_regions_provider.base_regions_provider) + assert list(cached_regions_provider) == regions + line_num = 0 + with open(cached_regions_provider.persist_path, 'r') as f: + for _ in f: + line_num += 1 + assert line_num == 1 + + @pytest.mark.parametrize( + 'cached_regions_provider', + [ + { + 'persist_path': os.path.join(tempfile.gettempdir(), 'test-base-provider.jsonl') + } + ], + indirect=True + ) + def test_should_provide_memo_expired_regions_when_base_provider_failed( + self, + cached_regions_provider, + bad_regions_provider + ): + expired_region = Region.from_region_id('z0') + expired_region.create_time = datetime.datetime.fromtimestamp(0) + expired_region.ttl = 1 + cached_regions_provider.set_regions([expired_region]) + cached_regions_provider.base_regions_provider = bad_regions_provider + regions = list(cached_regions_provider) + assert len(regions) > 0 + assert not regions[0].is_live + + @pytest.mark.parametrize( + 'cached_regions_provider', + [ + { + 'persist_path': os.path.join(tempfile.gettempdir(), 'test-base-provider.jsonl') + } + ], + indirect=True + ) + def test_should_provide_file_expired_regions_when_base_provider_failed( + self, + cached_regions_provider, + bad_regions_provider + ): + expired_region = Region.from_region_id('z0') + expired_region.create_time = datetime.datetime.fromtimestamp(0) + expired_region.ttl = 1 + cached_regions_provider.set_regions([expired_region]) + cached_regions_provider._cache_scope.memo_cache.clear() + cached_regions_provider.base_regions_provider = bad_regions_provider + regions = list(cached_regions_provider) + assert len(regions) > 0 + assert not regions[0].is_live + + @pytest.mark.parametrize( + 'cached_regions_provider', + [ + { + 'should_shrink_expired_regions': True + } + ], + indirect=True + ) + def test_shrink_with_expired_regions(self, cached_regions_provider): + expired_region = Region.from_region_id('z0') + expired_region.create_time = datetime.datetime.fromtimestamp(0) + expired_region.ttl = 1 + origin_cache_key = cached_regions_provider.cache_key + cached_regions_provider.set_regions([expired_region]) + cached_regions_provider.cache_key = 'another-cache-key' + list(cached_regions_provider) # trigger __shrink_cache() + assert len(cached_regions_provider._cache_scope.memo_cache[origin_cache_key]) == 0 + + def test_shrink_with_ignore_expired_regions(self, cached_regions_provider): + expired_region = Region.from_region_id('z0') + expired_region.create_time = datetime.datetime.fromtimestamp(0) + expired_region.ttl = 1 + origin_cache_key = cached_regions_provider.cache_key + cached_regions_provider.set_regions([expired_region]) + cached_regions_provider.cache_key = 'another-cache-key' + list(cached_regions_provider) # trigger __shrink_cache() + assert len(cached_regions_provider._cache_scope.memo_cache[origin_cache_key]) > 0 diff --git a/tests/cases/test_http/test_regions_retry_policy.py b/tests/cases/test_http/test_regions_retry_policy.py new file mode 100644 index 00000000..add39930 --- /dev/null +++ b/tests/cases/test_http/test_regions_retry_policy.py @@ -0,0 +1,263 @@ +import pytest + +from qiniu.http.endpoint import Endpoint +from qiniu.http.region import Region, ServiceName +from qiniu.http.regions_retry_policy import RegionsRetryPolicy +from qiniu.retry import Attempt + + +@pytest.fixture(scope='function') +def mocked_regions_provider(): + yield [ + Region.from_region_id('z0'), + Region.from_region_id('z1') + ] + + +class TestRegionsRetryPolicy: + def test_init(self, mocked_regions_provider): + regions_retry_policy = RegionsRetryPolicy( + regions_provider=mocked_regions_provider, + service_names=[ServiceName.UP] + ) + + mocked_context = {} + regions_retry_policy.init_context(mocked_context) + + assert mocked_context['region'] == mocked_regions_provider[0] + assert mocked_context['alternative_regions'] == mocked_regions_provider[1:] + assert mocked_context['service_name'] == ServiceName.UP + assert mocked_context['alternative_service_names'] == [] + assert mocked_context['endpoint'] == mocked_regions_provider[0].services[ServiceName.UP][0] + assert mocked_context['alternative_endpoints'] == mocked_regions_provider[0].services[ServiceName.UP][1:] + + @pytest.mark.parametrize( + 'regions,service_names,expect_should_retry,msg', + [ + ( + [ + Region.from_region_id('z0'), + Region.from_region_id('z1') + ], + [ServiceName.UP], + True, + 'Should retry when there are alternative regions' + ), + ( + [ + Region.from_region_id( + 'z0', + extended_services={ + ServiceName.UP_ACC: [ + Endpoint('python-sdk.kodo-accelerate.cn-east-1.qiniucs.com') + ] + } + ) + ], + [ServiceName.UP_ACC, ServiceName.UP], + True, + 'Should retry when there are alternative services' + ), + ( + [ + Region.from_region_id('z0') + ], + [ServiceName.UP_ACC, ServiceName.UP], + False, + 'Should not retry when there are no alternative regions or empty endpoint in services' + ), + ( + [ + Region.from_region_id('z0') + ], + [ServiceName.UP], + False, + 'Should not retry when there are no alternative regions or services' + ), + ], + ids=lambda v: v if type(v) is str else '' + ) + def test_should_retry( + self, + regions, + service_names, + expect_should_retry, + msg + ): + regions_retry_policy = RegionsRetryPolicy( + regions_provider=regions, + service_names=service_names + ) + + mocked_attempt = Attempt() + regions_retry_policy.init_context(mocked_attempt.context) + + assert regions_retry_policy.should_retry(mocked_attempt) == expect_should_retry, msg + + @pytest.mark.parametrize( + 'regions,service_names', + [ + ( + [ + Region.from_region_id('z0'), + Region.from_region_id('z1') + ], + [ServiceName.UP] + ), + ( + [ + Region.from_region_id( + 'z0', + extended_services={ + ServiceName.UP_ACC: [ + Endpoint('python-sdk.kodo-accelerate.cn-east-1.qiniucs.com') + ] + } + ) + ], + [ServiceName.UP_ACC, ServiceName.UP] + ) + ] + ) + def test_prepare_retry(self, regions, service_names): + mocked_attempt = Attempt() + + regions_retry_policy = RegionsRetryPolicy( + regions_provider=regions, + service_names=service_names + ) + regions_retry_policy.init_context(mocked_attempt.context) + + actual_tried_endpoints = [ + mocked_attempt.context.get('endpoint') + ] + while regions_retry_policy.should_retry(mocked_attempt): + regions_retry_policy.prepare_retry(mocked_attempt) + actual_tried_endpoints.append(mocked_attempt.context.get('endpoint')) + + # There is no endpoints retry policy, + # so just the first endpoint will be tried + expect_tried_endpoints = [ + r.services[sn][0] + for r in regions + for sn in service_names + if sn in r.services and r.services[sn] + ] + + print(actual_tried_endpoints) + print(expect_tried_endpoints) + + assert [ + e.get_value() + for e in actual_tried_endpoints + ] == [ + e.get_value() + for e in expect_tried_endpoints + ] + + @pytest.mark.parametrize( + 'regions,service_names,expect_change_region_times', + [ + # tow region, retry once + ( + [ + Region.from_region_id('z0'), + Region.from_region_id('z1') + ], + [ServiceName.UP], + 1 + ), + # one region, tow service, retry service once, region zero + ( + [ + Region.from_region_id( + 'z0', + extended_services={ + ServiceName.UP_ACC: [ + Endpoint('python-sdk.kodo-accelerate.cn-east-1.qiniucs.com') + ] + } + ) + ], + [ServiceName.UP_ACC, ServiceName.UP], + 0 + ), + # tow region, tow service, retry service once, region once + ( + [ + Region.from_region_id( + 'z0', + extended_services={ + ServiceName.UP_ACC: [ + Endpoint('python-sdk.kodo-accelerate.cn-east-1.qiniucs.com') + ] + } + ), + Region.from_region_id('z1') + ], + [ServiceName.UP_ACC, ServiceName.UP], + 1 + ) + ] + ) + def test_on_change_region_option( + self, + regions, + service_names, + expect_change_region_times, + use_ref + ): + actual_change_region_times_ref = use_ref(0) + + def handle_change_region(_context): + actual_change_region_times_ref.value += 1 + + regions_retry_policy = RegionsRetryPolicy( + regions_provider=regions, + service_names=service_names, + on_change_region=handle_change_region + ) + + mocked_attempt = Attempt() + regions_retry_policy.init_context(mocked_attempt.context) + + while regions_retry_policy.should_retry(mocked_attempt): + regions_retry_policy.prepare_retry(mocked_attempt) + + assert actual_change_region_times_ref.value == expect_change_region_times + + def test_init_with_preferred_endpoints_option_new_temp_region(self, mocked_regions_provider): + preferred_endpoints = [ + Endpoint('python-sdk.kodo-accelerate.cn-east-1.qiniucs.com') + ] + regions_retry_policy = RegionsRetryPolicy( + regions_provider=mocked_regions_provider, + service_names=[ServiceName.UP], + preferred_endpoints_provider=preferred_endpoints + ) + + mocked_context = {} + regions_retry_policy.init_context(mocked_context) + + assert mocked_context['region'].region_id == 'preferred_region' + assert mocked_context['region'].services[ServiceName.UP] == preferred_endpoints + assert mocked_context['alternative_regions'] == list(mocked_regions_provider) + + def test_init_with_preferred_endpoints_option_reorder_regions(self, mocked_regions_provider): + mocked_regions = list(mocked_regions_provider) + preferred_region_index = 1 + preferred_endpoints = [ + mocked_regions[preferred_region_index].services[ServiceName.UP][0] + ] + regions_retry_policy = RegionsRetryPolicy( + regions_provider=mocked_regions_provider, + service_names=[ServiceName.UP], + preferred_endpoints_provider=preferred_endpoints + ) + + mocked_context = {} + regions_retry_policy.init_context(mocked_context) + + assert mocked_context['region'] == mocked_regions[preferred_region_index] + mocked_regions.pop(preferred_region_index) + assert mocked_context['alternative_regions'] == mocked_regions diff --git a/tests/cases/test_retry/__init__.py b/tests/cases/test_retry/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/test_retry/test_retrier.py b/tests/cases/test_retry/test_retrier.py new file mode 100644 index 00000000..c4d1cd9b --- /dev/null +++ b/tests/cases/test_retry/test_retrier.py @@ -0,0 +1,142 @@ +import qiniu.retry +import qiniu.retry.abc + + +class MaxRetryPolicy(qiniu.retry.abc.RetryPolicy): + def __init__(self, max_times): + super(MaxRetryPolicy, self).__init__() + self.max_times = max_times + + def is_important(self, attempt): + return attempt.context[self]['retriedTimes'] >= self.max_times + + def init_context(self, context): + context[self] = { + 'retriedTimes': 0 + } + + def should_retry(self, attempt): + if not attempt.exception: + return False + return attempt.context[self]['retriedTimes'] < self.max_times + + def prepare_retry(self, attempt): + pass + + def after_retry(self, attempt, policy): + attempt.context[self]['retriedTimes'] += 1 + + +class TestRetry: + def test_retrier_with_code_block(self, use_ref): + retried_times_ref = use_ref(0) + + def handle_before_retry(_attempt, _policy): + retried_times_ref.value += 1 + return True + + max_retry_times = 3 + retrier = qiniu.retry.Retrier( + policies=[ + MaxRetryPolicy(max_times=max_retry_times) + ], + before_retry=handle_before_retry + ) + + tried_times = 0 + try: + for attempt in retrier: + with attempt: + tried_times += 1 + raise Exception('mocked error') + except Exception as err: + assert str(err) == 'mocked error' + + assert tried_times == max_retry_times + 1 + assert retried_times_ref.value == max_retry_times + + def test_retrier_with_try_do(self, use_ref): + retried_times_ref = use_ref(0) + + def handle_before_retry(_attempt, _policy): + retried_times_ref.value += 1 + return True + + max_retry_times = 3 + retrier = qiniu.retry.Retrier( + policies=[ + MaxRetryPolicy(max_times=max_retry_times) + ], + before_retry=handle_before_retry + ) + + tried_times_ref = use_ref(0) + + def add_one(n): + tried_times_ref.value += 1 + if tried_times_ref.value <= 3: + raise Exception('mock error') + return n + 1 + + result = retrier.try_do(add_one, 1) + assert result == 2 + assert tried_times_ref.value == max_retry_times + 1 + assert retried_times_ref.value == max_retry_times + + def test_retrier_with_decorator(self, use_ref): + retried_times_ref = use_ref(0) + + def handle_before_retry(_attempt, _policy): + retried_times_ref.value += 1 + return True + + max_retry_times = 3 + retrier = qiniu.retry.Retrier( + policies=[ + MaxRetryPolicy(max_times=max_retry_times) + ], + before_retry=handle_before_retry + ) + + tried_times_ref = use_ref(0) + + @retrier.retry + def add_one(n): + tried_times_ref.value += 1 + if tried_times_ref.value <= 3: + raise Exception('mock error') + return n + 1 + + result = add_one(1) + assert result == 2 + assert tried_times_ref.value == max_retry_times + 1 + assert retried_times_ref.value == max_retry_times + + def test_retrier_with_no_need_retry_err(self, use_ref): + retried_times_ref = use_ref(0) + + def handle_before_retry(_attempt, _policy): + retried_times_ref.value += 1 + return True + + max_retry_times = 3 + retrier = qiniu.retry.Retrier( + policies=[ + MaxRetryPolicy(max_times=max_retry_times) + ], + before_retry=handle_before_retry + ) + + tried_times = 0 + try: + for attempt in retrier: + with attempt: + tried_times += 1 + err = Exception('mocked error') + err.no_need_retry = True + raise err + except Exception as err: + assert str(err) == 'mocked error' + + assert tried_times == 1 + assert retried_times_ref.value == 0 diff --git a/tests/cases/test_services/test_processing/__init__.py b/tests/cases/test_services/test_processing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/test_services/test_processing/test_pfop.py b/tests/cases/test_services/test_processing/test_pfop.py new file mode 100644 index 00000000..003be43f --- /dev/null +++ b/tests/cases/test_services/test_processing/test_pfop.py @@ -0,0 +1,49 @@ +import pytest + + +from qiniu import PersistentFop, op_save + + +persistent_id = None + + +class TestPersistentFop: + def test_pfop_execute(self, qn_auth): + pfop = PersistentFop(qn_auth, 'testres', 'sdktest') + op = op_save('avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', 'pythonsdk', 'pfoptest') + ops = [ + op + ] + ret, resp = pfop.execute('sintel_trailer.mp4', ops, 1) + assert resp.status_code == 200, resp + assert ret['persistentId'] is not None, resp + global persistent_id + persistent_id = ret['persistentId'] + + def test_pfop_get_status(self, qn_auth): + assert persistent_id is not None + pfop = PersistentFop(qn_auth, 'testres', 'sdktest') + ret, resp = pfop.get_status(persistent_id) + assert resp.status_code == 200, resp + assert ret is not None, resp + + def test_pfop_idle_time_task(self, set_conf_default, qn_auth): + persistence_key = 'python-sdk-pfop-test/test-pfop-by-api' + + key = 'sintel_trailer.mp4' + pfop = PersistentFop(qn_auth, 'testres') + ops = [ + op_save( + op='avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', + bucket='pythonsdk', + key=persistence_key + ) + ] + ret, resp = pfop.execute(key, ops, force=1, persistent_type=1) + assert resp.status_code == 200, resp + assert 'persistentId' in ret, resp + + ret, resp = pfop.get_status(ret['persistentId']) + assert resp.status_code == 200, resp + assert ret['type'] == 1, resp + assert ret['creationDate'] is not None, resp diff --git a/tests/cases/test_services/test_storage/conftest.py b/tests/cases/test_services/test_storage/conftest.py index a791092c..64b81b20 100644 --- a/tests/cases/test_services/test_storage/conftest.py +++ b/tests/cases/test_services/test_storage/conftest.py @@ -1,8 +1,140 @@ +import os +from collections import namedtuple +from hashlib import new as hashlib_new +import tempfile + import pytest +import requests + from qiniu import BucketManager +from qiniu.utils import io_md5 +from qiniu.config import QUERY_REGION_HOST, QUERY_REGION_BACKUP_HOSTS +from qiniu.http.endpoint import Endpoint +from qiniu.http.regions_provider import Region, ServiceName, get_default_regions_provider -@pytest.fixture() +@pytest.fixture(scope='session') def bucket_manager(qn_auth): yield BucketManager(qn_auth) + + +@pytest.fixture(scope='session') +def get_remote_object_headers_and_md5(download_domain): + def fetch_calc_md5(key=None, scheme=None, url=None): + if not key and not url: + raise TypeError('Must provide key or url') + + scheme = scheme if scheme is not None else 'http' + download_url = '{}://{}/{}'.format(scheme, download_domain, key) + if url: + download_url = url + + resp = requests.get(download_url, stream=True) + resp.raise_for_status() + + return resp.headers, io_md5(resp.iter_content(chunk_size=8192)) + + yield fetch_calc_md5 + + +@pytest.fixture(scope='session') +def get_real_regions(): + def _get_real_regions(access_key, bucket_name): + regions = list( + get_default_regions_provider( + query_endpoints_provider=[ + Endpoint.from_host(h) + for h in [QUERY_REGION_HOST] + QUERY_REGION_BACKUP_HOSTS + ], + access_key=access_key, + bucket_name=bucket_name + ) + ) + + if not regions: + raise RuntimeError('No regions found') + + return regions + + yield _get_real_regions + + +@pytest.fixture(scope='function') +def regions_with_real_endpoints(access_key, bucket_name, get_real_regions): + yield get_real_regions(access_key, bucket_name) + + +@pytest.fixture(scope='function') +def regions_with_fake_endpoints(regions_with_real_endpoints): + """ + Returns + ------- + list[Region] + The first element is the fake region with fake endpoints for every service. + The second element is the real region with first fake endpoint for every service. + The rest elements are real regions with real endpoints if exists. + """ + regions = regions_with_real_endpoints + + regions[0].services = { + sn: [ + Endpoint('fake-{0}.python-sdk.qiniu.com'.format(sn.value)) + ] + endpoints + for sn, endpoints in regions[0].services.items() + } + + regions.insert(0, Region( + 'fake-id', + 'fake-s3-id', + services={ + sn: [ + Endpoint('fake-region-{0}.python-sdk.qiniu.com'.format(sn.value)) + ] + for sn in ServiceName + } + )) + + yield regions + + +TempFile = namedtuple( + 'TempFile', + [ + 'path', + 'md5', + 'name', + 'size' + ] +) + + +@pytest.fixture(scope='function') +def temp_file(request): + size = 4 * 1024 + if hasattr(request, 'param'): + size = request.param + + tmp_file_path = tempfile.mktemp() + chunk_size = 4 * 1024 + + md5_hasher = hashlib_new('md5') + with open(tmp_file_path, 'wb') as f: + remaining_bytes = size + while remaining_bytes > 0: + chunk = os.urandom(min(chunk_size, remaining_bytes)) + f.write(chunk) + md5_hasher.update(chunk) + remaining_bytes -= len(chunk) + + yield TempFile( + path=tmp_file_path, + md5=md5_hasher.hexdigest(), + name=os.path.basename(tmp_file_path), + size=size + ) + + try: + os.remove(tmp_file_path) + except Exception: + pass diff --git a/tests/cases/test_services/test_storage/test_bucket_manager.py b/tests/cases/test_services/test_storage/test_bucket_manager.py new file mode 100644 index 00000000..68455652 --- /dev/null +++ b/tests/cases/test_services/test_storage/test_bucket_manager.py @@ -0,0 +1,205 @@ +import pytest + +from qiniu.services.storage.bucket import BucketManager +from qiniu.region import LegacyRegion +from qiniu import build_batch_restore_ar + + +@pytest.fixture(scope='function') +def object_key(bucket_manager, bucket_name, rand_string): + key_to = 'copyto_' + rand_string(8) + bucket_manager.copy( + bucket=bucket_name, + key='copyfrom', + bucket_to=bucket_name, + key_to=key_to, + force='true' + ) + + yield key_to + + bucket_manager.delete(bucket_name, key_to) + + +class TestBucketManager: + # TODO(lihs): Move other test cases to here from test_qiniu.py + def test_restore_ar(self, bucket_manager, bucket_name, object_key): + ret, resp = bucket_manager.restore_ar(bucket_name, object_key, 7) + assert not resp.ok(), resp + ret, resp = bucket_manager.change_type(bucket_name, object_key, 2) + assert resp.ok(), resp + ret, resp = bucket_manager.restore_ar(bucket_name, object_key, 7) + assert resp.ok(), resp + + @pytest.mark.parametrize( + 'cond,expect_ok', + [ + ( + None, True + ), + ( + { + 'mime': 'text/plain' + }, + True + ), + ( + { + 'mime': 'application/json' + }, + False + ) + ] + ) + def test_change_status( + self, + bucket_manager, + bucket_name, + object_key, + cond, + expect_ok + ): + ret, resp = bucket_manager.change_status(bucket_name, object_key, 1, cond) + assert resp.ok() == expect_ok, resp + + def test_mkbucketv3(self, bucket_manager, rand_string): + # tested manually, no drop bucket API to auto cleanup + # ret, resp = bucket_manager.mkbucketv3('py-test-' + rand_string(8).lower(), 'z0') + # assert resp.ok(), resp + pass + + def test_list_bucket(self, bucket_manager, bucket_name): + ret, resp = bucket_manager.list_bucket('na0') + assert resp.ok(), resp + assert any(b.get('tbl') == bucket_name for b in ret) + + def test_bucket_info(self, bucket_manager, bucket_name): + ret, resp = bucket_manager.bucket_info(bucket_name) + assert resp.ok(), resp + for k in [ + 'protected', + 'private' + ]: + assert k in ret + + def test_change_bucket_permission(self, bucket_manager, bucket_name): + ret, resp = bucket_manager.bucket_info(bucket_name) + assert resp.ok(), resp + original_private = ret['private'] + ret, resp = bucket_manager.change_bucket_permission( + bucket_name, + 1 if original_private == 1 else 0 + ) + assert resp.ok(), resp + ret, resp = bucket_manager.change_bucket_permission( + bucket_name, + original_private + ) + assert resp.ok(), resp + + def test_batch_restore_ar( + self, + bucket_manager, + bucket_name, + object_key + ): + bucket_manager.change_type(bucket_name, object_key, 2) + ops = build_batch_restore_ar( + bucket_name, + { + object_key: 7 + } + ) + ret, resp = bucket_manager.batch(ops) + assert resp.status_code == 200, resp + assert len(ret) > 0 + assert ret[0].get('code') == 200, ret[0] + + def test_compatible_with_zone(self, qn_auth, bucket_name, regions_with_real_endpoints): + r = LegacyRegion( + io_host='https://fake-io.python-sdk.qiniu.com', + rs_host='https://fake-rs.python-sdk.qiniu.com', + rsf_host='https://fake-rsf.python-sdk.qiniu.com', + api_host='https://fake-api.python-sdk.qiniu.com' + ) + bucket_manager = BucketManager( + qn_auth, + zone=r + ) + + # rs host + ret, resp = bucket_manager.stat(bucket_name, 'python-sdk.html') + assert resp.status_code == -1 + assert ret is None + + # rsf host + ret, _eof, resp = bucket_manager.list(bucket_name, '', limit=10) + assert resp.status_code == -1 + assert ret is None + + # io host + ret, info = bucket_manager.prefetch(bucket_name, 'python-sdk.html') + assert resp.status_code == -1 + assert ret is None + + # api host + # no API method to test + + @pytest.mark.parametrize( + 'preferred_scheme', + [ + None, # default 'http' + 'http', + 'https' + ] + ) + def test_preferred_scheme( + self, + qn_auth, + bucket_name, + preferred_scheme + ): + bucket_manager = BucketManager( + auth=qn_auth, + preferred_scheme=preferred_scheme + ) + + ret, resp = bucket_manager.stat(bucket_name, 'python-sdk.html') + + assert ret is not None, resp + assert resp.ok(), resp + + expect_scheme = preferred_scheme if preferred_scheme else 'http' + assert resp.url.startswith(expect_scheme + '://'), resp.url + + def test_operation_with_regions_and_retrier( + self, + qn_auth, + bucket_name, + regions_with_fake_endpoints + ): + bucket_manager = BucketManager( + auth=qn_auth, + regions=regions_with_fake_endpoints, + ) + + ret, resp = bucket_manager.stat(bucket_name, 'python-sdk.html') + + assert ret is not None, resp + assert resp.ok(), resp + + def test_uc_service_with_retrier( + self, + qn_auth, + bucket_name, + regions_with_fake_endpoints + ): + bucket_manager = BucketManager( + auth=qn_auth, + regions=regions_with_fake_endpoints + ) + + ret, resp = bucket_manager.list_bucket('na0') + assert resp.ok(), resp + assert len(ret) > 0, resp + assert any(b.get('tbl') for b in ret), ret diff --git a/tests/cases/test_services/test_storage/test_upload_pfop.py b/tests/cases/test_services/test_storage/test_upload_pfop.py new file mode 100644 index 00000000..78818ba4 --- /dev/null +++ b/tests/cases/test_services/test_storage/test_upload_pfop.py @@ -0,0 +1,66 @@ +import pytest + +import qiniu + + +KB = 1024 +MB = 1024 * KB +GB = 1024 * MB + + +# set a bucket lifecycle manually to delete prefix `test-pfop`! +# or this test will continue to occupy bucket space. +class TestPersistentFopByUpload: + @pytest.mark.parametrize('temp_file', [10 * MB], indirect=True) + @pytest.mark.parametrize('persistent_type', [None, 0, 1]) + def test_pfop_with_upload( + self, + set_conf_default, + qn_auth, + bucket_name, + temp_file, + persistent_type + ): + key = 'test-pfop-upload-file' + persistent_key = '_'.join([ + 'test-pfop-by-upload', + 'type', + str(persistent_type) + ]) + persistent_ops = ';'.join([ + qiniu.op_save( + op='avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', + bucket=bucket_name, + key=persistent_key + ) + ]) + + upload_policy = { + 'persistentOps': persistent_ops + } + + if persistent_type is not None: + upload_policy['persistentType'] = persistent_type + + token = qn_auth.upload_token( + bucket_name, + key, + policy=upload_policy + ) + ret, resp = qiniu.put_file( + token, + key, + temp_file.path, + check_crc=True + ) + + assert ret is not None, resp + assert ret['key'] == key, resp + assert 'persistentId' in ret, resp + + pfop = qiniu.PersistentFop(qn_auth, bucket_name) + ret, resp = pfop.get_status(ret['persistentId']) + assert resp.status_code == 200, resp + if persistent_type == 1: + assert ret['type'] == 1, resp + assert ret['creationDate'] is not None, resp diff --git a/tests/cases/test_services/test_storage/test_uploader.py b/tests/cases/test_services/test_storage/test_uploader.py index 44b153e6..158f111d 100644 --- a/tests/cases/test_services/test_storage/test_uploader.py +++ b/tests/cases/test_services/test_storage/test_uploader.py @@ -1,20 +1,21 @@ -import os from collections import namedtuple -import tempfile import pytest from qiniu.compat import json, is_py2 from qiniu import ( Zone, - etag, + config as qn_config, set_default, put_file, put_data, - put_stream + put_stream, + build_batch_delete ) -from qiniu import config as qn_config +from qiniu.http.endpoint import Endpoint +from qiniu.http.region import ServiceName from qiniu.services.storage.uploader import _form_put +from qiniu.services.storage.uploaders.abc import UploaderBase KB = 1024 MB = 1024 * KB @@ -54,11 +55,43 @@ def commonly_options(request): 'x-qn-meta-age': '18' } ) - if hasattr(request, 'params'): - res = res._replace(**request.params) + if hasattr(request, 'param'): + res = res._replace(**request.param) yield res +@pytest.fixture(scope='class') +def auto_remove(bucket_manager): + grouped_keys_by_bucket_name = {} + + def _auto_remove(bucket_name, key): + if bucket_name not in grouped_keys_by_bucket_name: + grouped_keys_by_bucket_name[bucket_name] = [] + grouped_keys_by_bucket_name[bucket_name].append(key) + return key + + yield _auto_remove + + for bkt_name, keys in grouped_keys_by_bucket_name.items(): + try: + delete_ops = build_batch_delete(bkt_name, keys) + bucket_manager.batch(delete_ops) + except Exception as err: + print('Failed to delete {0} keys: {1} by {2}'.format(bkt_name, keys, err)) + + +@pytest.fixture(scope='class') +def get_key(bucket_name, rand_string, auto_remove): + def _get_key(key, no_rand_trail=False): + result = key + '-' + rand_string(8) + if no_rand_trail: + result = key + auto_remove(bucket_name, result) + return result + + yield _get_key + + @pytest.fixture(scope='function') def set_default_up_host_zone(request, valid_up_host): zone_args = { @@ -77,41 +110,17 @@ def set_default_up_host_zone(request, valid_up_host): qn_config._is_customized_default['default_zone'] = False -@pytest.fixture(scope='function') -def temp_file(request): - size = 4 * KB - if hasattr(request, 'param'): - size = request.param - - tmp_file_path = tempfile.mktemp() - chunk_size = 4 * KB - - with open(tmp_file_path, 'wb') as f: - remaining_bytes = size - while remaining_bytes > 0: - chunk = os.urandom(min(chunk_size, remaining_bytes)) - f.write(chunk) - remaining_bytes -= len(chunk) - - yield tmp_file_path - - try: - os.remove(tmp_file_path) - except Exception: - pass - - class TestUploadFuncs: - def test_put(self, qn_auth, bucket_name): - key = 'a\\b\\c"hello' + def test_put(self, qn_auth, bucket_name, get_key): + key = get_key('a\\b\\c"hello', no_rand_trail=True) data = 'hello bubby!' token = qn_auth.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) assert ret['key'] == key - def test_put_crc(self, qn_auth, bucket_name): - key = '' + def test_put_crc(self, qn_auth, bucket_name, get_key): + key = get_key('', no_rand_trail=True) data = 'hello bubby!' token = qn_auth.upload_token(bucket_name, key) ret, info = put_data(token, key, data, check_crc=True) @@ -119,23 +128,34 @@ def test_put_crc(self, qn_auth, bucket_name): assert ret['key'] == key @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) - def test_put_file(self, qn_auth, bucket_name, temp_file, commonly_options): - key = 'test_file' + def test_put_file( + self, + qn_auth, + bucket_name, + temp_file, + commonly_options, + get_remote_object_headers_and_md5, + get_key + ): + key = get_key('test_file') token = qn_auth.upload_token(bucket_name, key) ret, info = put_file( token, key, - temp_file, + temp_file.path, mime_type=commonly_options.mime_type, check_crc=True ) - print(info) - assert ret['key'] == key - assert ret['hash'] == etag(temp_file) - def test_put_with_invalid_crc(self, qn_auth, bucket_name): - key = 'test_invalid' + _, actual_md5 = get_remote_object_headers_and_md5(key=key) + + assert ret is not None, info + assert ret['key'] == key, info + assert actual_md5 == temp_file.md5 + + def test_put_with_invalid_crc(self, qn_auth, bucket_name, get_key): + key = get_key('test_invalid') data = 'hello bubby!' crc32 = 'wrong crc32' token = qn_auth.upload_token(bucket_name) @@ -143,13 +163,14 @@ def test_put_with_invalid_crc(self, qn_auth, bucket_name): assert ret is None, info assert info.status_code == 400, info - def test_put_without_key(self, qn_auth, bucket_name): + def test_put_without_key(self, qn_auth, bucket_name, get_key): key = None data = 'hello bubby!' token = qn_auth.upload_token(bucket_name) ret, info = put_data(token, key, data) - print(info) - assert ret['hash'] == ret['key'] + assert 'key' in ret, info + get_key(ret['key'], no_rand_trail=True) # auto remove the file + assert ret['hash'] == ret['key'], info data = 'hello bubby!' token = qn_auth.upload_token(bucket_name, 'nokey2') @@ -166,8 +187,8 @@ def test_put_without_key(self, qn_auth, bucket_name): ], indirect=True ) - def test_without_read_without_seek_retry(self, set_default_up_host_zone, qn_auth, bucket_name): - key = 'retry' + def test_without_read_without_seek_retry(self, set_default_up_host_zone, qn_auth, bucket_name, get_key): + key = get_key('retry') data = 'hello retry!' token = qn_auth.upload_token(bucket_name) ret, info = put_data(token, key, data) @@ -181,12 +202,13 @@ def test_put_data_without_fname( qn_auth, bucket_name, is_travis, - temp_file + temp_file, + get_key ): if is_travis: return - key = 'test_putData_without_fname' - with open(temp_file, 'rb') as input_stream: + key = get_key('test_putData_without_fname') + with open(temp_file.path, 'rb') as input_stream: token = qn_auth.upload_token(bucket_name, key) ret, info = put_data(token, key, input_stream) print(info) @@ -199,12 +221,13 @@ def test_put_data_with_empty_fname( bucket_name, is_travis, temp_file, - commonly_options + commonly_options, + get_key ): if is_travis: return - key = 'test_putData_without_fname1' - with open(temp_file, 'rb') as input_stream: + key = get_key('test_putData_without_fname1') + with open(temp_file.path, 'rb') as input_stream: token = qn_auth.upload_token(bucket_name, key) ret, info = put_data( token, @@ -226,12 +249,13 @@ def test_put_data_with_space_only_fname( bucket_name, is_travis, temp_file, - commonly_options + commonly_options, + get_key ): if is_travis: return - key = 'test_putData_without_fname2' - with open(temp_file, 'rb') as input_stream: + key = get_key('test_putData_without_fname2') + with open(temp_file.path, 'rb') as input_stream: token = qn_auth.upload_token(bucket_name, key) ret, info = put_data( token, @@ -253,13 +277,17 @@ def test_put_file_with_metadata( bucket_name, temp_file, commonly_options, - bucket_manager + bucket_manager, + get_remote_object_headers_and_md5, + get_key ): - key = 'test_file_with_metadata' + key = get_key('test_file_with_metadata') token = qn_auth.upload_token(bucket_name, key) - ret, info = put_file(token, key, temp_file, metadata=commonly_options.metadata) + ret, info = put_file(token, key, temp_file.path, metadata=commonly_options.metadata) + _, actual_md5 = get_remote_object_headers_and_md5(key=key) assert ret['key'] == key - assert ret['hash'] == etag(temp_file) + assert actual_md5 == temp_file.md5 + ret, info = bucket_manager.stat(bucket_name, key) assert 'x-qn-meta' in ret assert ret['x-qn-meta']['name'] == 'qiniu' @@ -270,9 +298,10 @@ def test_put_data_with_metadata( qn_auth, bucket_name, commonly_options, - bucket_manager + bucket_manager, + get_key ): - key = 'put_data_with_metadata' + key = get_key('put_data_with_metadata') data = 'hello metadata!' token = qn_auth.upload_token(bucket_name, key) ret, info = put_data(token, key, data, metadata=commonly_options.metadata) @@ -290,9 +319,11 @@ def test_put_file_with_callback( temp_file, commonly_options, bucket_manager, - upload_callback_url + upload_callback_url, + get_remote_object_headers_and_md5, + get_key ): - key = 'test_file_with_callback' + key = get_key('test_file_with_callback') policy = { 'callbackUrl': upload_callback_url, 'callbackBody': '{"custom_vars":{"a":$(x:a)},"key":$(key),"hash":$(etag)}', @@ -302,27 +333,52 @@ def test_put_file_with_callback( ret, info = put_file( token, key, - temp_file, + temp_file.path, metadata=commonly_options.metadata, params=commonly_options.params, ) + _, actual_md5 = get_remote_object_headers_and_md5(key=key) assert ret['key'] == key - assert ret['hash'] == etag(temp_file) + assert actual_md5 == temp_file.md5 assert ret['custom_vars']['a'] == 'a' + ret, info = bucket_manager.stat(bucket_name, key) assert 'x-qn-meta' in ret assert ret['x-qn-meta']['name'] == 'qiniu' assert ret['x-qn-meta']['age'] == '18' + @pytest.mark.parametrize('temp_file', [64 * KB, 10 * MB], indirect=True) + def test_put_file_with_regions_retry( + self, + qn_auth, + bucket_name, + temp_file, + regions_with_fake_endpoints, + get_remote_object_headers_and_md5, + get_key + ): + key = get_key('test_file_with_form_regions_retry') + token = qn_auth.upload_token(bucket_name, key) + ret, info = put_file( + token, + key, + temp_file.path, + regions=regions_with_fake_endpoints + ) + _, actual_md5 = get_remote_object_headers_and_md5(key=key) + assert ret['key'] == key + assert actual_md5 == temp_file.md5 + def test_put_data_with_callback( self, qn_auth, bucket_name, commonly_options, bucket_manager, - upload_callback_url + upload_callback_url, + get_key ): - key = 'put_data_with_metadata' + key = get_key('put_data_with_metadata') data = 'hello metadata!' policy = { 'callbackUrl': upload_callback_url, @@ -347,17 +403,16 @@ def test_put_data_with_callback( class TestResumableUploader: @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) - def test_put_stream(self, qn_auth, bucket_name, temp_file, commonly_options): - key = 'test_file_r' - size = os.stat(temp_file).st_size - with open(temp_file, 'rb') as input_stream: + def test_put_stream(self, qn_auth, bucket_name, temp_file, commonly_options, get_key): + key = get_key('test_file_r') + with open(temp_file.path, 'rb') as input_stream: token = qn_auth.upload_token(bucket_name, key) ret, info = put_stream( token, key, input_stream, - os.path.basename(temp_file), - size, + temp_file.name, + temp_file.size, None, commonly_options.params, commonly_options.mime_type, @@ -368,17 +423,16 @@ def test_put_stream(self, qn_auth, bucket_name, temp_file, commonly_options): assert ret['key'] == key @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) - def test_put_stream_v2_without_bucket_name(self, qn_auth, bucket_name, temp_file, commonly_options): - key = 'test_file_r' - size = os.stat(temp_file).st_size - with open(temp_file, 'rb') as input_stream: + def test_put_stream_v2_without_bucket_name(self, qn_auth, bucket_name, temp_file, commonly_options, get_key): + key = get_key('test_file_r') + with open(temp_file.path, 'rb') as input_stream: token = qn_auth.upload_token(bucket_name, key) ret, info = put_stream( token, key, input_stream, - os.path.basename(temp_file), - size, + temp_file.name, + temp_file.size, None, commonly_options.params, commonly_options.mime_type, @@ -401,17 +455,16 @@ def test_put_stream_v2_without_bucket_name(self, qn_auth, bucket_name, temp_file ], indirect=True ) - def test_put_stream_v2(self, qn_auth, bucket_name, temp_file, commonly_options): - key = 'test_file_r' - size = os.stat(temp_file).st_size - with open(temp_file, 'rb') as input_stream: + def test_put_stream_v2(self, qn_auth, bucket_name, temp_file, commonly_options, get_key): + key = get_key('test_file_r') + with open(temp_file.path, 'rb') as input_stream: token = qn_auth.upload_token(bucket_name, key) ret, info = put_stream( token, key, input_stream, - os.path.basename(temp_file), - size, + temp_file.name, + temp_file.size, None, commonly_options.params, commonly_options.mime_type, @@ -422,18 +475,17 @@ def test_put_stream_v2(self, qn_auth, bucket_name, temp_file, commonly_options): assert ret['key'] == key @pytest.mark.parametrize('temp_file', [4 * MB + 1], indirect=True) - def test_put_stream_v2_without_key(self, qn_auth, bucket_name, temp_file, commonly_options): + def test_put_stream_v2_without_key(self, qn_auth, bucket_name, temp_file, commonly_options, get_key): part_size = 4 * MB key = None - size = os.stat(temp_file).st_size - with open(temp_file, 'rb') as input_stream: + with open(temp_file.path, 'rb') as input_stream: token = qn_auth.upload_token(bucket_name, key) ret, info = put_stream( token, key, input_stream, - os.path.basename(temp_file), - size, + temp_file.name, + temp_file.size, None, commonly_options.params, commonly_options.mime_type, @@ -441,21 +493,22 @@ def test_put_stream_v2_without_key(self, qn_auth, bucket_name, temp_file, common version='v2', bucket_name=bucket_name ) - assert ret['key'] == ret['hash'] + assert 'key' in ret + get_key(ret['key'], no_rand_trail=True) # auto remove the file + assert ret['key'] == ret['hash'] @pytest.mark.parametrize('temp_file', [4 * MB + 1], indirect=True) - def test_put_stream_v2_with_empty_return_body(self, qn_auth, bucket_name, temp_file, commonly_options): + def test_put_stream_v2_with_empty_return_body(self, qn_auth, bucket_name, temp_file, commonly_options, get_key): part_size = 4 * MB - key = 'test_file_empty_return_body' - size = os.stat(temp_file).st_size - with open(temp_file, 'rb') as input_stream: + key = get_key('test_file_empty_return_body') + with open(temp_file.path, 'rb') as input_stream: token = qn_auth.upload_token(bucket_name, key, policy={'returnBody': ' '}) ret, info = put_stream( token, key, input_stream, - os.path.basename(temp_file), - size, + temp_file.name, + temp_file.size, None, commonly_options.params, commonly_options.mime_type, @@ -467,14 +520,14 @@ def test_put_stream_v2_with_empty_return_body(self, qn_auth, bucket_name, temp_f assert ret == {} @pytest.mark.parametrize('temp_file', [4 * MB + 1], indirect=True) - def test_big_file(self, qn_auth, bucket_name, temp_file, commonly_options): - key = 'big' + def test_big_file(self, qn_auth, bucket_name, temp_file, commonly_options, get_key): + key = get_key('big') token = qn_auth.upload_token(bucket_name, key) ret, info = put_file( token, key, - temp_file, + temp_file.path, commonly_options.params, commonly_options.mime_type, progress_handler=lambda progress, total: progress @@ -491,32 +544,40 @@ def test_big_file(self, qn_auth, bucket_name, temp_file, commonly_options): indirect=True ) @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) - def test_retry(self, set_default_up_host_zone, qn_auth, bucket_name, temp_file, commonly_options): - key = 'test_file_r_retry' + def test_legacy_retry( + self, + set_default_up_host_zone, + qn_auth, + bucket_name, + temp_file, + commonly_options, + get_remote_object_headers_and_md5, + get_key + ): + key = get_key('test_file_r_retry') token = qn_auth.upload_token(bucket_name, key) ret, info = put_file( token, key, - temp_file, + temp_file.path, commonly_options.params, commonly_options.mime_type ) - print(info) - assert ret['key'] == key - assert ret['hash'] == etag(temp_file) + _, actual_md5 = get_remote_object_headers_and_md5(key=key) + assert ret['key'] == key, info + assert actual_md5 == temp_file.md5 @pytest.mark.parametrize('temp_file', [64 * KB], indirect=True) - def test_put_stream_with_key_limits(self, qn_auth, bucket_name, temp_file, commonly_options): - key = 'test_file_r' - size = os.stat(temp_file).st_size - with open(temp_file, 'rb') as input_stream: + def test_put_stream_with_key_limits(self, qn_auth, bucket_name, temp_file, commonly_options, get_key): + key = get_key('test_file_r') + with open(temp_file.path, 'rb') as input_stream: token = qn_auth.upload_token(bucket_name, key, policy={'keylimit': ['test_file_d']}) ret, info = put_stream( token, key, input_stream, - os.path.basename(temp_file), - size, + temp_file.name, + temp_file.size, None, commonly_options.params, commonly_options.mime_type @@ -525,14 +586,14 @@ def test_put_stream_with_key_limits(self, qn_auth, bucket_name, temp_file, commo token = qn_auth.upload_token( bucket_name, key, - policy={'keylimit': ['test_file_d', 'test_file_r']} + policy={'keylimit': ['test_file_d', key]} ) ret, info = put_stream( token, key, input_stream, - os.path.basename(temp_file), - size, + temp_file.name, + temp_file.size, None, commonly_options.params, commonly_options.mime_type @@ -546,18 +607,18 @@ def test_put_stream_with_metadata( bucket_name, temp_file, commonly_options, - bucket_manager + bucket_manager, + get_key ): - key = 'test_put_stream_with_metadata' - size = os.stat(temp_file).st_size - with open(temp_file, 'rb') as input_stream: + key = get_key('test_put_stream_with_metadata') + with open(temp_file.path, 'rb') as input_stream: token = qn_auth.upload_token(bucket_name, key) ret, info = put_stream( token, key, input_stream, - os.path.basename(temp_file), - size, + temp_file.name, + temp_file.size, None, commonly_options.params, commonly_options.mime_type, @@ -579,19 +640,19 @@ def test_put_stream_v2_with_metadata( bucket_name, temp_file, commonly_options, - bucket_manager + bucket_manager, + get_key ): part_size = 4 * MB - key = 'test_put_stream_v2_with_metadata' - size = os.stat(temp_file).st_size - with open(temp_file, 'rb') as input_stream: + key = get_key('test_put_stream_v2_with_metadata') + with open(temp_file.path, 'rb') as input_stream: token = qn_auth.upload_token(bucket_name, key) ret, info = put_stream( token, key, input_stream, - os.path.basename(temp_file), - size, + temp_file.name, + temp_file.size, None, commonly_options.params, commonly_options.mime_type, @@ -614,11 +675,11 @@ def test_put_stream_with_callback( temp_file, commonly_options, bucket_manager, - upload_callback_url + upload_callback_url, + get_key ): - key = 'test_put_stream_with_callback' - size = os.stat(temp_file).st_size - with open(temp_file, 'rb') as input_stream: + key = get_key('test_put_stream_with_callback') + with open(temp_file.path, 'rb') as input_stream: policy = { 'callbackUrl': upload_callback_url, 'callbackBody': '{"custom_vars":{"a":$(x:a)},"key":$(key),"hash":$(etag)}', @@ -629,8 +690,8 @@ def test_put_stream_with_callback( token, key, input_stream, - os.path.basename(temp_file), - size, + temp_file.name, + temp_file.size, None, commonly_options.params, commonly_options.mime_type, @@ -654,12 +715,12 @@ def test_put_stream_v2_with_callback( temp_file, commonly_options, bucket_manager, - upload_callback_url + upload_callback_url, + get_key ): part_size = 4 * MB - key = 'test_put_stream_v2_with_metadata' - size = os.stat(temp_file).st_size - with open(temp_file, 'rb') as input_stream: + key = get_key('test_put_stream_v2_with_metadata') + with open(temp_file.path, 'rb') as input_stream: policy = { 'callbackUrl': upload_callback_url, 'callbackBody': '{"custom_vars":{"a":$(x:a)},"key":$(key),"hash":$(etag)}', @@ -670,8 +731,8 @@ def test_put_stream_v2_with_callback( token, key, input_stream, - os.path.basename(temp_file), - size, + temp_file.name, + temp_file.size, None, commonly_options.params, commonly_options.mime_type, @@ -689,9 +750,8 @@ def test_put_stream_v2_with_callback( @pytest.mark.parametrize('temp_file', [30 * MB], indirect=True) @pytest.mark.parametrize('version', ['v1', 'v2']) - def test_resume_upload(self, bucket_name, qn_auth, temp_file, version): - key = 'test_resume_upload_{}'.format(version) - size = os.stat(temp_file).st_size + def test_resume_upload(self, bucket_name, qn_auth, temp_file, version, get_key): + key = get_key('test_resume_upload_' + version) part_size = 4 * MB def mock_fail(uploaded_size, _total_size): @@ -704,7 +764,7 @@ def mock_fail(uploaded_size, _total_size): _ret, _into = put_file( up_token=token, key=key, - file_path=temp_file, + file_path=temp_file.path, hostscache_dir=None, part_size=part_size, version=version, @@ -727,7 +787,7 @@ def should_start_from_resume(uploaded_size, _total_size): ret, into = put_file( up_token=token, key=key, - file_path=temp_file, + file_path=temp_file.path, hostscache_dir=None, part_size=part_size, version=version, @@ -735,3 +795,115 @@ def should_start_from_resume(uploaded_size, _total_size): progress_handler=should_start_from_resume ) assert ret['key'] == key + + @pytest.mark.parametrize('temp_file', [ + 64 * KB, # form + 10 * MB # resume + ], indirect=True) + @pytest.mark.parametrize('version', ['v1', 'v2']) + def test_upload_acc_normally(self, bucket_name, qn_auth, temp_file, version, get_key): + key = get_key('test_upload_acc_normally') + + token = qn_auth.upload_token(bucket_name, key) + ret, resp = put_file( + up_token=token, + key=key, + file_path=temp_file.path, + version=version, + accelerate_uploading=True + ) + + assert ret['key'] == key, resp + assert 'kodo-accelerate' in resp.url, resp + + @pytest.mark.parametrize('temp_file', [ + 64 * KB, # form + 10 * MB # resume + ], indirect=True) + @pytest.mark.parametrize('version', ['v1', 'v2']) + def test_upload_acc_fallback_src_by_network_err( + self, + bucket_name, + qn_auth, + temp_file, + version, + get_key, + get_real_regions + ): + regions = get_real_regions(qn_auth.get_access_key(), bucket_name) + r = regions[0] + r.services[ServiceName.UP_ACC] = [ + Endpoint('qiniu-acc.fake.qiniu.com') + ] + + key = get_key('test_upload_acc_fallback_src_by_network_err') + + token = qn_auth.upload_token(bucket_name, key) + ret, resp = put_file( + up_token=token, + key=key, + file_path=temp_file.path, + version=version, + regions=[r], + accelerate_uploading=True + ) + + assert ret['key'] == key, resp + + @pytest.mark.parametrize('temp_file', [ + 64 * KB, # form + 10 * MB # resume + ], indirect=True) + @pytest.mark.parametrize('version', ['v1', 'v2']) + def test_upload_acc_fallback_src_by_acc_unavailable( + self, + no_acc_bucket_name, + qn_auth, + temp_file, + version, + rand_string, + auto_remove, + get_real_regions + ): + regions = get_real_regions(qn_auth.get_access_key(), no_acc_bucket_name) + + region = regions[0] + region.services[ServiceName.UP_ACC] = [ + Endpoint('{0}.kodo-accelerate.{1}.qiniucs.com'.format(no_acc_bucket_name, region.s3_region_id)), + Endpoint('fake-acc.python-sdk.qiniu.com') + ] + + key = 'test_upload_acc_fallback_src_by_acc_unavailable-' + rand_string(8) + auto_remove(no_acc_bucket_name, key) + + token = qn_auth.upload_token(no_acc_bucket_name, key) + ret, resp = put_file( + up_token=token, + key=key, + file_path=temp_file.path, + version=version, + accelerate_uploading=True + ) + + assert ret['key'] == key, resp + + def test_uploader_base_compatible(self, qn_auth, bucket_name): + if is_py2: + class MockUploader(UploaderBase): + def upload( + self, + **kwargs + ): + pass + uploader = MockUploader( + bucket_name=bucket_name, + auth=qn_auth + ) + else: + uploader = UploaderBase( + bucket_name=bucket_name, + auth=qn_auth + ) + + up_hosts = uploader._get_up_hosts() + assert len(up_hosts) > 0 diff --git a/tests/cases/test_services/test_storage/test_uploaders_default_retrier.py b/tests/cases/test_services/test_storage/test_uploaders_default_retrier.py new file mode 100644 index 00000000..2aaa83ee --- /dev/null +++ b/tests/cases/test_services/test_storage/test_uploaders_default_retrier.py @@ -0,0 +1,235 @@ +import pytest + +import os + +from qiniu.http.region import ServiceName, Region +from qiniu.retry import Attempt +from qiniu.services.storage.uploaders._default_retrier import ( + ProgressRecord, + TokenExpiredRetryPolicy, + AccUnavailableRetryPolicy +) + + +@pytest.fixture( + scope='function', + params=[ + {'api_version': 'v1'}, + {'api_version': 'v2'} + ] +) +def fake_progress_record(request): + api_version = request.param.get('api_version') + file_path = os.path.join(os.getcwd(), 'fake-progress-record') + + with open(file_path, 'w'): + pass + + def _delete(): + try: + os.remove(file_path) + except OSError: + pass + + def _exists(): + return os.path.exists(file_path) + + yield ProgressRecord( + upload_api_version=api_version, + exists=_exists, + delete=_delete + ) + + _delete() + + +class MockResponse: + def __init__(self, status_code, text_body=None): + self.status_code = status_code + self.text_body = text_body + + +class TestTokenExpiredRetryPolicy: + def test_should_retry(self, fake_progress_record): + policy = TokenExpiredRetryPolicy( + upload_api_version=fake_progress_record.upload_api_version, + record_delete_handler=fake_progress_record.delete, + record_exists_handler=fake_progress_record.exists + ) + + attempt = Attempt() + policy.init_context(attempt.context) + + if fake_progress_record.upload_api_version == 'v1': + mocked_resp = MockResponse(status_code=701) + else: + mocked_resp = MockResponse(status_code=612) + attempt.result = (None, mocked_resp) + + assert policy.should_retry(attempt) + + def test_should_not_retry_by_no_result(self, fake_progress_record): + policy = TokenExpiredRetryPolicy( + upload_api_version=fake_progress_record.upload_api_version, + record_delete_handler=fake_progress_record.delete, + record_exists_handler=fake_progress_record.exists + ) + attempt = Attempt() + policy.init_context(attempt.context) + + assert not policy.should_retry(attempt) + + def test_should_not_retry_by_default_max_retried_times(self, fake_progress_record): + policy = TokenExpiredRetryPolicy( + upload_api_version=fake_progress_record.upload_api_version, + record_delete_handler=fake_progress_record.delete, + record_exists_handler=fake_progress_record.exists + ) + attempt = Attempt() + policy.init_context(attempt.context) + if fake_progress_record.upload_api_version == 'v1': + mocked_resp = MockResponse(status_code=701) + else: + mocked_resp = MockResponse(status_code=612) + attempt.result = (None, mocked_resp) + attempt.context[policy] = attempt.context[policy]._replace(retried_times=1) + + assert not policy.should_retry(attempt) + + def test_should_not_retry_by_file_no_exists(self, fake_progress_record): + policy = TokenExpiredRetryPolicy( + upload_api_version=fake_progress_record.upload_api_version, + record_delete_handler=fake_progress_record.delete, + record_exists_handler=fake_progress_record.exists + ) + + attempt = Attempt() + policy.init_context(attempt.context) + if fake_progress_record.upload_api_version == 'v1': + mocked_resp = MockResponse(status_code=701) + else: + mocked_resp = MockResponse(status_code=612) + attempt.result = (None, mocked_resp) + fake_progress_record.delete() + + assert not policy.should_retry(attempt) + + def test_prepare_retry(self, fake_progress_record): + policy = TokenExpiredRetryPolicy( + upload_api_version=fake_progress_record.upload_api_version, + record_delete_handler=fake_progress_record.delete, + record_exists_handler=fake_progress_record.exists + ) + + attempt = Attempt() + policy.init_context(attempt.context) + if fake_progress_record.upload_api_version == 'v1': + mocked_resp = MockResponse(status_code=701) + else: + mocked_resp = MockResponse(status_code=612) + attempt.result = (None, mocked_resp) + + policy.prepare_retry(attempt) + + assert not fake_progress_record.exists() + + +class TestAccUnavailableRetryPolicy: + def test_should_retry(self): + policy = AccUnavailableRetryPolicy() + attempt = Attempt() + + attempt.context['service_name'] = ServiceName.UP_ACC + attempt.context['alternative_service_names'] = [ServiceName.UP] + attempt.context['region'] = Region.from_region_id('z0') + + mocked_resp = MockResponse( + status_code=400, + text_body='{"error":"transfer acceleration is not configured on this bucket"}' + ) + attempt.result = (None, mocked_resp) + + assert policy.should_retry(attempt) + + def test_should_not_retry_by_no_result(self): + policy = AccUnavailableRetryPolicy() + attempt = Attempt() + + attempt.context['service_name'] = ServiceName.UP_ACC + attempt.context['alternative_service_names'] = [ServiceName.UP] + attempt.context['region'] = Region.from_region_id('z0') + + assert not policy.should_retry(attempt) + + def test_should_not_retry_by_no_alternative_services(self): + policy = AccUnavailableRetryPolicy() + attempt = Attempt() + + attempt.context['service_name'] = ServiceName.UP + attempt.context['alternative_service_names'] = [] + attempt.context['region'] = Region.from_region_id('z0') + + mocked_resp = MockResponse( + status_code=400, + text_body='{"error":"transfer acceleration is not configured on this bucket"}' + ) + attempt.result = (None, mocked_resp) + + assert not policy.should_retry(attempt) + + def test_should_not_retry_by_no_alternative_endpoints(self): + policy = AccUnavailableRetryPolicy() + attempt = Attempt() + + attempt.context['service_name'] = ServiceName.UP_ACC + attempt.context['alternative_service_names'] = [ServiceName.UP] + attempt.context['region'] = Region.from_region_id('z0') + attempt.context['region'].services[ServiceName.UP] = [] + + mocked_resp = MockResponse( + status_code=400, + text_body='{"error":"transfer acceleration is not configured on this bucket"}' + ) + attempt.result = (None, mocked_resp) + + assert not policy.should_retry(attempt) + + def test_should_not_retry_by_other_error(self): + policy = AccUnavailableRetryPolicy() + attempt = Attempt() + + attempt.context['service_name'] = ServiceName.UP_ACC + attempt.context['alternative_service_names'] = [ServiceName.UP] + attempt.context['region'] = Region.from_region_id('z0') + + mocked_resp = MockResponse( + status_code=400, + text_body='{"error":"Bad Request"}' + ) + attempt.result = (None, mocked_resp) + + assert not policy.should_retry(attempt) + + def test_prepare_retry(self): + policy = AccUnavailableRetryPolicy() + attempt = Attempt() + region = Region.from_region_id('z0') + + attempt.context['service_name'] = ServiceName.UP_ACC + attempt.context['alternative_service_names'] = [ServiceName.UP] + attempt.context['region'] = region + + mocked_resp = MockResponse( + status_code=400, + text_body='{"error":"transfer acceleration is not configured on this bucket"}' + ) + attempt.result = (None, mocked_resp) + + policy.prepare_retry(attempt) + + assert attempt.context['service_name'] == ServiceName.UP + assert ( + [attempt.context['endpoint']] + attempt.context['alternative_endpoints'] + == + region.services[ServiceName.UP] + ) diff --git a/tests/cases/test_zone/__init__.py b/tests/cases/test_zone/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/test_zone/test_lagacy_region.py b/tests/cases/test_zone/test_lagacy_region.py new file mode 100644 index 00000000..0a8de93d --- /dev/null +++ b/tests/cases/test_zone/test_lagacy_region.py @@ -0,0 +1,111 @@ +import pytest + +from qiniu.http.region import Region, ServiceName +from qiniu.region import LegacyRegion +from qiniu.compat import json, is_py2 + + +@pytest.fixture +def mocked_hosts(): + mocked_hosts = { + ServiceName.UP: ['https://up.python-example.qiniu.com', 'https://up-2.python-example.qiniu.com'], + ServiceName.IO: ['https://io.python-example.qiniu.com'], + ServiceName.RS: ['https://rs.python-example.qiniu.com'], + ServiceName.RSF: ['https://rsf.python-example.qiniu.com'], + ServiceName.API: ['https://api.python-example.qiniu.com'] + } + yield mocked_hosts + + +@pytest.fixture +def mock_legacy_region(mocked_hosts): + region = LegacyRegion( + up_host=mocked_hosts[ServiceName.UP][0], + up_host_backup=mocked_hosts[ServiceName.UP][1], + io_host=mocked_hosts[ServiceName.IO][0], + rs_host=mocked_hosts[ServiceName.RS][0], + rsf_host=mocked_hosts[ServiceName.RSF][0], + api_host=mocked_hosts[ServiceName.API][0] + ) + yield region + + +class TestLegacyRegion: + def test_get_hosts_from_self(self, mocked_hosts, mock_legacy_region, qn_auth, bucket_name): + cases = [ + # up will always query from the old version, + # which version implements the `get_up_host_*` method + ( + mock_legacy_region.get_io_host(qn_auth.get_access_key(), None), + mocked_hosts[ServiceName.IO][0] + ), + ( + mock_legacy_region.get_rs_host(qn_auth.get_access_key(), None), + mocked_hosts[ServiceName.RS][0] + ), + ( + mock_legacy_region.get_rsf_host(qn_auth.get_access_key(), None), + mocked_hosts[ServiceName.RSF][0] + ), + ( + mock_legacy_region.get_api_host(qn_auth.get_access_key(), None), + mocked_hosts[ServiceName.API][0] + ) + ] + for actual, expect in cases: + assert actual == expect + + def test_get_hosts_from_query(self, qn_auth, bucket_name): + up_token = qn_auth.upload_token(bucket_name) + region = LegacyRegion() + up_host = region.get_up_host_by_token(up_token, None) + up_host_backup = region.get_up_host_backup_by_token(up_token, None) + if is_py2: + up_host = up_host.encode() + up_host_backup = up_host_backup.encode() + assert type(up_host) is str and len(up_host) > 0 + assert type(up_host_backup) is str and len(up_host_backup) > 0 + assert up_host != up_host_backup + + def test_compatible_with_http_region(self, mocked_hosts, mock_legacy_region): + assert isinstance(mock_legacy_region, Region) + assert mocked_hosts == { + k: [ + e.get_value() + for e in mock_legacy_region.services[k] + ] + for k in mocked_hosts + } + + def test_get_bucket_hosts(self, access_key, bucket_name): + region = LegacyRegion() + bucket_hosts = region.get_bucket_hosts(access_key, bucket_name) + for k in [ + 'upHosts', + 'ioHosts', + 'rsHosts', + 'rsfHosts', + 'apiHosts' + ]: + assert all(h.startswith('http') for h in bucket_hosts[k]), bucket_hosts[k] + + def test_bucket_hosts(self, access_key, bucket_name): + region = LegacyRegion() + bucket_hosts_str = region.bucket_hosts(access_key, bucket_name) + bucket_hosts = json.loads(bucket_hosts_str) + + region_hosts = bucket_hosts.get('hosts', []) + + assert len(region_hosts) > 0 + + for r in region_hosts: + for k in [ + 'up', + 'io', + 'rs', + 'rsf', + 'api' + ]: + service_hosts = r[k].get('domains') + assert len(service_hosts) > 0 + assert all(len(h) for h in service_hosts) diff --git a/tests/cases/test_zone/test_qiniu_conf.py b/tests/cases/test_zone/test_qiniu_conf.py new file mode 100644 index 00000000..c6bce5b8 --- /dev/null +++ b/tests/cases/test_zone/test_qiniu_conf.py @@ -0,0 +1,99 @@ +import pytest + +from qiniu import Zone +from qiniu.config import get_default + +TEST_RS_HOST = 'rs.test.region.compatible.config.qiniu.com' +TEST_RSF_HOST = 'rsf.test.region.compatible.config.qiniu.com' +TEST_API_HOST = 'api.test.region.compatible.config.qiniu.com' + + +class TestQiniuConfWithZone: + """ + Test qiniu.conf with Zone(aka LegacyRegion) + """ + + @pytest.mark.parametrize( + 'set_conf_default', + [ + { + 'default_uc_backup_hosts': [], + }, + { + 'default_uc_backup_hosts': [], + 'default_query_region_backup_hosts': [] + } + ], + indirect=True + ) + def test_disable_backup_hosts(self, set_conf_default): + assert get_default('default_uc_backup_hosts') == [] + assert get_default('default_query_region_backup_hosts') == [] + + @pytest.mark.parametrize( + 'set_conf_default', + [ + { + 'default_rs_host': TEST_RS_HOST, + 'default_rsf_host': TEST_RSF_HOST, + 'default_api_host': TEST_API_HOST + } + ], + indirect=True + ) + def test_config_compatible(self, set_conf_default): + zone = Zone() + assert zone.get_rs_host("mock_ak", "mock_bucket") == TEST_RS_HOST + assert zone.get_rsf_host("mock_ak", "mock_bucket") == TEST_RSF_HOST + assert zone.get_api_host("mock_ak", "mock_bucket") == TEST_API_HOST + + @pytest.mark.parametrize( + 'set_conf_default', + [ + { + 'default_query_region_host': 'https://fake-uc.phpsdk.qiniu.com' + } + ], + indirect=True + ) + def test_query_region_with_custom_domain(self, access_key, bucket_name, set_conf_default): + with pytest.raises(Exception) as exc: + zone = Zone() + zone.bucket_hosts(access_key, bucket_name) + assert 'HTTP Status Code -1' in str(exc) + + @pytest.mark.parametrize( + 'set_conf_default', + [ + { + 'default_query_region_host': 'https://fake-uc.phpsdk.qiniu.com', + 'default_query_region_backup_hosts': [ + 'unavailable-uc.phpsdk.qiniu.com', + 'uc.qbox.me' + ] + } + ], + indirect=True + ) + def test_query_region_with_backup_domains(self, access_key, bucket_name, set_conf_default): + zone = Zone() + data = zone.bucket_hosts(access_key, bucket_name) + assert data != 'null' + + @pytest.mark.parametrize( + 'set_conf_default', + [ + { + 'default_uc_host': 'https://fake-uc.phpsdk.qiniu.com', + 'default_query_region_backup_hosts': [ + 'unavailable-uc.phpsdk.qiniu.com', + 'uc.qbox.me' + ] + } + ], + indirect=True + ) + def test_query_region_with_uc_and_backup_domains(self, access_key, bucket_name, set_conf_default): + zone = Zone() + data = zone.bucket_hosts(access_key, bucket_name) + assert data != 'null' From fca7849eccaf2690321d5aeb85e235a847e7acd0 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Mon, 21 Oct 2024 14:33:17 +0800 Subject: [PATCH 466/478] Bump to v7.15.0 (#452) --- .github/workflows/ci-test.yml | 101 +++++++++++- .gitignore | 3 +- CHANGELOG.md | 8 +- codecov.yml | 16 +- qiniu/__init__.py | 2 +- qiniu/auth.py | 3 +- qiniu/http/regions_provider.py | 11 +- qiniu/region.py | 4 +- qiniu/services/processing/pfop.py | 19 ++- qiniu/utils.py | 30 +++- setup.py | 4 - test_qiniu.py | 135 ++-------------- tests/cases/test_http/test_region.py | 2 +- .../cases/test_http/test_regions_provider.py | 6 +- .../test_processing/test_pfop.py | 76 +++++++-- .../test_storage/test_upload_pfop.py | 66 +++++--- tests/cases/test_utils.py | 145 ++++++++++++++++++ 17 files changed, 437 insertions(+), 194 deletions(-) create mode 100644 tests/cases/test_utils.py diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index c705019b..52195876 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -60,14 +60,107 @@ jobs: QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}} QINIU_TEST_ENV: "travis" MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" - PYTHONPATH: "$PYTHONPATH:." run: | flake8 --show-source --max-line-length=160 ./qiniu - coverage run -m pytest ./test_qiniu.py ./tests/cases - ocular --data-file .coverage - codecov + python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml + - name: Post Setup mock server + if: ${{ always() }} + shell: bash + run: | + set +e cat mock-server.pid | xargs kill + rm mock-server.pid - name: Print mock server log if: ${{ failure() }} run: | cat py-mock-server.log + - name: Upload results to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + test-win: + strategy: + fail-fast: false + max-parallel: 1 + matrix: + python_version: ['2.7', '3.5', '3.9'] + runs-on: windows-2019 + # make sure only one test running, + # remove this when cases could run in parallel. + needs: test + steps: + - name: Checkout repo + uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge + python-version: ${{ matrix.python_version }} + activate-environment: qiniu-sdk + auto-activate-base: false + - name: Setup pip + env: + PYTHON_VERSION: ${{ matrix.python_version }} + PIP_BOOTSTRAP_SCRIPT_PREFIX: https://bootstrap.pypa.io/pip + run: | + # reinstall pip by some python(<3.7) not compatible + $pyversion = [Version]"$ENV:PYTHON_VERSION" + if ($pyversion -lt [Version]"3.7") { + Invoke-WebRequest "$ENV:PIP_BOOTSTRAP_SCRIPT_PREFIX/$($pyversion.Major).$($pyversion.Minor)/get-pip.py" -OutFile "$ENV:TEMP\get-pip.py" + python $ENV:TEMP\get-pip.py --user + Remove-Item -Path "$ENV:TEMP\get-pip.py" + } + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -I -e ".[dev]" + - name: Run cases + env: + QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} + QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} + QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }} + QINIU_TEST_NO_ACC_BUCKET: ${{ secrets.QINIU_TEST_NO_ACC_BUCKET }} + QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }} + QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}} + QINIU_TEST_ENV: "github" + MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" + PYTHONPATH: "$PYTHONPATH:." + run: | + Write-Host "======== Setup Mock Server =========" + conda create -y -n mock-server python=3.10 + conda activate mock-server + python --version + $processOptions = @{ + FilePath="python" + ArgumentList="tests\mock_server\main.py", "--port", "9000" + PassThru=$true + RedirectStandardOutput="py-mock-server.log" + } + $mocksrvp = Start-Process @processOptions + $mocksrvp.Id | Out-File -FilePath "mock-server.pid" + conda deactivate + Sleep 3 + Write-Host "======== Running Test =========" + python --version + python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml + - name: Post Setup mock server + if: ${{ always() }} + run: | + Try { + $mocksrvpid = Get-Content -Path "mock-server.pid" + Stop-Process -Id $mocksrvpid + Remove-Item -Path "mock-server.pid" + } Catch { + Write-Host -Object $_ + } + - name: Print mock server log + if: ${{ failure() }} + run: | + Get-Content -Path "py-mock-server.log" | Write-Host + - name: Upload results to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 93665221..261e665c 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ pip-log.txt .coverage .tox nosetests.xml +coverage.xml # Translations *.mo @@ -45,4 +46,4 @@ nosetests.xml .project .pydevproject /.idea -/.venv \ No newline at end of file +/.venv* diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bf26f46..5722b56b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 7.15.0 +* 对象存储，持久化处理支持工作流模版 +* 对象存储，修复 Windows 平台兼容性问题 + ## 7.14.0 * 对象存储，空间管理、上传文件新增备用域名重试逻辑 * 对象存储，调整查询区域主备域名 @@ -44,7 +48,7 @@ ## 7.9.0（2022-07-20） * 对象存储，支持使用时不配置区域信息，SDK 自动获取； * 对象存储，新增 list_domains API 用于查询空间绑定的域名 -* 对象存储，上传 API 新增支持设置自定义元数据，详情见 put_data, put_file, put_stream API +* 对象存储，上传 API 新增支持设置自定义元数据，详情见 put_data, put_file, put_stream API * 解决部分已知问题 ## 7.8.0（2022-06-08） @@ -237,5 +241,3 @@ * 代码覆盖度报告 * policy改为dict, 便于灵活增加，并加入过期字段检查 * 文件列表支持目录形式 - - diff --git a/codecov.yml b/codecov.yml index 3f36c50a..0aab28d3 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,14 +1,14 @@ codecov: ci: - - prow.qiniu.io # prow 里面运行需添加，其他 CI 不要 - require_ci_to_pass: no # 改为 no，否则 codecov 会等待其他 GitHub 上所有 CI 通过才会留言。 + - prow.qiniu.io # prow need this. seems useless + require_ci_to_pass: no # `no` means the bot will comment on the PR even before all ci passed -github_checks: #关闭github checks +github_checks: # close github checks annotations: false comment: layout: "reach, diff, flags, files" - behavior: new # 默认是更新旧留言，改为 new，删除旧的，增加新的。 + behavior: new # `new` means the bot will comment a new message instead of edit the old one require_changes: false # if true: only post the comment if coverage changes require_base: no # [yes :: must have a base report to post] require_head: yes # [yes :: must have a head report to post] @@ -16,13 +16,13 @@ comment: - "master" coverage: - status: # 评判 pr 通过的标准 + status: # check coverage status to pass or fail patch: off - project: # project 统计所有代码x + project: # project analyze all code in the project default: # basic - target: 73.5% # 总体通过标准 - threshold: 3% # 允许单次下降的幅度 + target: 73.5% # the minimum coverage ratio that the commit must meet + threshold: 3% # allow the coverage to drop base: auto if_not_found: success if_ci_failed: error diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 55acfb25..1ae68c00 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.14.0' +__version__ = '7.15.0' from .auth import Auth, QiniuMacAuth diff --git a/qiniu/auth.py b/qiniu/auth.py index d3e0a055..1647199e 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -34,10 +34,11 @@ str('fsizeMin'), # 上传文件最少字节数 str('keylimit'), # 设置允许上传的key列表，字符串数组类型，数组长度不可超过20个，如果设置了这个字段，上传时必须提供key - str('persistentOps'), # 持久化处理操作 + str('persistentOps'), # 持久化处理操作，与 persistentWorkflowTemplateID 二选一 str('persistentNotifyUrl'), # 持久化处理结果通知URL str('persistentPipeline'), # 持久化处理独享队列 str('persistentType'), # 为 `1` 时，开启闲时任务，必须是 int 类型 + str('persistentWorkflowTemplateID'), # 工作流模板 ID，与 persistentOps 二选一 str('deleteAfterDays'), # 文件多少天后自动删除 str('fileType'), # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储，4为归档直读存储 diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py index 8b52822c..13d1800a 100644 --- a/qiniu/http/regions_provider.py +++ b/qiniu/http/regions_provider.py @@ -5,9 +5,10 @@ import logging import tempfile import os +import shutil from qiniu.compat import json, b as to_bytes -from qiniu.utils import io_md5 +from qiniu.utils import io_md5, dt2ts from .endpoint import Endpoint from .region import Region, ServiceName @@ -264,7 +265,7 @@ def _persist_region(region): }, ttl=region.ttl, # use datetime.datetime.timestamp() when min version of python >= 3 - createTime=int(float(region.create_time.strftime('%s.%f')) * 1000) + createTime=dt2ts(region.create_time) )._asdict() @@ -338,8 +339,10 @@ def _walk_persist_cache_file(persist_path, ignore_parse_error=False): with open(persist_path, 'r') as f: for line in f: + if not line.strip(): + continue try: - cache_key, regions = _parse_persisted_regions(line) + cache_key, regions = _parse_persisted_regions(line.strip()) yield cache_key, regions except Exception as err: if not ignore_parse_error: @@ -655,7 +658,7 @@ def __shrink_cache(self): ) # rename file - os.rename(shrink_file_path, self._cache_scope.persist_path) + shutil.move(shrink_file_path, self._cache_scope.persist_path) except FileAlreadyLocked: pass finally: diff --git a/qiniu/region.py b/qiniu/region.py index 09ac791d..a59d488e 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -6,7 +6,7 @@ from .compat import json, s as str_from_bytes -from .utils import urlsafe_base64_decode +from .utils import urlsafe_base64_decode, dt2ts from .config import UC_HOST, is_customized_default, get_default from .http.endpoint import Endpoint as _HTTPEndpoint from .http.regions_provider import Region as _HTTPRegion, ServiceName, get_default_regions_provider @@ -190,7 +190,7 @@ def get_bucket_hosts(self, ak, bucket, home_dir=None, force=False): ttl = region.ttl if region.ttl > 0 else 24 * 3600 # 1 day # use datetime.datetime.timestamp() when min version of python >= 3 - create_time = int(float(region.create_time.strftime('%s.%f')) * 1000) + create_time = dt2ts(region.create_time) bucket_hosts['deadline'] = create_time + ttl return bucket_hosts diff --git a/qiniu/services/processing/pfop.py b/qiniu/services/processing/pfop.py index 346e6277..4b2641e2 100644 --- a/qiniu/services/processing/pfop.py +++ b/qiniu/services/processing/pfop.py @@ -24,7 +24,7 @@ def __init__(self, auth, bucket, pipeline=None, notify_url=None): self.pipeline = pipeline self.notify_url = notify_url - def execute(self, key, fops, force=None, persistent_type=None): + def execute(self, key, fops=None, force=None, persistent_type=None, workflow_template_id=None): """ 执行持久化处理 @@ -32,28 +32,39 @@ def execute(self, key, fops, force=None, persistent_type=None): ---------- key: str 待处理的源文件 - fops: list[str] + fops: list[str], optional 处理详细操作，规格详见 https://developer.qiniu.com/dora/manual/1291/persistent-data-processing-pfop + 与 template_id 二选一 force: int or str, optional 强制执行持久化处理开关 persistent_type: int or str, optional 持久化处理类型，为 '1' 时开启闲时任务 + template_id: str, optional + 与 fops 二选一 Returns ------- ret: dict 持久化处理的 persistentId，类似 {"persistentId": 5476bedf7823de4068253bae}; resp: ResponseInfo """ - ops = ';'.join(fops) - data = {'bucket': self.bucket, 'key': key, 'fops': ops} + if not fops and not workflow_template_id: + raise ValueError('Must provide one of fops or template_id') + data = { + 'bucket': self.bucket, + 'key': key, + } if self.pipeline: data['pipeline'] = self.pipeline if self.notify_url: data['notifyURL'] = self.notify_url + if fops: + data['fops'] = ';'.join(fops) if force == 1 or force == '1': data['force'] = str(force) if persistent_type and type(int(persistent_type)) is int: data['type'] = str(persistent_type) + if workflow_template_id: + data['workflowTemplateID'] = workflow_template_id url = '{0}/pfop'.format(config.get_default('default_api_host')) return http._post_with_auth(url, data, self.auth) diff --git a/qiniu/utils.py b/qiniu/utils.py index f8517e35..197b8813 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- from hashlib import sha1, new as hashlib_new from base64 import urlsafe_b64encode, urlsafe_b64decode -from datetime import datetime +from datetime import datetime, tzinfo, timedelta + from .compat import b, s try: @@ -236,3 +237,30 @@ def canonical_mime_header_key(field_name): result += ch upper = ch == "-" return result + + +class _UTC_TZINFO(tzinfo): + def utcoffset(self, dt): + return timedelta(hours=0) + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return timedelta(0) + + +def dt2ts(dt): + """ + converte datetime to timestamp + + Parameters + ---------- + dt: datetime.datetime + """ + if not dt.tzinfo: + st = (dt - datetime(1970, 1, 1)).total_seconds() + else: + st = (dt - datetime(1970, 1, 1, tzinfo=_UTC_TZINFO())).total_seconds() + + return int(st) diff --git a/setup.py b/setup.py index cf97eae2..fa920d45 100644 --- a/setup.py +++ b/setup.py @@ -42,10 +42,8 @@ def find_version(*file_paths): 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', @@ -66,8 +64,6 @@ def find_version(*file_paths): 'pytest', 'pytest-cov', 'freezegun', - 'scrutinizer-ocular', - 'codecov' ] }, diff --git a/test_qiniu.py b/test_qiniu.py index 2b71aa22..c8dce456 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -65,129 +65,6 @@ def remove_temp_file(file): except OSError: pass - -class UtilsTest(unittest.TestCase): - def test_urlsafe(self): - a = 'hello\x96' - u = urlsafe_base64_encode(a) - assert b(a) == urlsafe_base64_decode(u) - - def test_canonical_mime_header_key(self): - field_names = [ - ":status", - ":x-test-1", - ":x-Test-2", - "content-type", - "CONTENT-LENGTH", - "oRiGin", - "ReFer", - "Last-Modified", - "acCePt-ChArsEt", - "x-test-3", - "cache-control", - ] - expect_canonical_field_names = [ - ":status", - ":x-test-1", - ":x-Test-2", - "Content-Type", - "Content-Length", - "Origin", - "Refer", - "Last-Modified", - "Accept-Charset", - "X-Test-3", - "Cache-Control", - ] - assert len(field_names) == len(expect_canonical_field_names) - for i in range(len(field_names)): - assert canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i] - - def test_entry(self): - case_list = [ - { - 'msg': 'normal', - 'bucket': 'qiniuphotos', - 'key': 'gogopher.jpg', - 'expect': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' - }, - { - 'msg': 'key empty', - 'bucket': 'qiniuphotos', - 'key': '', - 'expect': 'cWluaXVwaG90b3M6' - }, - { - 'msg': 'key undefined', - 'bucket': 'qiniuphotos', - 'key': None, - 'expect': 'cWluaXVwaG90b3M=' - }, - { - 'msg': 'key need replace plus symbol', - 'bucket': 'qiniuphotos', - 'key': '012ts>a', - 'expect': 'cWluaXVwaG90b3M6MDEydHM-YQ==' - }, - { - 'msg': 'key need replace slash symbol', - 'bucket': 'qiniuphotos', - 'key': '012ts?a', - 'expect': 'cWluaXVwaG90b3M6MDEydHM_YQ==' - } - ] - for c in case_list: - assert c.get('expect') == entry(c.get('bucket'), c.get('key')), c.get('msg') - - def test_decode_entry(self): - case_list = [ - { - 'msg': 'normal', - 'expect': { - 'bucket': 'qiniuphotos', - 'key': 'gogopher.jpg' - }, - 'entry': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' - }, - { - 'msg': 'key empty', - 'expect': { - 'bucket': 'qiniuphotos', - 'key': '' - }, - 'entry': 'cWluaXVwaG90b3M6' - }, - { - 'msg': 'key undefined', - 'expect': { - 'bucket': 'qiniuphotos', - 'key': None - }, - 'entry': 'cWluaXVwaG90b3M=' - }, - { - 'msg': 'key need replace plus symbol', - 'expect': { - 'bucket': 'qiniuphotos', - 'key': '012ts>a' - }, - 'entry': 'cWluaXVwaG90b3M6MDEydHM-YQ==' - }, - { - 'msg': 'key need replace slash symbol', - 'expect': { - 'bucket': 'qiniuphotos', - 'key': '012ts?a' - }, - 'entry': 'cWluaXVwaG90b3M6MDEydHM_YQ==' - } - ] - for c in case_list: - bucket, key = decode_entry(c.get('entry')) - assert bucket == c.get('expect').get('bucket'), c.get('msg') - assert key == c.get('expect').get('key'), c.get('msg') - - class BucketTestCase(unittest.TestCase): q = Auth(access_key, secret_key) bucket = BucketManager(q) @@ -408,7 +285,11 @@ def test_invalid_x_qiniu_date_with_disable_date_sign(self): def test_invalid_x_qiniu_date_env(self): os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = 'True' ret, info = self.bucket.stat(bucket_name, 'python-sdk.html') - os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE') + if hasattr(os, 'unsetenv'): + os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE') + else: + # fix unsetenv not exists in earlier python on windows + os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = '' assert 'hash' in ret @freeze_time("1970-01-01") @@ -417,7 +298,11 @@ def test_invalid_x_qiniu_date_env_be_ignored(self): q = Auth(access_key, secret_key, disable_qiniu_timestamp_signature=False) bucket = BucketManager(q) ret, info = bucket.stat(bucket_name, 'python-sdk.html') - os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE') + if hasattr(os, 'unsetenv'): + os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE') + else: + # fix unsetenv not exists in earlier python on windows + os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = '' assert ret is None assert info.status_code == 403 diff --git a/tests/cases/test_http/test_region.py b/tests/cases/test_http/test_region.py index a66b16c9..976d2619 100644 --- a/tests/cases/test_http/test_region.py +++ b/tests/cases/test_http/test_region.py @@ -36,7 +36,7 @@ def test_custom_options(self): k in region.services for k in chain(ServiceName, ['custom-service']) ) - assert datetime.now() - region.create_time > timedelta(days=1) + assert datetime.now() - region.create_time >= timedelta(days=1) assert region.ttl == 3600 assert not region.is_live diff --git a/tests/cases/test_http/test_regions_provider.py b/tests/cases/test_http/test_regions_provider.py index 163f19d2..7289f5ca 100644 --- a/tests/cases/test_http/test_regions_provider.py +++ b/tests/cases/test_http/test_regions_provider.py @@ -186,8 +186,10 @@ def test_getter_with_base_regions_provider(self, cached_regions_provider): assert list(cached_regions_provider) == regions line_num = 0 with open(cached_regions_provider.persist_path, 'r') as f: - for _ in f: - line_num += 1 + for l in f: + # ignore empty line + if l.strip(): + line_num += 1 assert line_num == 1 @pytest.mark.parametrize( diff --git a/tests/cases/test_services/test_processing/test_pfop.py b/tests/cases/test_services/test_processing/test_pfop.py index 003be43f..ebaf18f4 100644 --- a/tests/cases/test_services/test_processing/test_pfop.py +++ b/tests/cases/test_services/test_processing/test_pfop.py @@ -1,6 +1,5 @@ import pytest - from qiniu import PersistentFop, op_save @@ -16,6 +15,7 @@ def test_pfop_execute(self, qn_auth): ] ret, resp = pfop.execute('sintel_trailer.mp4', ops, 1) assert resp.status_code == 200, resp + assert ret is not None, resp assert ret['persistentId'] is not None, resp global persistent_id persistent_id = ret['persistentId'] @@ -27,23 +27,71 @@ def test_pfop_get_status(self, qn_auth): assert resp.status_code == 200, resp assert ret is not None, resp - def test_pfop_idle_time_task(self, set_conf_default, qn_auth): - persistence_key = 'python-sdk-pfop-test/test-pfop-by-api' + @pytest.mark.parametrize( + 'persistent_options', + ( + # included by above test_pfop_execute + # { + # 'persistent_type': None, + # }, + { + 'persistent_type': 0, + }, + { + 'persistent_type': 1, + }, + { + 'workflow_template_id': 'test-workflow', + }, + ) + ) + def test_pfop_idle_time_task( + self, + set_conf_default, + qn_auth, + bucket_name, + persistent_options, + ): + persistent_type = persistent_options.get('persistent_type') + workflow_template_id = persistent_options.get('workflow_template_id', None) + + execute_opts = {} + if workflow_template_id: + execute_opts['workflow_template_id'] = workflow_template_id + else: + persistent_key = '_'.join([ + 'test-pfop/test-pfop-by-api', + 'type', + str(persistent_type) + ]) + execute_opts['fops'] = [ + op_save( + op='avinfo', + bucket=bucket_name, + key=persistent_key + ) + ] + + if persistent_type is not None: + execute_opts['persistent_type'] = persistent_type + + pfop = PersistentFop(qn_auth, bucket_name) + key = 'qiniu.png' + ret, resp = pfop.execute( + key, + **execute_opts + ) - key = 'sintel_trailer.mp4' - pfop = PersistentFop(qn_auth, 'testres') - ops = [ - op_save( - op='avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', - bucket='pythonsdk', - key=persistence_key - ) - ] - ret, resp = pfop.execute(key, ops, force=1, persistent_type=1) assert resp.status_code == 200, resp + assert ret is not None assert 'persistentId' in ret, resp ret, resp = pfop.get_status(ret['persistentId']) assert resp.status_code == 200, resp - assert ret['type'] == 1, resp + assert ret is not None assert ret['creationDate'] is not None, resp + + if persistent_id == 1: + assert ret['type'] == 1, resp + elif workflow_template_id: + assert workflow_template_id in ret['taskFrom'], resp diff --git a/tests/cases/test_services/test_storage/test_upload_pfop.py b/tests/cases/test_services/test_storage/test_upload_pfop.py index 78818ba4..3effa9c7 100644 --- a/tests/cases/test_services/test_storage/test_upload_pfop.py +++ b/tests/cases/test_services/test_storage/test_upload_pfop.py @@ -12,36 +12,60 @@ # or this test will continue to occupy bucket space. class TestPersistentFopByUpload: @pytest.mark.parametrize('temp_file', [10 * MB], indirect=True) - @pytest.mark.parametrize('persistent_type', [None, 0, 1]) + @pytest.mark.parametrize( + 'persistent_options', + ( + { + 'persistent_type': None, + }, + { + 'persistent_type': 0, + }, + { + 'persistent_type': 1, + }, + { + 'persistent_workflow_template_id': 'test-workflow', + }, + ) + ) def test_pfop_with_upload( self, set_conf_default, qn_auth, bucket_name, temp_file, - persistent_type + persistent_options, ): - key = 'test-pfop-upload-file' - persistent_key = '_'.join([ - 'test-pfop-by-upload', - 'type', - str(persistent_type) - ]) - persistent_ops = ';'.join([ - qiniu.op_save( - op='avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', - bucket=bucket_name, - key=persistent_key - ) - ]) + key = 'test-pfop/upload-file' + persistent_type = persistent_options.get('persistent_type') + persistent_workflow_template_id = persistent_options.get('persistent_workflow_template_id') + + upload_policy = {} - upload_policy = { - 'persistentOps': persistent_ops - } + # set pfops or tmplate id + if persistent_workflow_template_id: + upload_policy['persistentWorkflowTemplateID'] = persistent_workflow_template_id + else: + persistent_key = '_'.join([ + 'test-pfop/test-pfop-by-upload', + 'type', + str(persistent_type) + ]) + persistent_ops = ';'.join([ + qiniu.op_save( + op='avinfo', + bucket=bucket_name, + key=persistent_key + ) + ]) + upload_policy['persistentOps'] = persistent_ops + # set persistent type if persistent_type is not None: upload_policy['persistentType'] = persistent_type + # upload token = qn_auth.upload_token( bucket_name, key, @@ -61,6 +85,10 @@ def test_pfop_with_upload( pfop = qiniu.PersistentFop(qn_auth, bucket_name) ret, resp = pfop.get_status(ret['persistentId']) assert resp.status_code == 200, resp + assert ret is not None, resp + assert ret['creationDate'] is not None, resp + if persistent_type == 1: assert ret['type'] == 1, resp - assert ret['creationDate'] is not None, resp + elif persistent_workflow_template_id: + assert persistent_workflow_template_id in ret['taskFrom'], resp diff --git a/tests/cases/test_utils.py b/tests/cases/test_utils.py new file mode 100644 index 00000000..11d9db77 --- /dev/null +++ b/tests/cases/test_utils.py @@ -0,0 +1,145 @@ +from datetime import datetime, timedelta, tzinfo + +from qiniu import utils, compat + + +class _CN_TZINFO(tzinfo): + def utcoffset(self, dt): + return timedelta(hours=8) + + def tzname(self, dt): + return "CST" + + def dst(self, dt): + return timedelta(0) + + +class TestUtils: + def test_urlsafe(self): + a = 'hello\x96' + u = utils.urlsafe_base64_encode(a) + assert compat.b(a) == utils.urlsafe_base64_decode(u) + + def test_canonical_mime_header_key(self): + field_names = [ + ":status", + ":x-test-1", + ":x-Test-2", + "content-type", + "CONTENT-LENGTH", + "oRiGin", + "ReFer", + "Last-Modified", + "acCePt-ChArsEt", + "x-test-3", + "cache-control", + ] + expect_canonical_field_names = [ + ":status", + ":x-test-1", + ":x-Test-2", + "Content-Type", + "Content-Length", + "Origin", + "Refer", + "Last-Modified", + "Accept-Charset", + "X-Test-3", + "Cache-Control", + ] + assert len(field_names) == len(expect_canonical_field_names) + for i in range(len(field_names)): + assert utils.canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i] + + def test_entry(self): + case_list = [ + { + 'msg': 'normal', + 'bucket': 'qiniuphotos', + 'key': 'gogopher.jpg', + 'expect': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' + }, + { + 'msg': 'key empty', + 'bucket': 'qiniuphotos', + 'key': '', + 'expect': 'cWluaXVwaG90b3M6' + }, + { + 'msg': 'key undefined', + 'bucket': 'qiniuphotos', + 'key': None, + 'expect': 'cWluaXVwaG90b3M=' + }, + { + 'msg': 'key need replace plus symbol', + 'bucket': 'qiniuphotos', + 'key': '012ts>a', + 'expect': 'cWluaXVwaG90b3M6MDEydHM-YQ==' + }, + { + 'msg': 'key need replace slash symbol', + 'bucket': 'qiniuphotos', + 'key': '012ts?a', + 'expect': 'cWluaXVwaG90b3M6MDEydHM_YQ==' + } + ] + for c in case_list: + assert c.get('expect') == utils.entry(c.get('bucket'), c.get('key')), c.get('msg') + + def test_decode_entry(self): + case_list = [ + { + 'msg': 'normal', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': 'gogopher.jpg' + }, + 'entry': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' + }, + { + 'msg': 'key empty', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': '' + }, + 'entry': 'cWluaXVwaG90b3M6' + }, + { + 'msg': 'key undefined', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': None + }, + 'entry': 'cWluaXVwaG90b3M=' + }, + { + 'msg': 'key need replace plus symbol', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': '012ts>a' + }, + 'entry': 'cWluaXVwaG90b3M6MDEydHM-YQ==' + }, + { + 'msg': 'key need replace slash symbol', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': '012ts?a' + }, + 'entry': 'cWluaXVwaG90b3M6MDEydHM_YQ==' + } + ] + for c in case_list: + bucket, key = utils.decode_entry(c.get('entry')) + assert bucket == c.get('expect', {}).get('bucket'), c.get('msg') + assert key == c.get('expect', {}).get('key'), c.get('msg') + + def test_dt2ts(self): + dt = datetime(year=2011, month=8, day=3, tzinfo=_CN_TZINFO()) + expect = 1312300800 + assert utils.dt2ts(dt) == expect + + base_dt = datetime(year=2011, month=8, day=3) + now_dt = datetime.now() + assert int((now_dt - base_dt).total_seconds()) == utils.dt2ts(now_dt) - utils.dt2ts(base_dt) From f3a0b9b49518d414c5f4444dc17c2d94467bf5f7 Mon Sep 17 00:00:00 2001 From: yuansl <yuanshenglong@qiniu.com> Date: Mon, 9 Dec 2024 11:31:37 +0800 Subject: [PATCH 467/478] =?UTF-8?q?DP-5290=20python-sdk/cdn:=20=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E5=9F=9F=E5=90=8D=E5=B8=A6=E5=AE=BD=EF=BC=8C=E6=94=AF?= =?UTF-8?q?=E6=8C=81type=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cdn/manager: add DataType class(enum), add optional parameter 'data_type' to `CdnManager.get_bandwidth_data` & `CdnManager.get_flux_data` - see https://jira.qiniu.io/browse/DP-5290 --- examples/cdn_bandwidth.py | 8 +++++++- qiniu/__init__.py | 2 +- qiniu/services/cdn/manager.py | 20 +++++++++++++++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/examples/cdn_bandwidth.py b/examples/cdn_bandwidth.py index ad4e97a8..32fbb40b 100644 --- a/examples/cdn_bandwidth.py +++ b/examples/cdn_bandwidth.py @@ -5,7 +5,7 @@ 查询指定域名指定时间段内的带宽 """ import qiniu -from qiniu import CdnManager +from qiniu import CdnManager, DataType # 账户ak，sk @@ -31,3 +31,9 @@ print(ret) print(info) + +ret, info = cdn_manager.get_bandwidth_data( + urls, startDate, endDate, granularity, data_type=DataType.BANDWIDTH) + +print(ret) +print(info) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 1ae68c00..a18aac2c 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -21,7 +21,7 @@ build_batch_stat, build_batch_delete, build_batch_restoreAr, build_batch_restore_ar from .services.storage.uploader import put_data, put_file, put_stream from .services.storage.upload_progress_recorder import UploadProgressRecorder -from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url, DomainManager +from .services.cdn.manager import CdnManager, DataType, create_timestamp_anti_leech_url, DomainManager from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save from .services.compute.app import AccountClient diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index bf107926..8ba944e5 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -5,9 +5,17 @@ from qiniu.compat import is_py2 from qiniu.compat import is_py3 +from enum import Enum import hashlib +class DataType(Enum): + BANDWIDTH = 'bandwidth' + X302BANDWIDTH = '302bandwidth' + X302MBANDWIDTH = '302mbandwidth' + FLOW = 'flow' + X302FLOW = '302flow' + X302MFLOW = '302mflow' def urlencode(str): if is_py2: @@ -60,7 +68,7 @@ def refresh_urls_and_dirs(self, urls, dirs): Returns: 一个dict变量和一个ResponseInfo对象 参考代码 examples/cdn_manager.py - """ + """ req = {} if urls is not None and len(urls) > 0: req.update({"urls": urls}) @@ -89,7 +97,7 @@ def prefetch_urls(self, urls): url = '{0}/v2/tune/prefetch'.format(self.server) return self.__post(url, body) - def get_bandwidth_data(self, domains, start_date, end_date, granularity): + def get_bandwidth_data(self, domains, start_date, end_date, granularity, data_type=None): """ 查询带宽数据，文档 https://developer.qiniu.com/fusion/api/traffic-bandwidth @@ -98,6 +106,7 @@ def get_bandwidth_data(self, domains, start_date, end_date, granularity): start_date: 起始日期 end_date: 结束日期 granularity: 数据间隔 + data_type: 计量数据类型, see class DataType.XXXBANDWIDTH Returns: 一个dict变量和一个ResponseInfo对象 @@ -108,12 +117,14 @@ def get_bandwidth_data(self, domains, start_date, end_date, granularity): req.update({"startDate": start_date}) req.update({"endDate": end_date}) req.update({"granularity": granularity}) + if data_type is not None: + req.update({'type': data_type.value}) # should be one of 'bandwidth', '302bandwidth', '302mbandwidth' body = json.dumps(req) url = '{0}/v2/tune/bandwidth'.format(self.server) return self.__post(url, body) - def get_flux_data(self, domains, start_date, end_date, granularity): + def get_flux_data(self, domains, start_date, end_date, granularity, data_type=None): """ 查询流量数据，文档 https://developer.qiniu.com/fusion/api/traffic-bandwidth @@ -122,6 +133,7 @@ def get_flux_data(self, domains, start_date, end_date, granularity): start_date: 起始日期 end_date: 结束日期 granularity: 数据间隔 + data_type: 计量数据类型, see class DataType.XXXFLOW Returns: 一个dict变量和一个ResponseInfo对象 @@ -132,6 +144,8 @@ def get_flux_data(self, domains, start_date, end_date, granularity): req.update({"startDate": start_date}) req.update({"endDate": end_date}) req.update({"granularity": granularity}) + if data_type is not None: + req.update({'type': data_type.value}) # should be one of 'flow', '302flow', '302mflow' body = json.dumps(req) url = '{0}/v2/tune/flux'.format(self.server) From 1ca9520e869e1f283411787e1b5fd177369bd779 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Tue, 10 Dec 2024 16:35:00 +0800 Subject: [PATCH 468/478] fix: improve file cache of regions in concurrent scenarios --- qiniu/compat.py | 7 + qiniu/http/regions_provider.py | 215 +++++++++++++----- qiniu/http/response.py | 1 + .../cases/test_http/test_regions_provider.py | 41 +++- 4 files changed, 210 insertions(+), 54 deletions(-) diff --git a/qiniu/compat.py b/qiniu/compat.py index 6a418c99..079aef70 100644 --- a/qiniu/compat.py +++ b/qiniu/compat.py @@ -14,6 +14,13 @@ # because of u'...' Unicode literals. import json # noqa +# ------- +# Platform +# ------- + +is_windows = sys.platform == 'win32' +is_linux = sys.platform == 'linux' +is_macos = sys.platform == 'darwin' # ------- # Pythons diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py index 13d1800a..aa22568c 100644 --- a/qiniu/http/regions_provider.py +++ b/qiniu/http/regions_provider.py @@ -6,8 +6,9 @@ import tempfile import os import shutil +import threading -from qiniu.compat import json, b as to_bytes +from qiniu.compat import json, b as to_bytes, is_windows, is_linux, is_macos from qiniu.utils import io_md5, dt2ts from .endpoint import Endpoint @@ -24,7 +25,7 @@ def __iter__(self): """ Returns ------- - list[Region] + Generator[Region, None, None] """ @@ -137,27 +138,112 @@ def __init__(self, message): super(FileAlreadyLocked, self).__init__(message) -class _FileLocker: - def __init__(self, origin_file): - self._origin_file = origin_file +_file_threading_lockers_lock = threading.Lock() +_file_threading_lockers = {} + + +class _FileThreadingLocker: + def __init__(self, fd): + self._fd = fd def __enter__(self): - if os.access(self.lock_file_path, os.R_OK | os.W_OK): - raise FileAlreadyLocked('File {0} already locked'.format(self._origin_file)) - with open(self.lock_file_path, 'w'): - pass + with _file_threading_lockers_lock: + global _file_threading_lockers + threading_lock = _file_threading_lockers.get(self._file_path, threading.Lock()) + # Could use keyword style `acquire(blocking=False)` when min version of python update to >= 3 + if not threading_lock.acquire(False): + raise FileAlreadyLocked('File {0} already locked'.format(self._file_path)) + _file_threading_lockers[self._file_path] = threading_lock def __exit__(self, exc_type, exc_val, exc_tb): - os.remove(self.lock_file_path) + with _file_threading_lockers_lock: + global _file_threading_lockers + threading_lock = _file_threading_lockers.get(self._file_path) + if threading_lock and threading_lock.locked(): + threading_lock.release() + del _file_threading_lockers[self._file_path] @property - def lock_file_path(self): - """ - Returns - ------- - str - """ - return self._origin_file + '.lock' + def _file_path(self): + return self._fd.name + + +if is_linux or is_macos: + import fcntl + + # Use subclass of _FileThreadingLocker when min version of python update to >= 3 + class _FileLocker: + def __init__(self, fd): + self._fd = fd + + def __enter__(self): + try: + fcntl.lockf(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + # Use `raise ... from ...` when min version of python update to >= 3 + raise FileAlreadyLocked('File {0} already locked'.format(self._file_path)) + + def __exit__(self, exc_type, exc_val, exc_tb): + fcntl.lockf(self._fd, fcntl.LOCK_UN) + + @property + def _file_path(self): + return self._fd.name + +elif is_windows: + import msvcrt + + + class _FileLocker: + def __init__(self, fd): + self._fd = fd + + def __enter__(self): + try: + # TODO(lihs): set `_nbyte` bigger? + msvcrt.locking(self._fd, msvcrt.LK_LOCK | msvcrt.LK_NBLCK, 1) + except OSError: + raise FileAlreadyLocked('File {0} already locked'.format(self._file_path)) + + def __exit__(self, exc_type, exc_val, exc_tb): + msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) + + @property + def _file_path(self): + return self._fd.name + +else: + class _FileLocker: + def __init__(self, fd): + self._fd = fd + + def __enter__(self): + try: + # Atomic file creation + open_flags = os.O_EXCL | os.O_RDWR | os.O_CREAT + fd = os.open(self.lock_file_path, open_flags) + os.close(fd) + except FileExistsError: + raise FileAlreadyLocked('File {0} already locked'.format(self._file_path)) + + def __exit__(self, exc_type, exc_val, exc_tb): + try: + os.remove(self.lock_file_path) + except FileNotFoundError: + pass + + @property + def _file_path(self): + return self._fd.name + + @property + def lock_file_path(self): + """ + Returns + ------- + str + """ + return self._file_path + '.lock' # use dataclass instead namedtuple if min version of python update to 3.7 @@ -168,7 +254,8 @@ def lock_file_path(self): 'persist_path', 'last_shrink_at', 'shrink_interval', - 'should_shrink_expired_regions' + 'should_shrink_expired_regions', + 'memo_cache_lock' ] ) @@ -177,11 +264,12 @@ def lock_file_path(self): memo_cache={}, persist_path=os.path.join( tempfile.gettempdir(), - 'qn-regions-cache.jsonl' + 'qn-py-sdk-regions-cache.jsonl' ), last_shrink_at=datetime.datetime.fromtimestamp(0), - shrink_interval=datetime.timedelta(-1), # useless for now - should_shrink_expired_regions=False + shrink_interval=datetime.timedelta(days=1), + should_shrink_expired_regions=False, + memo_cache_lock=threading.Lock() ) @@ -323,7 +411,7 @@ def _parse_persisted_regions(persisted_data): return parsed_data.get('cacheKey'), regions -def _walk_persist_cache_file(persist_path, ignore_parse_error=False): +def _walk_persist_cache_file(persist_path, ignore_parse_error=True): """ Parameters ---------- @@ -394,23 +482,24 @@ def __init__( self.base_regions_provider = base_regions_provider persist_path = kwargs.get('persist_path', None) + last_shrink_at = datetime.datetime.fromtimestamp(0) if persist_path is None: persist_path = _global_cache_scope.persist_path + last_shrink_at = _global_cache_scope.last_shrink_at shrink_interval = kwargs.get('shrink_interval', None) if shrink_interval is None: - shrink_interval = datetime.timedelta(days=1) + shrink_interval = _global_cache_scope.shrink_interval should_shrink_expired_regions = kwargs.get('should_shrink_expired_regions', None) if should_shrink_expired_regions is None: - should_shrink_expired_regions = False + should_shrink_expired_regions = _global_cache_scope.should_shrink_expired_regions - self._cache_scope = CacheScope( - memo_cache=_global_cache_scope.memo_cache, + self._cache_scope = _global_cache_scope._replace( persist_path=persist_path, - last_shrink_at=datetime.datetime.fromtimestamp(0), + last_shrink_at=last_shrink_at, shrink_interval=shrink_interval, - should_shrink_expired_regions=should_shrink_expired_regions, + should_shrink_expired_regions=should_shrink_expired_regions ) def __iter__(self): @@ -423,7 +512,7 @@ def __iter__(self): self.__get_regions_from_base_provider ] - regions = None + regions = [] for get_regions in get_regions_fns: regions = get_regions(fallback=regions) if regions and all(r.is_live for r in regions): @@ -439,7 +528,8 @@ def set_regions(self, regions): ---------- regions: list[Region] """ - self._cache_scope.memo_cache[self.cache_key] = regions + with self._cache_scope.memo_cache_lock: + self._cache_scope.memo_cache[self.cache_key] = regions if not self._cache_scope.persist_path: return @@ -469,8 +559,11 @@ def persist_path(self, value): ---------- value: str """ + if value == self._cache_scope.persist_path: + return self._cache_scope = self._cache_scope._replace( - persist_path=value + persist_path=value, + last_shrink_at=datetime.datetime.fromtimestamp(0) ) @property @@ -586,7 +679,6 @@ def __get_regions_from_base_provider(self, fallback=None): def __flush_file_cache_to_memo(self): for cache_key, regions in _walk_persist_cache_file( persist_path=self._cache_scope.persist_path - # ignore_parse_error=True ): if cache_key not in self._cache_scope.memo_cache: self._cache_scope.memo_cache[cache_key] = regions @@ -609,12 +701,18 @@ def __should_shrink(self): def __shrink_cache(self): # shrink memory cache if self._cache_scope.should_shrink_expired_regions: - kept_memo_cache = {} - for k, regions in self._cache_scope.memo_cache.items(): - live_regions = [r for r in regions if r.is_live] - if live_regions: - kept_memo_cache[k] = live_regions - self._cache_scope = self._cache_scope._replace(memo_cache=kept_memo_cache) + memo_cache_old = self._cache_scope.memo_cache.copy() + # Could use keyword style `acquire(blocking=False)` when min version of python update to >= 3 + if self._cache_scope.memo_cache_lock.acquire(False): + try: + for k, regions in memo_cache_old.items(): + live_regions = [r for r in regions if r.is_live] + if live_regions: + self._cache_scope.memo_cache[k] = live_regions + else: + del self._cache_scope.memo_cache[k] + finally: + self._cache_scope.memo_cache_lock.release() # shrink file cache if not self._cache_scope.persist_path: @@ -625,7 +723,7 @@ def __shrink_cache(self): shrink_file_path = self._cache_scope.persist_path + '.shrink' try: - with _FileLocker(shrink_file_path): + with open(shrink_file_path, 'a') as f, _FileThreadingLocker(f), _FileLocker(f): # filter data shrunk_cache = {} for cache_key, regions in _walk_persist_cache_file( @@ -646,25 +744,36 @@ def __shrink_cache(self): ) # write data - with open(shrink_file_path, 'a') as f: - for cache_key, regions in shrunk_cache.items(): - f.write( - json.dumps( - { - 'cacheKey': cache_key, - 'regions': [_persist_region(r) for r in regions] - } - ) + os.linesep - ) + for cache_key, regions in shrunk_cache.items(): + f.write( + json.dumps( + { + 'cacheKey': cache_key, + 'regions': [_persist_region(r) for r in regions] + } + ) + os.linesep + ) + + # make the cache file available for all users + if is_linux or is_macos: + os.chmod(shrink_file_path, 0o666) # rename file shutil.move(shrink_file_path, self._cache_scope.persist_path) + + # update last shrink time + self._cache_scope = self._cache_scope._replace( + last_shrink_at=datetime.datetime.now() + ) + global _global_cache_scope + if _global_cache_scope.persist_path == self._cache_scope.persist_path: + _global_cache_scope = _global_cache_scope._replace( + last_shrink_at=self._cache_scope.last_shrink_at + ) + except FileAlreadyLocked: + # skip file shrink by another running pass - finally: - self._cache_scope = self._cache_scope._replace( - last_shrink_at=datetime.datetime.now() - ) def get_default_regions_provider( diff --git a/qiniu/http/response.py b/qiniu/http/response.py index 6450438d..cbfcf034 100644 --- a/qiniu/http/response.py +++ b/qiniu/http/response.py @@ -57,6 +57,7 @@ def need_retry(self): ]): return False # https://developer.qiniu.com/fusion/kb/1352/the-http-request-return-a-status-code + # https://developer.qiniu.com/kodo/3928/error-responses if self.status_code in [ 501, 509, 573, 579, 608, 612, 614, 616, 618, 630, 631, 632, 640, 701 ]: diff --git a/tests/cases/test_http/test_regions_provider.py b/tests/cases/test_http/test_regions_provider.py index 7289f5ca..ba84faec 100644 --- a/tests/cases/test_http/test_regions_provider.py +++ b/tests/cases/test_http/test_regions_provider.py @@ -1,7 +1,9 @@ import os import datetime import tempfile +import time import json +from multiprocessing.pool import ThreadPool import pytest @@ -9,7 +11,15 @@ from qiniu.config import QUERY_REGION_HOST, QUERY_REGION_BACKUP_HOSTS from qiniu.http.endpoint import Endpoint from qiniu.http.region import Region -from qiniu.http.regions_provider import QueryRegionsProvider, CachedRegionsProvider, _global_cache_scope, _persist_region +from qiniu.http.regions_provider import ( + CachedRegionsProvider, + FileAlreadyLocked, + QueryRegionsProvider, + _FileThreadingLocker, + _FileLocker, + _global_cache_scope, + _persist_region, +) @pytest.fixture(scope='session') @@ -32,6 +42,16 @@ def query_regions_provider(access_key, bucket_name, query_regions_endpoints_prov yield query_regions_provider +@pytest.fixture(scope='function') +def temp_file_path(rand_string): + p = os.path.join(tempfile.gettempdir(), rand_string(16)) + yield p + try: + os.remove(p) + except FileNotFoundError: + pass + + class TestQueryRegionsProvider: def test_getter(self, query_regions_provider): ret = list(query_regions_provider) @@ -267,3 +287,22 @@ def test_shrink_with_ignore_expired_regions(self, cached_regions_provider): cached_regions_provider.cache_key = 'another-cache-key' list(cached_regions_provider) # trigger __shrink_cache() assert len(cached_regions_provider._cache_scope.memo_cache[origin_cache_key]) > 0 + + def test_file_locker(self, temp_file_path): + handled_cnt = 0 + skipped_cnt = 0 + + + def process_file(_n): + nonlocal handled_cnt, skipped_cnt + try: + with open(temp_file_path, 'w') as f, _FileThreadingLocker(f), _FileLocker(f): + time.sleep(1) + handled_cnt += 1 + except FileAlreadyLocked: + skipped_cnt += 1 + + + ThreadPool(4).map(process_file, range(20)) + assert handled_cnt + skipped_cnt == 20 + assert 0 < handled_cnt <= 4 From 642c71397a29ed7ccf107ee013ba6e0420628c0a Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Tue, 10 Dec 2024 17:10:20 +0800 Subject: [PATCH 469/478] feat: add single flight and apply to QueryRegionsProvider --- qiniu/http/regions_provider.py | 14 ++++- qiniu/http/single_flight.py | 50 +++++++++++++++++ tests/cases/test_http/test_single_flight.py | 59 +++++++++++++++++++++ 3 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 qiniu/http/single_flight.py create mode 100644 tests/cases/test_http/test_single_flight.py diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py index aa22568c..a521a471 100644 --- a/qiniu/http/regions_provider.py +++ b/qiniu/http/regions_provider.py @@ -15,6 +15,7 @@ from .region import Region, ServiceName from .default_client import qn_http_client from .middleware import RetryDomainsMiddleware +from .single_flight import SingleFlight class RegionsProvider: @@ -70,6 +71,9 @@ def _get_region_from_query(data, **kwargs): ) +_query_regions_single_flight = SingleFlight() + + class QueryRegionsProvider(RegionsProvider): def __init__( self, @@ -95,7 +99,15 @@ def __init__( self.max_retry_times_per_endpoint = max_retry_times_per_endpoint def __iter__(self): - regions = self.__fetch_regions() + endpoints_md5 = io_md5([ + to_bytes(e.host) for e in self.endpoints_provider + ]) + flight_key = ':'.join([ + endpoints_md5, + self.access_key, + self.bucket_name + ]) + regions = _query_regions_single_flight.do(flight_key, self.__fetch_regions) # change to `yield from` when min version of python update to >= 3.3 for r in regions: yield r diff --git a/qiniu/http/single_flight.py b/qiniu/http/single_flight.py new file mode 100644 index 00000000..28536de0 --- /dev/null +++ b/qiniu/http/single_flight.py @@ -0,0 +1,50 @@ +import threading + + +class _FlightLock: + """ + Do not use dataclass which caused the event created only once + """ + def __init__(self): + self.event = threading.Event() + self.result = None + self.error = None + + +class SingleFlight: + def __init__(self): + self._locks = {} + self._lock = threading.Lock() + + def do(self, key, fn, *args, **kwargs): + # here does not use `with` statement + # because need to wait by another object if it exists, + # and reduce the `acquire` times if it not exists + self._lock.acquire() + if key in self._locks: + flight_lock = self._locks[key] + + self._lock.release() + flight_lock.event.wait() + + if flight_lock.error: + raise flight_lock.error + return flight_lock.result + + flight_lock = _FlightLock() + self._locks[key] = flight_lock + self._lock.release() + + try: + flight_lock.result = fn(*args, **kwargs) + except Exception as e: + flight_lock.error = e + finally: + flight_lock.event.set() + + with self._lock: + del self._locks[key] + + if flight_lock.error: + raise flight_lock.error + return flight_lock.result diff --git a/tests/cases/test_http/test_single_flight.py b/tests/cases/test_http/test_single_flight.py new file mode 100644 index 00000000..48748ecd --- /dev/null +++ b/tests/cases/test_http/test_single_flight.py @@ -0,0 +1,59 @@ +import pytest +import time +from multiprocessing.pool import ThreadPool + +from qiniu.http.single_flight import SingleFlight + +class TestSingleFlight: + def test_single_flight_success(self): + sf = SingleFlight() + + def fn(): + return "result" + + result = sf.do("key1", fn) + assert result == "result" + + def test_single_flight_exception(self): + sf = SingleFlight() + + def fn(): + raise ValueError("error") + + with pytest.raises(ValueError, match="error"): + sf.do("key2", fn) + + def test_single_flight_concurrent(self): + sf = SingleFlight() + share_state = [] + results = [] + + def fn(): + time.sleep(1) + share_state.append('share_state') + return "result" + + def worker(_n): + result = sf.do("key3", fn) + results.append(result) + + ThreadPool(2).map(worker, range(5)) + + assert len(share_state) == 3 + assert all(result == "result" for result in results) + + def test_single_flight_different_keys(self): + sf = SingleFlight() + results = [] + + def fn(): + time.sleep(1) + return "result" + + def worker(n): + result = sf.do("key{}".format(n), fn) + results.append(result) + + ThreadPool(2).map(worker, range(2)) + assert len(results) == 2 + assert all(result == "result" for result in results) From ae5773041eed273370a735cd55430b28c5caf885 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Wed, 11 Dec 2024 14:19:10 +0800 Subject: [PATCH 470/478] test: fix test cases --- qiniu/http/regions_provider.py | 3 ++- tests/cases/test_http/test_regions_provider.py | 8 +++++++- tests/cases/test_zone/test_qiniu_conf.py | 10 +++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py index a521a471..de7f1350 100644 --- a/qiniu/http/regions_provider.py +++ b/qiniu/http/regions_provider.py @@ -524,7 +524,8 @@ def __iter__(self): self.__get_regions_from_base_provider ] - regions = [] + # set the fallback to None for raise errors when failed + regions = None for get_regions in get_regions_fns: regions = get_regions(fallback=regions) if regions and all(r.is_live for r in regions): diff --git a/tests/cases/test_http/test_regions_provider.py b/tests/cases/test_http/test_regions_provider.py index ba84faec..fb13c17d 100644 --- a/tests/cases/test_http/test_regions_provider.py +++ b/tests/cases/test_http/test_regions_provider.py @@ -275,7 +275,13 @@ def test_shrink_with_expired_regions(self, cached_regions_provider): origin_cache_key = cached_regions_provider.cache_key cached_regions_provider.set_regions([expired_region]) cached_regions_provider.cache_key = 'another-cache-key' - list(cached_regions_provider) # trigger __shrink_cache() + + # trigger __shrink_cache() + cached_regions_provider._cache_scope = cached_regions_provider._cache_scope._replace( + last_shrink_at=datetime.datetime.fromtimestamp(0) + ) + list(cached_regions_provider) + assert len(cached_regions_provider._cache_scope.memo_cache[origin_cache_key]) == 0 def test_shrink_with_ignore_expired_regions(self, cached_regions_provider): diff --git a/tests/cases/test_zone/test_qiniu_conf.py b/tests/cases/test_zone/test_qiniu_conf.py index c6bce5b8..0c05dfaf 100644 --- a/tests/cases/test_zone/test_qiniu_conf.py +++ b/tests/cases/test_zone/test_qiniu_conf.py @@ -51,7 +51,7 @@ def test_config_compatible(self, set_conf_default): 'set_conf_default', [ { - 'default_query_region_host': 'https://fake-uc.phpsdk.qiniu.com' + 'default_query_region_host': 'https://fake-uc.pysdk.qiniu.com' } ], indirect=True @@ -66,9 +66,9 @@ def test_query_region_with_custom_domain(self, access_key, bucket_name, set_conf 'set_conf_default', [ { - 'default_query_region_host': 'https://fake-uc.phpsdk.qiniu.com', + 'default_query_region_host': 'https://fake-uc.pysdk.qiniu.com', 'default_query_region_backup_hosts': [ - 'unavailable-uc.phpsdk.qiniu.com', + 'unavailable-uc.pysdk.qiniu.com', 'uc.qbox.me' ] } @@ -78,13 +78,13 @@ def test_query_region_with_custom_domain(self, access_key, bucket_name, set_conf def test_query_region_with_backup_domains(self, access_key, bucket_name, set_conf_default): zone = Zone() data = zone.bucket_hosts(access_key, bucket_name) - assert data != 'null' + assert data != 'null' and len(data) > 0 @pytest.mark.parametrize( 'set_conf_default', [ { - 'default_uc_host': 'https://fake-uc.phpsdk.qiniu.com', + 'default_uc_host': 'https://fake-uc.pysdk.qiniu.com', 'default_query_region_backup_hosts': [ 'unavailable-uc.phpsdk.qiniu.com', 'uc.qbox.me' From cd942fc67d96b9f9673c9ee885c09f0093e19a01 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Wed, 11 Dec 2024 14:46:10 +0800 Subject: [PATCH 471/478] fix: python2 compatible and flake8 style --- qiniu/http/regions_provider.py | 5 ++--- tests/cases/test_http/test_regions_provider.py | 15 +++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py index de7f1350..57a34e44 100644 --- a/qiniu/http/regions_provider.py +++ b/qiniu/http/regions_provider.py @@ -205,7 +205,6 @@ def _file_path(self): elif is_windows: import msvcrt - class _FileLocker: def __init__(self, fd): self._fd = fd @@ -235,13 +234,13 @@ def __enter__(self): open_flags = os.O_EXCL | os.O_RDWR | os.O_CREAT fd = os.open(self.lock_file_path, open_flags) os.close(fd) - except FileExistsError: + except IOError: raise FileAlreadyLocked('File {0} already locked'.format(self._file_path)) def __exit__(self, exc_type, exc_val, exc_tb): try: os.remove(self.lock_file_path) - except FileNotFoundError: + except IOError: pass @property diff --git a/tests/cases/test_http/test_regions_provider.py b/tests/cases/test_http/test_regions_provider.py index fb13c17d..73dca89d 100644 --- a/tests/cases/test_http/test_regions_provider.py +++ b/tests/cases/test_http/test_regions_provider.py @@ -294,21 +294,20 @@ def test_shrink_with_ignore_expired_regions(self, cached_regions_provider): list(cached_regions_provider) # trigger __shrink_cache() assert len(cached_regions_provider._cache_scope.memo_cache[origin_cache_key]) > 0 - def test_file_locker(self, temp_file_path): - handled_cnt = 0 - skipped_cnt = 0 + def test_file_locker(self, temp_file_path, use_ref): + handled_cnt = use_ref(0) + skipped_cnt = use_ref(0) def process_file(_n): - nonlocal handled_cnt, skipped_cnt try: with open(temp_file_path, 'w') as f, _FileThreadingLocker(f), _FileLocker(f): time.sleep(1) - handled_cnt += 1 + handled_cnt.value += 1 except FileAlreadyLocked: - skipped_cnt += 1 + skipped_cnt.value += 1 ThreadPool(4).map(process_file, range(20)) - assert handled_cnt + skipped_cnt == 20 - assert 0 < handled_cnt <= 4 + assert handled_cnt.value + skipped_cnt.value == 20 + assert 0 < handled_cnt.value <= 4 From b8e29a779226d020b08084a95cffb419a62ce6e8 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Wed, 11 Dec 2024 14:49:13 +0800 Subject: [PATCH 472/478] docs: update version and changelogs --- CHANGELOG.md | 4 ++++ qiniu/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5722b56b..40f8fc7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 7.16.0 +* 对象存储，优化并发场景的区域查询 +* CDN，查询域名带宽，支持 `data_type` 参数 + ## 7.15.0 * 对象存储，持久化处理支持工作流模版 * 对象存储，修复 Windows 平台兼容性问题 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index a18aac2c..d097fdb8 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.15.0' +__version__ = '7.16.0' from .auth import Auth, QiniuMacAuth From e33c0e19e23ba9a54ffff5d6f10e26df9e508b8e Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Wed, 11 Dec 2024 14:53:05 +0800 Subject: [PATCH 473/478] styles: fix flake8 styles --- qiniu/services/cdn/manager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 8ba944e5..6700ecaf 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -9,6 +9,7 @@ import hashlib + class DataType(Enum): BANDWIDTH = 'bandwidth' X302BANDWIDTH = '302bandwidth' @@ -17,6 +18,7 @@ class DataType(Enum): X302FLOW = '302flow' X302MFLOW = '302mflow' + def urlencode(str): if is_py2: import urllib2 @@ -118,7 +120,7 @@ def get_bandwidth_data(self, domains, start_date, end_date, granularity, data_ty req.update({"endDate": end_date}) req.update({"granularity": granularity}) if data_type is not None: - req.update({'type': data_type.value}) # should be one of 'bandwidth', '302bandwidth', '302mbandwidth' + req.update({'type': data_type.value}) # should be one of 'bandwidth', '302bandwidth', '302mbandwidth' body = json.dumps(req) url = '{0}/v2/tune/bandwidth'.format(self.server) @@ -145,7 +147,7 @@ def get_flux_data(self, domains, start_date, end_date, granularity, data_type=No req.update({"endDate": end_date}) req.update({"granularity": granularity}) if data_type is not None: - req.update({'type': data_type.value}) # should be one of 'flow', '302flow', '302mflow' + req.update({'type': data_type.value}) # should be one of 'flow', '302flow', '302mflow' body = json.dumps(req) url = '{0}/v2/tune/flux'.format(self.server) From b6fd793836a5f3f7c9f61d96c84bce4fbcc6eb14 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Thu, 12 Dec 2024 11:14:14 +0800 Subject: [PATCH 474/478] fix: file lock on windows --- qiniu/http/regions_provider.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py index 57a34e44..cbcb7228 100644 --- a/qiniu/http/regions_provider.py +++ b/qiniu/http/regions_provider.py @@ -211,13 +211,12 @@ def __init__(self, fd): def __enter__(self): try: - # TODO(lihs): set `_nbyte` bigger? - msvcrt.locking(self._fd, msvcrt.LK_LOCK | msvcrt.LK_NBLCK, 1) + msvcrt.locking(self._fd.fileno(), msvcrt.LK_LOCK | msvcrt.LK_NBLCK, 1) except OSError: raise FileAlreadyLocked('File {0} already locked'.format(self._file_path)) def __exit__(self, exc_type, exc_val, exc_tb): - msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) + msvcrt.locking(self._fd.fileno(), msvcrt.LK_UNLCK, 1) @property def _file_path(self): From 166be8f7b062a747a13ba4c7920254cb2d768264 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Thu, 12 Dec 2024 20:40:18 +0800 Subject: [PATCH 475/478] fix: windows locker --- qiniu/http/regions_provider.py | 44 ++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py index cbcb7228..c4415973 100644 --- a/qiniu/http/regions_provider.py +++ b/qiniu/http/regions_provider.py @@ -208,38 +208,64 @@ def _file_path(self): class _FileLocker: def __init__(self, fd): self._fd = fd + self._lock_fd = None + self._already_locked = False def __enter__(self): try: - msvcrt.locking(self._fd.fileno(), msvcrt.LK_LOCK | msvcrt.LK_NBLCK, 1) + self._lock_fd = open(self._lock_file_path, 'w') + msvcrt.locking(self._lock_fd.fileno(), msvcrt.LK_LOCK | msvcrt.LK_NBLCK, 1) except OSError: + self._already_locked = True raise FileAlreadyLocked('File {0} already locked'.format(self._file_path)) def __exit__(self, exc_type, exc_val, exc_tb): - msvcrt.locking(self._fd.fileno(), msvcrt.LK_UNLCK, 1) + if self._already_locked: + if self._lock_fd: + self._lock_fd.close() + return + + try: + msvcrt.locking(self._lock_fd.fileno(), msvcrt.LK_UNLCK, 1) + finally: + self._lock_fd.close() + os.remove(self._lock_file_path) @property def _file_path(self): return self._fd.name + @property + def _lock_file_path(self): + """ + Returns + ------- + str + """ + return self._file_path + '.lock' + else: class _FileLocker: def __init__(self, fd): self._fd = fd + self._already_locked = False def __enter__(self): try: # Atomic file creation open_flags = os.O_EXCL | os.O_RDWR | os.O_CREAT - fd = os.open(self.lock_file_path, open_flags) + fd = os.open(self._lock_file_path, open_flags) os.close(fd) - except IOError: + except OSError: + self._already_locked = True raise FileAlreadyLocked('File {0} already locked'.format(self._file_path)) def __exit__(self, exc_type, exc_val, exc_tb): + if self._already_locked: + return try: - os.remove(self.lock_file_path) - except IOError: + os.remove(self._lock_file_path) + except OSError: pass @property @@ -247,7 +273,7 @@ def _file_path(self): return self._fd.name @property - def lock_file_path(self): + def _lock_file_path(self): """ Returns ------- @@ -770,6 +796,10 @@ def __shrink_cache(self): os.chmod(shrink_file_path, 0o666) # rename file + if is_windows: + # windows must close first, or will raise permission error + # be careful to do something with the file after this + f.close() shutil.move(shrink_file_path, self._cache_scope.persist_path) # update last shrink time From dcc0da4128d2f9f9529d2aa25fe389a4f022a268 Mon Sep 17 00:00:00 2001 From: lihsai0 <lihsai0@gmail.com> Date: Mon, 16 Dec 2024 11:50:58 +0800 Subject: [PATCH 476/478] feat: no raise error if regions cache shrink failed --- qiniu/http/regions_provider.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py index c4415973..e75b6457 100644 --- a/qiniu/http/regions_provider.py +++ b/qiniu/http/regions_provider.py @@ -540,7 +540,10 @@ def __init__( def __iter__(self): if self.__should_shrink: - self.__shrink_cache() + try: + self.__shrink_cache() + except Exception as err: + logging.warning('failed to shrink cache', err) get_regions_fns = [ self.__get_regions_from_memo, From 132c9d2893c80646405a40818ea9bc61560b6a84 Mon Sep 17 00:00:00 2001 From: Jie Liu <eirture@gmail.com> Date: Fri, 14 Feb 2025 15:50:30 +0800 Subject: [PATCH 477/478] fix: error messages format --- qiniu/http/regions_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py index e75b6457..19d85161 100644 --- a/qiniu/http/regions_provider.py +++ b/qiniu/http/regions_provider.py @@ -543,7 +543,7 @@ def __iter__(self): try: self.__shrink_cache() except Exception as err: - logging.warning('failed to shrink cache', err) + logging.warning('failed to shrink cache. error: %s', err) get_regions_fns = [ self.__get_regions_from_memo, @@ -581,7 +581,7 @@ def set_regions(self, regions): 'regions': [_persist_region(r) for r in regions] }) + os.linesep) except Exception as err: - logging.warning('failed to cache regions result to file', err) + logging.warning('failed to cache regions result to file. error: %s', err) @property def persist_path(self): From 322adecbc8804d483a48267afb6c047b4d61f3ad Mon Sep 17 00:00:00 2001 From: Jie Liu <eirture@gmail.com> Date: Mon, 17 Feb 2025 20:44:06 +0800 Subject: [PATCH 478/478] fix: permissions of the region provider cache file --- qiniu/http/regions_provider.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py index 19d85161..c451d166 100644 --- a/qiniu/http/regions_provider.py +++ b/qiniu/http/regions_provider.py @@ -1,5 +1,6 @@ import abc import datetime +import errno import itertools from collections import namedtuple import logging @@ -300,7 +301,8 @@ def _lock_file_path(self): memo_cache={}, persist_path=os.path.join( tempfile.gettempdir(), - 'qn-py-sdk-regions-cache.jsonl' + 'qn-py-sdk', + 'regions-cache.jsonl' ), last_shrink_at=datetime.datetime.fromtimestamp(0), shrink_interval=datetime.timedelta(days=1), @@ -520,8 +522,24 @@ def __init__( persist_path = kwargs.get('persist_path', None) last_shrink_at = datetime.datetime.fromtimestamp(0) if persist_path is None: - persist_path = _global_cache_scope.persist_path - last_shrink_at = _global_cache_scope.last_shrink_at + cache_dir = os.path.dirname(_global_cache_scope.persist_path) + try: + # make sure the cache dir is available for all users. + # we can not use the '/tmp' dir directly on linux, + # because the permission is 0o1777 + if not os.path.exists(cache_dir): + # os.makedirs have no exists_ok parameter in python 2.7 + os.makedirs(cache_dir) + os.chmod(cache_dir, 0o777) + persist_path = _global_cache_scope.persist_path + last_shrink_at = _global_cache_scope.last_shrink_at + except Exception as err: + if isinstance(err, OSError) and err.errno == errno.EEXIST: + persist_path = _global_cache_scope.persist_path + last_shrink_at = _global_cache_scope.last_shrink_at + else: + logging.warning( + 'failed to create cache dir %s. error: %s', cache_dir, err) shrink_interval = kwargs.get('shrink_interval', None) if shrink_interval is None: