Merge pull request #95 from qiniu/develop

Release v6.1.4

Author: longbai

CHANGELOG.md

@@ -1,5 +1,15 @@
 ## CHANGE LOG
 
+### v6.1.4
+
+2013-10-24 issue [#95](https://github.com/qiniu/python-sdk/pull/95)
+
+- [#78] 增加 putpolicy 选项:saveKey,insertOnly,detectMime,fsizeLimit,persistentNotifyUrl,persistentOps
+- [#80] 增加 gettoken 过期时间参数，增加 rsf 返回为空的EOF判断
+- [#86] 修正 断点续传的bug
+- [#93] 修正 4M 分块计算bug
+- [#96] 修正 mime_type typo
+
 ### v6.1.3
 
 2013-10-24 issue [#77](https://github.com/qiniu/python-sdk/pull/77)

docs/README.md

@@ -2,6 +2,8 @@
 title: Python SDK 使用指南
 ---
 
+# Python SDK 使用指南
+
 此 Python SDK 适用于2.x版本，基于 [七牛云存储官方API](http://docs.qiniu.com/) 构建。使用此 SDK 构建您的网络应用程序，能让您以非常便捷地方式将数据安全地存储到七牛云存储上。无论您的网络应用是一个网站程序，还是包括从云端（服务端程序）到终端（手持设备应用）的架构的服务或应用，通过七牛云存储及其 SDK，都能让您应用程序的终端用户高速上传和下载，同时也让您的服务端更加轻盈。
 
 SDK 下载地址：<https://github.com/qiniu/python-sdk/tags>

qiniu/auth/up.py

@@ -5,10 +5,12 @@
 
 class Client(rpc.Client):
 	up_token = None
-	
+
 	def __init__(self, up_token, host=None):
 		if host is None:
 			host = conf.UP_HOST
+		if host.startswith("http://"):
+			host = host[7:]
 		self.up_token = up_token
 		super(Client, self).__init__(host)
 

qiniu/resumable_io.py

@@ -14,6 +14,7 @@
 _chunk_size = 256 * 1024
 _try_times = 3
 _block_size = 4 * 1024 * 1024
+_block_mask = _block_size - 1
 
 class Error(Exception):
 	value = None
@@ -84,15 +85,14 @@ def put(uptoken, key, f, fsize, extra):
 	if extra.chunk_size is None:
 		extra.chunk_size = _chunk_size
 
-	client = auth.up.Client(uptoken)
 	for i in xrange(0, block_cnt):
 		try_time = extra.try_times
 		read_length = _block_size
 		if (i+1)*_block_size > fsize:
 			read_length = fsize - i*_block_size
 		data_slice = f.read(read_length)
 		while True:
-			err = resumable_block_put(client, data_slice, i, extra)
+			err = resumable_block_put(data_slice, i, extra, uptoken)
 			if err is None:
 				break
 
@@ -101,34 +101,38 @@ def put(uptoken, key, f, fsize, extra):
 				return None, err_put_failed
 			print err, ".. retry"
 
-	return mkfile(client, key, fsize, extra)
+	mkfile_client = auth.up.Client(uptoken, extra.progresses[-1]["host"])
+	return mkfile(mkfile_client, key, fsize, extra)
 
 # ----------------------------------------------------------
 
-def resumable_block_put(client, block, index, extra):
+def resumable_block_put(block, index, extra, uptoken):
 	block_size = len(block)
 
+	mkblk_client = auth.up.Client(uptoken, conf.UP_HOST)
 	if extra.progresses[index] is None or "ctx" not in extra.progresses[index]:
 		end_pos = extra.chunk_size-1
 		if block_size < extra.chunk_size:
 			end_pos = block_size-1
 		chunk = block[: end_pos]
 		crc32 = gen_crc32(chunk)
 		chunk = bytearray(chunk)
-		extra.progresses[index], err = mkblock(client, block_size, chunk)
+		extra.progresses[index], err = mkblock(mkblk_client, block_size, chunk)
 		if not extra.progresses[index]["crc32"] == crc32:
 			return err_unmatched_checksum
 		if err is not None:
 			extra.notify_err(index, end_pos + 1, err)
 			return err
 		extra.notify(index, end_pos + 1, extra.progresses[index])
 
+	bput_client = auth.up.Client(uptoken, extra.progresses[index]["host"])
 	while extra.progresses[index]["offset"] < block_size:
 		offset = extra.progresses[index]["offset"]
 		chunk = block[offset: offset+extra.chunk_size-1]
 		crc32 = gen_crc32(chunk)
 		chunk = bytearray(chunk)
-		extra.progresses[index], err = putblock(client, extra.progresses[index], chunk)
+
+		extra.progresses[index], err = putblock(bput_client, extra.progresses[index], chunk)
 		if not extra.progresses[index]["crc32"] == crc32:
 			return err_unmatched_checksum
 		if err is not None:
@@ -138,7 +142,7 @@ def resumable_block_put(client, block, index, extra):
 
 def block_count(size):
 	global _block_size
-	return size / _block_size + 1
+	return (size + _block_mask) / _block_size
 
 def mkblock(client, block_size, first_chunk):
 	url = "http://%s/mkblk/%s" % (conf.UP_HOST, block_size)

qiniu/rpc.py

@@ -112,7 +112,7 @@ def encode_multipart_formdata(self, fields, files):
 			disposition = "Content-Disposition: form-data;"
 			filename = _qiniu_escape(file_info.get('filename'))
 			L.append('%s name="file"; filename="%s"' % (disposition, filename))
-			L.append('Content-Type: %s' % file_info.get('content_type', 'application/octet-stream'))
+			L.append('Content-Type: %s' % file_info.get('mime_type', 'application/octet-stream'))
 			L.append('')
 			L.append('')
 			b2 = CRLF.join(L)

qiniu/rs/rs_token.py

@@ -77,8 +77,8 @@ def token(self, mac=None):
 
 class GetPolicy(object):
 	expires = 3600
-	def __init__(self):
-		pass
+	def __init__(self, expires=3600):
+		self.expires = expires
 
 	def make_request(self, base_url, mac=None):
 		'''

qiniu/rs/test/rs_token_test.py

@@ -22,11 +22,38 @@ class TestToken(unittest.TestCase):
 	def test_put_policy(self):
 		policy = rs.PutPolicy(bucket_name)
 		policy.endUser = "hello!"
+		policy.returnUrl = "http://localhost:1234/path?query=hello"
+		policy.returnBody = "$(sha1)"
+		# Do not specify the returnUrl and callbackUrl at the same time
+		policy.callbackUrl = "http://1.2.3.4/callback"
+		policy.callbackBody = "$(bucket)"
+
+		policy.saveKey = "$(sha1)"
+		policy.insertOnly = 1
+		policy.detectMime = 1
+		policy.fsizeLimit = 1024
+		policy.persistentNotifyUrl = "http://4.3.2.1/persistentNotifyUrl"
+		policy.persistentOps = "avthumb/flash"
+
 		tokens = policy.token().split(':')
+
+		# chcek first part of token
 		self.assertEqual(conf.ACCESS_KEY, tokens[0])
 		data = json.loads(decode(tokens[2]))
+
+		# check if same
 		self.assertEqual(data["scope"], bucket_name)
 		self.assertEqual(data["endUser"], policy.endUser)
+		self.assertEqual(data["returnUrl"], policy.returnUrl)
+		self.assertEqual(data["returnBody"], policy.returnBody)
+		self.assertEqual(data["callbackUrl"], policy.callbackUrl)
+		self.assertEqual(data["callbackBody"], policy.callbackBody)
+		self.assertEqual(data["saveKey"], policy.saveKey)
+		self.assertEqual(data["exclusive"], policy.insertOnly)
+		self.assertEqual(data["detectMime"], policy.detectMime)
+		self.assertEqual(data["fsizeLimit"], policy.fsizeLimit)
+		self.assertEqual(data["persistentNotifyUrl"], policy.persistentNotifyUrl)
+		self.assertEqual(data["persistentOps"], policy.persistentOps)
 
 		new_hmac = encode(hmac.new(conf.SECRET_KEY, tokens[2], sha1).digest())
 		self.assertEqual(new_hmac, tokens[1])

qiniu/rsf.py

@@ -36,6 +36,6 @@ def list_prefix(self, bucket, prefix=None, marker=None, limit=None):
 			ops['prefix'] = prefix
 		url = '%s?%s' % ('/list', urllib.urlencode(ops))
 		ret, err = self.conn.call_with(url, body=None, content_type='application/x-www-form-urlencoded')
-		if not ret.get('marker'):
+		if ret and not ret.get('marker'):
 			err = EOF
 		return ret, err

qiniu/test/resumable_io_test.py

@@ -77,6 +77,31 @@ def test_put(self):
 		self.assertEqual(ret["hash"], "FnyTMUqPNRTdk1Wou7oLqDHkBm_p", "hash not match")
 		rs.Client().delete(bucket, key)
 
+	def test_put_4m(self):
+		ostype = platform.system()
+		if ostype.lower().find("windows") != -1:
+			tmpf = "".join([os.getcwd(), os.tmpnam()])
+		else:
+			tmpf = os.tmpnam()
+		dst = open(tmpf, 'wb')
+		dst.write("abcd" * 1024 * 1024)
+		dst.flush()
+
+		policy = rs.PutPolicy(bucket)
+		extra = resumable_io.PutExtra(bucket)
+		extra.bucket = bucket
+		extra.params = {"x:foo": "test"}
+		key = "sdk_py_resumable_block_6_%s" % r(9)
+		localfile = dst.name
+		ret, err = resumable_io.put_file(policy.token(), key, localfile, extra)
+		assert ret.get("x:foo") == "test", "return data not contains 'x:foo'"
+		dst.close()
+		os.remove(tmpf)
+
+		assert err is None, err
+		self.assertEqual(ret["hash"], "FnIVmMd_oaUV3MLDM6F9in4RMz2U", "hash not match")
+		rs.Client().delete(bucket, key)
+
 
 if __name__ == "__main__":
 	unittest.main()

qiniu/test/rsf_test.py

@@ -11,9 +11,9 @@
 class TestRsf(unittest.TestCase):
 	def test_list_prefix(self):
 		c = rsf.Client()
-		ret, err = c.list_prefix(bucket_name, limit = 1)
+		ret, err = c.list_prefix(bucket_name, limit = 4)
 		self.assertEqual(err is rsf.EOF or err is None, True)
-		assert len(ret.get('items')) == 1
+		assert len(ret.get('items')) == 4
 
 
 if __name__ == "__main__":