Skip to content

Commit 97b75df

Browse files
xwen-winniexwen-winnie
authored
Merge pull request #522 from qiniu/ys_dev
支持多区域上传
2 parents c1ef0b1 + 4f5f606 commit 97b75df

14 files changed

+1012
-285
lines changed

‎src/main/java/com/qiniu/storage/AutoRegion.java

Lines changed: 182 additions & 157 deletions
Large diffs are not rendered by default.

‎src/main/java/com/qiniu/storage/BaseUploader.java

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package com.qiniu.storage;
2+
3+
import com.qiniu.common.QiniuException;
4+
import com.qiniu.http.Client;
5+
import com.qiniu.http.Response;
6+
7+
public abstract class BaseUploader {
8+
9+
protected final Client client;
10+
protected final String key;
11+
protected final String upToken;
12+
protected final ConfigHelper configHelper;
13+
protected final Configuration config;
14+
15+
BaseUploader(Client client, String upToken, String key, Configuration config) {
16+
this.client = client;
17+
this.key = key;
18+
this.upToken = upToken;
19+
if (config == null) {
20+
this.config = new Configuration();
21+
} else {
22+
this.config = config.clone();
23+
}
24+
this.configHelper = new ConfigHelper(this.config);
25+
}
26+
27+
public Response upload() throws QiniuException {
28+
if (this.config == null) {
29+
throw QiniuException.unrecoverable("config can't be empty");
30+
}
31+
return uploadWithRegionRetry();
32+
}
33+
34+
private Response uploadWithRegionRetry() throws QiniuException {
35+
Response response = null;
36+
while (true) {
37+
try {
38+
response = uploadFlows();
39+
if (!couldSwitchRegionAndRetry(response, null)
40+
|| !couldReloadSource() || !reloadSource()
41+
|| config.region == null || !config.region.switchRegion(new UploadToken(upToken))) {
42+
break;
43+
}
44+
} catch (QiniuException e) {
45+
if (!couldSwitchRegionAndRetry(null, e)
46+
|| !couldReloadSource() || !reloadSource()
47+
|| config.region == null || !config.region.switchRegion(new UploadToken(upToken))) {
48+
throw e;
49+
}
50+
}
51+
}
52+
return response;
53+
}
54+
55+
abstract Response uploadFlows() throws QiniuException;
56+
57+
abstract boolean couldReloadSource();
58+
59+
abstract boolean reloadSource();
60+
61+
private boolean couldSwitchRegionAndRetry(Response response, QiniuException exception) {
62+
Response checkResponse = response;
63+
if (checkResponse == null && exception != null) {
64+
checkResponse = exception.response;
65+
}
66+
67+
if (checkResponse != null) {
68+
int statusCode = checkResponse.statusCode;
69+
return (statusCode > -2 && statusCode < 200) || (statusCode > 299
70+
&& statusCode != 401 && statusCode != 413 && statusCode != 419
71+
&& statusCode != 608 && statusCode != 614 && statusCode != 630);
72+
}
73+
74+
return exception == null || !exception.isUnrecoverable();
75+
}
76+
}

‎src/main/java/com/qiniu/storage/FixBlockUploader.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import java.util.concurrent.locks.Lock;
1919
import java.util.concurrent.locks.ReentrantLock;
2020

21+
@Deprecated
2122
public class FixBlockUploader {
2223
private final int blockSize;
2324
private final ConfigHelper configHelper;
@@ -883,6 +884,7 @@ class Record {
883884
long size;
884885
long blockSize;
885886
List<EtagIdx> etagIdxes;
887+
886888
// 用于区分记录是 V1 还是 V2
887889
boolean isValid() {
888890
return uploadId != null && etagIdxes != null && etagIdxes.size() > 0;

‎src/main/java/com/qiniu/storage/FormUploader.java

Lines changed: 24 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,8 @@
1414
* 该类封装了七牛提供的表单上传机制
1515
* 参考文档：<a href="https://developer.qiniu.com/kodo/manual/form-upload">表单上传</a>
1616
*/
17-
public final class FormUploader {
17+
public final class FormUploader extends BaseUploader {
1818

19-
private final String token;
20-
private final String key;
2119
private final File file;
2220
private final byte[] data;
2321
private final String mime;
@@ -45,9 +43,9 @@ public FormUploader(Client client, String upToken, String key, File file, String
4543

4644
private FormUploader(Client client, String upToken, String key, byte[] data, File file, StringMap params,
4745
String mime, boolean checkCrc, Configuration configuration) {
46+
super(client, upToken, key, configuration);
47+
4848
this.client = client;
49-
token = upToken;
50-
this.key = key;
5149
this.file = file;
5250
this.data = data;
5351
this.params = params;
@@ -56,23 +54,21 @@ private FormUploader(Client client, String upToken, String key, byte[] data, Fil
5654
this.configHelper = new ConfigHelper(configuration);
5755
}
5856

59-
/**
60-
* 同步上传文件
61-
*/
62-
public Response upload() throws QiniuException {
57+
@Override
58+
Response uploadFlows() throws QiniuException {
6359
buildParams();
64-
String host = configHelper.upHost(token);
60+
String host = configHelper.upHost(upToken);
6561
try {
6662
if (data != null) {
67-
return client.multipartPost(configHelper.upHost(token), params, "file", filename, data,
63+
return client.multipartPost(configHelper.upHost(upToken), params, "file", filename, data,
6864
mime, new StringMap());
6965
} else {
70-
return client.multipartPost(configHelper.upHost(token), params, "file", filename, file,
66+
return client.multipartPost(configHelper.upHost(upToken), params, "file", filename, file,
7167
mime, new StringMap());
7268
}
7369
} catch (QiniuException e) {
7470
if (e.response == null || e.response.needSwitchServer()) {
75-
changeHost(token, host);
71+
changeHost(upToken, host);
7672
}
7773
throw e;
7874
}
@@ -83,32 +79,42 @@ public Response upload() throws QiniuException {
8379
*/
8480
public void asyncUpload(final UpCompletionHandler handler) throws IOException {
8581
buildParams();
86-
final String host = configHelper.upHost(token);
82+
final String host = configHelper.upHost(upToken);
8783
if (data != null) {
8884
client.asyncMultipartPost(host, params, "file", filename,
8985
data, mime, new StringMap(), new AsyncCallback() {
9086
@Override
9187
public void complete(Response res) {
9288
if (res != null && res.needSwitchServer()) {
93-
changeHost(token, host);
89+
changeHost(upToken, host);
9490
}
9591
handler.complete(key, res);
9692
}
9793
});
9894
return;
9995
}
100-
client.asyncMultipartPost(configHelper.upHost(token), params, "file", filename,
96+
client.asyncMultipartPost(configHelper.upHost(upToken), params, "file", filename,
10197
file, mime, new StringMap(), new AsyncCallback() {
10298
@Override
10399
public void complete(Response res) {
104100
if (res != null && res.needSwitchServer()) {
105-
changeHost(token, host);
101+
changeHost(upToken, host);
106102
}
107103
handler.complete(key, res);
108104
}
109105
});
110106
}
111107

108+
@Override
109+
boolean couldReloadSource() {
110+
return true;
111+
}
112+
113+
@Override
114+
boolean reloadSource() {
115+
return true;
116+
}
117+
112118
private void changeHost(String upToken, String host) {
113119
try {
114120
configHelper.tryChangeUpHost(upToken, host);
@@ -120,7 +126,7 @@ private void changeHost(String upToken, String host) {
120126

121127
private void buildParams() throws QiniuException {
122128
if (params == null) return;
123-
params.put("token", token);
129+
params.put("token", upToken);
124130

125131
if (key != null) {
126132
params.put("key", key);

‎src/main/java/com/qiniu/storage/Region.java

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77

88
public class Region {
99

10+
// 有效时间戳，过了有效期，region会无效，此处只在取时缓存判断； -1 为无限期
11+
private long timestamp = -1;
1012
// 区域名称：z0 华东 z1 华北 z2 华南 na0 北美 as0 东南亚
1113
private String region = "z0";
1214

@@ -25,6 +27,23 @@ public class Region {
2527
private String rsHost = "rs.qbox.me";
2628
private String rsfHost = "rsf.qbox.me";
2729
private String apiHost = "api.qiniu.com";
30+
private String ucHost = "uc.qbox.me";
31+
32+
Region() {
33+
}
34+
35+
Region(long timestamp, String region, List<String> srcUpHosts, List<String> accUpHosts, String iovipHost,
36+
String rsHost, String rsfHost, String apiHost, String ucHost) {
37+
this.timestamp = timestamp;
38+
this.region = region;
39+
this.srcUpHosts = srcUpHosts;
40+
this.accUpHosts = accUpHosts;
41+
this.iovipHost = iovipHost;
42+
this.rsHost = rsHost;
43+
this.rsfHost = rsfHost;
44+
this.apiHost = apiHost;
45+
this.ucHost = ucHost;
46+
}
2847

2948
/**
3049
* 华东机房相关域名
@@ -211,10 +230,18 @@ public static Region autoRegion(String ucServer) {
211230
return new Builder().autoRegion(ucServer);
212231
}
213232

233+
boolean switchRegion(RegionReqInfo regionReqInfo) {
234+
return false;
235+
}
236+
214237
String getRegion(RegionReqInfo regionReqInfo) {
215238
return this.region;
216239
}
217240

241+
Region getCurrentRegion(RegionReqInfo regionReqInfo) {
242+
return this;
243+
}
244+
218245
List<String> getSrcUpHost(RegionReqInfo regionReqInfo) throws QiniuException {
219246
return this.srcUpHosts;
220247
}
@@ -239,6 +266,14 @@ String getApiHost(RegionReqInfo regionReqInfo) throws QiniuException {
239266
return apiHost;
240267
}
241268

269+
boolean isValid() {
270+
if (timestamp < 0) {
271+
return true;
272+
} else {
273+
return System.currentTimeMillis() < timestamp * 1000;
274+
}
275+
}
276+
242277
/**
243278
* 域名构造器
244279
*/

‎src/main/java/com/qiniu/storage/RegionGroup.java

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
package com.qiniu.storage;
2+
3+
import com.qiniu.common.QiniuException;
4+
5+
import java.util.ArrayList;
6+
import java.util.List;
7+
8+
public class RegionGroup extends Region {
9+
10+
private Region currentRegion = null;
11+
private int currentRegionIndex = 0;
12+
private final List<Region> regionList = new ArrayList<>();
13+
14+
15+
public boolean addRegion(Region region) {
16+
if (region == null) {
17+
return false;
18+
}
19+
20+
regionList.add(region);
21+
22+
if (currentRegion == null) {
23+
updateCurrentRegion();
24+
}
25+
26+
return true;
27+
}
28+
29+
@Override
30+
boolean switchRegion(RegionReqInfo regionReqInfo) {
31+
if (currentRegion != null && currentRegion.isValid() && currentRegion.switchRegion(regionReqInfo)) {
32+
return true;
33+
}
34+
35+
if ((currentRegionIndex + 1) < regionList.size()) {
36+
currentRegionIndex += 1;
37+
updateCurrentRegion();
38+
return true;
39+
} else {
40+
return false;
41+
}
42+
}
43+
44+
String getRegion(RegionReqInfo regionReqInfo) {
45+
if (currentRegion == null) {
46+
return "";
47+
} else {
48+
return currentRegion.getRegion(regionReqInfo);
49+
}
50+
}
51+
52+
List<String> getSrcUpHost(RegionReqInfo regionReqInfo) throws QiniuException {
53+
if (currentRegion == null) {
54+
return null;
55+
} else {
56+
return currentRegion.getSrcUpHost(regionReqInfo);
57+
}
58+
}
59+
60+
List<String> getAccUpHost(RegionReqInfo regionReqInfo) throws QiniuException {
61+
if (currentRegion == null) {
62+
return null;
63+
} else {
64+
return currentRegion.getAccUpHost(regionReqInfo);
65+
}
66+
}
67+
68+
String getIovipHost(RegionReqInfo regionReqInfo) throws QiniuException {
69+
if (currentRegion == null) {
70+
return null;
71+
} else {
72+
return currentRegion.getIovipHost(regionReqInfo);
73+
}
74+
}
75+
76+
String getRsHost(RegionReqInfo regionReqInfo) throws QiniuException {
77+
if (currentRegion == null) {
78+
return null;
79+
} else {
80+
return currentRegion.getRsHost(regionReqInfo);
81+
}
82+
}
83+
84+
String getRsfHost(RegionReqInfo regionReqInfo) throws QiniuException {
85+
if (currentRegion == null) {
86+
return null;
87+
} else {
88+
return currentRegion.getRsfHost(regionReqInfo);
89+
}
90+
}
91+
92+
String getApiHost(RegionReqInfo regionReqInfo) throws QiniuException {
93+
if (currentRegion == null) {
94+
return null;
95+
} else {
96+
return currentRegion.getApiHost(regionReqInfo);
97+
}
98+
}
99+
100+
Region getCurrentRegion(RegionReqInfo regionReqInfo) {
101+
if (currentRegion == null) {
102+
return null;
103+
} else if (currentRegion instanceof AutoRegion || currentRegion instanceof RegionGroup) {
104+
return currentRegion.getCurrentRegion(regionReqInfo);
105+
} else {
106+
return currentRegion;
107+
}
108+
}
109+
110+
@Override
111+
boolean isValid() {
112+
if (currentRegion == null) {
113+
return false;
114+
}
115+
// 只判断当前的
116+
return currentRegion.isValid();
117+
}
118+
119+
private void updateCurrentRegion() {
120+
if (regionList.size() == 0) {
121+
return;
122+
}
123+
124+
if (currentRegionIndex < regionList.size()) {
125+
currentRegion = regionList.get(currentRegionIndex);
126+
}
127+
}
128+
}

0 commit comments

Comments
 (0)