Skip to content

Commit 21f78a6

Browse files
dingxin-techdingxin-tech
committed
prepare 0.51.3-public
1 parent 3f0fb8d commit 21f78a6

File tree

6 files changed

+186
-23
lines changed

6 files changed

+186
-23
lines changed

‎odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/commons/proto/ProtobufRecordStreamWriter.java

Lines changed: 37 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -21,31 +21,13 @@
2121

2222
import static com.aliyun.odps.data.ArrayRecord.DEFAULT_CALENDAR;
2323

24-
import com.aliyun.odps.Column;
25-
import com.aliyun.odps.TableSchema;
26-
import com.aliyun.odps.commons.util.DateUtils;
27-
import com.aliyun.odps.data.*;
28-
import com.aliyun.odps.tunnel.io.Checksum;
29-
import com.aliyun.odps.tunnel.io.CompressOption;
30-
import com.aliyun.odps.tunnel.io.ProtobufRecordPack;
31-
import com.aliyun.odps.type.ArrayTypeInfo;
32-
import com.aliyun.odps.type.MapTypeInfo;
33-
import com.aliyun.odps.type.StructTypeInfo;
34-
import com.aliyun.odps.type.TypeInfo;
35-
import com.google.protobuf.CodedOutputStream;
36-
import com.google.protobuf.WireFormat;
37-
import org.apache.commons.io.output.CountingOutputStream;
38-
import org.xerial.snappy.SnappyFramedOutputStream;
39-
import net.jpountz.lz4.LZ4FrameOutputStream;
40-
4124
import java.io.IOException;
4225
import java.io.OutputStream;
4326
import java.math.BigDecimal;
4427
import java.sql.Timestamp;
4528
import java.time.Instant;
4629
import java.time.LocalDate;
4730
import java.time.LocalDateTime;
48-
import java.time.ZoneId;
4931
import java.time.ZoneOffset;
5032
import java.time.ZonedDateTime;
5133
import java.util.ArrayList;
@@ -56,6 +38,36 @@
5638
import java.util.zip.Deflater;
5739
import java.util.zip.DeflaterOutputStream;
5840

41+
import org.apache.commons.io.output.CountingOutputStream;
42+
import org.xerial.snappy.SnappyFramedOutputStream;
43+
44+
import com.aliyun.odps.Column;
45+
import com.aliyun.odps.TableSchema;
46+
import com.aliyun.odps.commons.util.DateUtils;
47+
import com.aliyun.odps.data.AbstractChar;
48+
import com.aliyun.odps.data.Binary;
49+
import com.aliyun.odps.data.IntervalDayTime;
50+
import com.aliyun.odps.data.IntervalYearMonth;
51+
import com.aliyun.odps.data.OdpsTypeTransformer;
52+
import com.aliyun.odps.data.Record;
53+
import com.aliyun.odps.data.RecordPack;
54+
import com.aliyun.odps.data.RecordReader;
55+
import com.aliyun.odps.data.RecordWriter;
56+
import com.aliyun.odps.data.SimpleJsonValue;
57+
import com.aliyun.odps.data.Struct;
58+
import com.aliyun.odps.exceptions.SchemaMismatchException;
59+
import com.aliyun.odps.tunnel.io.Checksum;
60+
import com.aliyun.odps.tunnel.io.CompressOption;
61+
import com.aliyun.odps.tunnel.io.ProtobufRecordPack;
62+
import com.aliyun.odps.type.ArrayTypeInfo;
63+
import com.aliyun.odps.type.MapTypeInfo;
64+
import com.aliyun.odps.type.StructTypeInfo;
65+
import com.aliyun.odps.type.TypeInfo;
66+
import com.google.protobuf.CodedOutputStream;
67+
import com.google.protobuf.WireFormat;
68+
69+
import net.jpountz.lz4.LZ4FrameOutputStream;
70+
5971
/**
6072
* @author chao.liu
6173
*/
@@ -117,9 +129,14 @@ public void write(Record r) throws IOException {
117129
int recordValues = r.getColumnCount();
118130
int columnCount = columns.length;
119131
if (recordValues > columnCount) {
120-
throw new IOException("record values more than schema.");
132+
throw new SchemaMismatchException(
133+
String.format("Record values are more than schema. "
134+
+ "The record written has %d columns but the current session only has %d columns. "
135+
+ "If you just performed Schema Evolution, please recreate the session and try again.",
136+
recordValues, columnCount),
137+
null
138+
);
121139
}
122-
123140
int i = 0;
124141
for (; i < columnCount && i < recordValues; i++) {
125142

‎odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/sqa/v2/SQLExecutorImpl.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -666,4 +666,8 @@ public boolean isSelect(String sql) throws OdpsException {
666666
throw new OdpsException("Sql isSelect failed", e);
667667
}
668668
}
669+
670+
public void setProject(String project) {
671+
odps.setDefaultProject(project);
672+
}
669673
}

‎odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/utils/TeaUtils.java

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
package com.aliyun.odps.utils;
2+
3+
import java.nio.charset.StandardCharsets;
4+
import java.text.SimpleDateFormat;
5+
import java.util.Arrays;
6+
import java.util.Date;
7+
import java.util.Locale;
8+
import java.util.Map;
9+
import java.util.SimpleTimeZone;
10+
import java.util.TreeMap;
11+
12+
import javax.crypto.Mac;
13+
import javax.crypto.spec.SecretKeySpec;
14+
15+
import org.apache.commons.codec.binary.Base64;
16+
17+
/**
18+
* @author dingxin ([email protected])
19+
*/
20+
public class TeaUtils {
21+
22+
/**
23+
* get timestamp format like 'Fri, 13 Dec 2024 02:57:00 GMT'
24+
*
25+
* @return timestamp string
26+
*/
27+
public static String getApiTimestamp() {
28+
SimpleDateFormat rfc822DateFormat = new SimpleDateFormat(
29+
"EEE, dd MMM yyyy HH:mm:ss z", Locale.US);
30+
rfc822DateFormat.setTimeZone(new SimpleTimeZone(0, "GMT"));
31+
return rfc822DateFormat.format(new Date());
32+
}
33+
34+
static final String CONTENT_MD5 = "Content-MD5";
35+
static final String CONTENT_TYPE = "Content-Type";
36+
static final String DATE = "Date";
37+
static final String PREFIX = "x-odps-";
38+
39+
public static String buildCanonicalString(String method, String resource,
40+
Map<String, String> params,
41+
Map<String, String> headers) {
42+
StringBuilder builder = new StringBuilder();
43+
builder.append(method + "\n");
44+
TreeMap<String, String> headersToSign = new TreeMap<String, String>();
45+
if (headers != null) {
46+
for (Map.Entry<String, String> header : headers.entrySet()) {
47+
if (header.getKey() == null) {
48+
continue;
49+
}
50+
String lowerKey = header.getKey().toLowerCase();
51+
if (lowerKey.equals(CONTENT_MD5.toLowerCase())
52+
|| lowerKey.equals(CONTENT_TYPE.toLowerCase())
53+
|| lowerKey.equals(DATE.toLowerCase()) || lowerKey.startsWith(PREFIX)) {
54+
headersToSign.put(lowerKey, header.getValue());
55+
}
56+
}
57+
}
58+
if (!headersToSign.containsKey(CONTENT_TYPE.toLowerCase())) {
59+
headersToSign.put(CONTENT_TYPE.toLowerCase(), "");
60+
}
61+
if (!headersToSign.containsKey(CONTENT_MD5.toLowerCase())) {
62+
headersToSign.put(CONTENT_MD5.toLowerCase(), "");
63+
}
64+
// Add params that have the prefix "x-odps-"
65+
if (params != null) {
66+
for (Map.Entry<String, String> p : params.entrySet()) {
67+
if (p.getKey().startsWith(PREFIX)) {
68+
headersToSign.put(p.getKey(), p.getValue());
69+
}
70+
}
71+
}
72+
// Add all headers to sign to the builder
73+
for (Map.Entry<String, String> entry : headersToSign.entrySet()) {
74+
String key = entry.getKey();
75+
Object value = entry.getValue();
76+
if (key.startsWith(PREFIX)) {
77+
// null key will error in jdk.
78+
builder.append(key);
79+
builder.append(':');
80+
if (value != null) {
81+
builder.append(value);
82+
}
83+
} else {
84+
builder.append(value);
85+
}
86+
builder.append("\n");
87+
}
88+
// Add canonical resource
89+
builder.append(buildCanonicalResource(resource, params));
90+
return builder.toString();
91+
}
92+
93+
private static String buildCanonicalResource(String resource, Map<String, String> params) {
94+
StringBuilder builder = new StringBuilder();
95+
builder.append(resource);
96+
97+
if (params != null && params.size() > 0) {
98+
String[] names = params.keySet().toArray(new String[params.size()]);
99+
Arrays.sort(names);
100+
char separater = '?';
101+
for (String name : names) {
102+
103+
builder.append(separater);
104+
builder.append(name);
105+
String paramValue = params.get(name);
106+
if (paramValue != null && paramValue.length() > 0) {
107+
builder.append("=").append(paramValue);
108+
}
109+
separater = '&';
110+
}
111+
}
112+
return builder.toString();
113+
}
114+
115+
public static String getSignature(String strToSign, String accessKeyId, String accessKeySecret) {
116+
byte[] crypto;
117+
crypto = hmacsha1Signature(strToSign.getBytes(StandardCharsets.UTF_8),
118+
accessKeySecret.getBytes());
119+
120+
String signature = Base64.encodeBase64String(crypto).trim();
121+
return "ODPS " + accessKeyId + ":" + signature;
122+
}
123+
124+
private static byte[] hmacsha1Signature(byte[] data, byte[] key) {
125+
try {
126+
SecretKeySpec signingKey = new SecretKeySpec(key, "HmacSHA1");
127+
Mac mac = Mac.getInstance("HmacSHA1");
128+
mac.init(signingKey);
129+
return mac.doFinal(data);
130+
} catch (Exception e) {
131+
throw new RuntimeException(e.getMessage(), e);
132+
}
133+
}
134+
}

‎odps-sdk/odps-sdk-core/src/test/java/com/aliyun/odps/TableTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ public class TableTest extends TestBase {
106106
@BeforeClass
107107
public static void setUp() throws Exception {
108108
odps = OdpsTestUtils.newDefaultOdps();
109+
cleanUp();
109110
OdpsTestUtils.createTableForTest(SOURCE_TABLE_NAME);
110111

111112
schema = new TableSchema();
@@ -165,7 +166,6 @@ public static void setUp() throws Exception {
165166
partitionedTable.createPartition(new PartitionSpec("p1=1,p2=baz"), true);
166167
}
167168

168-
@AfterClass
169169
public static void cleanUp() throws Exception {
170170
odps.tables().delete(TABLE_NAME, true);
171171
odps.tables().delete(TABLE_NAME_2, true);

‎odps-sdk/odps-sdk-core/src/test/java/com/aliyun/odps/TablesTest.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -402,6 +402,8 @@ public void testBatchLoading() throws OdpsException {
402402
public void testBatchLoadingCrossProjects() throws OdpsException, IOException {
403403
String crossProject = OdpsTestUtils.loadConfig().getProperty("security.project");
404404
assertNotEquals(odps.getDefaultProject(), crossProject);
405+
// make sure at least one table
406+
odps.tables().create(crossProject, "test_table", SCHEMA, true);
405407

406408
List<Table> expectedTables = getTables(odps, crossProject, 10);
407409
assertTrue(expectedTables.size() > 0);

‎odps-sdk/odps-sdk-core/src/test/java/com/aliyun/odps/test/LogViewTest.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,10 @@ public void testLogView() throws OdpsException {
3838
try {
3939
LogView log = odps.logview();
4040
Iterator<Instance> instIter = odps.instances().iterator();
41-
instIter.hasNext();
41+
boolean hasNext = instIter.hasNext();
42+
if (!hasNext) {
43+
return;
44+
}
4245
Instance i = instIter.next();
4346
String logview = log.generateLogView(i, 7 * 24);
4447
System.out.println(logview);
@@ -63,7 +66,10 @@ public void testLogViewHost() throws OdpsException {
6366
LogView log = odps.logview();
6467
log.setLogViewHost("http://test.a.b.c");
6568
Iterator<Instance> instIter = odps.instances().iterator();
66-
instIter.hasNext();
69+
boolean hasNext = instIter.hasNext();
70+
if (!hasNext) {
71+
return;
72+
}
6773
Instance i = instIter.next();
6874
assertTrue(
6975
log.generateLogView(i, 7 * 24).startsWith("http://test.a.b.c/logview"));

0 commit comments

Comments
 (0)