Skip to content
This repository was archived by the owner on Feb 13, 2025. It is now read-only.

Commit bd0ab1a

Browse files
author
Anselm Kruis
committed
merge branch 3.5
2 parents c7705e0 + cb76496 commit bd0ab1a

File tree

1 file changed

+234
-41
lines changed

1 file changed

+234
-41
lines changed

‎Lib/test/pickletester.py

Lines changed: 234 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ def create_dynamic_class(name, bases):
145145
result.reduce_args = (name, bases)
146146
return result
147147

148-
# DATA0 .. DATA2 are the pickles we expect under the various protocols, for
148+
# DATA0 .. DATA4 are the pickles we expect under the various protocols, for
149149
# the object returned by create_data().
150150

151151
DATA0 = (
@@ -401,27 +401,176 @@ def create_dynamic_class(name, bases):
401401
highest protocol among opcodes = 2
402402
"""
403403

404+
DATA3 = (
405+
b'\x80\x03]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
406+
b'builtins\ncomplex\nq\x01G'
407+
b'@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00\x86q\x02'
408+
b'Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff'
409+
b'\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7f'
410+
b'J\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00abcq'
411+
b'\x04h\x04c__main__\nC\nq\x05)\x81q'
412+
b'\x06}q\x07(X\x03\x00\x00\x00barq\x08K\x02X\x03\x00'
413+
b'\x00\x00fooq\tK\x01ubh\x06tq\nh\nK\x05'
414+
b'e.'
415+
)
416+
417+
# Disassembly of DATA3
418+
DATA3_DIS = """\
419+
0: \x80 PROTO 3
420+
2: ] EMPTY_LIST
421+
3: q BINPUT 0
422+
5: ( MARK
423+
6: K BININT1 0
424+
8: K BININT1 1
425+
10: G BINFLOAT 2.0
426+
19: c GLOBAL 'builtins complex'
427+
37: q BINPUT 1
428+
39: G BINFLOAT 3.0
429+
48: G BINFLOAT 0.0
430+
57: \x86 TUPLE2
431+
58: q BINPUT 2
432+
60: R REDUCE
433+
61: q BINPUT 3
434+
63: K BININT1 1
435+
65: J BININT -1
436+
70: K BININT1 255
437+
72: J BININT -255
438+
77: J BININT -256
439+
82: M BININT2 65535
440+
85: J BININT -65535
441+
90: J BININT -65536
442+
95: J BININT 2147483647
443+
100: J BININT -2147483647
444+
105: J BININT -2147483648
445+
110: ( MARK
446+
111: X BINUNICODE 'abc'
447+
119: q BINPUT 4
448+
121: h BINGET 4
449+
123: c GLOBAL '__main__ C'
450+
135: q BINPUT 5
451+
137: ) EMPTY_TUPLE
452+
138: \x81 NEWOBJ
453+
139: q BINPUT 6
454+
141: } EMPTY_DICT
455+
142: q BINPUT 7
456+
144: ( MARK
457+
145: X BINUNICODE 'bar'
458+
153: q BINPUT 8
459+
155: K BININT1 2
460+
157: X BINUNICODE 'foo'
461+
165: q BINPUT 9
462+
167: K BININT1 1
463+
169: u SETITEMS (MARK at 144)
464+
170: b BUILD
465+
171: h BINGET 6
466+
173: t TUPLE (MARK at 110)
467+
174: q BINPUT 10
468+
176: h BINGET 10
469+
178: K BININT1 5
470+
180: e APPENDS (MARK at 5)
471+
181: . STOP
472+
highest protocol among opcodes = 2
473+
"""
474+
475+
DATA4 = (
476+
b'\x80\x04\x95\xa8\x00\x00\x00\x00\x00\x00\x00]\x94(K\x00K\x01G@'
477+
b'\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x07'
478+
b'complex\x94\x93\x94G@\x08\x00\x00\x00\x00\x00\x00G'
479+
b'\x00\x00\x00\x00\x00\x00\x00\x00\x86\x94R\x94K\x01J\xff\xff\xff\xffK'
480+
b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ'
481+
b'\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80('
482+
b'\x8c\x03abc\x94h\x06\x8c\x08__main__\x94\x8c'
483+
b'\x01C\x94\x93\x94)\x81\x94}\x94(\x8c\x03bar\x94K\x02\x8c'
484+
b'\x03foo\x94K\x01ubh\nt\x94h\x0eK\x05e.'
485+
)
486+
487+
# Disassembly of DATA4
488+
DATA4_DIS = """\
489+
0: \x80 PROTO 4
490+
2: \x95 FRAME 168
491+
11: ] EMPTY_LIST
492+
12: \x94 MEMOIZE
493+
13: ( MARK
494+
14: K BININT1 0
495+
16: K BININT1 1
496+
18: G BINFLOAT 2.0
497+
27: \x8c SHORT_BINUNICODE 'builtins'
498+
37: \x94 MEMOIZE
499+
38: \x8c SHORT_BINUNICODE 'complex'
500+
47: \x94 MEMOIZE
501+
48: \x93 STACK_GLOBAL
502+
49: \x94 MEMOIZE
503+
50: G BINFLOAT 3.0
504+
59: G BINFLOAT 0.0
505+
68: \x86 TUPLE2
506+
69: \x94 MEMOIZE
507+
70: R REDUCE
508+
71: \x94 MEMOIZE
509+
72: K BININT1 1
510+
74: J BININT -1
511+
79: K BININT1 255
512+
81: J BININT -255
513+
86: J BININT -256
514+
91: M BININT2 65535
515+
94: J BININT -65535
516+
99: J BININT -65536
517+
104: J BININT 2147483647
518+
109: J BININT -2147483647
519+
114: J BININT -2147483648
520+
119: ( MARK
521+
120: \x8c SHORT_BINUNICODE 'abc'
522+
125: \x94 MEMOIZE
523+
126: h BINGET 6
524+
128: \x8c SHORT_BINUNICODE '__main__'
525+
138: \x94 MEMOIZE
526+
139: \x8c SHORT_BINUNICODE 'C'
527+
142: \x94 MEMOIZE
528+
143: \x93 STACK_GLOBAL
529+
144: \x94 MEMOIZE
530+
145: ) EMPTY_TUPLE
531+
146: \x81 NEWOBJ
532+
147: \x94 MEMOIZE
533+
148: } EMPTY_DICT
534+
149: \x94 MEMOIZE
535+
150: ( MARK
536+
151: \x8c SHORT_BINUNICODE 'bar'
537+
156: \x94 MEMOIZE
538+
157: K BININT1 2
539+
159: \x8c SHORT_BINUNICODE 'foo'
540+
164: \x94 MEMOIZE
541+
165: K BININT1 1
542+
167: u SETITEMS (MARK at 150)
543+
168: b BUILD
544+
169: h BINGET 10
545+
171: t TUPLE (MARK at 119)
546+
172: \x94 MEMOIZE
547+
173: h BINGET 14
548+
175: K BININT1 5
549+
177: e APPENDS (MARK at 13)
550+
178: . STOP
551+
highest protocol among opcodes = 4
552+
"""
553+
404554
# set([1,2]) pickled from 2.x with protocol 2
405-
DATA3 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.'
555+
DATA_SET = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.'
406556

407557
# xrange(5) pickled from 2.x with protocol 2
408-
DATA4 = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.'
558+
DATA_XRANGE = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.'
409559
if stackless:
410-
DATA4_SLP = b'\x80\x02cstackless._wrap\nrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02)b.'
560+
DATA_XRANGE_SLP = b'\x80\x02cstackless._wrap\nrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02)b.'
411561
else:
412-
DATA4_SLP = DATA4
413-
562+
DATA_XRANGE_SLP = DATA_XRANGE
414563

415564
# a SimpleCookie() object pickled from 2.x with protocol 2
416-
DATA5 = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key'
417-
b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U'
418-
b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07'
419-
b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U'
420-
b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b'
421-
b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.')
565+
DATA_COOKIE = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key'
566+
b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U'
567+
b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07'
568+
b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U'
569+
b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b'
570+
b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.')
422571

423572
# set([3]) pickled from 2.x with protocol 2
424-
DATA6 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.'
573+
DATA_SET2 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.'
425574

426575
python2_exceptions_without_args = (
427576
ArithmeticError,
@@ -473,20 +622,10 @@ def create_dynamic_class(name, bases):
473622

474623
exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.'
475624

476-
# Exception objects without arguments pickled from 2.x with protocol 2
477-
DATA7 = {
478-
exception :
479-
exception_pickle.replace(b'?', exception.__name__.encode("ascii"))
480-
for exception in python2_exceptions_without_args
481-
}
482-
483-
# StandardError is mapped to Exception, test that separately
484-
DATA8 = exception_pickle.replace(b'?', b'StandardError')
485-
486625
# UnicodeEncodeError object pickled from 2.x with protocol 2
487-
DATA9 = (b'\x80\x02cexceptions\nUnicodeEncodeError\n'
488-
b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01'
489-
b'U\x03badq\x03tq\x04Rq\x05.')
626+
DATA_UEERR = (b'\x80\x02cexceptions\nUnicodeEncodeError\n'
627+
b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01'
628+
b'U\x03badq\x03tq\x04Rq\x05.')
490629

491630

492631
def create_data():
@@ -542,6 +681,12 @@ def test_load_from_data1(self):
542681
def test_load_from_data2(self):
543682
self.assert_is_copy(self._testdata, self.loads(DATA2))
544683

684+
def test_load_from_data3(self):
685+
self.assert_is_copy(self._testdata, self.loads(DATA3))
686+
687+
def test_load_from_data4(self):
688+
self.assert_is_copy(self._testdata, self.loads(DATA4))
689+
545690
def test_load_classic_instance(self):
546691
# See issue5180. Test loading 2.x pickles that
547692
# contain an instance of old style class.
@@ -599,11 +744,6 @@ def test_load_classic_instance(self):
599744
b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
600745
self.assert_is_copy(X(*args), self.loads(pickle2))
601746

602-
def test_get(self):
603-
self.assertRaises(KeyError, self.loads, b'g0\np0')
604-
self.assert_is_copy([(100,), (100,)],
605-
self.loads(b'((Kdtp0\nh\x00l.))'))
606-
607747
def test_maxint64(self):
608748
maxint64 = (1 << 63) - 1
609749
data = b'I' + str(maxint64).encode("ascii") + b'\n.'
@@ -621,24 +761,29 @@ def test_pop_empty_stack(self):
621761

622762
def test_unpickle_from_2x(self):
623763
# Unpickle non-trivial data from Python 2.x.
624-
loaded = self.loads(DATA3)
764+
loaded = self.loads(DATA_SET)
625765
self.assertEqual(loaded, set([1, 2]))
626-
loaded = self.loads(DATA4)
627-
self.assertEqual(type(loaded), type(range(0)))
766+
loaded = self.loads(DATA_XRANGE_SLP)
767+
if not stackless:
768+
# Stackless-Python provides a fake range for unpickling
769+
self.assertEqual(type(loaded), type(range(0)))
628770
self.assertEqual(list(loaded), list(range(5)))
629-
loaded = self.loads(DATA5)
771+
loaded = self.loads(DATA_COOKIE)
630772
self.assertEqual(type(loaded), SimpleCookie)
631773
self.assertEqual(list(loaded.keys()), ["key"])
632774
self.assertEqual(loaded["key"].value, "value")
633775

634-
for (exc, data) in DATA7.items():
776+
# Exception objects without arguments pickled from 2.x with protocol 2
777+
for exc in python2_exceptions_without_args:
778+
data = exception_pickle.replace(b'?', exc.__name__.encode("ascii"))
635779
loaded = self.loads(data)
636780
self.assertIs(type(loaded), exc)
637781

638-
loaded = self.loads(DATA8)
782+
# StandardError is mapped to Exception, test that separately
783+
loaded = self.loads(exception_pickle.replace(b'?', b'StandardError'))
639784
self.assertIs(type(loaded), Exception)
640785

641-
loaded = self.loads(DATA9)
786+
loaded = self.loads(DATA_UEERR)
642787
self.assertIs(type(loaded), UnicodeEncodeError)
643788
self.assertEqual(loaded.object, "foo")
644789
self.assertEqual(loaded.encoding, "ascii")
@@ -675,11 +820,26 @@ def test_load_long_python2_str_as_bytes(self):
675820
b'x' * 300 + pickle.STOP,
676821
encoding='bytes'), b'x' * 300)
677822

823+
def test_constants(self):
824+
self.assertIsNone(self.loads(b'N.'))
825+
self.assertIs(self.loads(b'\x88.'), True)
826+
self.assertIs(self.loads(b'\x89.'), False)
827+
self.assertIs(self.loads(b'I01\n.'), True)
828+
self.assertIs(self.loads(b'I00\n.'), False)
829+
678830
def test_empty_bytestring(self):
679831
# issue 11286
680832
empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r')
681833
self.assertEqual(empty, '')
682834

835+
def test_short_binbytes(self):
836+
dumped = b'\x80\x03C\x04\xe2\x82\xac\x00.'
837+
self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
838+
839+
def test_binbytes(self):
840+
dumped = b'\x80\x03B\x04\x00\x00\x00\xe2\x82\xac\x00.'
841+
self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
842+
683843
@requires_32b
684844
def test_negative_32b_binbytes(self):
685845
# On 32-bit builds, a BINBYTES of 2**31 or more is refused
@@ -694,6 +854,39 @@ def test_negative_32b_binunicode(self):
694854
with self.assertRaises((pickle.UnpicklingError, OverflowError)):
695855
self.loads(dumped)
696856

857+
def test_short_binunicode(self):
858+
dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.'
859+
self.assertEqual(self.loads(dumped), '\u20ac\x00')
860+
861+
def test_misc_get(self):
862+
self.assertRaises(KeyError, self.loads, b'g0\np0')
863+
self.assert_is_copy([(100,), (100,)],
864+
self.loads(b'((Kdtp0\nh\x00l.))'))
865+
866+
def test_get(self):
867+
pickled = b'((lp100000\ng100000\nt.'
868+
unpickled = self.loads(pickled)
869+
self.assertEqual(unpickled, ([],)*2)
870+
self.assertIs(unpickled[0], unpickled[1])
871+
872+
def test_binget(self):
873+
pickled = b'(]q\xffh\xfft.'
874+
unpickled = self.loads(pickled)
875+
self.assertEqual(unpickled, ([],)*2)
876+
self.assertIs(unpickled[0], unpickled[1])
877+
878+
def test_long_binget(self):
879+
pickled = b'(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.'
880+
unpickled = self.loads(pickled)
881+
self.assertEqual(unpickled, ([],)*2)
882+
self.assertIs(unpickled[0], unpickled[1])
883+
884+
def test_dup(self):
885+
pickled = b'((l2t.'
886+
unpickled = self.loads(pickled)
887+
self.assertEqual(unpickled, ([],)*2)
888+
self.assertIs(unpickled[0], unpickled[1])
889+
697890
def test_negative_put(self):
698891
# Issue #12847
699892
dumped = b'Va\np-1\n.'
@@ -1506,9 +1699,9 @@ def test_pickle_to_2x(self):
15061699
# NOTE: this test is a bit too strong since we can produce different
15071700
# bytecode that 2.x will still understand.
15081701
dumped = self.dumps(range(5), 2)
1509-
self.assertEqual(dumped, DATA4)
1702+
self.assertEqual(dumped, DATA_XRANGE)
15101703
dumped = self.dumps(set([3]), 2)
1511-
self.assertEqual(dumped, DATA6)
1704+
self.assertEqual(dumped, DATA_SET2)
15121705

15131706
def test_large_pickles(self):
15141707
# Test the correctness of internal buffering routines when handling
@@ -2405,7 +2598,7 @@ def reduce_2(obj):
24052598
# Print some stuff that can be used to rewrite DATA{0,1,2}
24062599
from pickletools import dis
24072600
x = create_data()
2408-
for i in range(3):
2601+
for i in range(pickle.HIGHEST_PROTOCOL+1):
24092602
p = pickle.dumps(x, i)
24102603
print("DATA{0} = (".format(i))
24112604
for j in range(0, len(p), 20):

0 commit comments

Comments
 (0)