Skip to content
This repository was archived by the owner on Feb 13, 2025. It is now read-only.

Commit cb76496

Browse files
serhiy-storchakaserhiy-storchaka
committed
Added additional unpickling tests.
2 parents 104de66 + b8b951f commit cb76496

File tree

1 file changed

+229
-37
lines changed

1 file changed

+229
-37
lines changed

‎Lib/test/pickletester.py

Lines changed: 229 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ def create_dynamic_class(name, bases):
145145
result.reduce_args = (name, bases)
146146
return result
147147

148-
# DATA0 .. DATA2 are the pickles we expect under the various protocols, for
148+
# DATA0 .. DATA4 are the pickles we expect under the various protocols, for
149149
# the object returned by create_data().
150150

151151
DATA0 = (
@@ -401,22 +401,172 @@ def create_dynamic_class(name, bases):
401401
highest protocol among opcodes = 2
402402
"""
403403

404+
DATA3 = (
405+
b'\x80\x03]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
406+
b'builtins\ncomplex\nq\x01G'
407+
b'@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00\x86q\x02'
408+
b'Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff'
409+
b'\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7f'
410+
b'J\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00abcq'
411+
b'\x04h\x04c__main__\nC\nq\x05)\x81q'
412+
b'\x06}q\x07(X\x03\x00\x00\x00barq\x08K\x02X\x03\x00'
413+
b'\x00\x00fooq\tK\x01ubh\x06tq\nh\nK\x05'
414+
b'e.'
415+
)
416+
417+
# Disassembly of DATA3
418+
DATA3_DIS = """\
419+
0: \x80 PROTO 3
420+
2: ] EMPTY_LIST
421+
3: q BINPUT 0
422+
5: ( MARK
423+
6: K BININT1 0
424+
8: K BININT1 1
425+
10: G BINFLOAT 2.0
426+
19: c GLOBAL 'builtins complex'
427+
37: q BINPUT 1
428+
39: G BINFLOAT 3.0
429+
48: G BINFLOAT 0.0
430+
57: \x86 TUPLE2
431+
58: q BINPUT 2
432+
60: R REDUCE
433+
61: q BINPUT 3
434+
63: K BININT1 1
435+
65: J BININT -1
436+
70: K BININT1 255
437+
72: J BININT -255
438+
77: J BININT -256
439+
82: M BININT2 65535
440+
85: J BININT -65535
441+
90: J BININT -65536
442+
95: J BININT 2147483647
443+
100: J BININT -2147483647
444+
105: J BININT -2147483648
445+
110: ( MARK
446+
111: X BINUNICODE 'abc'
447+
119: q BINPUT 4
448+
121: h BINGET 4
449+
123: c GLOBAL '__main__ C'
450+
135: q BINPUT 5
451+
137: ) EMPTY_TUPLE
452+
138: \x81 NEWOBJ
453+
139: q BINPUT 6
454+
141: } EMPTY_DICT
455+
142: q BINPUT 7
456+
144: ( MARK
457+
145: X BINUNICODE 'bar'
458+
153: q BINPUT 8
459+
155: K BININT1 2
460+
157: X BINUNICODE 'foo'
461+
165: q BINPUT 9
462+
167: K BININT1 1
463+
169: u SETITEMS (MARK at 144)
464+
170: b BUILD
465+
171: h BINGET 6
466+
173: t TUPLE (MARK at 110)
467+
174: q BINPUT 10
468+
176: h BINGET 10
469+
178: K BININT1 5
470+
180: e APPENDS (MARK at 5)
471+
181: . STOP
472+
highest protocol among opcodes = 2
473+
"""
474+
475+
DATA4 = (
476+
b'\x80\x04\x95\xa8\x00\x00\x00\x00\x00\x00\x00]\x94(K\x00K\x01G@'
477+
b'\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x07'
478+
b'complex\x94\x93\x94G@\x08\x00\x00\x00\x00\x00\x00G'
479+
b'\x00\x00\x00\x00\x00\x00\x00\x00\x86\x94R\x94K\x01J\xff\xff\xff\xffK'
480+
b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ'
481+
b'\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80('
482+
b'\x8c\x03abc\x94h\x06\x8c\x08__main__\x94\x8c'
483+
b'\x01C\x94\x93\x94)\x81\x94}\x94(\x8c\x03bar\x94K\x02\x8c'
484+
b'\x03foo\x94K\x01ubh\nt\x94h\x0eK\x05e.'
485+
)
486+
487+
# Disassembly of DATA4
488+
DATA4_DIS = """\
489+
0: \x80 PROTO 4
490+
2: \x95 FRAME 168
491+
11: ] EMPTY_LIST
492+
12: \x94 MEMOIZE
493+
13: ( MARK
494+
14: K BININT1 0
495+
16: K BININT1 1
496+
18: G BINFLOAT 2.0
497+
27: \x8c SHORT_BINUNICODE 'builtins'
498+
37: \x94 MEMOIZE
499+
38: \x8c SHORT_BINUNICODE 'complex'
500+
47: \x94 MEMOIZE
501+
48: \x93 STACK_GLOBAL
502+
49: \x94 MEMOIZE
503+
50: G BINFLOAT 3.0
504+
59: G BINFLOAT 0.0
505+
68: \x86 TUPLE2
506+
69: \x94 MEMOIZE
507+
70: R REDUCE
508+
71: \x94 MEMOIZE
509+
72: K BININT1 1
510+
74: J BININT -1
511+
79: K BININT1 255
512+
81: J BININT -255
513+
86: J BININT -256
514+
91: M BININT2 65535
515+
94: J BININT -65535
516+
99: J BININT -65536
517+
104: J BININT 2147483647
518+
109: J BININT -2147483647
519+
114: J BININT -2147483648
520+
119: ( MARK
521+
120: \x8c SHORT_BINUNICODE 'abc'
522+
125: \x94 MEMOIZE
523+
126: h BINGET 6
524+
128: \x8c SHORT_BINUNICODE '__main__'
525+
138: \x94 MEMOIZE
526+
139: \x8c SHORT_BINUNICODE 'C'
527+
142: \x94 MEMOIZE
528+
143: \x93 STACK_GLOBAL
529+
144: \x94 MEMOIZE
530+
145: ) EMPTY_TUPLE
531+
146: \x81 NEWOBJ
532+
147: \x94 MEMOIZE
533+
148: } EMPTY_DICT
534+
149: \x94 MEMOIZE
535+
150: ( MARK
536+
151: \x8c SHORT_BINUNICODE 'bar'
537+
156: \x94 MEMOIZE
538+
157: K BININT1 2
539+
159: \x8c SHORT_BINUNICODE 'foo'
540+
164: \x94 MEMOIZE
541+
165: K BININT1 1
542+
167: u SETITEMS (MARK at 150)
543+
168: b BUILD
544+
169: h BINGET 10
545+
171: t TUPLE (MARK at 119)
546+
172: \x94 MEMOIZE
547+
173: h BINGET 14
548+
175: K BININT1 5
549+
177: e APPENDS (MARK at 13)
550+
178: . STOP
551+
highest protocol among opcodes = 4
552+
"""
553+
404554
# set([1,2]) pickled from 2.x with protocol 2
405-
DATA3 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.'
555+
DATA_SET = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.'
406556

407557
# xrange(5) pickled from 2.x with protocol 2
408-
DATA4 = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.'
558+
DATA_XRANGE = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.'
409559

410560
# a SimpleCookie() object pickled from 2.x with protocol 2
411-
DATA5 = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key'
412-
b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U'
413-
b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07'
414-
b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U'
415-
b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b'
416-
b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.')
561+
DATA_COOKIE = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key'
562+
b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U'
563+
b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07'
564+
b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U'
565+
b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b'
566+
b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.')
417567

418568
# set([3]) pickled from 2.x with protocol 2
419-
DATA6 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.'
569+
DATA_SET2 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.'
420570

421571
python2_exceptions_without_args = (
422572
ArithmeticError,
@@ -468,20 +618,10 @@ def create_dynamic_class(name, bases):
468618

469619
exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.'
470620

471-
# Exception objects without arguments pickled from 2.x with protocol 2
472-
DATA7 = {
473-
exception :
474-
exception_pickle.replace(b'?', exception.__name__.encode("ascii"))
475-
for exception in python2_exceptions_without_args
476-
}
477-
478-
# StandardError is mapped to Exception, test that separately
479-
DATA8 = exception_pickle.replace(b'?', b'StandardError')
480-
481621
# UnicodeEncodeError object pickled from 2.x with protocol 2
482-
DATA9 = (b'\x80\x02cexceptions\nUnicodeEncodeError\n'
483-
b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01'
484-
b'U\x03badq\x03tq\x04Rq\x05.')
622+
DATA_UEERR = (b'\x80\x02cexceptions\nUnicodeEncodeError\n'
623+
b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01'
624+
b'U\x03badq\x03tq\x04Rq\x05.')
485625

486626

487627
def create_data():
@@ -537,6 +677,12 @@ def test_load_from_data1(self):
537677
def test_load_from_data2(self):
538678
self.assert_is_copy(self._testdata, self.loads(DATA2))
539679

680+
def test_load_from_data3(self):
681+
self.assert_is_copy(self._testdata, self.loads(DATA3))
682+
683+
def test_load_from_data4(self):
684+
self.assert_is_copy(self._testdata, self.loads(DATA4))
685+
540686
def test_load_classic_instance(self):
541687
# See issue5180. Test loading 2.x pickles that
542688
# contain an instance of old style class.
@@ -594,11 +740,6 @@ def test_load_classic_instance(self):
594740
b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
595741
self.assert_is_copy(X(*args), self.loads(pickle2))
596742

597-
def test_get(self):
598-
self.assertRaises(KeyError, self.loads, b'g0\np0')
599-
self.assert_is_copy([(100,), (100,)],
600-
self.loads(b'((Kdtp0\nh\x00l.))'))
601-
602743
def test_maxint64(self):
603744
maxint64 = (1 << 63) - 1
604745
data = b'I' + str(maxint64).encode("ascii") + b'\n.'
@@ -616,24 +757,27 @@ def test_pop_empty_stack(self):
616757

617758
def test_unpickle_from_2x(self):
618759
# Unpickle non-trivial data from Python 2.x.
619-
loaded = self.loads(DATA3)
760+
loaded = self.loads(DATA_SET)
620761
self.assertEqual(loaded, set([1, 2]))
621-
loaded = self.loads(DATA4)
762+
loaded = self.loads(DATA_XRANGE)
622763
self.assertEqual(type(loaded), type(range(0)))
623764
self.assertEqual(list(loaded), list(range(5)))
624-
loaded = self.loads(DATA5)
765+
loaded = self.loads(DATA_COOKIE)
625766
self.assertEqual(type(loaded), SimpleCookie)
626767
self.assertEqual(list(loaded.keys()), ["key"])
627768
self.assertEqual(loaded["key"].value, "value")
628769

629-
for (exc, data) in DATA7.items():
770+
# Exception objects without arguments pickled from 2.x with protocol 2
771+
for exc in python2_exceptions_without_args:
772+
data = exception_pickle.replace(b'?', exc.__name__.encode("ascii"))
630773
loaded = self.loads(data)
631774
self.assertIs(type(loaded), exc)
632775

633-
loaded = self.loads(DATA8)
776+
# StandardError is mapped to Exception, test that separately
777+
loaded = self.loads(exception_pickle.replace(b'?', b'StandardError'))
634778
self.assertIs(type(loaded), Exception)
635779

636-
loaded = self.loads(DATA9)
780+
loaded = self.loads(DATA_UEERR)
637781
self.assertIs(type(loaded), UnicodeEncodeError)
638782
self.assertEqual(loaded.object, "foo")
639783
self.assertEqual(loaded.encoding, "ascii")
@@ -670,11 +814,26 @@ def test_load_long_python2_str_as_bytes(self):
670814
b'x' * 300 + pickle.STOP,
671815
encoding='bytes'), b'x' * 300)
672816

817+
def test_constants(self):
818+
self.assertIsNone(self.loads(b'N.'))
819+
self.assertIs(self.loads(b'\x88.'), True)
820+
self.assertIs(self.loads(b'\x89.'), False)
821+
self.assertIs(self.loads(b'I01\n.'), True)
822+
self.assertIs(self.loads(b'I00\n.'), False)
823+
673824
def test_empty_bytestring(self):
674825
# issue 11286
675826
empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r')
676827
self.assertEqual(empty, '')
677828

829+
def test_short_binbytes(self):
830+
dumped = b'\x80\x03C\x04\xe2\x82\xac\x00.'
831+
self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
832+
833+
def test_binbytes(self):
834+
dumped = b'\x80\x03B\x04\x00\x00\x00\xe2\x82\xac\x00.'
835+
self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
836+
678837
@requires_32b
679838
def test_negative_32b_binbytes(self):
680839
# On 32-bit builds, a BINBYTES of 2**31 or more is refused
@@ -689,6 +848,39 @@ def test_negative_32b_binunicode(self):
689848
with self.assertRaises((pickle.UnpicklingError, OverflowError)):
690849
self.loads(dumped)
691850

851+
def test_short_binunicode(self):
852+
dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.'
853+
self.assertEqual(self.loads(dumped), '\u20ac\x00')
854+
855+
def test_misc_get(self):
856+
self.assertRaises(KeyError, self.loads, b'g0\np0')
857+
self.assert_is_copy([(100,), (100,)],
858+
self.loads(b'((Kdtp0\nh\x00l.))'))
859+
860+
def test_get(self):
861+
pickled = b'((lp100000\ng100000\nt.'
862+
unpickled = self.loads(pickled)
863+
self.assertEqual(unpickled, ([],)*2)
864+
self.assertIs(unpickled[0], unpickled[1])
865+
866+
def test_binget(self):
867+
pickled = b'(]q\xffh\xfft.'
868+
unpickled = self.loads(pickled)
869+
self.assertEqual(unpickled, ([],)*2)
870+
self.assertIs(unpickled[0], unpickled[1])
871+
872+
def test_long_binget(self):
873+
pickled = b'(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.'
874+
unpickled = self.loads(pickled)
875+
self.assertEqual(unpickled, ([],)*2)
876+
self.assertIs(unpickled[0], unpickled[1])
877+
878+
def test_dup(self):
879+
pickled = b'((l2t.'
880+
unpickled = self.loads(pickled)
881+
self.assertEqual(unpickled, ([],)*2)
882+
self.assertIs(unpickled[0], unpickled[1])
883+
692884
def test_negative_put(self):
693885
# Issue #12847
694886
dumped = b'Va\np-1\n.'
@@ -1501,9 +1693,9 @@ def test_pickle_to_2x(self):
15011693
# NOTE: this test is a bit too strong since we can produce different
15021694
# bytecode that 2.x will still understand.
15031695
dumped = self.dumps(range(5), 2)
1504-
self.assertEqual(dumped, DATA4)
1696+
self.assertEqual(dumped, DATA_XRANGE)
15051697
dumped = self.dumps(set([3]), 2)
1506-
self.assertEqual(dumped, DATA6)
1698+
self.assertEqual(dumped, DATA_SET2)
15071699

15081700
def test_large_pickles(self):
15091701
# Test the correctness of internal buffering routines when handling
@@ -2398,7 +2590,7 @@ def reduce_2(obj):
23982590
# Print some stuff that can be used to rewrite DATA{0,1,2}
23992591
from pickletools import dis
24002592
x = create_data()
2401-
for i in range(3):
2593+
for i in range(pickle.HIGHEST_PROTOCOL+1):
24022594
p = pickle.dumps(x, i)
24032595
print("DATA{0} = (".format(i))
24042596
for j in range(0, len(p), 20):

0 commit comments

Comments
 (0)