Skip to content

Commit cf61f1a

Browse files
mtarldmtarld
authored
chore: improve lexer (#2)
1 parent 6977292 commit cf61f1a

File tree

325 files changed

+678
-128
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

325 files changed

+678
-128
lines changed

‎DependencyInjection/MarshallerExtension.php

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
use Symfony\Component\Marshaller\Context\Generation\TypeFormatterContextBuilder;
1818
use Symfony\Component\Marshaller\Context\Marshal\JsonEncodeFlagsContextBuilder;
1919
use Symfony\Component\Marshaller\Context\Marshal\TypeContextBuilder;
20-
use Symfony\Component\Marshaller\Context\UnmarshalContextBuilderInterface;
2120
use Symfony\Component\Marshaller\Marshaller;
2221
use Symfony\Component\Marshaller\MarshallerInterface;
2322
use Symfony\Component\Marshaller\Type\PhpstanTypeExtractor;
@@ -65,45 +64,44 @@ public function load(array $configs, ContainerBuilder $container): void
6564
//
6665
// Generation context builders
6766
//
68-
$container->register('marshaller.builder.generation.hook', HookContextBuilder::class)
67+
$container->register('.marshaller.builder.generation.hook', HookContextBuilder::class)
6968
->addTag('marshaller.context.builder.generation', ['priority' => -128]);
7069

71-
$container->register('marshaller.builder.generation.type_formatter', TypeFormatterContextBuilder::class)
70+
$container->register('.marshaller.builder.generation.type_formatter', TypeFormatterContextBuilder::class)
7271
->addTag('marshaller.context.builder.generation', ['priority' => -128]);
7372

74-
$container->register('marshaller.builder.generation.name_attribute', NameAttributeContextBuilder::class)
73+
$container->register('.marshaller.builder.generation.name_attribute', NameAttributeContextBuilder::class)
7574
->addTag('marshaller.context.builder.generation', ['priority' => -128]);
7675

77-
$container->register('marshaller.builder.generation.formatter_attribute', FormatterAttributeContextBuilder::class)
76+
$container->register('.marshaller.builder.generation.formatter_attribute', FormatterAttributeContextBuilder::class)
7877
->addTag('marshaller.context.builder.generation', ['priority' => -128]);
7978

8079
//
8180
// Marshal context builders
8281
//
83-
$container->register('marshaller.builder.marshal.type', TypeContextBuilder::class)
82+
$container->register('.marshaller.builder.marshal.type', TypeContextBuilder::class)
8483
->addTag('marshaller.context.builder.marshal', ['priority' => -128]);
8584

86-
$container->register('marshaller.builder.marshal.json_encode_flags', JsonEncodeFlagsContextBuilder::class)
85+
$container->register('.marshaller.builder.marshal.json_encode_flags', JsonEncodeFlagsContextBuilder::class)
8786
->addTag('marshaller.context.builder.marshal', ['priority' => -128]);
8887

8988
//
9089
// Unmarshal context builders
9190
//
92-
$container->register('marshaller.builder.unmarshal.formatter_attribute', FormatterAttributeContextBuilder::class)
93-
->addTag('marshaller.context.builder.unmarshal', ['priority' => -128])
94-
->addTag('proxy', ['interface' => UnmarshalContextBuilderInterface::class]);
91+
$container->register('.marshaller.builder.unmarshal.formatter_attribute', FormatterAttributeContextBuilder::class)
92+
->addTag('marshaller.context.builder.unmarshal', ['priority' => -128]);
9593

9694
//
9795
// Cache
9896
//
99-
$container->register('marshaller.cache.warmable_resolver', WarmableResolver::class)
97+
$container->register('.marshaller.cache.warmable_resolver', WarmableResolver::class)
10098
->setArguments([
10199
new Parameter('marshaller.warmable_paths'),
102100
]);
103101

104-
$container->register('marshaller.cache.template_warmer', TemplateCacheWarmer::class)
102+
$container->register('.marshaller.cache.template_warmer', TemplateCacheWarmer::class)
105103
->setArguments([
106-
new Reference('marshaller.cache.warmable_resolver'),
104+
new Reference('.marshaller.cache.warmable_resolver'),
107105
new Reference('marshaller'),
108106
new Parameter('marshaller.cache_dir'),
109107
new Parameter('marshaller.warmable_formats'),

‎Internal/Lexer/JsonLexer.php

Lines changed: 138 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -6,45 +6,113 @@
66

77
final class JsonLexer implements LexerInterface
88
{
9-
public function tokens(mixed $resource): \Iterator
9+
private const DICT_START = 1;
10+
private const DICT_END = 2;
11+
12+
private const LIST_START = 4;
13+
private const LIST_END = 8;
14+
15+
private const KEY = 16;
16+
private const COLUMN = 32;
17+
private const COMMA = 64;
18+
19+
private const SCALAR = 128;
20+
21+
private const END = 256;
22+
23+
private const VALUE = self::DICT_START | self::LIST_START | self::SCALAR;
24+
25+
public function tokens(mixed $resource, array $context): \Generator
1026
{
11-
return (new \IteratorIterator($this->tokenize($resource)))->getInnerIterator();
27+
$expectedType = self::VALUE;
28+
$structureStack = new \SplStack();
29+
30+
foreach ($this->tokenize($resource) as [$type, $token]) {
31+
if ('' === $token) {
32+
continue;
33+
}
34+
35+
if (!($type & $expectedType)) {
36+
throw new \RuntimeException('Invalid JSON.');
37+
}
38+
39+
if (self::SCALAR === $type) {
40+
json_decode($token, flags: $context['json_decode_flags'] ?? 0);
41+
42+
if (JSON_ERROR_NONE !== json_last_error()) {
43+
throw new \RuntimeException('Invalid JSON.');
44+
}
45+
}
46+
47+
if (self::KEY === $type && !(str_starts_with($token, '"') && str_ends_with($token, '"'))) {
48+
throw new \RuntimeException('Invalid JSON.');
49+
}
50+
51+
yield $token;
52+
53+
if (self::DICT_START === $type) {
54+
$structureStack->push('dict');
55+
} elseif (self::LIST_START === $type) {
56+
$structureStack->push('list');
57+
} elseif ($type & (self::DICT_END | self::LIST_END)) {
58+
$structureStack->pop();
59+
}
60+
61+
$currentStructure = !$structureStack->isEmpty() ? $structureStack->top() : null;
62+
63+
$expectedType = match (true) {
64+
self::DICT_START === $type => self::KEY | self::DICT_END,
65+
self::LIST_START === $type => self::VALUE | self::LIST_END,
66+
67+
self::KEY === $type => self::COLUMN,
68+
self::COLUMN === $type => self::VALUE,
69+
70+
self::COMMA === $type && 'dict' === $currentStructure => self::KEY,
71+
self::COMMA === $type && 'list' === $currentStructure => self::VALUE,
72+
73+
0 !== ($type & (self::DICT_END | self::LIST_END | self::SCALAR)) && 'dict' === $currentStructure => self::COMMA | self::DICT_END,
74+
0 !== ($type & (self::DICT_END | self::LIST_END | self::SCALAR)) && 'list' === $currentStructure => self::COMMA | self::LIST_END,
75+
0 !== ($type & (self::DICT_END | self::LIST_END | self::SCALAR)) => self::END,
76+
77+
default => throw new \RuntimeException('Invalid JSON.'),
78+
};
79+
}
80+
81+
if (self::END !== $expectedType) {
82+
throw new \RuntimeException('Invalid JSON.');
83+
}
1284
}
1385

1486
/**
1587
* @param resource $resource
1688
*
17-
* @return \Traversable<string>
89+
* @return \Generator<array{int, string}>
1890
*/
19-
private function tokenize(mixed $resource): \Traversable
91+
private function tokenize(mixed $resource): \Generator
2092
{
21-
$structureBoundaries = ['{' => true, '}' => true, '[' => true, ']' => true, ':' => true, ',' => true];
22-
23-
$buffer = '';
93+
$token = '';
2494
$inString = false;
2595
$escaping = false;
2696

2797
while (!feof($resource)) {
28-
if (false === $line = stream_get_line($resource, 4096, "\n")) {
29-
yield $buffer;
30-
31-
return;
98+
if (false === $buffer = stream_get_contents($resource, 4096)) {
99+
throw new \RuntimeException('Cannot read JSON resource.');
32100
}
33101

34-
$length = \strlen($line);
102+
$length = \strlen($buffer);
35103

36104
for ($i = 0; $i < $length; ++$i) {
37-
$byte = $line[$i];
105+
$byte = $buffer[$i];
38106

39107
if ($escaping) {
40108
$escaping = false;
41-
$buffer .= $byte;
109+
$token .= $byte;
42110

43111
continue;
44112
}
45113

46114
if ($inString) {
47-
$buffer .= $byte;
115+
$token .= $byte;
48116

49117
if ('"' === $byte) {
50118
$inString = false;
@@ -56,32 +124,76 @@ private function tokenize(mixed $resource): \Traversable
56124
}
57125

58126
if ('"' === $byte) {
59-
$buffer .= $byte;
127+
$token .= $byte;
60128
$inString = true;
61129

62130
continue;
63131
}
64132

65-
if (isset($structureBoundaries[$byte])) {
66-
if ('' !== $buffer) {
67-
yield $buffer;
68-
$buffer = '';
69-
}
133+
if (',' === $byte) {
134+
yield [self::SCALAR, $token];
135+
yield [self::COMMA, $byte];
136+
137+
$token = '';
138+
139+
continue;
140+
}
141+
142+
if (':' === $byte) {
143+
yield [self::KEY, $token];
144+
yield [self::COLUMN, $byte];
145+
146+
$token = '';
147+
148+
continue;
149+
}
150+
151+
if ('{' === $byte) {
152+
yield [self::SCALAR, $token];
153+
yield [self::DICT_START, $byte];
154+
155+
$token = '';
156+
157+
continue;
158+
}
159+
160+
if ('[' === $byte) {
161+
yield [self::SCALAR, $token];
162+
yield [self::LIST_START, $byte];
163+
164+
$token = '';
165+
166+
continue;
167+
}
70168

71-
yield $byte;
169+
if ('}' === $byte) {
170+
yield [self::SCALAR, $token];
171+
yield [self::DICT_END, $byte];
172+
173+
$token = '';
72174

73175
continue;
74176
}
75177

76-
// TODO other kind of spaces
77-
if (' ' === $byte) {
178+
if (']' === $byte) {
179+
yield [self::SCALAR, $token];
180+
yield [self::LIST_END, $byte];
181+
182+
$token = '';
183+
78184
continue;
79185
}
80186

81-
$buffer .= $byte;
187+
if ('' === $token && in_array($byte, [' ', "\r", "\t", "\n"], true)) {
188+
continue;
189+
}
190+
191+
$token .= $byte;
82192
}
83193
}
84194

85-
yield $buffer;
195+
if (!$inString && !$escaping) {
196+
yield [self::SCALAR, $token];
197+
}
86198
}
87199
}

‎Internal/Lexer/LexerInterface.php

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@
77
interface LexerInterface
88
{
99
/**
10-
* @param resource $resource
10+
* @param resource $resource
11+
* @param array<string, mixed> $context
1112
*
12-
* @return \Iterator<string>
13+
* @return \Generator<string>
1314
*/
14-
public function tokens(mixed $resource): \Iterator;
15+
public function tokens(mixed $resource, array $context): \Generator;
1516
}

‎Marshaller.php

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,15 @@ public function marshal(mixed $data, string $format, StreamInterface $output, Co
3737

3838
// if template does not exist, it'll be generated therefore raw context must be filled accordingly
3939
if (!file_exists(sprintf('%s/%s.%s.php', $this->cacheDir, md5($type), $format))) {
40-
$rawContext = $this->buildGenerateContext($type, $context, $rawContext);
40+
$rawContext = $this->buildGenerationContext($type, $context, $rawContext);
4141
}
4242

4343
marshal($data, $output->stream(), $format, $rawContext);
4444
}
4545

4646
public function generate(string $type, string $format, Context $context = null): string
4747
{
48-
return marshal_generate($type, $format, $this->buildGenerateContext($type, $context));
48+
return marshal_generate($type, $format, $this->buildGenerationContext($type, $context));
4949
}
5050

5151
public function unmarshal(StreamInterface $input, string $type, string $format, Context $context = null): mixed
@@ -73,7 +73,7 @@ private function buildMarshalContext(?Context $context): array
7373
*
7474
* @return array<string, mixed>
7575
*/
76-
private function buildGenerateContext(string $type, ?Context $context, array $rawContext = []): array
76+
private function buildGenerationContext(string $type, ?Context $context, array $rawContext = []): array
7777
{
7878
$context = $context ?? new Context();
7979

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
- create dedicated exceptions and wrap native ones
55

66
## Unmarshal
7+
- tests
78
- if constructor -> newInstanceWithoutConstructor but set defaults
89
- else classic new instance
910

‎Resources/functions.php

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ function marshal_generate(string $type, string $format, array $context = []): st
8080
*/
8181
function unmarshal($resource, string $type, string $format, array $context = []): mixed
8282
{
83-
return ParserFactory::create($format)->parse(LexerFactory::create($format)->tokens($resource), Type::createFromString($type), $context);
83+
$tokens = LexerFactory::create($format)->tokens($resource, $context);
84+
85+
return ParserFactory::create($format)->parse($tokens, Type::createFromString($type), $context);
8486
}
8587
}

‎Tests/Cache/TemplateCacheWarmerTest.php

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@
88
use Symfony\Component\Marshaller\Cache\TemplateCacheWarmer;
99
use Symfony\Component\Marshaller\Cache\WarmableResolver;
1010
use Symfony\Component\Marshaller\MarshallerInterface;
11-
use Symfony\Component\Marshaller\Tests\Fixtures\WarmableDummy;
12-
use Symfony\Component\Marshaller\Tests\Fixtures\WarmableNotNullableDummy;
13-
use Symfony\Component\Marshaller\Tests\Fixtures\WarmableNullableDummy;
11+
use Symfony\Component\Marshaller\Tests\Fixtures\Dto\WarmableDummy;
12+
use Symfony\Component\Marshaller\Tests\Fixtures\Dto\WarmableNotNullableDummy;
13+
use Symfony\Component\Marshaller\Tests\Fixtures\Dto\WarmableNullableDummy;
1414

1515
final class TemplateCacheWarmerTest extends TestCase
1616
{

‎Tests/Cache/WarmableResolverTest.php

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77
use PHPUnit\Framework\TestCase;
88
use Symfony\Component\Marshaller\Attribute\Warmable;
99
use Symfony\Component\Marshaller\Cache\WarmableResolver;
10-
use Symfony\Component\Marshaller\Tests\Fixtures\WarmableDummy;
11-
use Symfony\Component\Marshaller\Tests\Fixtures\WarmableNotNullableDummy;
12-
use Symfony\Component\Marshaller\Tests\Fixtures\WarmableNullableDummy;
10+
use Symfony\Component\Marshaller\Tests\Fixtures\Dto\WarmableDummy;
11+
use Symfony\Component\Marshaller\Tests\Fixtures\Dto\WarmableNotNullableDummy;
12+
use Symfony\Component\Marshaller\Tests\Fixtures\Dto\WarmableNullableDummy;
1313

1414
final class WarmableResolverTest extends TestCase
1515
{

‎Tests/Context/Generation/FormatterAttributeContextBuilderTest.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
use PHPUnit\Framework\TestCase;
88
use Symfony\Component\Marshaller\Context\Context;
99
use Symfony\Component\Marshaller\Context\Generation\FormatterAttributeContextBuilder;
10-
use Symfony\Component\Marshaller\Tests\Fixtures\DummyWithFormatterAttributes;
10+
use Symfony\Component\Marshaller\Tests\Fixtures\Dto\DummyWithFormatterAttributes;
1111

1212
final class FormatterAttributeContextBuilderTest extends TestCase
1313
{

‎Tests/Context/Generation/NameAttributeContextBuilderTest.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
use PHPUnit\Framework\TestCase;
88
use Symfony\Component\Marshaller\Context\Context;
99
use Symfony\Component\Marshaller\Context\Generation\NameAttributeContextBuilder;
10-
use Symfony\Component\Marshaller\Tests\Fixtures\DummyWithNameAttributes;
10+
use Symfony\Component\Marshaller\Tests\Fixtures\Dto\DummyWithNameAttributes;
1111

1212
final class NameAttributeContextBuilderTest extends TestCase
1313
{

0 commit comments

Comments
 (0)