remove bom once at start, add test

Author: dummdidumm

packages/svelte/src/compiler/index.js

@@ -20,6 +20,7 @@ export { default as preprocess } from './preprocess/index.js';
  * @returns {CompileResult}
  */
 export function compile(source, options) {
+	source = remove_bom(source);
 	state.reset_warning_filter(options.warningFilter);
 	const validated = validate_component_options(options, '');
 	state.reset(source, validated);
@@ -58,6 +59,7 @@ export function compile(source, options) {
  * @returns {CompileResult}
  */
 export function compileModule(source, options) {
+	source = remove_bom(source);
 	state.reset_warning_filter(options.warningFilter);
 	const validated = validate_module_options(options, '');
 	state.reset(source, validated);
@@ -101,6 +103,7 @@ export function compileModule(source, options) {
  * @returns {AST.Root | LegacyRoot}
  */
 export function parse(source, { filename, rootDir, modern } = {}) {
+	source = remove_bom(source);
 	state.reset_warning_filter(() => false);
 	state.reset(source, { filename: filename ?? '(unknown)', rootDir });
 
@@ -140,6 +143,17 @@ function to_public_ast(source, ast, modern) {
 	return convert(source, ast);
 }
 
+/**
+ * Remove the byte order mark from a string if it's present since it would mess with our template generation logic
+ * @param {string} source
+ */
+function remove_bom(source) {
+	if (source.charCodeAt(0) === 0xfeff) {
+		return source.slice(1);
+	}
+	return source;
+}
+
 /**
  * @deprecated Replace this with `import { walk } from 'estree-walker'`
  * @returns {never}

packages/svelte/src/compiler/phases/1-parse/utils/html.js

@@ -38,10 +38,8 @@ const entity_pattern_attr_value = get_entity_pattern(true);
  * @param {boolean} is_attribute_value
  */
 export function decode_character_references(html, is_attribute_value) {
-	// Replace all BOM characters from the HTML with a standard space
-	const sanitised_html = html.replaceAll(String.fromCharCode(65279), ' ');
 	const entity_pattern = is_attribute_value ? entity_pattern_attr_value : entity_pattern_content;
-	return sanitised_html.replace(
+	return html.replace(
 		entity_pattern,
 		/**
 		 * @param {any} match
@@ -77,8 +75,8 @@ const NUL = 0;
 
 /** @param {number} code */
 function validate_code(code) {
-	// line feed becomes generic whitespace, as does a BOM character
-	if (code === 10 || code === 65279) {
+	// line feed becomes generic whitespace
+	if (code === 10) {
 		return 32;
 	}
 

packages/svelte/tests/parser-modern/test.ts

@@ -1,5 +1,5 @@
 import * as fs from 'node:fs';
-import { assert } from 'vitest';
+import { assert, it } from 'vitest';
 import { parse } from 'svelte/compiler';
 import { try_load_json } from '../helpers.js';
 import { suite, type BaseTest } from '../suite.js';
@@ -34,3 +34,24 @@ const { test, run } = suite<ParserTest>(async (config, cwd) => {
 export { test };
 
 await run(__dirname);
+
+it('Strips BOM from the input', () => {
+	const input = '\uFEFF<div></div>';
+	const actual = parse(input, { modern: true });
+	assert.deepEqual(JSON.parse(JSON.stringify(actual.fragment)), {
+		type: 'Fragment',
+		nodes: [
+			{
+				attributes: [],
+				end: 11,
+				fragment: {
+					nodes: [],
+					type: 'Fragment'
+				},
+				name: 'div',
+				start: 0,
+				type: 'RegularElement'
+			}
+		]
+	});
+});