Merge branch 'main' into fix-openai-inference-snippets

Author: Wauplin

.github/workflows/test.yml

@@ -61,11 +61,13 @@ jobs:
             echo "SINCE=$(git merge-base origin/${{ github.event.pull_request.base.ref }} ${{ github.sha }})" >> $GITHUB_OUTPUT
           fi
 
+      - run: google-chrome --version
+
       - run: npm install -g corepack@latest && corepack enable
 
       - uses: actions/setup-node@v3
         with:
-          node-version: "20"
+          node-version: "22"
           cache: "pnpm"
           cache-dependency-path: "**/pnpm-lock.yaml"
       - run: |

CONTRIBUTING.md

@@ -18,7 +18,11 @@ It's not a hard requirement, but please consider using an icon from [Gitmoji](ht
 
 ## Tests
 
-If you want to run only specific tests, you can do `pnpm test -- -t "test name"`
+If you want to run only specific tests, you can do `pnpm test -- -t "test name"`.
+
+You can also do `npx vitest ./packages/hub/src/utils/XetBlob.spec.ts` to run a specific test file.
+
+Or `cd packages/hub && npx vitest --browser.name=chrome --browser.headless --config vitest-browser.config.mts ./src/utils/XetBlob.spec.ts` to run browser tests on a specific file
 
 ## Adding a package
 

README.md

@@ -98,7 +98,7 @@ You can run our packages with vanilla JS, without any bundler, by using a CDN or
 ```html
 <script type="module">
     import { InferenceClient } from 'https://cdn.jsdelivr.net/npm/@huggingface/[email protected]/+esm';
-    import { createRepo, commit, deleteRepo, listFiles } from "https://cdn.jsdelivr.net/npm/@huggingface/hub@1.0.2/+esm";
+    import { createRepo, commit, deleteRepo, listFiles } from "https://cdn.jsdelivr.net/npm/@huggingface/hub@1.1.0/+esm";
 </script>
 ```
 

packages/hub/.eslintignore

@@ -1,2 +1,3 @@
 dist
 sha256.js
+src/vendor

packages/hub/README.md

@@ -174,3 +174,4 @@ Under the hood, `@huggingface/hub` uses a lazy blob implementation to load the f
 ## Dependencies
 
 - `@huggingface/tasks` : Typings only
+- `@huggingface/lz4` : URL join utility

packages/hub/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@huggingface/hub",
 	"packageManager": "[email protected]",
-	"version": "1.0.2",
+	"version": "1.1.0",
 	"description": "Utilities to interact with the Hugging Face hub",
 	"repository": "https://github.com/huggingface/huggingface.js.git",
 	"publishConfig": {

packages/hub/src/index.ts

@@ -22,3 +22,4 @@ export { HubApiError, InvalidApiResponseFormatError } from "./error";
  * Only exported for E2Es convenience
  */
 export { sha256 as __internal_sha256 } from "./utils/sha256";
+export { XetBlob as __internal_XetBlob } from "./utils/XetBlob";

packages/hub/src/lib/download-file-to-cache-dir.spec.ts

@@ -7,6 +7,7 @@ import type { Stats } from "node:fs";
 import { getHFHubCachePath, getRepoFolderName } from "./cache-management";
 import { toRepoId } from "../utils/toRepoId";
 import { downloadFileToCacheDir } from "./download-file-to-cache-dir";
+import { createSymlink } from "../utils/symlink";
 
 vi.mock("node:fs/promises", () => ({
 	writeFile: vi.fn(),
@@ -21,6 +22,10 @@ vi.mock("./paths-info", () => ({
 	pathsInfo: vi.fn(),
 }));
 
+vi.mock("../utils/symlink", () => ({
+	createSymlink: vi.fn(),
+}));
+
 const DUMMY_REPO: RepoId = {
 	name: "hello-world",
 	type: "model",
@@ -196,7 +201,7 @@ describe("downloadFileToCacheDir", () => {
 		expect(vi.mocked(lstat).mock.calls[0][0]).toBe(expectedBlob);
 
 		// symlink should have been created
-		expect(symlink).toHaveBeenCalledOnce();
+		expect(createSymlink).toHaveBeenCalledOnce();
 		// no download done
 		expect(fetchMock).not.toHaveBeenCalled();
 
@@ -283,6 +288,6 @@ describe("downloadFileToCacheDir", () => {
 		// 2. should rename the incomplete to the blob expected name
 		expect(rename).toHaveBeenCalledWith(incomplete, expectedBlob);
 		// 3. should create symlink pointing to blob
-		expect(symlink).toHaveBeenCalledWith(expectedBlob, expectPointer);
+		expect(createSymlink).toHaveBeenCalledWith(expectedBlob, expectPointer);
 	});
 });

packages/hub/src/lib/download-file-to-cache-dir.ts

@@ -1,11 +1,12 @@
 import { getHFHubCachePath, getRepoFolderName } from "./cache-management";
 import { dirname, join } from "node:path";
-import { writeFile, rename, symlink, lstat, mkdir, stat } from "node:fs/promises";
+import { writeFile, rename, lstat, mkdir, stat } from "node:fs/promises";
 import type { CommitInfo, PathInfo } from "./paths-info";
 import { pathsInfo } from "./paths-info";
 import type { CredentialsParams, RepoDesignation } from "../types/public";
 import { toRepoId } from "../utils/toRepoId";
 import { downloadFile } from "./download-file";
+import { createSymlink } from "../utils/symlink";
 
 export const REGEX_COMMIT_HASH: RegExp = new RegExp("^[0-9a-f]{40}$");
 
@@ -107,7 +108,7 @@ export async function downloadFileToCacheDir(
 	// shortcut the download if needed
 	if (await exists(blobPath)) {
 		// create symlinks in snapshot folder to blob object
-		await symlink(blobPath, pointerPath);
+		await createSymlink(blobPath, pointerPath);
 		return pointerPath;
 	}
 
@@ -127,6 +128,6 @@ export async function downloadFileToCacheDir(
 	// rename .incomplete file to expect blob
 	await rename(incomplete, blobPath);
 	// create symlinks in snapshot folder to blob object
-	await symlink(blobPath, pointerPath);
+	await createSymlink(blobPath, pointerPath);
 	return pointerPath;
 }

packages/hub/src/lib/list-files.spec.ts

@@ -60,7 +60,7 @@ describe("listFiles", () => {
 				type: "file",
 			},
 		]);
-	});
+	}, 30_000);
 
 	it("should fetch the list of files from the repo, including last commit", async () => {
 		const cursor = listFiles({
@@ -146,7 +146,7 @@ describe("listFiles", () => {
 				type: "file",
 			},
 		]);
-	});
+	}, 30_000);
 
 	it("should fetch the list of files from the repo, including subfolders", async () => {
 		const cursor = listFiles({
@@ -165,5 +165,5 @@ describe("listFiles", () => {
 		}
 
 		assert(files.some((file) => file.path === "data/XSUM-EMNLP18-Summary-Data-Original.tar.gz"));
-	});
+	}, 30_000);
 });

packages/hub/src/utils/WebBlob.spec.ts

@@ -58,7 +58,7 @@ describe("WebBlob", () => {
 		expect(webBlob).toBeInstanceOf(WebBlob);
 		expect(webBlob).toMatchObject({ url });
 		expect(await webBlob.slice(10, 22).text()).toBe("__metadata__");
-	});
+	}, 30_000);
 
 	it("should lazy load a Xet file hosted on Hugging Face", async () => {
 		const stableDiffusionUrl =
@@ -70,7 +70,7 @@ describe("WebBlob", () => {
 		expect(webBlob).toBeInstanceOf(WebBlob);
 		expect(webBlob).toMatchObject({ url });
 		expect(await webBlob.slice(10, 22).text()).toBe("__metadata__");
-	});
+	}, 30_000);
 
 	it("should create a slice on the file", async () => {
 		const expectedText = fullText.slice(10, 20);

packages/hub/src/utils/WebBlob.ts

@@ -60,7 +60,7 @@ export class WebBlob extends Blob {
 
 	override slice(start = 0, end = this.size): WebBlob {
 		if (start < 0 || end < 0) {
-			new TypeError("Unsupported negative start/end on FileBlob.slice");
+			new TypeError("Unsupported negative start/end on WebBlob.slice");
 		}
 
 		const slice = new WebBlob(

packages/hub/src/utils/XetBlob.spec.ts

@@ -0,0 +1,185 @@
+import { describe, expect, it } from "vitest";
+import { bg4_regoup_bytes, XetBlob } from "./XetBlob";
+
+describe("XetBlob", () => {
+	it("should lazy load the first 22 bytes", async () => {
+		const blob = new XetBlob({
+			repo: {
+				type: "model",
+				name: "celinah/xet-experiments",
+			},
+			hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b",
+			size: 5_234_139_343,
+		});
+
+		expect(await blob.slice(10, 22).text()).toBe("__metadata__");
+	}, 30_000);
+
+	it("should load the first chunk correctly", async () => {
+		let xorbCount = 0;
+		const blob = new XetBlob({
+			repo: {
+				type: "model",
+				name: "celinah/xet-experiments",
+			},
+			hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b",
+			size: 5_234_139_343,
+			fetch: async (url, opts) => {
+				if (typeof url === "string" && url.includes("/xorbs/")) {
+					xorbCount++;
+				}
+				return fetch(url, opts);
+			},
+		});
+
+		const xetDownload = await blob.slice(0, 29928).arrayBuffer();
+		const bridgeDownload = await fetch(
+			"https://huggingface.co/celinah/xet-experiments/resolve/main/model5GB.safetensors",
+			{
+				headers: {
+					Range: "bytes=0-29927",
+				},
+			}
+		).then((res) => res.arrayBuffer());
+
+		expect(new Uint8Array(xetDownload)).toEqual(new Uint8Array(bridgeDownload));
+		expect(xorbCount).toBe(1);
+	}, 30_000);
+
+	it("should load just past the first chunk correctly", async () => {
+		let xorbCount = 0;
+		const blob = new XetBlob({
+			repo: {
+				type: "model",
+				name: "celinah/xet-experiments",
+			},
+			hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b",
+			size: 5_234_139_343,
+			fetch: async (url, opts) => {
+				if (typeof url === "string" && url.includes("/xorbs/")) {
+					xorbCount++;
+				}
+				return fetch(url, opts);
+			},
+		});
+
+		const xetDownload = await blob.slice(0, 29929).arrayBuffer();
+		const bridgeDownload = await fetch(
+			"https://huggingface.co/celinah/xet-experiments/resolve/main/model5GB.safetensors",
+			{
+				headers: {
+					Range: "bytes=0-29928",
+				},
+			}
+		).then((res) => res.arrayBuffer());
+
+		expect(new Uint8Array(xetDownload)).toEqual(new Uint8Array(bridgeDownload));
+		expect(xetDownload.byteLength).toBe(29929);
+		expect(xorbCount).toBe(2);
+	});
+
+	// In github actions, this test doesn't work inside the browser, but it works locally
+	// inside both chrome and chromium browsers
+	// TODO: figure out why
+	if (typeof window === "undefined") {
+		it("should load correctly when loading far into a chunk range", async () => {
+			const blob = new XetBlob({
+				repo: {
+					type: "model",
+					name: "celinah/xet-experiments",
+				},
+				hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b",
+				size: 5_234_139_343,
+			});
+
+			const xetDownload = await blob.slice(10_000_000, 10_100_000).arrayBuffer();
+			const bridgeDownload = await fetch(
+				"https://huggingface.co/celinah/xet-experiments/resolve/main/model5GB.safetensors",
+				{
+					headers: {
+						Range: "bytes=10000000-10099999",
+					},
+				}
+			).then((res) => res.arrayBuffer());
+
+			console.log("xet", xetDownload.byteLength, "bridge", bridgeDownload.byteLength);
+			expect(new Uint8Array(xetDownload)).toEqual(new Uint8Array(bridgeDownload));
+		}, 30_000);
+	}
+
+	it("should load text correctly when offset_into_range starts in a chunk further than the first", async () => {
+		const blob = new XetBlob({
+			repo: {
+				type: "model",
+				name: "celinah/xet-experiments",
+			},
+			hash: "794efea76d8cb372bbe1385d9e51c3384555f3281e629903ecb6abeff7d54eec",
+			size: 62_914_580,
+		});
+
+		// Reconstruction info
+		// {
+		// 	"offset_into_first_range": 600000,
+		// 	"terms":
+		// 		[
+		// 			{
+		// 				"hash": "be748f77930d5929cabd510a15f2c30f2f460b639804ef79dea46affa04fd8b2",
+		// 				"unpacked_length": 655360,
+		// 				"range": { "start": 0, "end": 5 },
+		// 			},
+		// 			{
+		// 				"hash": "be748f77930d5929cabd510a15f2c30f2f460b639804ef79dea46affa04fd8b2",
+		// 				"unpacked_length": 655360,
+		// 				"range": { "start": 0, "end": 5 },
+		// 			},
+		// 		],
+		// 	"fetch_info":
+		// 		{
+		// 			"be748f77930d5929cabd510a15f2c30f2f460b639804ef79dea46affa04fd8b2":
+		// 				[
+		// 					{
+		// 						"range": { "start": 0, "end": 5 },
+		// 						"url": "...",
+		// 						"url_range": { "start": 0, "end": 2839 },
+		// 					},
+		// 				],
+		// 		},
+		// }
+
+		const text = await blob.slice(600_000, 700_000).text();
+		const bridgeDownload = await fetch("https://huggingface.co/celinah/xet-experiments/resolve/main/large_text.txt", {
+			headers: {
+				Range: "bytes=600000-699999",
+			},
+		}).then((res) => res.text());
+
+		console.log("xet", text.length, "bridge", bridgeDownload.length);
+		expect(text.length).toBe(bridgeDownload.length);
+	}, 30_000);
+
+	describe("bg4_regoup_bytes", () => {
+		it("should regroup bytes when the array is %4 length", () => {
+			expect(bg4_regoup_bytes(new Uint8Array([1, 5, 2, 6, 3, 7, 4, 8]))).toEqual(
+				new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8])
+			);
+		});
+
+		it("should regroup bytes when the array is %4 + 1 length", () => {
+			expect(bg4_regoup_bytes(new Uint8Array([1, 5, 9, 2, 6, 3, 7, 4, 8]))).toEqual(
+				new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9])
+			);
+		});
+
+		it("should regroup bytes when the array is %4 + 2 length", () => {
+			expect(bg4_regoup_bytes(new Uint8Array([1, 5, 9, 2, 6, 10, 3, 7, 4, 8]))).toEqual(
+				new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+			);
+		});
+
+		it("should regroup bytes when the array is %4 + 3 length", () => {
+			expect(bg4_regoup_bytes(new Uint8Array([1, 5, 9, 2, 6, 10, 3, 7, 11, 4, 8]))).toEqual(
+				new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
+			);
+		});
+	});
+});