[gguf] RE_GGUF_SHARD_FILE (#601)

I need on the frontend to detect which gguf file is sharded or not

cc: @ggerganov @madgetr

Author: Mishig

packages/gguf/src/gguf.spec.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { GGMLQuantizationType, gguf } from "./gguf";
+import { GGMLQuantizationType, RE_GGUF_SHARD_FILE, gguf } from "./gguf";
 
 const URL_LLAMA = "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/191239b/llama-2-7b-chat.Q2_K.gguf";
 const URL_MISTRAL_7B =
@@ -220,4 +220,13 @@ describe("gguf", () => {
 			dtype: GGMLQuantizationType.F32,
 		});
 	});
+
+	it("should detect sharded gguf filename", async () => {
+		const ggufPath = "grok-1/grok-1-q4_0-00003-of-00009.gguf"; // https://huggingface.co/ggml-org/models/blob/fcf344adb9686474c70e74dd5e55465e9e6176ef/grok-1/grok-1-q4_0-00003-of-00009.gguf
+		const match = ggufPath.match(RE_GGUF_SHARD_FILE);
+
+		expect(RE_GGUF_SHARD_FILE.test(ggufPath)).toEqual(true);
+		expect(match?.[1]).toEqual("00003");
+		expect(match?.[2]).toEqual("00009");
+	});
 });

packages/gguf/src/gguf.ts

@@ -5,6 +5,7 @@ export type { MetadataBaseValue, MetadataValue, Version, GGUFMetadata, GGUFTenso
 export { GGUFValueType, GGMLQuantizationType } from "./types";
 
 export const RE_GGUF_FILE = /\.gguf$/;
+export const RE_GGUF_SHARD_FILE = /-(\d{5})-of-(\d{5})\.gguf$/;
 
 const isVersion = (version: number): version is Version => version === 1 || version === 2 || version === 3;