Skip to content

Commit f280297

Browse files
coyotte508coyotte508
authored
✨ List-commits & count-commits (#647)
Fix #646
1 parent f7f442c commit f280297

File tree

7 files changed

+254
-3
lines changed

7 files changed

+254
-3
lines changed

‎packages/hub/src/error.ts

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ export async function createApiError(
66
): Promise<never> {
77
const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId);
88

9-
error.message = `Api error with status ${error.statusCode}.${opts?.message ? ` ${opts.message}.` : ""} Request ID: ${
10-
error.requestId
11-
}, url: ${error.url}`;
9+
error.message = `Api error with status ${error.statusCode}${opts?.message ? `. ${opts.message}` : ""}`;
10+
11+
const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : undefined]
12+
.filter(Boolean)
13+
.join(". ");
1214

1315
if (response.headers.get("Content-Type")?.startsWith("application/json")) {
1416
const json = await response.json();
@@ -18,6 +20,8 @@ export async function createApiError(
1820
error.data = { message: await response.text() };
1921
}
2022

23+
error.message += `. ${trailer}`;
24+
2125
throw error;
2226
}
2327

‎packages/hub/src/lib/count-commits.spec.ts

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import { assert, it, describe } from "vitest";
2+
import { countCommits } from "./count-commits";
3+
4+
describe("countCommits", () => {
5+
it("should fetch paginated commits from the repo", async () => {
6+
const count = await countCommits({
7+
repo: {
8+
name: "openai-community/gpt2",
9+
type: "model",
10+
},
11+
revision: "607a30d783dfa663caf39e06633721c8d4cfcd7e",
12+
});
13+
14+
assert.equal(count, 26);
15+
});
16+
});

‎packages/hub/src/lib/count-commits.ts

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import { HUB_URL } from "../consts";
2+
import { createApiError } from "../error";
3+
import type { Credentials, RepoDesignation } from "../types/public";
4+
import { checkCredentials } from "../utils/checkCredentials";
5+
import { toRepoId } from "../utils/toRepoId";
6+
7+
export async function countCommits(params: {
8+
credentials?: Credentials;
9+
repo: RepoDesignation;
10+
/**
11+
* Revision to list commits from. Defaults to the default branch.
12+
*/
13+
revision?: string;
14+
hubUrl?: string;
15+
fetch?: typeof fetch;
16+
}): Promise<number> {
17+
checkCredentials(params.credentials);
18+
const repoId = toRepoId(params.repo);
19+
20+
// Could upgrade to 1000 commits per page
21+
const url: string | undefined = `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commits/${
22+
params.revision ?? "main"
23+
}?limit=1`;
24+
25+
const res: Response = await (params.fetch ?? fetch)(url, {
26+
headers: params.credentials ? { Authorization: `Bearer ${params.credentials.accessToken}` } : {},
27+
});
28+
29+
if (!res.ok) {
30+
throw await createApiError(res);
31+
}
32+
33+
return parseInt(res.headers.get("x-total-count") ?? "0", 10);
34+
}

‎packages/hub/src/lib/index.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
export * from "./commit";
2+
export * from "./count-commits";
23
export * from "./create-repo";
34
export * from "./delete-file";
45
export * from "./delete-files";
56
export * from "./delete-repo";
67
export * from "./download-file";
78
export * from "./file-download-info";
89
export * from "./file-exists";
10+
export * from "./list-commits";
911
export * from "./list-datasets";
1012
export * from "./list-files";
1113
export * from "./list-models";

‎packages/hub/src/lib/list-commits.spec.ts

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import { assert, it, describe } from "vitest";
2+
import type { CommitData } from "./list-commits";
3+
import { listCommits } from "./list-commits";
4+
5+
describe("listCommits", () => {
6+
it("should fetch paginated commits from the repo", async () => {
7+
const commits: CommitData[] = [];
8+
for await (const commit of listCommits({
9+
repo: {
10+
name: "openai-community/gpt2",
11+
type: "model",
12+
},
13+
revision: "607a30d783dfa663caf39e06633721c8d4cfcd7e",
14+
batchSize: 5,
15+
})) {
16+
commits.push(commit);
17+
}
18+
19+
assert.equal(commits.length, 26);
20+
assert.deepEqual(commits.slice(0, 6), [
21+
{
22+
oid: "607a30d783dfa663caf39e06633721c8d4cfcd7e",
23+
title: "Adds the tokenizer configuration file (#80)",
24+
message: "\n\n\n- Adds tokenizer_config.json file (db6d57930088fb63e52c010bd9ac77c955ac55e7)\n\n",
25+
authors: [
26+
{
27+
username: "lysandre",
28+
avatarUrl:
29+
"https://cdn-avatars.huggingface.co/v1/production/uploads/1618450692745-5e3aec01f55e2b62848a5217.jpeg",
30+
},
31+
],
32+
date: new Date("2024-02-19T10:57:45.000Z"),
33+
},
34+
{
35+
oid: "11c5a3d5811f50298f278a704980280950aedb10",
36+
title: "Adding ONNX file of this model (#60)",
37+
message: "\n\n\n- Adding ONNX file of this model (9411f419c589519e1a46c94ac7789ea20fd7c322)\n\n",
38+
authors: [
39+
{
40+
username: "fxmarty",
41+
avatarUrl:
42+
"https://cdn-avatars.huggingface.co/v1/production/uploads/1651743336129-624c60cba8ec93a7ac188b56.png",
43+
},
44+
],
45+
date: new Date("2023-06-30T02:19:43.000Z"),
46+
},
47+
{
48+
oid: "e7da7f221d5bf496a48136c0cd264e630fe9fcc8",
49+
title: "Update generation_config.json",
50+
message: "",
51+
authors: [
52+
{
53+
username: "joaogante",
54+
avatarUrl: "https://cdn-avatars.huggingface.co/v1/production/uploads/1641203017724-noauth.png",
55+
},
56+
],
57+
date: new Date("2022-12-16T15:44:21.000Z"),
58+
},
59+
{
60+
oid: "f27b190eeac4c2302d24068eabf5e9d6044389ae",
61+
title: "Add note that this is the smallest version of the model (#18)",
62+
message:
63+
"\n\n\n- Add note that this is the smallest version of the model (611838ef095a5bb35bf2027d05e1194b7c9d37ac)\n\n\nCo-authored-by: helen <[email protected]>\n",
64+
authors: [
65+
{
66+
username: "sgugger",
67+
avatarUrl:
68+
"https://cdn-avatars.huggingface.co/v1/production/uploads/1593126474392-5ef50182b71947201082a4e5.jpeg",
69+
},
70+
{
71+
username: "mathemakitten",
72+
avatarUrl:
73+
"https://cdn-avatars.huggingface.co/v1/production/uploads/1658248499901-6079afe2d2cd8c150e6ae05e.jpeg",
74+
},
75+
],
76+
date: new Date("2022-11-23T12:55:26.000Z"),
77+
},
78+
{
79+
oid: "0dd7bcc7a64e4350d8859c9a2813132fbf6ae591",
80+
title: "Our very first generation_config.json (#17)",
81+
message:
82+
"\n\n\n- Our very first generation_config.json (671851b7e9d56ef062890732065d7bd5f4628bd6)\n\n\nCo-authored-by: Joao Gante <[email protected]>\n",
83+
authors: [
84+
{
85+
username: "sgugger",
86+
avatarUrl:
87+
"https://cdn-avatars.huggingface.co/v1/production/uploads/1593126474392-5ef50182b71947201082a4e5.jpeg",
88+
},
89+
{
90+
username: "joaogante",
91+
avatarUrl: "https://cdn-avatars.huggingface.co/v1/production/uploads/1641203017724-noauth.png",
92+
},
93+
],
94+
date: new Date("2022-11-18T18:19:30.000Z"),
95+
},
96+
{
97+
oid: "75e09b43581151bd1d9ef6700faa605df408979f",
98+
title: "Upload model.safetensors with huggingface_hub (#12)",
99+
message:
100+
"\n\n\n- Upload model.safetensors with huggingface_hub (ba2f794b2e4ea09ef932a6628fa0815dfaf09661)\n\n\nCo-authored-by: Nicolas Patry <[email protected]>\n",
101+
authors: [
102+
{
103+
username: "julien-c",
104+
avatarUrl:
105+
"https://cdn-avatars.huggingface.co/v1/production/uploads/5dd96eb166059660ed1ee413/NQtzmrDdbG0H8qkZvRyGk.jpeg",
106+
},
107+
{
108+
username: "Narsil",
109+
avatarUrl:
110+
"https://cdn-avatars.huggingface.co/v1/production/uploads/1608285816082-5e2967b819407e3277369b95.png",
111+
},
112+
],
113+
date: new Date("2022-10-20T09:34:54.000Z"),
114+
},
115+
]);
116+
});
117+
});

‎packages/hub/src/lib/list-commits.ts

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import { HUB_URL } from "../consts";
2+
import { createApiError } from "../error";
3+
import type { ApiCommitData } from "../types/api/api-commit";
4+
import type { Credentials, RepoDesignation } from "../types/public";
5+
import { checkCredentials } from "../utils/checkCredentials";
6+
import { parseLinkHeader } from "../utils/parseLinkHeader";
7+
import { toRepoId } from "../utils/toRepoId";
8+
9+
export interface CommitData {
10+
oid: string;
11+
title: string;
12+
message: string;
13+
authors: Array<{ username: string; avatarUrl: string }>;
14+
date: Date;
15+
}
16+
17+
export async function* listCommits(params: {
18+
credentials?: Credentials;
19+
repo: RepoDesignation;
20+
/**
21+
* Revision to list commits from. Defaults to the default branch.
22+
*/
23+
revision?: string;
24+
hubUrl?: string;
25+
/**
26+
* Number of commits to fetch from the hub each http call. Defaults to 100. Can be set to 1000.
27+
*/
28+
batchSize?: number;
29+
/**
30+
* Custom fetch function to use instead of the default one, for example to use a proxy or edit headers.
31+
*/
32+
fetch?: typeof fetch;
33+
}): AsyncGenerator<CommitData> {
34+
checkCredentials(params.credentials);
35+
const repoId = toRepoId(params.repo);
36+
37+
// Could upgrade to 1000 commits per page
38+
let url: string | undefined = `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commits/${
39+
params.revision ?? "main"
40+
}?limit=${params.batchSize ?? 100}`;
41+
42+
while (url) {
43+
const res: Response = await (params.fetch ?? fetch)(url, {
44+
headers: params.credentials ? { Authorization: `Bearer ${params.credentials.accessToken}` } : {},
45+
});
46+
47+
if (!res.ok) {
48+
throw await createApiError(res);
49+
}
50+
51+
const resJson: ApiCommitData[] = await res.json();
52+
for (const commit of resJson) {
53+
yield {
54+
oid: commit.id,
55+
title: commit.title,
56+
message: commit.message,
57+
authors: commit.authors.map((author) => ({
58+
username: author.user,
59+
avatarUrl: author.avatar,
60+
})),
61+
date: new Date(commit.date),
62+
};
63+
}
64+
65+
const linkHeader = res.headers.get("Link");
66+
67+
url = linkHeader ? parseLinkHeader(linkHeader).next : undefined;
68+
}
69+
}

‎packages/hub/src/types/api/api-commit.d.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,3 +180,12 @@ export type ApiCommitOperation =
180180
key: "deletedFile";
181181
value: ApiCommitDeletedEntry;
182182
};
183+
184+
export interface ApiCommitData {
185+
id: string;
186+
title: string;
187+
message: string;
188+
authors: Array<{ user: string; avatar: string }>;
189+
date: string;
190+
formatted?: string;
191+
}

0 commit comments

Comments
 (0)