Skip to content

Commit 1645684

Browse files
yim-leeyim-lee
authored
[Collections] Fetch GitHub repo languages (#3411)
We can obtain a repo's programming languages via GitHub API. It would be nice to include that in package metadata.
1 parent 5f80bd7 commit 1645684

File tree

9 files changed

+41
-11
lines changed

9 files changed

+41
-11
lines changed

‎Fixtures/Collections/GitHub/languages.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"Swift": 3836385,
3+
"Shell": 96027,
4+
"C": 87433,
5+
}

‎Sources/PackageCollections/Model/Package.swift

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/*
22
This source file is part of the Swift.org open source project
33

4-
Copyright (c) 2020 Apple Inc. and the Swift project authors
4+
Copyright (c) 2020-2021 Apple Inc. and the Swift project authors
55
Licensed under Apache License v2.0 with Runtime Library Exception
66

77
See http://swift.org/LICENSE.txt for license information
@@ -80,7 +80,10 @@ extension PackageCollectionsModel {
8080
/// Package authors
8181
public let authors: [Author]?
8282

83-
/// Initializes a `PackageMetadata`
83+
/// The package's programming languages
84+
public let languages: Set<String>?
85+
86+
/// Initializes a `Package`
8487
init(
8588
repository: RepositorySpecifier,
8689
summary: String?,
@@ -89,7 +92,8 @@ extension PackageCollectionsModel {
8992
watchersCount: Int?,
9093
readmeURL: URL?,
9194
license: License?,
92-
authors: [Author]?
95+
authors: [Author]?,
96+
languages: Set<String>?
9397
) {
9498
self.reference = .init(repository: repository)
9599
self.repository = repository
@@ -100,6 +104,7 @@ extension PackageCollectionsModel {
100104
self.readmeURL = readmeURL
101105
self.license = license
102106
self.authors = authors
107+
self.languages = languages
103108
}
104109
}
105110
}

‎Sources/PackageCollections/PackageCollections.swift

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -550,7 +550,8 @@ public struct PackageCollections: PackageCollectionsProtocol {
550550
watchersCount: basicMetadata?.watchersCount,
551551
readmeURL: basicMetadata?.readmeURL ?? package.readmeURL,
552552
license: basicMetadata?.license ?? package.license,
553-
authors: basicMetadata?.authors
553+
authors: basicMetadata?.authors ?? package.authors,
554+
languages: basicMetadata?.languages ?? package.languages
554555
)
555556
}
556557
}

‎Sources/PackageCollections/Providers/GitHubPackageMetadataProvider.swift

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
5757
let contributorsURL = baseURL.appendingPathComponent("contributors")
5858
let readmeURL = baseURL.appendingPathComponent("readme")
5959
let licenseURL = baseURL.appendingPathComponent("license")
60+
let languagesURL = baseURL.appendingPathComponent("languages")
6061

6162
let sync = DispatchGroup()
6263
let results = ThreadSafeKeyValueStore<URL, Result<HTTPClientResponse, Error>>()
@@ -90,7 +91,7 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
9091
self.diagnosticsEngine?.emit(warning: "Approaching API limits on \(metadataURL.host ?? metadataURL.absoluteString) (\(apiRemaining)/\(apiLimit)), consider configuring an API token for this service.")
9192
}
9293
// if successful, fan out multiple API calls
93-
[releasesURL, contributorsURL, readmeURL, licenseURL].forEach { url in
94+
[releasesURL, contributorsURL, readmeURL, licenseURL, languagesURL].forEach { url in
9495
sync.enter()
9596
var headers = HTTPClientHeaders()
9697
headers.add(name: "Accept", value: "application/vnd.github.v3+json")
@@ -123,6 +124,7 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
123124
let contributors = try results[contributorsURL]?.success?.decodeBody([Contributor].self, using: self.decoder)
124125
let readme = try results[readmeURL]?.success?.decodeBody(Readme.self, using: self.decoder)
125126
let license = try results[licenseURL]?.success?.decodeBody(License.self, using: self.decoder)
127+
let languages = try results[languagesURL]?.success?.decodeBody([String: Int].self, using: self.decoder)?.keys
126128

127129
let model = Model.PackageBasicMetadata(
128130
summary: metadata.description,
@@ -138,6 +140,7 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
138140
readmeURL: readme?.downloadURL,
139141
license: license.flatMap { .init(type: Model.LicenseType(string: $0.license.spdxID), url: $0.downloadURL) },
140142
authors: contributors?.map { .init(username: $0.login, url: $0.url, service: .init(name: "GitHub")) },
143+
languages: languages.flatMap(Set.init) ?? metadata.language.map { [$0] },
141144
processedAt: Date()
142145
)
143146

‎Sources/PackageCollections/Providers/JSONPackageCollectionProvider.swift

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,8 @@ struct JSONPackageCollectionProvider: PackageCollectionProvider {
255255
watchersCount: nil,
256256
readmeURL: package.readmeURL,
257257
license: package.license.flatMap { Model.License(from: $0) },
258-
authors: nil)
258+
authors: nil,
259+
languages: nil)
259260
}
260261

261262
if !serializationOkay {

‎Sources/PackageCollections/Providers/PackageMetadataProvider.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ extension Model {
3535
let readmeURL: Foundation.URL?
3636
let license: PackageCollectionsModel.License?
3737
let authors: [PackageCollectionsModel.Package.Author]?
38+
let languages: Set<String>?
3839
let processedAt: Date
3940
}
4041

‎Tests/PackageCollectionsTests/GitHubPackageMetadataProviderTests.swift

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/*
22
This source file is part of the Swift.org open source project
33

4-
Copyright (c) 2020 Apple Inc. and the Swift project authors
4+
Copyright (c) 2020-2021 Apple Inc. and the Swift project authors
55
Licensed under Apache License v2.0 with Runtime Library Exception
66

77
See http://swift.org/LICENSE.txt for license information
@@ -80,6 +80,12 @@ class GitHubPackageMetadataProviderTests: XCTestCase {
8080
completion(.success(.init(statusCode: 200,
8181
headers: .init([.init(name: "Content-Length", value: "\(data.count)")]),
8282
body: data)))
83+
case (.get, apiURL.appendingPathComponent("languages")):
84+
let path = directoryPath.appending(components: "GitHub", "languages.json")
85+
let data = Data(try! localFileSystem.readFileContents(path).contents)
86+
completion(.success(.init(statusCode: 200,
87+
headers: .init([.init(name: "Content-Length", value: "\(data.count)")]),
88+
body: data)))
8389
default:
8490
XCTFail("method and url should match")
8591
}
@@ -104,6 +110,7 @@ class GitHubPackageMetadataProviderTests: XCTestCase {
104110
XCTAssertEqual(metadata.license?.type, PackageCollectionsModel.LicenseType.MIT)
105111
XCTAssertEqual(metadata.license?.url, URL(string: "https://raw.githubusercontent.com/benbalter/gman/master/LICENSE?lab=true"))
106112
XCTAssertEqual(metadata.watchersCount, 80)
113+
XCTAssertEqual(metadata.languages, ["Swift", "Shell", "C"])
107114
}
108115
}
109116

‎Tests/PackageCollectionsTests/PackageCollectionsTests.swift

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -606,7 +606,8 @@ final class PackageCollectionsTests: XCTestCase {
606606
watchersCount: nil,
607607
readmeURL: nil,
608608
license: nil,
609-
authors: nil)
609+
authors: nil,
610+
languages: nil)
610611

611612
let mockCollection = PackageCollectionsModel.Collection(source: .init(type: .json, url: URL(string: "https://feed.mock/\(UUID().uuidString)")!),
612613
name: UUID().uuidString,
@@ -770,7 +771,8 @@ final class PackageCollectionsTests: XCTestCase {
770771
watchersCount: nil,
771772
readmeURL: nil,
772773
license: nil,
773-
authors: nil)
774+
authors: nil,
775+
languages: nil)
774776

775777
let mockCollection = PackageCollectionsModel.Collection(source: .init(type: .json, url: URL(string: "https://feed.mock/\(UUID().uuidString)")!),
776778
name: UUID().uuidString,
@@ -1201,7 +1203,8 @@ final class PackageCollectionsTests: XCTestCase {
12011203
watchersCount: Int.random(in: 0 ... 50),
12021204
readmeURL: URL(string: "https://package-\(packageId)-readme")!,
12031205
license: PackageCollectionsModel.License(type: .Apache2_0, url: URL(string: "http://apache.license")!),
1204-
authors: (0 ..< Int.random(in: 1 ... 10)).map { .init(username: "\($0)", url: nil, service: nil) })
1206+
authors: (0 ..< Int.random(in: 1 ... 10)).map { .init(username: "\($0)", url: nil, service: nil) },
1207+
languages: nil)
12051208

12061209
let mockMetadata = PackageCollectionsModel.PackageBasicMetadata(summary: "\(mockPackage.summary!) 2",
12071210
keywords: mockPackage.keywords.flatMap { $0.map { "\($0)-2" } },
@@ -1210,6 +1213,7 @@ final class PackageCollectionsTests: XCTestCase {
12101213
readmeURL: URL(string: "\(mockPackage.readmeURL!.absoluteString)-2")!,
12111214
license: PackageCollectionsModel.License(type: .Apache2_0, url: URL(string: "\(mockPackage.license!.url.absoluteString)-2")!),
12121215
authors: mockPackage.authors.flatMap { $0.map { .init(username: "\($0.username + "2")", url: nil, service: nil) } },
1216+
languages: ["Swift"],
12131217
processedAt: Date())
12141218

12151219
let metadata = PackageCollections.mergedPackageMetadata(package: mockPackage, basicMetadata: mockMetadata)
@@ -1243,6 +1247,7 @@ final class PackageCollectionsTests: XCTestCase {
12431247
XCTAssertEqual(metadata.readmeURL, mockMetadata.readmeURL, "readmeURL should match")
12441248
XCTAssertEqual(metadata.license, mockMetadata.license, "license should match")
12451249
XCTAssertEqual(metadata.authors, mockMetadata.authors, "authors should match")
1250+
XCTAssertEqual(metadata.languages, mockMetadata.languages, "languages should match")
12461251
}
12471252

12481253
func testFetchMetadataNotFoundInCollections() throws {

‎Tests/PackageCollectionsTests/Utility.swift

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,8 @@ func makeMockCollections(count: Int = Int.random(in: 50 ... 100), maxPackages: I
8383
watchersCount: Int.random(in: 1 ... 1000),
8484
readmeURL: URL(string: "https://package-\(packageIndex)-readme")!,
8585
license: PackageCollectionsModel.License(type: .Apache2_0, url: URL(string: "https://\(packageIndex).license")!),
86-
authors: nil)
86+
authors: nil,
87+
languages: nil)
8788
}
8889

8990
var signature: PackageCollectionsModel.SignatureData?
@@ -116,6 +117,7 @@ func makeMockPackageBasicMetadata() -> PackageCollectionsModel.PackageBasicMetad
116117
readmeURL: URL(string: "https://package-readme")!,
117118
license: PackageCollectionsModel.License(type: .Apache2_0, url: URL(string: "https://package-license")!),
118119
authors: (0 ..< Int.random(in: 1 ... 10)).map { .init(username: "\($0)", url: nil, service: nil) },
120+
languages: ["Swift"],
119121
processedAt: Date())
120122
}
121123

0 commit comments

Comments
 (0)