From 95c2dc6715e434a747d7fcbd3e8b17f3f11474de Mon Sep 17 00:00:00 2001 From: Dave Verwer Date: Wed, 8 Nov 2023 17:31:10 +0000 Subject: [PATCH] # This is a combination of 16 commits. # This is the 1st commit message: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WIP # This is the commit message #2: Made `Funding` Codable. # This is the commit message #3: Return the funding from `fetchFunding`. # This is the commit message #4: Added `funding` column migration. # This is the commit message #5: Store the funding information in `Repository`. # This is the commit message #6: I’m not sure how these got removed! # This is the commit message #7: Tested ingestion with `fetchFunding`. # This is the commit message #8: Switched over to using the GraphQL query to fetch parsed funding links. # This is the commit message #9: Removed a bit of left over code from previous method and reduce noise in diff. # This is the commit message #10: Corrected the column definition for `fundingLinks`. # This is the commit message #11: Made the incoming funding link data from GitHub optional. # This is the commit message #12: Simplify repository.releases assignment # This is the commit message #13: Move FundingLink to top level type # This is the commit message #14: Extend test_fetchMetadata to cover funding links # This is the commit message #15: Simplify decoder # This is the commit message #16: Disconnect FundingLink type from GH specific internal representation --- Sources/App/Commands/Ingest.swift | 4 +- Sources/App/Core/Github.swift | 24 ++++++ .../070/AddFundingToRepositories.swift | 30 ++++++++ Sources/App/Models/FundingLink.swift | 73 +++++++++++++++++++ Sources/App/Models/Repository.swift | 6 ++ Sources/App/configure.swift | 3 + .../Fixtures/github-graphql-resource.json | 10 +++ Tests/AppTests/GithubTests.swift | 4 + Tests/AppTests/IngestorTests.swift | 10 +++ .../AppTests/Mocks/GithubMetadata+mock.swift | 2 + 10 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 Sources/App/Migrations/070/AddFundingToRepositories.swift create mode 100644 Sources/App/Models/FundingLink.swift diff --git a/Sources/App/Commands/Ingest.swift b/Sources/App/Commands/Ingest.swift index f455312574..610865de1e 100644 --- a/Sources/App/Commands/Ingest.swift +++ b/Sources/App/Commands/Ingest.swift @@ -200,6 +200,7 @@ func updateRepository(on database: Database, repository.defaultBranch = repoMetadata.defaultBranch repository.forks = repoMetadata.forkCount + repository.fundingLinks = repoMetadata.fundingLinks?.map(FundingLink.init(from:)) ?? [] repository.homepageUrl = repoMetadata.homepageUrl?.trimmed repository.isArchived = repoMetadata.isArchived repository.isInOrganization = repoMetadata.isInOrganization @@ -216,8 +217,7 @@ func updateRepository(on database: Database, repository.ownerAvatarUrl = repoMetadata.owner.avatarUrl repository.s3Readme = s3Readme repository.readmeHtmlUrl = readmeInfo?.htmlUrl - repository.releases = metadata.repository?.releases.nodes - .map(Release.init(from:)) ?? [] + repository.releases = repoMetadata.releases.nodes.map(Release.init(from:)) repository.stars = repoMetadata.stargazerCount repository.summary = repoMetadata.description diff --git a/Sources/App/Core/Github.swift b/Sources/App/Core/Github.swift index c2e6a90438..ee3c46020a 100644 --- a/Sources/App/Core/Github.swift +++ b/Sources/App/Core/Github.swift @@ -276,6 +276,10 @@ extension Github { } description forkCount + fundingLinks { + platform + url + } homepageUrl isArchived isFork @@ -338,6 +342,7 @@ extension Github { var defaultBranchRef: DefaultBranchRef? var description: String? var forkCount: Int + var fundingLinks: [FundingLinkNode]? var homepageUrl: String? var isArchived: Bool // periphery:ignore @@ -365,6 +370,25 @@ extension Github { } } + struct FundingLinkNode: Codable, Equatable { + enum Platform: String, Codable { + case communityBridge = "COMMUNITYBRIDGE" + case customUrl = "CUSTOMURL" + case gitHub = "GITHUB" + case issueHunt = "ISSUEHUNT" + case koFi = "KOFI" + case lfxCrowdfunding = "LFXCROWDFUNDING" + case liberaPay = "LIBERAPAY" + case openCollective = "OPENCOLLECTIVE" + case otechie = "OTECHIE" + case patreon = "PATREON" + case tideLift = "TIDELIFT" + } + + var platform: Platform + var url: String + } + struct IssueNodes: Decodable, Equatable { var nodes: [IssueNode] diff --git a/Sources/App/Migrations/070/AddFundingToRepositories.swift b/Sources/App/Migrations/070/AddFundingToRepositories.swift new file mode 100644 index 0000000000..b0a317c417 --- /dev/null +++ b/Sources/App/Migrations/070/AddFundingToRepositories.swift @@ -0,0 +1,30 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Fluent + + +struct AddFundingToRepositories: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema("repositories") + .field("funding_links", .array(of: .json), .sql(.default("{}"))) + .update() + } + + func revert(on database: Database) async throws { + try await database.schema("repositories") + .deleteField("funding_links") + .update() + } +} diff --git a/Sources/App/Models/FundingLink.swift b/Sources/App/Models/FundingLink.swift new file mode 100644 index 0000000000..4c7dc462c0 --- /dev/null +++ b/Sources/App/Models/FundingLink.swift @@ -0,0 +1,73 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation + + +struct FundingLink: Codable, Equatable { + enum Platform: String, Codable { + case communityBridge + case customUrl + case gitHub + case issueHunt + case koFi + case lfxCrowdfunding + case liberaPay + case openCollective + case otechie + case patreon + case tideLift + } + + var platform: Platform + var url: String +} + + +extension FundingLink { + init(from node: Github.Metadata.FundingLinkNode) { + platform = .init(from: node.platform) + url = node.url + } +} + + +extension FundingLink.Platform { + init(from platform: Github.Metadata.FundingLinkNode.Platform) { + switch platform { + case .communityBridge: + self = .communityBridge + case .customUrl: + self = .customUrl + case .gitHub: + self = .gitHub + case .issueHunt: + self = .issueHunt + case .koFi: + self = .koFi + case .lfxCrowdfunding: + self = .lfxCrowdfunding + case .liberaPay: + self = .liberaPay + case .openCollective: + self = .openCollective + case .otechie: + self = .otechie + case .patreon: + self = .patreon + case .tideLift: + self = .tideLift + } + } +} diff --git a/Sources/App/Models/Repository.swift b/Sources/App/Models/Repository.swift index cdcee640e8..89a075121f 100644 --- a/Sources/App/Models/Repository.swift +++ b/Sources/App/Models/Repository.swift @@ -57,6 +57,9 @@ final class Repository: Model, Content { @Field(key: "forks") var forks: Int + @Field(key: "funding_links") + var fundingLinks: [FundingLink] + @Field(key: "homepage_url") var homepageUrl: String? @@ -131,6 +134,7 @@ final class Repository: Model, Content { defaultBranch: String? = nil, firstCommitDate: Date? = nil, forks: Int = 0, + fundingLinks: [FundingLink] = [], forkedFrom: Repository? = nil, homepageUrl: String? = nil, isArchived: Bool = false, @@ -163,6 +167,7 @@ final class Repository: Model, Content { if let forkId = forkedFrom?.id { self.$forkedFrom.id = forkId } + self.fundingLinks = fundingLinks self.homepageUrl = homepageUrl self.isArchived = isArchived self.isInOrganization = isInOrganization @@ -192,6 +197,7 @@ final class Repository: Model, Content { self.defaultBranch = nil self.firstCommitDate = nil self.forks = 0 + self.fundingLinks = [] self.homepageUrl = nil self.isArchived = false self.isInOrganization = false diff --git a/Sources/App/configure.swift b/Sources/App/configure.swift index 3fa1641632..d6e237374b 100644 --- a/Sources/App/configure.swift +++ b/Sources/App/configure.swift @@ -306,6 +306,9 @@ public func configure(_ app: Application) throws -> String { do { // Migration 069 - add builder_version to builds app.migrations.add(UpdateBuildAddBuilderVersion()) } + do { // Migration 070 - Add `funding` JSON field to `repositories` + app.migrations.add(AddFundingToRepositories()) + } app.commands.use(Analyze.Command(), as: "analyze") app.commands.use(CreateRestfileCommand(), as: "create-restfile") diff --git a/Tests/AppTests/Fixtures/github-graphql-resource.json b/Tests/AppTests/Fixtures/github-graphql-resource.json index cdf14a175c..9463219ecf 100644 --- a/Tests/AppTests/Fixtures/github-graphql-resource.json +++ b/Tests/AppTests/Fixtures/github-graphql-resource.json @@ -614,6 +614,16 @@ }, "description": "Elegant HTTP Networking in Swift", "forkCount": 6727, + "fundingLinks": [ + { + "platform": "GITHUB", + "url": "https://github.com/Alamofire" + }, + { + "platform": "GITHUB", + "url": "https://github.com/jshier" + } + ], "isArchived": false, "isFork": false, "licenseInfo": { diff --git a/Tests/AppTests/GithubTests.swift b/Tests/AppTests/GithubTests.swift index 8015981766..0fa8ae2d1f 100644 --- a/Tests/AppTests/GithubTests.swift +++ b/Tests/AppTests/GithubTests.swift @@ -136,6 +136,10 @@ class GithubTests: AppTestCase { XCTAssertEqual(res.repository?.closedPullRequests.nodes.first!.closedAt, iso8601.date(from: "2021-05-28T15:50:17Z")) XCTAssertEqual(res.repository?.forkCount, 6727) + XCTAssertEqual(res.repository?.fundingLinks, [ + .init(platform: .gitHub, url: "https://github.com/Alamofire"), + .init(platform: .gitHub, url: "https://github.com/jshier"), + ]) XCTAssertEqual(res.repository?.mergedPullRequests.nodes.first!.closedAt, iso8601.date(from: "2021-06-07T22:47:01Z")) XCTAssertEqual(res.repository?.name, "Alamofire") diff --git a/Tests/AppTests/IngestorTests.swift b/Tests/AppTests/IngestorTests.swift index 23eff2fdb5..92f14c1899 100644 --- a/Tests/AppTests/IngestorTests.swift +++ b/Tests/AppTests/IngestorTests.swift @@ -110,6 +110,11 @@ class IngestorTests: AppTestCase { let repo = Repository(packageId: try pkg.requireID()) let md: Github.Metadata = .init(defaultBranch: "main", forks: 1, + fundingLinks: [ + .init(platform: .gitHub, url: "https://github.com/username"), + .init(platform: .customUrl, url: "https://example.com/username1"), + .init(platform: .customUrl, url: "https://example.com/username2") + ], homepageUrl: "https://swiftpackageindex.com/Alamofire/Alamofire", isInOrganization: true, issuesClosedAtDates: [ @@ -153,6 +158,11 @@ class IngestorTests: AppTestCase { let repo = try await Repository.query(on: app.db).first().unwrap() XCTAssertEqual(repo.defaultBranch, "main") XCTAssertEqual(repo.forks, 1) + XCTAssertEqual(repo.fundingLinks, [ + .init(platform: .gitHub, url: "https://github.com/username"), + .init(platform: .customUrl, url: "https://example.com/username1"), + .init(platform: .customUrl, url: "https://example.com/username2") + ]) XCTAssertEqual(repo.homepageUrl, "https://swiftpackageindex.com/Alamofire/Alamofire") XCTAssertEqual(repo.isInOrganization, true) XCTAssertEqual(repo.keywords, ["bar", "baz", "foo"]) diff --git a/Tests/AppTests/Mocks/GithubMetadata+mock.swift b/Tests/AppTests/Mocks/GithubMetadata+mock.swift index 1fefbf359e..561977b1a6 100644 --- a/Tests/AppTests/Mocks/GithubMetadata+mock.swift +++ b/Tests/AppTests/Mocks/GithubMetadata+mock.swift @@ -55,6 +55,7 @@ extension Github.Metadata { init(defaultBranch: String, forks: Int, + fundingLinks: [FundingLinkNode] = [], homepageUrl: String?, isInOrganization: Bool, issuesClosedAtDates: [Date], @@ -81,6 +82,7 @@ extension Github.Metadata { defaultBranchRef: .init(name: defaultBranch), description: summary, forkCount: forks, + fundingLinks: fundingLinks, homepageUrl: homepageUrl, isArchived: false, isFork: false,