Merge pull request #2712 from SwiftPackageIndex/fetch-funding

Fetch funding information and store in the database

Sources/App/Commands/Ingest.swift

@@ -200,6 +200,7 @@ func updateRepository(on database: Database,
 
     repository.defaultBranch = repoMetadata.defaultBranch
     repository.forks = repoMetadata.forkCount
+    repository.fundingLinks = repoMetadata.fundingLinks?.map(FundingLink.init(from:)) ?? []
     repository.homepageUrl = repoMetadata.homepageUrl?.trimmed
     repository.isArchived = repoMetadata.isArchived
     repository.isInOrganization = repoMetadata.isInOrganization
@@ -216,8 +217,7 @@ func updateRepository(on database: Database,
     repository.ownerAvatarUrl = repoMetadata.owner.avatarUrl
     repository.s3Readme = s3Readme
     repository.readmeHtmlUrl = readmeInfo?.htmlUrl
-    repository.releases = metadata.repository?.releases.nodes
-        .map(Release.init(from:)) ?? []
+    repository.releases = repoMetadata.releases.nodes.map(Release.init(from:))
     repository.stars = repoMetadata.stargazerCount
     repository.summary = repoMetadata.description
 

Sources/App/Core/Github.swift

@@ -276,6 +276,10 @@ extension Github {
                     }
                     description
                     forkCount
+                    fundingLinks {
+                      platform
+                      url
+                    }
                     homepageUrl
                     isArchived
                     isFork
@@ -338,6 +342,7 @@ extension Github {
             var defaultBranchRef: DefaultBranchRef?
             var description: String?
             var forkCount: Int
+            var fundingLinks: [FundingLinkNode]?
             var homepageUrl: String?
             var isArchived: Bool
             // periphery:ignore
@@ -365,6 +370,25 @@ extension Github {
             }
         }
 
+        struct FundingLinkNode: Codable, Equatable {
+            enum Platform: String, Codable {
+                case communityBridge = "COMMUNITY_BRIDGE"
+                case customUrl = "CUSTOM"
+                case gitHub = "GITHUB"
+                case issueHunt = "ISSUEHUNT"
+                case koFi = "KO_FI"
+                case lfxCrowdfunding = "LFX_CROWDFUNDING"
+                case liberaPay = "LIBERAPAY"
+                case openCollective = "OPEN_COLLECTIVE"
+                case otechie = "OTECHIE"
+                case patreon = "PATREON"
+                case tideLift = "TIDELIFT"
+            }
+
+            var platform: Platform
+            var url: String
+        }
+
         struct IssueNodes: Decodable, Equatable {
             var nodes: [IssueNode]
 

Sources/App/Migrations/070/AddFundingToRepositories.swift

@@ -0,0 +1,30 @@
+// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Fluent
+
+
+struct AddFundingToRepositories: AsyncMigration {
+    func prepare(on database: Database) async throws {
+        try await database.schema("repositories")
+            .field("funding_links", .array(of: .json), .sql(.default("{}")))
+            .update()
+    }
+
+    func revert(on database: Database) async throws {
+        try await database.schema("repositories")
+            .deleteField("funding_links")
+            .update()
+    }
+}

Sources/App/Models/FundingLink.swift

@@ -0,0 +1,73 @@
+// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+
+struct FundingLink: Codable, Equatable {
+    enum Platform: String, Codable {
+        case communityBridge
+        case customUrl
+        case gitHub
+        case issueHunt
+        case koFi
+        case lfxCrowdfunding
+        case liberaPay
+        case openCollective
+        case otechie
+        case patreon
+        case tideLift
+    }
+
+    var platform: Platform
+    var url: String
+}
+
+
+extension FundingLink {
+    init(from node: Github.Metadata.FundingLinkNode) {
+        platform = .init(from: node.platform)
+        url = node.url
+    }
+}
+
+
+extension FundingLink.Platform {
+    init(from platform: Github.Metadata.FundingLinkNode.Platform) {
+        switch platform {
+            case .communityBridge:
+                self = .communityBridge
+            case .customUrl:
+                self = .customUrl
+            case .gitHub:
+                self = .gitHub
+            case .issueHunt:
+                self = .issueHunt
+            case .koFi:
+                self = .koFi
+            case .lfxCrowdfunding:
+                self = .lfxCrowdfunding
+            case .liberaPay:
+                self = .liberaPay
+            case .openCollective:
+                self = .openCollective
+            case .otechie:
+                self = .otechie
+            case .patreon:
+                self = .patreon
+            case .tideLift:
+                self = .tideLift
+        }
+    }
+}

Sources/App/Models/Repository.swift

@@ -57,6 +57,9 @@ final class Repository: Model, Content {
     @Field(key: "forks")
     var forks: Int
 
+    @Field(key: "funding_links")
+    var fundingLinks: [FundingLink]
+
     @Field(key: "homepage_url")
     var homepageUrl: String?
 
@@ -131,6 +134,7 @@ final class Repository: Model, Content {
          defaultBranch: String? = nil,
          firstCommitDate: Date? = nil,
          forks: Int = 0,
+         fundingLinks: [FundingLink] = [],
          forkedFrom: Repository? = nil,
          homepageUrl: String? = nil,
          isArchived: Bool = false,
@@ -163,6 +167,7 @@ final class Repository: Model, Content {
         if let forkId = forkedFrom?.id {
             self.$forkedFrom.id = forkId
         }
+        self.fundingLinks = fundingLinks
         self.homepageUrl = homepageUrl
         self.isArchived = isArchived
         self.isInOrganization = isInOrganization
@@ -192,6 +197,7 @@ final class Repository: Model, Content {
         self.defaultBranch = nil
         self.firstCommitDate = nil
         self.forks = 0
+        self.fundingLinks = []
         self.homepageUrl = nil
         self.isArchived = false
         self.isInOrganization = false

Sources/App/configure.swift

@@ -306,6 +306,9 @@ public func configure(_ app: Application) throws -> String {
     do { // Migration 069 - add builder_version to builds
         app.migrations.add(UpdateBuildAddBuilderVersion())
     }
+    do { // Migration 070 - Add `funding` JSON field to `repositories`
+        app.migrations.add(AddFundingToRepositories())
+    }
 
     app.commands.use(Analyze.Command(), as: "analyze")
     app.commands.use(CreateRestfileCommand(), as: "create-restfile")

Tests/AppTests/Fixtures/github-graphql-resource.json

@@ -614,6 +614,16 @@
       },
       "description": "Elegant HTTP Networking in Swift",
       "forkCount": 6727,
+      "fundingLinks": [
+        {
+          "platform": "GITHUB",
+          "url": "https://github.com/Alamofire"
+        },
+        {
+          "platform": "LFX_CROWDFUNDING",
+          "url": "https://crowdfunding.lfx.linuxfoundation.org/projects/alamofire"
+        }
+      ],
       "isArchived": false,
       "isFork": false,
       "licenseInfo": {

Tests/AppTests/GithubTests.swift

@@ -136,6 +136,10 @@ class GithubTests: AppTestCase {
         XCTAssertEqual(res.repository?.closedPullRequests.nodes.first!.closedAt,
                        iso8601.date(from: "2021-05-28T15:50:17Z"))
         XCTAssertEqual(res.repository?.forkCount, 6727)
+        XCTAssertEqual(res.repository?.fundingLinks, [
+            .init(platform: .gitHub, url: "https://github.com/Alamofire"),
+            .init(platform: .lfxCrowdfunding, url: "https://crowdfunding.lfx.linuxfoundation.org/projects/alamofire"),
+        ])
         XCTAssertEqual(res.repository?.mergedPullRequests.nodes.first!.closedAt,
                        iso8601.date(from: "2021-06-07T22:47:01Z"))
         XCTAssertEqual(res.repository?.name, "Alamofire")

Tests/AppTests/IngestorTests.swift

@@ -110,6 +110,11 @@ class IngestorTests: AppTestCase {
         let repo = Repository(packageId: try pkg.requireID())
         let md: Github.Metadata = .init(defaultBranch: "main",
                                         forks: 1,
+                                        fundingLinks: [
+                                            .init(platform: .gitHub, url: "https://github.com/username"),
+                                            .init(platform: .customUrl, url: "https://example.com/username1"),
+                                            .init(platform: .customUrl, url: "https://example.com/username2")
+                                        ],
                                         homepageUrl: "https://swiftpackageindex.com/Alamofire/Alamofire",
                                         isInOrganization: true,
                                         issuesClosedAtDates: [
@@ -153,6 +158,11 @@ class IngestorTests: AppTestCase {
             let repo = try await Repository.query(on: app.db).first().unwrap()
             XCTAssertEqual(repo.defaultBranch, "main")
             XCTAssertEqual(repo.forks, 1)
+            XCTAssertEqual(repo.fundingLinks, [
+                .init(platform: .gitHub, url: "https://github.com/username"),
+                .init(platform: .customUrl, url: "https://example.com/username1"),
+                .init(platform: .customUrl, url: "https://example.com/username2")
+            ])
             XCTAssertEqual(repo.homepageUrl, "https://swiftpackageindex.com/Alamofire/Alamofire")
             XCTAssertEqual(repo.isInOrganization, true)
             XCTAssertEqual(repo.keywords, ["bar", "baz", "foo"])

Tests/AppTests/Mocks/GithubMetadata+mock.swift

@@ -55,6 +55,7 @@ extension Github.Metadata {
 
     init(defaultBranch: String,
          forks: Int,
+         fundingLinks: [FundingLinkNode] = [],
          homepageUrl: String?,
          isInOrganization: Bool,
          issuesClosedAtDates: [Date],
@@ -81,6 +82,7 @@ extension Github.Metadata {
                               defaultBranchRef: .init(name: defaultBranch),
                               description: summary,
                               forkCount: forks,
+                              fundingLinks: fundingLinks,
                               homepageUrl: homepageUrl,
                               isArchived: false,
                               isFork: false,