Add endpoint to receive token found request from GitHub

This adds an endpoint that allows GitHub to report when a token is
found within code on GH from their secret scanning program[1].

When a token is reported, we disable it, then send the user who owns
the token an email letting them know.

[1]: https://developer.github.com/partnerships/secret-scanning/

dev-resources/ecdsa-key-pub.pem

@@ -0,0 +1,4 @@
+-----BEGIN PUBLIC KEY-----
+MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEUloBIEfuiCgASICzlMGgxcbAMejL
+TO6FUgzXNYGoYZtnYMFcRQruWJNZ8z0weh5phOewpuWeY9T4CeMKryFD8g==
+-----END PUBLIC KEY-----

dev-resources/ecdsa-key.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHgCAQEEIQCQ3pY8T1UdFtylrXXN7MQsZrVRbGEYvg9cW/pfKJLy/6AKBggqhkjO
+PQMBB6FEA0IABFJaASBH7ogoAEiAs5TBoMXGwDHoy0zuhVIM1zWBqGGbZ2DBXEUK
+7liTWfM9MHoeaYTnsKblnmPU+AnjCq8hQ/I=
+-----END EC PRIVATE KEY-----

project.clj

@@ -2,7 +2,8 @@
   :min-lein-version "2.0.0"
   :dependencies [[org.clojure/clojure "1.10.1"]
                  [org.clojure/core.memoize "0.8.2"]
-                 [raven-clj "1.4.3"]
+                 [raven-clj "1.4.3"
+                  :exclusions [cheshire]]
                  [org.apache.maven/maven-model "3.0.4"
                   :exclusions
                   [org.codehaus.plexus/plexus-utils]]
@@ -31,6 +32,8 @@
                                org.clojure/core.cache
                                ring/ring-core
                                slingshot]]
+                 [buddy/buddy-core "1.6.0"
+                  :exclusions [commons-codec]]
                  [clj-stacktrace "0.2.8"]
                  [clj-time "0.11.0"]
                  [ring/ring-anti-forgery "1.0.1"

resources/queries/queryfile.sql

@@ -4,6 +4,12 @@ FROM users
 WHERE "user" = :username
 LIMIT 1;
 
+--name: find-user-by-id
+SELECT *
+FROM users
+WHERE id = :id
+LIMIT 1;
+
 --name: find-user-by-user-or-email
 SELECT *
 FROM users
@@ -39,6 +45,10 @@ select *
 FROM deploy_tokens
 WHERE id = :id;
 
+--name: all-tokens
+select *
+FROM deploy_tokens;
+
 --name: find-groupnames
 SELECT name
 FROM groups

src/clojars/db.clj

@@ -5,6 +5,7 @@
             [clojars.config :refer [config]]
             [clojars.db.sql :as sql]
             [clojars.maven :as mvn]
+            [clojars.util :refer [filter-some]]
             [clojure.edn :as edn]
             [clojure.set :as set]
             [clojure.string :as str])
@@ -63,6 +64,7 @@
     "terms"
     "test"
     "testing"
+    "token-breach"
     "tokens"
     "upload"
     "user"
@@ -93,6 +95,11 @@
                                         {:connection db
                                          :result-set-fn first}))
 
+(defn find-user-by-id [db id]
+  (sql/find-user-by-id {:id id}
+                       {:connection db
+                        :result-set-fn first}))
+
 (defn find-user-tokens-by-username [db username]
   (sql/find-user-tokens-by-username {:username username}
     {:connection db}))
@@ -102,6 +109,14 @@
     {:connection db
      :result-set-fn first}))
 
+(defn find-token-by-value
+  "Finds a token with the matching value. This is somewhat expensive,
+  since it scans all tokens."
+  [db token-value]
+  (sql/all-tokens {}
+                  {:connection db
+                   :result-set-fn (partial filter-some #(creds/bcrypt-verify token-value (:token %)))}))
+
 (defn find-groupnames [db username]
   (sql/find-groupnames {:username username}
                        {:connection db

src/clojars/routes/token_breach.clj

@@ -0,0 +1,68 @@
+(ns clojars.routes.token-breach
+  (:require
+   [buddy.core.codecs.base64 :as base64]
+   [buddy.core.dsa :as dsa]
+   [buddy.core.keys :as keys]
+   [cheshire.core :as json]
+   [clj-http.client :as client]
+   [clojars.db :as db]
+   [compojure.core :as compojure :refer [POST]]
+   [ring.util.response :as response]))
+
+(defn- get-github-key
+  "Retrieves the public key text from the github api for the key
+  identifier, then converts the key text to a key object."
+  [identifier]
+  (->> (client/get "https://api.github.com/meta/public_keys/token_scanning"
+                   {:as :json})
+       :body
+       :public_keys
+       (some (fn [{:keys [key_identifier key]}]
+               (when (= identifier key_identifier)
+                 key)))
+       (keys/str->public-key)))
+
+(defn- valid-github-request?
+  "Verifies the request was signed using GitHub's key.
+  https://developer.github.com/partnerships/secret-scanning/"
+  [headers body-str]
+  (let [key-id (get headers "github-public-key-identifier")
+        key-sig (get headers "github-public-key-signature")
+        key (get-github-key key-id)
+        sig (base64/decode key-sig)]
+    (dsa/verify body-str sig {:key key :alg :ecdsa+sha256})))
+
+(defn- send-email
+  [mailer {:as _user :keys [email]} {:as _token :keys [disabled name]} url]
+  (mailer
+   email
+   "Deploy token found on GitHub"
+   (->> ["Hello,"
+         (format
+          "We received a notice from GitHub that your deploy token named '%s' was found by their secret scanning service."
+          name)
+         (format "The commit was found at: %s" url)
+         (if disabled
+           "The token was already disabled, so we took no further action."
+           "This token has been disabled to prevent malicious use.")]
+        (interpose "\n\n")
+        (apply str))))
+
+(defn- handle-github-token-breach
+  [db mailer {:as _request :keys [headers body]}]
+  (let [body-str (slurp body)]
+    (if (valid-github-request? headers body-str)
+      (let [data (json/parse-string body-str true)]
+        (doseq [{:keys [token url]} data]
+          (when-let [{:as db-token :keys [id disabled user_id]}
+                     (db/find-token-by-value db token)]
+            (when (not disabled)
+              (db/disable-deploy-token db id))
+            (send-email mailer (db/find-user-by-id db user_id) db-token url)))
+        (response/status 200))
+      (response/status 422))))
+
+(defn routes [db mailer]
+  (compojure/routes
+   (POST "/token-breach/github" request
+         (handle-github-token-breach db mailer request))))

src/clojars/util.clj

@@ -0,0 +1,6 @@
+(ns clojars.util)
+
+(defn filter-some
+  "Returns the first x in coll where (pred x) returns logical true, else nil."
+  [pred coll]
+  (some #(when (pred %) %) coll))

src/clojars/web.clj

@@ -11,14 +11,14 @@
              [http-utils :refer [wrap-x-frame-options wrap-secure-session]]
              [middleware :refer [wrap-ignore-trailing-slash]]]
             [clojars.friend.registration :as registration]
-            [clojars.routes
-             [api :as api]
-             [artifact :as artifact]
-             [group :as group]
-             [repo :as repo]
-             [session :as session]
-             [token :as token]
-             [user :as user]]
+            [clojars.routes.api :as api]
+            [clojars.routes.artifact :as artifact]
+            [clojars.routes.group :as group]
+            [clojars.routes.repo :as repo]
+            [clojars.routes.session :as session]
+            [clojars.routes.token :as token]
+            [clojars.routes.token-breach :as token-breach]
+            [clojars.routes.user :as user]
             [clojars.web
              [browse :refer [browse]]
              [common :refer [html-doc]]
@@ -132,7 +132,9 @@
             (repo/wrap-exceptions reporter)
             (repo/wrap-file (:repo (config)))
             (repo/wrap-reject-double-dot)))
-      (wrap-secure-session))
+        (wrap-secure-session))
+    (-> (token-breach/routes db mailer)
+        (wrap-exceptions reporter))
    (-> (main-routes db reporter stats search mailer)
        (friend/authenticate
         {:credential-fn (password-credential-fn db)

test/clojars/unit/web/token_breach_test.clj

@@ -0,0 +1,72 @@
+(ns clojars.unit.web.token-breach-test
+  (:require
+   [buddy.core.codecs.base64 :as base64]
+   [buddy.core.dsa :as dsa]
+   [buddy.core.keys :as keys]
+   [cheshire.core :as json]
+   [clj-http.client :as client]
+   [clojars.db :as db]
+   [clojars.test-helper :as help]
+   [clojars.util :refer [filter-some]]
+   [clojure.java.io :as io]
+   [clojure.test :refer [deftest is testing use-fixtures]]
+   [ring.mock.request :refer [body header request]]))
+
+(use-fixtures :each
+  help/default-fixture
+  help/with-clean-database)
+
+(def privkey (keys/private-key (io/resource "ecdsa-key.pem")))
+(def pubkey-str (slurp (io/resource "ecdsa-key-pub.pem")))
+(def github-response {:public_keys [{:key_identifier "abcd"
+                                     :key pubkey-str
+                                     :is_current true}]})
+
+(defn- build-breach-request
+  [token-value]
+  (let [payload [{:token token-value
+                  :type "whatever"
+                  :url "https://github.com/foo/bar"}]
+        payload-str (json/encode payload)
+        sig (dsa/sign payload-str {:key privkey :alg :ecdsa+sha256})
+        sig-b64 (String. (base64/encode sig))]
+    (-> (request :post "/token-breach/github")
+        (body payload-str)
+        (header "Content-Type" "application/json")
+        (header "GITHUB-PUBLIC-KEY-IDENTIFIER" "abcd")
+        (header "GITHUB-PUBLIC-KEY-SIGNATURE" sig-b64))))
+
+(defn- find-token [username token-name]
+  (filter-some #(= token-name (:name %))
+               (db/find-user-tokens-by-username help/*db* username)))
+
+(deftest test-github-token-breach-reporting-works
+  (let [_user (db/add-user help/*db* "[email protected]" "ham" "biscuit")
+        mailer-args (atom nil)
+        app (help/app {:mailer (fn [& args] (reset! mailer-args args))})]
+    (with-redefs [client/get (constantly {:body github-response})]
+      (testing "when token is enabled"
+        (let [token (db/add-deploy-token help/*db* "ham" "a token")
+              res (app (build-breach-request (:token token)))
+              db-token (find-token "ham" "a token")
+              [to subject message] @mailer-args]
+          (is (= 200 (:status res)))
+          (is (:disabled db-token))
+          (is (= "[email protected]" to))
+          (is (= "Deploy token found on GitHub" subject))
+          (is (re-find #"'a token'" message))
+          (is (re-find #"https://github.com/foo/bar" message))
+          (is (re-find #"has been disabled" message))))
+
+      (testing "when token is disabled"
+        (let [token (db/add-deploy-token help/*db* "ham" "another token")
+              db-token (find-token "ham" "another token")
+              _ (db/disable-deploy-token help/*db* (:id db-token))
+              res (app (build-breach-request (:token token)))
+              [to subject message] @mailer-args]
+          (is (= 200 (:status res)))
+          (is (= "[email protected]" to))
+          (is (= "Deploy token found on GitHub" subject))
+          (is (re-find #"'another token'" message))
+          (is (re-find #"https://github.com/foo/bar" message))
+          (is (re-find #"was already disabled" message)))))))