Move feed generation into main codebase [#456]

This integrates code from an old POC rewrite of clojars that only
existed on the server, and was used to generate `/repo/feed.clj.gz`.

dev-resources/fake-0.0.3-SNAPSHOT/fake.pom

@@ -9,6 +9,7 @@
 
   <name>asdf</name>
 
+  <description>FAKE</description>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
   </properties>

dev-resources/test-0.0.3-SNAPSHOT/test.pom

@@ -8,6 +8,9 @@
   <packaging>jar</packaging>
   <name>asdf</name>
 
+  <description>TEST</description>
+  <url>http://example.com</url>
+
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
   </properties>

src/clojars/maven.clj

@@ -5,27 +5,45 @@
             [clojars.errors :refer [report-error]])
   (:import org.apache.maven.model.io.xpp3.MavenXpp3Reader
            org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader
-           java.io.IOException))
-
-(defn model-to-map [model]
-  {:name (or (.getArtifactId model)
-             (-> model .getParent .getArtifactId))
-   :group (or (.getGroupId model)
-              (-> model .getParent .getGroupId))
-   :version (or (.getVersion model)
-                (-> model .getParent .getVersion))
-   :description (.getDescription model)
-   :homepage (.getUrl model)
-   :url (.getUrl model)
-   :licenses (.getLicenses model)
-   :scm (.getScm model)
-   :authors (vec (map #(.getName %) (.getContributors model)))
-   :dependencies (vec (map
-                       (fn [d] {:group_name (.getGroupId d)
-                                :jar_name (.getArtifactId d)
-                                :version (.getVersion d)
-                                :scope (or (.getScope d) "compile")})
-                       (.getDependencies model)))})
+           java.io.IOException
+           (org.apache.maven.model Scm Model)))
+(defn without-nil-values
+  "Prunes a map of pairs that have nil values."
+  [m]
+  (reduce (fn [m entry]
+            (if (nil? (val entry))
+              m
+              (conj m entry)))
+          (empty m) m))
+
+(defn model-to-map [^Model model]
+  (without-nil-values
+    {:name         (or (.getArtifactId model)
+                       (-> model .getParent .getArtifactId))
+     :group        (or (.getGroupId model)
+                       (-> model .getParent .getGroupId))
+     :version      (or (.getVersion model)
+                       (-> model .getParent .getVersion))
+     :description  (.getDescription model)
+     :homepage     (.getUrl model)
+     :url          (.getUrl model)
+     :licenses     (.getLicenses model)
+     :scm          (.getScm model)
+     :authors      (vec (map #(.getName %) (.getContributors model)))
+     :dependencies (vec (map
+                          (fn [d] {:group_name (.getGroupId d)
+                                   :jar_name   (.getArtifactId d)
+                                   :version    (.getVersion d)
+                                   :scope      (or (.getScope d) "compile")})
+                          (.getDependencies model)))}))
+
+(defn scm-to-map [^Scm scm]
+  (when scm
+    (without-nil-values
+      {:connection           (.getConnection scm)
+       :developer-connection (.getDeveloperConnection scm)
+       :tag                  (.getTag scm)
+       :url                  (.getUrl scm)})))
 
 (defn read-pom
   "Reads a pom file returning a maven Model object."
@@ -80,3 +98,64 @@
         url (and scm (.getUrl scm))
         base-url (re-find #"https?://github.com/[^/]+/[^/]+" (str url))]
     (if (and base-url (.getTag scm)) (str base-url "/commit/" (.getTag scm)))))
+
+(defn parse-int [^String s]
+  (when s
+    (Integer/parseInt s)))
+
+(defn parse-version
+  "Parse a Maven-style version number.
+
+  The basic format is major[.minor[.increment]][(-|.)(buildNumber|qualifier)]
+
+  The major, minor, increment and buildNumber are numeric with leading zeros
+  disallowed (except plain 0).  If the value after the first - is non-numeric
+  then it is assumed to be a qualifier.  If the format does not match then we
+  just treat the whole thing as a qualifier."
+  [s]
+  (let [[match major minor incremental _ _ build-number qualifier]
+        (re-matches #"(0|[1-9][0-9]*)(?:\.(0|[1-9][0-9]*)(?:\.(0|[1-9][0-9]*))?)?(?:(-|\.)((0|[1-9][0-9]*)|(.*)))?" s)]
+    (try
+      (without-nil-values
+        {:major        (parse-int major)
+         :minor        (parse-int minor)
+         :incremental  (parse-int incremental)
+         :build-number (parse-int build-number)
+         :qualifier    (if match qualifier s)})
+      (catch NumberFormatException _
+        {:qualifier s}))))
+
+(defmacro numeric-or
+  "Evaluates exprs one at a time.  Returns the first that returns non-zero."
+  ([x] x)
+  ([x & exprs]
+   `(let [value# ~x]
+      (if (zero? value#)
+        (numeric-or ~@exprs)
+        value#))))
+
+(defn compare-versions
+  "Compare two maven versions.  Accepts either the string or parsed
+  representation."
+  [x y]
+  (let [x (if (string? x) (parse-version x) x)
+        y (if (string? y) (parse-version y) y)]
+    (numeric-or
+      (compare (:major x 0) (:major y 0))
+      (compare (:minor x 0) (:minor y 0))
+      (compare (:incremental x 0) (:incremental y 0))
+      (let [qx (:qualifier x)
+            qy (:qualifier y)]
+        (if qx
+          (if qy
+            (if (and (> (count qx) (count qy)) (.startsWith qx qy))
+              -1                      ; x is longer, it's older
+              (if (and (< (count qx) (count qy)) (.startsWith qx qy))
+                1                     ; y is longer, it's older
+                (compare qx qy)))     ; same length, so string compare
+            -1)                       ; y has no qualifier, it's younger
+          (if qy
+            1                         ; x has no qualifier, it's younger
+            0)))                      ; no qualifiers
+      (compare (:build-number x 0) (:build-number y 0)))))
+

src/clojars/tools/generate_feed.clj

@@ -0,0 +1,45 @@
+(ns clojars.tools.generate-feed
+  (:require [clojure.java.io :as io]
+            [clojars.maven :as maven]
+            [clojure.set :as set])
+  (:import java.util.zip.GZIPOutputStream
+           (java.io FileOutputStream PrintWriter)))
+
+(defn pom-seq [repo]
+  (for [f (file-seq repo)
+        :when (and (not (re-matches #".*/\..*" (str f)))
+                   (.endsWith (.getName f) ".pom"))
+        :let [pom (try
+                    (-> (maven/pom-to-map f)
+                        (update :scm maven/scm-to-map)
+                        maven/without-nil-values)
+                    (catch Exception e (.printStackTrace e)))]
+        :when pom]
+    pom))
+
+(defn full-feed [repo]
+  (let [grouped-jars (->> (pom-seq repo)
+                          (map (comp #(select-keys % [:group-id :artifact-id :version
+                                                      :description :scm :url
+                                                      :homepage])
+                                     #(set/rename-keys % {:group :group-id
+                                                          :name  :artifact-id})))
+                          (group-by (juxt :group-id :artifact-id))
+                          (vals))]
+    (for [jars grouped-jars]
+      (let [jars (sort-by :version #(maven/compare-versions %2 %1) jars)]
+        (-> (first jars)
+            (dissoc :version)
+            (assoc :versions (vec (distinct (map :version jars)))))))))
+
+(defn write-feed! [feed f]
+  (with-open [w (-> (FileOutputStream. f)
+                    (GZIPOutputStream.)
+                    (PrintWriter.))]
+    (binding [*out* w]
+      (doseq [form feed]
+        (prn form)))))
+
+(defn -main [repo dest]
+  (write-feed! (full-feed (io/file repo)) (str dest "/feed.clj.gz")))
+

test/clojars/test/integration/api.clj

@@ -54,8 +54,8 @@
               :jar_name "test"
               :group_name "fake"
               :user "dantheman"
-              :description nil
-              :homepage nil
+              :description "TEST"
+              :homepage "http://example.com"
               :downloads 0}
              (first body)))))
 
@@ -70,8 +70,8 @@
               :jar_name "test"
               :group_name "fake"
               :user "dantheman"
-              :description nil
-              :homepage nil
+              :description "TEST"
+              :homepage "http://example.com"
               :downloads 0
               :recent_versions [{:downloads 0 :version "0.0.3-SNAPSHOT"}
                                 {:downloads 0 :version "0.0.2"}

test/clojars/test/unit/maven.clj

@@ -64,3 +64,16 @@
         (snapshot-pom-file-with {:group_name "fake"
                                  :jar_name "test"
                                  :version "0.2.1-alpha-SNAPSHOT"}))))
+
+;; this might be a good candidate for test.check
+(deftest comparing-versions
+  (are [op v1 v2] (op (compare-versions v1 v2) 0)
+    = "0.0.1"         "0.0.1"
+    > "0.0.1"         "0.0.1-SNAPSHOT"
+    > "0.0.1-alpha2"  "0.0.1-alpha1"
+    > "0.0.1-alpha22" "0.0.1-alpha1"
+    < "0.0.1"         "0.0.2"
+    < "1.0.0"         "2.0.0"
+    < "8.2.0.Final"   "10.0.0.CR5"
+    < "1.2.3.455"     "1.2.3.456"
+    < "1"             "2"))

test/clojars/test/unit/tools/generate_feed.clj

@@ -0,0 +1,35 @@
+(ns clojars.test.unit.tools.generate-feed
+  (:require [clojars.tools.generate-feed :refer :all]
+            [clojars.config :refer [config]]
+            [clojure.test :refer :all]
+            [clojars.test.test-helper :as help]
+            [clojure.java.io :as io]))
+
+(use-fixtures :each help/default-fixture)
+
+(defn setup-repo []
+  (doseq [name ["fake" "test"]
+          version ["0.0.1" "0.0.2" "0.0.3-SNAPSHOT"]
+          :let [dest-file (io/file (:repo config)
+                                   (format "%s/%s/%s/%s-%s.pom" name name version name version))]]
+    (.mkdirs (.getParentFile dest-file))
+    (io/copy (io/reader (io/resource (format "%s-%s/%s.pom" name version name)))
+             dest-file)))
+
+(deftest feed-generation-should-work
+  (setup-repo)
+  (let [expected [{:description "FAKE"
+                   :group-id    "fake"
+                   :artifact-id "fake"
+                   :versions    ["0.0.3-SNAPSHOT" "0.0.2" "0.0.1"]}
+                  {:description "TEST"
+                   :scm         {:connection           "scm:git:git://github.com/fake/test.git"
+                                 :developer-connection "scm:git:ssh://[email protected]/fake/test.git"
+                                 :tag                  "70470ff6ae74505bdbfe5955fca6797f613c113c"
+                                 :url                  "https://github.com/fake/test"}
+                   :group-id    "fake"
+                   :artifact-id "test"
+                   :url         "http://example.com"
+                   :homepage    "http://example.com"
+                   :versions    ["0.0.3-SNAPSHOT" "0.0.2" "0.0.1"]}]]
+    (is (= expected (full-feed (io/file (:repo config)))))))