From 0248f2da58e11ffb0bee2d76c4b248d9b3e37253 Mon Sep 17 00:00:00 2001 From: Nina Blanson Date: Sun, 6 Mar 2016 01:44:23 -0600 Subject: [PATCH 1/8] First pass at adding batch inserts --- project.clj | 2 +- src/yesql/generate.clj | 23 +++++++++++++++-------- test/yesql/acceptance_test.clj | 33 +++++++++++++++------------------ 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/project.clj b/project.clj index e9cb7cb..1dc4a42 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject yesql "0.5.2" +(defproject yesql "0.6.0" :description "A Clojure library for using SQL" :url "https://github.com/krisajenkins/yesql" :license {:name "Eclipse Public License" diff --git a/src/yesql/generate.clj b/src/yesql/generate.clj index 9ff489a..2856f1c 100644 --- a/src/yesql/generate.clj +++ b/src/yesql/generate.clj @@ -1,7 +1,7 @@ (ns yesql.generate (:require [clojure.java.jdbc :as jdbc] [clojure.set :as set] - [clojure.string :refer [join lower-case]] + [clojure.string :refer [join lower-case split]] [yesql.util :refer [create-root-var]] [yesql.types :refer [map->Query]] [yesql.statement-parser :refer [tokenize]]) @@ -72,21 +72,26 @@ ;; case, we only ever use one group, so we'll unpack the ;; single-element list with `first`. (defn execute-handler - [db sql-and-params call-options] - (first (jdbc/execute! db sql-and-params))) + [db sql params call-options] + (first (jdbc/execute! db (rewrite-query-for-jdbc sql params)))) (defn insert-handler - [db [statement & params] call-options] - (jdbc/db-do-prepared-return-keys db statement params)) + [db sql params call-options] + (let [table-string-position 0 + table-position 2 + table-keyword (keyword ((split (sql table-string-position) #" ") table-position))] + (if (vector? params) + (apply jdbc/insert! db table-keyword params) + (jdbc/insert! db table-keyword params)))) (defn query-handler - [db sql-and-params + [db sql params {:keys [row-fn result-set-fn identifiers] :or {identifiers lower-case row-fn identity result-set-fn doall} :as call-options}] - (jdbc/query db sql-and-params + (jdbc/query db (rewrite-query-for-jdbc sql params) :identifiers identifiers :row-fn row-fn :result-set-fn result-set-fn)) @@ -118,7 +123,9 @@ "Check the docs, and supply {:connection ...} as an option to the function call, or globally to the defquery declaration."]) name)) (jdbc-fn connection - (rewrite-query-for-jdbc tokens args) + tokens + args + ;(rewrite-query-for-jdbc tokens args) call-options))) [display-args generated-function] (let [named-args (if-let [as-vec (seq (mapv (comp symbol clojure.core/name) required-args))] diff --git a/test/yesql/acceptance_test.clj b/test/yesql/acceptance_test.clj index 069f9ae..e710dc0 100644 --- a/test/yesql/acceptance_test.clj +++ b/test/yesql/acceptance_test.clj @@ -26,16 +26,14 @@ (expect (create-person-table!)) ;; Insert -> Select. -(expect {:1 1M} (insert-person Select. (expect 1 (delete-person! {:name "Alice"})) -(expect 2 (count (find-older-than {:age 10}))) -(expect 1 (count (find-older-than {:age 30}))) -(expect 0 (count (find-older-than {:age 50}))) +(expect 4 (count (find-older-than {:age 10}))) +(expect 3 (count (find-older-than {:age 30}))) +(expect 1 (count (find-older-than {:age 50}))) ;; Failing transaction: Insert with abort. ;; Insert two rows in a transaction. The second throws a deliberate error, meaning no new rows created. -(expect 2 (count (find-older-than {:age 10}))) +(expect 4 (count (find-older-than {:age 10}))) (expect SQLException (jdbc/with-db-transaction [connection derby-db] @@ -70,8 +68,7 @@ :age 25} {:connection connection} ))) -(expect 2 - (count (find-older-than {:age 10}))) +(expect 4 (count (find-older-than {:age 10}))) ;;; Type error. (expect SQLDataException From be0bb7bc137c3dea28fa87a70bbdf11887c6737f Mon Sep 17 00:00:00 2001 From: Nina Blanson Date: Sun, 6 Mar 2016 01:57:14 -0600 Subject: [PATCH 2/8] Removing commented out code --- src/yesql/generate.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/yesql/generate.clj b/src/yesql/generate.clj index 2856f1c..de472a1 100644 --- a/src/yesql/generate.clj +++ b/src/yesql/generate.clj @@ -125,7 +125,6 @@ (jdbc-fn connection tokens args - ;(rewrite-query-for-jdbc tokens args) call-options))) [display-args generated-function] (let [named-args (if-let [as-vec (seq (mapv (comp symbol clojure.core/name) required-args))] From c2c1cd852b5d912df8beb3b82fbc1a47b2fdac04 Mon Sep 17 00:00:00 2001 From: Nina Blanson Date: Sun, 6 Mar 2016 02:06:46 -0600 Subject: [PATCH 3/8] Fixing unit tests --- test/yesql/acceptance_test.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/yesql/acceptance_test.clj b/test/yesql/acceptance_test.clj index e710dc0..43117b6 100644 --- a/test/yesql/acceptance_test.clj +++ b/test/yesql/acceptance_test.clj @@ -29,7 +29,7 @@ (expect (list {:1 1M}) (insert-person Date: Sun, 6 Mar 2016 02:31:35 -0600 Subject: [PATCH 4/8] Re-adding db-do-prepared-return-keys to retain backwards compatibility --- src/yesql/generate.clj | 13 +++++++------ test/yesql/acceptance_test.clj | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/yesql/generate.clj b/src/yesql/generate.clj index de472a1..f3676cc 100644 --- a/src/yesql/generate.clj +++ b/src/yesql/generate.clj @@ -77,12 +77,13 @@ (defn insert-handler [db sql params call-options] - (let [table-string-position 0 - table-position 2 - table-keyword (keyword ((split (sql table-string-position) #" ") table-position))] - (if (vector? params) - (apply jdbc/insert! db table-keyword params) - (jdbc/insert! db table-keyword params)))) + (if (vector? params) + (let [table-string-position 0 + table-position 2 + table-keyword (keyword ((split (sql table-string-position) #" ") table-position))] + (apply jdbc/insert! db table-keyword params)) + (let [[rewritten-sql & rewritten-params] (rewrite-query-for-jdbc sql params)] + (jdbc/db-do-prepared-return-keys db rewritten-sql rewritten-params)))) (defn query-handler [db sql params diff --git a/test/yesql/acceptance_test.clj b/test/yesql/acceptance_test.clj index 43117b6..5591fb8 100644 --- a/test/yesql/acceptance_test.clj +++ b/test/yesql/acceptance_test.clj @@ -26,9 +26,9 @@ (expect (create-person-table!)) ;; Insert -> Select. -(expect (list {:1 1M}) (insert-person Date: Mon, 14 Mar 2016 12:59:32 -0500 Subject: [PATCH 5/8] using regex to find table names --- src/yesql/generate.clj | 11 +++++------ src/yesql/statement_parser.clj | 2 ++ test/yesql/statement_parser_test.clj | 10 ++++++++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/yesql/generate.clj b/src/yesql/generate.clj index f3676cc..96bfd64 100644 --- a/src/yesql/generate.clj +++ b/src/yesql/generate.clj @@ -1,10 +1,10 @@ (ns yesql.generate (:require [clojure.java.jdbc :as jdbc] [clojure.set :as set] - [clojure.string :refer [join lower-case split]] + [clojure.string :refer [join lower-case trim]] [yesql.util :refer [create-root-var]] [yesql.types :refer [map->Query]] - [yesql.statement-parser :refer [tokenize]]) + [yesql.statement-parser :refer [tokenize insert-table-name-regex]]) (:import [yesql.types Query])) (def in-list-parameter? @@ -78,10 +78,9 @@ (defn insert-handler [db sql params call-options] (if (vector? params) - (let [table-string-position 0 - table-position 2 - table-keyword (keyword ((split (sql table-string-position) #" ") table-position))] - (apply jdbc/insert! db table-keyword params)) + (let [full-query (apply str sql) + table-name (re-find insert-table-name-regex full-query)] + (apply jdbc/insert! db table-name params)) (let [[rewritten-sql & rewritten-params] (rewrite-query-for-jdbc sql params)] (jdbc/db-do-prepared-return-keys db rewritten-sql rewritten-params)))) diff --git a/src/yesql/statement_parser.clj b/src/yesql/statement_parser.clj index 7d8b798..91762a7 100644 --- a/src/yesql/statement_parser.clj +++ b/src/yesql/statement_parser.clj @@ -6,6 +6,8 @@ [yesql.instaparse-util :refer [process-instaparse-result]]) (:import [yesql.types Query])) +(def insert-table-name-regex #"(?i)(?<=INSERT INTO\s).*?(?=\sVALUES|\s\()") + (def parser (instaparse/parser (io/resource "yesql/statement.bnf"))) diff --git a/test/yesql/statement_parser_test.clj b/test/yesql/statement_parser_test.clj index 7cd62ff..f793d36 100644 --- a/test/yesql/statement_parser_test.clj +++ b/test/yesql/statement_parser_test.clj @@ -4,6 +4,16 @@ [yesql.types :refer [map->Query]] [yesql.statement-parser :refer :all])) +(do-template [statement _ table-name-result] + (do (expect (quote table-name-result) + (re-find insert-table-name-regex statement))) + + "INSERT INTO table (:column1, :column2) VALUES (a, b)" => "table" + "insert into table (:column1, :column2) VALUES (a, b)" => "table" + "INSERT INTO table VALUES (a, b)" => "table" + "insert into table VALUES (a, b)" => "table" + "insert into table values (a, b)" => "table") + (do-template [statement _ split-result] (do (expect (quote split-result) (tokenize statement)) From e80b0b29143b7658618854a925b222fbfcf88985 Mon Sep 17 00:00:00 2001 From: Nina Blanson Date: Mon, 14 Mar 2016 13:07:29 -0500 Subject: [PATCH 6/8] removing trim reference removing trim reference since regex takes care of it now --- src/yesql/generate.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yesql/generate.clj b/src/yesql/generate.clj index 96bfd64..aa664d2 100644 --- a/src/yesql/generate.clj +++ b/src/yesql/generate.clj @@ -1,7 +1,7 @@ (ns yesql.generate (:require [clojure.java.jdbc :as jdbc] [clojure.set :as set] - [clojure.string :refer [join lower-case trim]] + [clojure.string :refer [join lower-case]] [yesql.util :refer [create-root-var]] [yesql.types :refer [map->Query]] [yesql.statement-parser :refer [tokenize insert-table-name-regex]]) From 81d26345aafa2f3bc4c1bef647a509b752c57917 Mon Sep 17 00:00:00 2001 From: Nina Blanson Date: Mon, 14 Mar 2016 13:59:36 -0500 Subject: [PATCH 7/8] move query assertions into separate fn moving query assertions, making it compatible with vec --- src/yesql/generate.clj | 48 +++++++++++++++++++--------------- test/yesql/acceptance_test.clj | 7 ++++- test/yesql/generate_test.clj | 17 ++++++++---- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/src/yesql/generate.clj b/src/yesql/generate.clj index aa664d2..4f30c71 100644 --- a/src/yesql/generate.clj +++ b/src/yesql/generate.clj @@ -32,10 +32,11 @@ expected-keys (conj expected-keys :?)))) -(defn rewrite-query-for-jdbc - [tokens initial-args] +(defn sane-query? [tokens initial-args] (let [{:keys [expected-keys expected-positional-count]} (analyse-statement-tokens tokens) - actual-keys (set (keys (dissoc initial-args :?))) + actual-keys (if (vector? initial-args) + (-> (mapcat keys initial-args) set (disj :?)) + (-> (keys initial-args) set (disj :?))) actual-positional-count (count (:? initial-args)) missing-keys (set/difference expected-keys actual-keys)] (assert (empty? missing-keys) @@ -48,24 +49,28 @@ ["Query argument mismatch." "Expected %d positional parameters. Got %d." "Supply positional parameters as {:? [...]}"]) - expected-positional-count actual-positional-count)) - (let [[final-query final-parameters consumed-args] - (reduce (fn [[query parameters args] token] - (cond - (string? token) [(str query token) - parameters - args] - (symbol? token) (let [[arg new-args] (if (= '? token) - [(first (:? args)) (update-in args [:?] rest)] - [(get args (keyword token)) args])] - [(str query (args-to-placeholders arg)) - (vec (if (in-list-parameter? arg) - (concat parameters arg) - (conj parameters arg))) - new-args]))) - ["" [] initial-args] - tokens)] - (concat [final-query] final-parameters)))) + expected-positional-count actual-positional-count)))) + +(defn rewrite-query-for-jdbc + [tokens initial-args] + (sane-query? tokens initial-args) + (let [[final-query final-parameters consumed-args] + (reduce (fn [[query parameters args] token] + (cond + (string? token) [(str query token) + parameters + args] + (symbol? token) (let [[arg new-args] (if (= '? token) + [(first (:? args)) (update-in args [:?] rest)] + [(get args (keyword token)) args])] + [(str query (args-to-placeholders arg)) + (vec (if (in-list-parameter? arg) + (concat parameters arg) + (conj parameters arg))) + new-args]))) + ["" [] initial-args] + tokens)] + (concat [final-query] final-parameters))) ;; Maintainer's note: clojure.java.jdbc.execute! returns a list of ;; rowcounts, because it takes a list of parameter groups. In our @@ -80,6 +85,7 @@ (if (vector? params) (let [full-query (apply str sql) table-name (re-find insert-table-name-regex full-query)] + (sane-query? sql params) (apply jdbc/insert! db table-name params)) (let [[rewritten-sql & rewritten-params] (rewrite-query-for-jdbc sql params)] (jdbc/db-do-prepared-return-keys db rewritten-sql rewritten-params)))) diff --git a/test/yesql/acceptance_test.clj b/test/yesql/acceptance_test.clj index 5591fb8..5e8f0bf 100644 --- a/test/yesql/acceptance_test.clj +++ b/test/yesql/acceptance_test.clj @@ -66,7 +66,7 @@ {:connection connection}) (insert-person ["SELECT * FROM users WHERE group_ids IN(?,?) AND parent_id = ?" 1 2 3]) + => ["SELECT * FROM users WHERE group_ids IN(?,?) AND parent_id = ?" 1 2 3]) ;;; Incorrect parameters. (expect AssertionError - (rewrite-query-for-jdbc (tokenize "SELECT age FROM users WHERE country = :country AND name = :name") - {:country "gb"})) + (sane-query? + (tokenize "SELECT age FROM users WHERE country = :country AND name = :name") + {:country "gb"})) (expect AssertionError - (rewrite-query-for-jdbc (tokenize "SELECT age FROM users WHERE country = ? AND name = ?") - {})) + (sane-query? + (tokenize "SELECT age FROM users WHERE country = ? AND name = ?") + {})) + +(expect AssertionError + (sane-query? + (tokenize "INSERT INTO users (country, name) VALUES (:country, :name)") + [{:country "gb"} {:country "us"}])) From 450b85d292f2f8c5f68bc042ca00429ff93681b5 Mon Sep 17 00:00:00 2001 From: Nina Blanson Date: Fri, 18 Mar 2016 23:36:52 -0500 Subject: [PATCH 8/8] updating readme --- README.md | 9 ++++++++- src/yesql/generate.clj | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9c9d347..822f773 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ in the REPL: ;=> ------------------------- ;=> user/users-by-country -;=> ([{:keys [country_code]}] +;=> ([{:keys [country_code]}] ;=> [{:keys [country_code]} {:keys [connection]}]) ;=> ;=> Counts the users in a given country. @@ -351,6 +351,13 @@ INSERT INTO person (name) VALUES (:name) ;=> {:name "Dave" :id 5} ``` +To batch insert, Yesql will take a vector of maps. + +```clojure +(create-person '({:name "Dave" :id 5} {:name "Jill" :id 6}) +``` + The exact return value will depend on your database driver. For example PostgreSQL returns the whole row, whereas Derby returns just `{:1 5M}`. diff --git a/src/yesql/generate.clj b/src/yesql/generate.clj index 4f30c71..341b30b 100644 --- a/src/yesql/generate.clj +++ b/src/yesql/generate.clj @@ -83,8 +83,8 @@ (defn insert-handler [db sql params call-options] (if (vector? params) - (let [full-query (apply str sql) - table-name (re-find insert-table-name-regex full-query)] + (let [full-query (apply str sql) + table-name (re-find insert-table-name-regex full-query)] (sane-query? sql params) (apply jdbc/insert! db table-name params)) (let [[rewritten-sql & rewritten-params] (rewrite-query-for-jdbc sql params)]