diff --git a/.gitignore b/.gitignore index 5b9a3ff..4f1e5d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,20 @@ -/target -/classes -/checkouts +target/ +classes/ +checkouts/ pom.xml pom.xml.asc *.jar *.class *.iml -/.lein-* -/.nrepl-port -/.project -/.classpath -/.settings -/.idea +.lein-* +.nrepl-port +.project +.classpath +.settings +.idea/ \#*\# .\#* +.prepl-port +.clj-kondo/ +.cpcache/ +.lsp/ diff --git a/.travis.yml b/.travis.yml index 8ec5ac9..e55ccee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,22 @@ -dist: precise -sudo: false language: clojure -lein: lein2 -script: lein2 midje -notifications: - irc: "irc.freenode.org#clj-postgresql" -jdk: - - oraclejdk8 +#lein: lein +script: + - lein sub install + - lein sub test +#notifications: +# irc: "irc.freenode.org#clj-postgresql" services: - postgresql addons: - postgresql: 9.4 + postgresql: "10" before_script: - psql -U postgres -c 'CREATE DATABASE clj_pg_test;' postgres - psql -U postgres -c 'CREATE EXTENSION postgis;' clj_pg_test after_failure: - psql -c '\d' -U postgres - - cat /var/log/postgresql/postgresql-9.4-main.log + - cat /var/log/postgresql/postgresql-10-main.log env: PGDATABASE='clj_pg_test' PGUSER='postgres' +#virt: lxd +#os: linux +#dist: focal +#arch: arm64 diff --git a/README.md b/README.md index 48a9409..9deefd6 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Add the following to the `:dependencies` section of your `project.clj` file: [Leiningen](https://github.com/technomancy/leiningen) dependency information: ```clj -[clj-postgresql "0.7.0"] +[clj-postgresql "1.0.0-SNAPSHOT"] ``` [Maven](http://maven.apache.org/) dependency information: @@ -26,7 +26,7 @@ Add the following to the `:dependencies` section of your `project.clj` file: clj-postgresql clj-postgresql - 0.7.0-SNAPSHOT + 1.0.0-SNAPSHOT ``` diff --git a/clj-postgresql-aws/project.clj b/clj-postgresql-aws/project.clj new file mode 100644 index 0000000..6895ae9 --- /dev/null +++ b/clj-postgresql-aws/project.clj @@ -0,0 +1,10 @@ +(defproject clj-postgresql/clj-postgresql-aws "1.0.1-SNAPSHOT" + :description "PostgreSQL helpers for Clojure projects." + :url "https://github.com/remodoy/clj-postgresql" + :license {:name "Two clause BSD license" + :url "http://github.com/remodoy/clj-postgresql/README.md"} + :scm {:dir ".."} + :dependencies [[org.clojure/clojure "1.12.0"] + [clj-postgresql/clj-postgresql-core "1.0.1-SNAPSHOT"] + [com.amazonaws/aws-java-sdk-rds "1.12.782"]] + :repl-options {:init-ns clj-postgresql.aws}) diff --git a/clj-postgresql-aws/src/clj_postgresql/aws.clj b/clj-postgresql-aws/src/clj_postgresql/aws.clj new file mode 100644 index 0000000..a3a5d39 --- /dev/null +++ b/clj-postgresql-aws/src/clj_postgresql/aws.clj @@ -0,0 +1,55 @@ +(ns clj-postgresql.aws + (:require [clj-postgresql.core :as pg]) + (:import (com.amazonaws.services.rds.auth RdsIamAuthTokenGenerator GetIamAuthTokenRequest) + (com.amazonaws.auth DefaultAWSCredentialsProviderChain) + (com.amazonaws.regions DefaultAwsRegionProviderChain))) + +(defn rds-env-spec + "Make a db spec map from RDS_* variables in environment. Elastic Beanstalk uses these variables." + [] + (let [rds-hostname (System/getenv "RDS_HOSTNAME") + rds-port (System/getenv "RDS_PORT") + rds-db-name (System/getenv "RDS_DB_NAME") + rds-username (System/getenv "RDS_USERNAME") + rds-password (System/getenv "RDS_PASSWORD")] + (cond-> {} + rds-db-name (assoc :dbname rds-db-name) + rds-hostname (assoc :host rds-hostname) + rds-port (assoc :port rds-port) + rds-username (assoc :user rds-username) + rds-password (assoc :password rds-password)))) + +(defn rds-spec + "Make a db spec that uses the RDS_* and PG* environment variables to connect to the database." + ([opts] + (let [rds-opts (rds-env-spec) + merged-opts (merge rds-opts opts)] + (pg/spec merged-opts))) + ([] + (rds-spec {}))) + +(defn make-auth-token [{:keys [host port user] :as opts}] + #_(println "opts:" opts) + (let [credentials-provider (DefaultAWSCredentialsProviderChain.) + region ^String (.getRegion (DefaultAwsRegionProviderChain.)) + token-generator (-> (RdsIamAuthTokenGenerator/builder) + (.credentials credentials-provider) + (.region region) + (.build)) + token-request (-> (GetIamAuthTokenRequest/builder) + (.hostname host) + (.port (or (when port (Integer/parseInt port)) 5432)) + (.userName user) + (.build))] + #_(println "region:" region) + (-> token-generator + (.getAuthToken token-request)))) + +(defn rds-iam-spec + "Make a db spec with IAM auth token (if no password given)." + ([opts] + (let [spec-opts (rds-spec opts)] + (cond-> spec-opts + (not (contains? spec-opts :password)) (assoc :password (make-auth-token spec-opts))))) + ([] + (rds-iam-spec {}))) diff --git a/clj-postgresql-aws/test/clj_postgresql/clj_postgresql_aws_test.clj b/clj-postgresql-aws/test/clj_postgresql/clj_postgresql_aws_test.clj new file mode 100644 index 0000000..1c4d0b1 --- /dev/null +++ b/clj-postgresql-aws/test/clj_postgresql/clj_postgresql_aws_test.clj @@ -0,0 +1,7 @@ +(ns clj-postgresql.clj-postgresql-aws-test + (:require [clojure.test :refer :all] + [clj-postgresql.clj-postgresql-aws :refer :all])) + +(deftest a-test + (testing "FIXME, I fail." + (is (= 0 1)))) diff --git a/clj-postgresql-core/project.clj b/clj-postgresql-core/project.clj new file mode 100644 index 0000000..b3964d1 --- /dev/null +++ b/clj-postgresql-core/project.clj @@ -0,0 +1,11 @@ +(defproject clj-postgresql/clj-postgresql-core "1.0.1-SNAPSHOT" + :description "PostgreSQL helpers for Clojure projects." + :url "https://github.com/remodoy/clj-postgresql" + :license {:name "Two clause BSD license" + :url "http://github.com/remodoy/clj-postgresql/README.md"} + :scm {:dir ".."} + :dependencies [[org.clojure/clojure "1.12.0"] + [org.postgresql/postgresql "42.7.5"] + [cheshire "5.13.0"] + [org.clojure/java.jdbc "0.7.11"]] + :repl-options {:init-ns clj-postgresql.core}) diff --git a/clj-postgresql-core/src/clj_postgresql/core.clj b/clj-postgresql-core/src/clj_postgresql/core.clj new file mode 100644 index 0000000..cb46702 --- /dev/null +++ b/clj-postgresql-core/src/clj_postgresql/core.clj @@ -0,0 +1,71 @@ +(ns clj-postgresql.core + "Allow using PostgreSQL from Clojure as effortlessly as possible by reading connection parameter defaults from + PostgreSQL environment variables PGDATABASE, PGHOST, PGPORT, PGUSER and by reading password from ~/.pgpass if available." + (:require [clj-postgresql.pgpass :as pgpass] + [clojure.java.jdbc :as jdbc])) + +(defn- getenv->map + "Convert crazy non-map thingy which comes from (System/getenv) into a keywordized map. + If no argument given, fetch env with (System/getenv)." + ([x] + {:post [(map? %)]} + (zipmap + (map keyword (keys x)) + (vals x))) + ([] + (getenv->map (System/getenv)))) + +(defn default-spec + "Reasonable defaults as with the psql command line tool. + Use username for user and db. Don't use host." + [] + (let [username (java.lang.System/getProperty "user.name")] + {:dbtype "postgresql" + :user username + :dbname username})) + +(defn env-spec + "Get db spec by reading PG* variables from the environment." + [{:keys [PGDATABASE PGHOST PGPORT PGUSER PGPASS] :as env}] + {:pre [(map? env)] + :post [(map? %)]} + (cond-> {} + PGDATABASE (assoc :dbname PGDATABASE) + PGHOST (assoc :host PGHOST) + PGPORT (assoc :port PGPORT) + PGUSER (assoc :user PGUSER) + PGPASS (assoc :password PGPASS))) + +(defn spec + "Create database spec for PostgreSQL. Uses PG* environment variables by default + and acceps options in the form: + (spec {:dbname ... :host ... :port ... :user ... :password ...})" + ([opts] + {:post [(contains? % :dbname) + (contains? % :user)]} + (let [default-spec-opts (default-spec) + env-spec-opts (env-spec (getenv->map (System/getenv))) + spec-opts (select-keys opts [:dbname :host :port :user :password]) + extra-opts (dissoc opts :dbname :host :port :user :password) + db-spec (merge default-spec-opts env-spec-opts spec-opts) + password (when-not (:password db-spec) + (pgpass/pgpass-lookup db-spec))] + (cond-> (merge extra-opts db-spec) + password (assoc :password password)))) + ([] + (spec {}))) + +(defn close! + "Close db-spec if possible. Return true if the datasource was closeable and closed." + [{:keys [datasource]}] + (when (instance? java.io.Closeable datasource) + (.close ^java.io.Closeable datasource) + true)) + +(defn tables + [db] + (jdbc/with-db-metadata [md db] + (->> (doall (jdbc/metadata-result (.getTables md nil nil nil (into-array ["TABLE"])))) + (map :table_name) + (map keyword) + (set)))) diff --git a/src/clj_postgresql/pgpass.clj b/clj-postgresql-core/src/clj_postgresql/pgpass.clj similarity index 74% rename from src/clj_postgresql/pgpass.clj rename to clj-postgresql-core/src/clj_postgresql/pgpass.clj index 5cd28f4..8de1487 100644 --- a/src/clj_postgresql/pgpass.clj +++ b/clj-postgresql-core/src/clj_postgresql/pgpass.clj @@ -8,8 +8,8 @@ Return a map of fields {:pg-hostname \"*\" ...}" [s] (zipmap - [:pg-hostname :pg-port :pg-database :pg-username :pg-password] - (str/split s #":"))) + [:pg-hostname :pg-port :pg-database :pg-username :pg-password] + (str/split s #":"))) (defn read-pgpass "Find ~/.pgpass, read it and parse lines into maps" @@ -27,11 +27,11 @@ "(filter (partial pgpass-matches? spec) pgpass-lines)" [{:keys [host port dbname user]} {:keys [pg-hostname pg-port pg-database pg-username pg-password]}] (when - (and - (or (= pg-hostname "*") (= pg-hostname host) (and (= pg-hostname "localhost") (nil? host))) - (or (= pg-port "*") (= pg-port port) (and (= pg-port "5432") (nil? port))) - (or (= pg-database "*") (= pg-database dbname)) - (or (= pg-username "*") (= pg-username user))) + (and + (or (= pg-hostname "*") (= pg-hostname host) (and (= pg-hostname "localhost") (nil? host))) + (or (= pg-port "*") (= pg-port port) (and (= pg-port "5432") (nil? port))) + (or (= pg-database "*") (= pg-database dbname)) + (or (= pg-username "*") (= pg-username user))) pg-password)) (defn pgpass-lookup diff --git a/clj-postgresql-core/src/clj_postgresql/types.clj b/clj-postgresql-core/src/clj_postgresql/types.clj new file mode 100644 index 0000000..e29fc95 --- /dev/null +++ b/clj-postgresql-core/src/clj_postgresql/types.clj @@ -0,0 +1,43 @@ +(ns clj-postgresql.types + "Participate in clojure.java.jdbc's ISQLValue and IResultSetReadColumn protocols + to allow using PostGIS geometry types without the PGgeometry wrapper, support the + PGjson type and allow coercing clojure structures into PostGIS types. + + clojure.java.jdbc/ISQLParameter protocol's set-parameter is used to set statement parameters. + Per default it just delegates to clojure.java.jdbc/ISQLValue protocol's sql-value method. + Thus if we have a special Clojure/Java type like org.postgis.Geometry, we can just implement + ISQLValue for that type. But if we want to convert generic maps, vectors, etc. to special database + types we need to implement ISQLParameter for the generic type and peek into statement's metadata + to figure out what the target type in database is. + + For parameter handling we implement: + - map->parameter (IPersistentMap) + - vec->parameter (IPersistentVector, Sequable) + - num->parameter (Number) + + Extend clojure.java.jdbc's protocol for converting query parameters to SQL values. + We try to determine which SQL type is correct for which clojure structure. + 1. See query parameter meta data. JDBC might already know what PostgreSQL wants. + 2. Look into parameter's clojure metadata for type hints + " + (:require [clj-postgresql.types maps vectors numbers json inet]) + (:import (org.postgresql.util PGobject PGInterval PGmoney))) + + +(defn object + "Make a custom PGobject, e.g. (pg/object \"json\" \"{}\")" + [type value] + (doto (PGobject.) + (.setType (name type)) + (.setValue (str value)))) + +(defn interval + "Create a PGinterval. (pg/interval :hours 2)" + [& {:keys [years months days hours minutes seconds] + :or {years 0 months 0 days 0 hours 0 minutes 0 seconds 0.0}}] + (PGInterval. years months days hours minutes ^double seconds)) + +(defn money + "Create PGmoney object" + [amount] + (PGmoney. ^double amount)) diff --git a/clj-postgresql-core/src/clj_postgresql/types/geometric.clj b/clj-postgresql-core/src/clj_postgresql/types/geometric.clj new file mode 100644 index 0000000..915622f --- /dev/null +++ b/clj-postgresql-core/src/clj_postgresql/types/geometric.clj @@ -0,0 +1,76 @@ +(ns clj-postgresql.types.geometric + (:import (org.postgresql.geometric PGpoint PGbox PGcircle PGline PGlseg PGpath PGpolygon))) + +;; +;; Constructors for geometric Types +;; + +(defn point + "Create a PGpoint object" + ([x y] + (PGpoint. x y)) + ([obj] + (cond + (instance? PGpoint obj) obj + (coll? obj) (point (first obj) (second obj)) + :else (PGpoint. (str obj))))) + +(defn box + "Create a PGbox object" + ([p1 p2] + (PGbox. (point p1) (point p2))) + ([x1 y1 x2 y2] + (PGbox. x1 y1 x2 y2)) + ([obj] + (if (instance? PGbox obj) + obj + (PGbox. (str obj))))) + +(defn circle + "Create a PGcircle object" + ([x y r] + (PGcircle. x y r)) + ([center-point r] + (PGcircle. (point center-point) r)) + ([obj] + (if (instance? PGcircle obj) + obj + (PGcircle. (str obj))))) + +(defn line + "Create a PGline object" + ([x1 y1 x2 y2] + (PGline. x1 y1 x2 y2)) + ([p1 p2] + (PGline. (point p1) (point p2))) + ([obj] + (if (instance? PGline obj) + obj + (PGline. (str obj))))) + +(defn lseg + "Create a PGlseg object" + ([x1 y1 x2 y2] + (PGlseg. x1 y1 x2 y2)) + ([p1 p2] + (PGlseg. (point p1) (point p2))) + ([obj] + (if (instance? PGlseg obj) + obj + (PGlseg. (str obj))))) + +(defn path + "Create a PGpath object" + ([points open?] + (PGpath. (into-array PGpoint (map point points)) open?)) + ([obj] + (if (instance? PGpath obj) + obj + (PGpath. (str obj))))) + +(defn polygon + "Create a PGpolygon object" + [points-or-str] + (if (coll? points-or-str) + (PGpolygon. ^"[Lorg.postgresql.geometric.PGpoint;" (into-array PGpoint (map point points-or-str))) + (PGpolygon. ^String (str points-or-str)))) diff --git a/clj-postgresql-core/src/clj_postgresql/types/inet.clj b/clj-postgresql-core/src/clj_postgresql/types/inet.clj new file mode 100644 index 0000000..f09985c --- /dev/null +++ b/clj-postgresql-core/src/clj_postgresql/types/inet.clj @@ -0,0 +1,25 @@ +(ns clj-postgresql.types.inet + (:require [clj-postgresql.types.vectors :refer [vec->parameter]]) + (:import (java.net InetAddress) + (org.postgresql.util PGobject))) + +;; +;; Convert all InetAddress object to string presentation. +;; +(extend-protocol clojure.java.jdbc/ISQLValue + InetAddress + (^PGobject sql-value [inet-addr] + (doto (PGobject.) + (.setType "inet") + (.setValue (.getHostAddress inet-addr))))) + +;; +;; Convert vectors like [192 168 0 1] into inet object. +;; +(defmethod vec->parameter :inet + [v _] + (if (= (count v) 4) + (doto (PGobject.) + (.setType "inet") + (.setValue (clojure.string/join "." v))) + v)) diff --git a/clj-postgresql-core/src/clj_postgresql/types/json.clj b/clj-postgresql-core/src/clj_postgresql/types/json.clj new file mode 100644 index 0000000..a0313c3 --- /dev/null +++ b/clj-postgresql-core/src/clj_postgresql/types/json.clj @@ -0,0 +1,51 @@ +(ns clj-postgresql.types.json + "Extensions to transparently convert between clojure maps and PostgreSQL JSON fields." + (:require [clj-postgresql.types.maps :refer [map->parameter]] + [clj-postgresql.types.vectors :refer [vec->parameter]] + [clj-postgresql.types.pgobject :refer [read-pgobject]] + [cheshire.core :as json]) + (:import (org.postgresql.util PGobject))) + +;; +;; Parameters to PostgreSQL +;; + +(def ^:dynamic *json-opts* {}) + +(defmethod map->parameter :json + [m _] + (doto (PGobject.) + (.setType "json") + (.setValue (json/generate-string m *json-opts*)))) + +(defmethod map->parameter :jsonb + [m _] + (doto (PGobject.) + (.setType "jsonb") + (.setValue (json/generate-string m *json-opts*)))) + +(defmethod vec->parameter :json + [v _] + (doto (PGobject.) + (.setType "json") + (.setValue (json/generate-string v *json-opts*)))) + +(defmethod vec->parameter :jsonb + [v _] + (doto (PGobject.) + (.setType "jsonb") + (.setValue (json/generate-string v *json-opts*)))) + +;; +;; Result columns from PostgreSQL +;; + +(defmethod read-pgobject :json + [^PGobject x] + (when-let [val (.getValue x)] + (json/parse-string val))) + +(defmethod read-pgobject :jsonb + [^PGobject x] + (when-let [val (.getValue x)] + (json/parse-string val))) \ No newline at end of file diff --git a/clj-postgresql-core/src/clj_postgresql/types/maps.clj b/clj-postgresql-core/src/clj_postgresql/types/maps.clj new file mode 100644 index 0000000..e283f30 --- /dev/null +++ b/clj-postgresql-core/src/clj_postgresql/types/maps.clj @@ -0,0 +1,18 @@ +(ns clj-postgresql.types.maps + (:require [clojure.java.jdbc :as jdbc]) + (:import (clojure.lang IPersistentMap) + (java.sql PreparedStatement))) + +(defmulti map->parameter #(keyword %2)) + +(defmethod map->parameter :default + [m _] + (jdbc/sql-value m)) + +(extend-protocol jdbc/ISQLParameter + IPersistentMap + (set-parameter [m ^PreparedStatement stmt ^long ix] + (let [meta (.getParameterMetaData stmt)] + (if-let [type-name (keyword (.getParameterTypeName meta ix))] + (.setObject stmt ix (map->parameter m type-name)) + (.setObject stmt ix (jdbc/sql-value m)))))) diff --git a/clj-postgresql-core/src/clj_postgresql/types/numbers.clj b/clj-postgresql-core/src/clj_postgresql/types/numbers.clj new file mode 100644 index 0000000..da35999 --- /dev/null +++ b/clj-postgresql-core/src/clj_postgresql/types/numbers.clj @@ -0,0 +1,27 @@ +(ns clj-postgresql.types.numbers + "Convert numbers to SQL parameter values. + Conversion is done for target types like timestamp for which it makes sense to accept numeric values." + (:require [clojure.java.jdbc :as jdbc]) + (:import (java.sql Timestamp PreparedStatement))) + +(defmulti num->parameter #(keyword %2)) + +(defmethod num->parameter :timestamptz + [number _] + (Timestamp. number)) + +(defmethod num->parameter :timestamp + [number _] + (Timestamp. number)) + +(defmethod num->parameter :default + [number _] + (jdbc/sql-value number)) + +(extend-protocol clojure.java.jdbc/ISQLParameter + Number + (set-parameter [num ^PreparedStatement stmt ^long ix] + (let [meta (.getParameterMetaData stmt)] + (if-let [type-name (.getParameterTypeName meta ix)] + (.setObject stmt ix (num->parameter num type-name)) + (.setObject stmt ix (jdbc/sql-value num)))))) diff --git a/clj-postgresql-core/src/clj_postgresql/types/pgobject.clj b/clj-postgresql-core/src/clj_postgresql/types/pgobject.clj new file mode 100644 index 0000000..ee8ada2 --- /dev/null +++ b/clj-postgresql-core/src/clj_postgresql/types/pgobject.clj @@ -0,0 +1,17 @@ +(ns clj-postgresql.types.pgobject + "PGobject parsing magic" + (:require [clojure.java.jdbc :as jdbc]) + (:import (org.postgresql.util PGobject))) + +(defmulti read-pgobject + "Convert returned PGobject to Clojure value. Dispatch by PGobject type." + #(keyword (when % (.getType ^PGobject %)))) + +(defmethod read-pgobject :default + [^PGobject x] + (.getValue x)) + +(extend-protocol jdbc/IResultSetReadColumn + PGobject + (result-set-read-column [val _ _] + (read-pgobject val))) diff --git a/clj-postgresql-core/src/clj_postgresql/types/vectors.clj b/clj-postgresql-core/src/clj_postgresql/types/vectors.clj new file mode 100644 index 0000000..7cec7b1 --- /dev/null +++ b/clj-postgresql-core/src/clj_postgresql/types/vectors.clj @@ -0,0 +1,77 @@ +(ns clj-postgresql.types.vectors + "Parameters: + Convert Clojure vectors into SQL arrays if target type is e.g. _int. + For non array target types, dispatch vec->parameter multimethod for converting + Clojure vector specific target type. Convert any non vector Sequable parameters to vector. + + Query results: + Convert array to vector of objects. + " + (:require [clojure.java.jdbc :as jdbc] + [clj-postgresql.types.pgobject :refer [read-pgobject]]) + (:import (clojure.lang IPersistentVector Seqable) + (java.sql PreparedStatement Array) + (org.postgresql.util PGobject))) + +(defmulti vec->parameter #(keyword %2)) + +(defmethod vec->parameter :default + [v _] + (jdbc/sql-value v)) + +(extend-protocol jdbc/ISQLParameter + IPersistentVector + (set-parameter [v ^PreparedStatement stmt ^long ix] + (let [conn (.getConnection stmt) + meta (.getParameterMetaData stmt) + type-name (.getParameterTypeName meta ix)] + (if-let [elem-type (when type-name (second (re-find #"^_(.*)" type-name)))] + (.setObject stmt ix (.createArrayOf conn elem-type (to-array v))) + (.setObject stmt ix (vec->parameter v type-name))))) + Seqable + (set-parameter [seqable ^PreparedStatement stmt ^long ix] + (jdbc/set-parameter (vec (seq seqable)) stmt ix))) + +;; +;; Handle java.sql.Array results +;; + +(extend-protocol jdbc/IResultSetReadColumn + ;; Covert java.sql.Array to Clojure vector + Array + (result-set-read-column [val _ _] + (vec (.getArray val)))) + +;; +;; Handle arrays and vectors coming in as PGobject +;; + +(defn- read-pg-vector + "oidvector, int2vector, etc. are space separated lists" + [s] + (when (seq s) + (clojure.string/split s #"\s+"))) + +(defn- read-pg-array + "Arrays are of form {1,2,3}" + [s] + (when (seq s) + (when-let [[_ content] (re-matches #"^\{(.+)\}$" s)] + (if-not (empty? content) + (clojure.string/split content #"\s*,\s*") + [])))) + +(defmethod read-pgobject :oidvector + [^PGobject x] + (when-let [val (.getValue x)] + (mapv read-string (read-pg-vector val)))) + +(defmethod read-pgobject :int2vector + [^PGobject x] + (when-let [val (.getValue x)] + (mapv read-string (read-pg-vector val)))) + +(defmethod read-pgobject :anyarray + [^PGobject x] + (when-let [val (.getValue x)] + (vec (read-pg-array val)))) diff --git a/clj-postgresql-core/src/clj_postgresql/types/xml.clj b/clj-postgresql-core/src/clj_postgresql/types/xml.clj new file mode 100644 index 0000000..b083ff2 --- /dev/null +++ b/clj-postgresql-core/src/clj_postgresql/types/xml.clj @@ -0,0 +1,27 @@ +(ns clj-postgresql.types.xml + "PostgreSQL XML fields will be returned as SQLXML objects. Parse these to clojure.xml maps." + (:require [clojure.java.jdbc :as jdbc] + [clojure.xml :as xml] + [clj-postgresql.types.maps :refer [map->parameter]]) + (:import (java.sql SQLXML) + (org.postgresql.util PGobject))) + +;; +;; Extend clojure.jdbc's protocols to parse XML fields into clojure xml maps. +;; +(extend-protocol jdbc/IResultSetReadColumn + ;; Parse SQLXML to a Clojure map representing the XML content. + SQLXML + (result-set-read-column [val _ _] + (xml/parse (.getBinaryStream val)))) + +;; +;; Handle a Clojure map parameter destined to XML field in database. +;; Convert into PGobject. +;; +(defmethod map->parameter :xml + [m _] + (doto (PGobject.) + (.setType "xml") + (.setValue (with-out-str + (xml/emit m))))) diff --git a/clj-postgresql-core/src/clj_postgresql/util.clj b/clj-postgresql-core/src/clj_postgresql/util.clj new file mode 100644 index 0000000..d5a6d2b --- /dev/null +++ b/clj-postgresql-core/src/clj_postgresql/util.clj @@ -0,0 +1,37 @@ +(ns clj-postgresql.util + (:import (java.sql ParameterMetaData ResultSetMetaData))) + +(defn parameter-metadata->map + "Convert ParameterMetaData to a map." + [^ParameterMetaData md i] + {:parameter-class (.getParameterClassName md i) + :parameter-mode (.getParameterMode md i) + :parameter-type (.getParameterType md i) + :parameter-type-name (.getParameterTypeName md i) + :precision (.getPrecision md i) + :scale (.getScale md i) + :nullable? (.isNullable md i) + :signed? (.isSigned md i)}) + +(defn result-set-metadata->map + "Convert ResultSetMetaData to a map." + [^ResultSetMetaData md i] + {:catalog-name (.getCatalogName md i) + :column-class-name (.getColumnClassName md i) + :column-display-size (.getColumnDisplaySize md i) + :column-label (.getColumnLabel md i) + :column-type (.getColumnType md i) + :column-type-name (.getColumnTypeName md i) + :precision (.getPrecision md i) + :scale (.getScale md i) + :schema-name (.getSchemaName md i) + :table-name (.getTableName md i) + :auto-increment? (.isAutoIncrement md i) + :case-sensitive? (.isCaseSensitive md i) + :currency? (.isCurrency md i) + :definitely-writable? (.isDefinitelyWritable md i) + :nullable? (.isNullable md i) + :read-only? (.isReadOnly md i) + :searchable? (.isSearchable md i) + :signed? (.isSigned md i) + :writable? (.isWritable md i)}) \ No newline at end of file diff --git a/clj-postgresql-core/test/clj_postgresql/t_core.clj b/clj-postgresql-core/test/clj_postgresql/t_core.clj new file mode 100644 index 0000000..0473ba5 --- /dev/null +++ b/clj-postgresql-core/test/clj_postgresql/t_core.clj @@ -0,0 +1,41 @@ +(ns clj-postgresql.t-core + (:require [clojure.test :refer :all] + [clj-postgresql.core :as pg] + [clj-postgresql.types] + [clojure.java.jdbc :as jdbc]) + (:import [java.net InetAddress])) + +(defn query + [& args] + (jdbc/query (pg/spec) args)) + +(defn query1 + [& args] + (let [result (apply query args)] + (first result))) + +(deftest core-test + (testing "Parsing data types works" + (is (= (query1 "SELECT true AS x") {:x true})) + (is (= (query1 "SELECT false AS x") {:x false})) + (is (= (query1 "SELECT false AS x, true AS y") {:x false :y true})) + (is (= (query1 "SELECT '1 2 3'::oidvector AS x") {:x [1 2 3]})) + (is (= (query1 "SELECT '{a,b}'::text[] AS x") {:x ["a" "b"]})) + (is (= (query1 "SELECT '{a,b}'::text[]::anyarray AS x") {:x ["a" "b"]})) + (is (= (query1 "SELECT '{\"foo\":1}'::json AS x") {:x {"foo" 1}})) + (is (= (query1 "SELECT '{\"foo\":1}'::jsonb AS x") {:x {"foo" 1}})) + (is (= (query1 "SELECT 'CaMeL' AS x") {:x "CaMeL"}))) + + (testing "Data type parameters work" + (is (= (query1 "SELECT true AS x WHERE true = ?" true) {:x true})) + (is (= (query1 "SELECT true AS x WHERE false = ?" false) {:x true})) + (is (= (query1 "SELECT true AS x WHERE 'a'::text = ?" "a") {:x true})) + (is (= (query1 "SELECT ?::json AS x" {"foo" {"bar" 1}}) {:x {"foo" {"bar" 1}}})) + (is (= (query1 "SELECT ?::json AS x" {:foo {:bar 1}}) {:x {"foo" {"bar" 1}}})) + (is (= (query1 "SELECT ?::jsonb AS x" {"foo" {"bar" 1}}) {:x {"foo" {"bar" 1}}})) + (is (= (query1 "SELECT ?::jsonb AS x" {:foo {:bar 1}}) {:x {"foo" {"bar" 1}}})) + (is (= (query1 "SELECT ?::int[] AS x" [1 2 7 6 5]) {:x [1 2 7 6 5]})) + (is (= (query1 "SELECT ?::text[] AS x" '("a" "b" "c" "d" "e")) {:x ["a" "b" "c" "d" "e"]})) + (is (= (query1 "SELECT ?::varchar[] AS x" ["a" 1 "B" 2.0]) {:x ["a" "1" "B" "2.0"]})) + (is (= (query1 "SELECT ?::inet AS x" (InetAddress/getByName "127.0.0.1")) {:x "127.0.0.1"})) + (is (= (query1 "SELECT ?::inet AS x" (InetAddress/getByName "::1")) {:x "::1"})))) diff --git a/clj-postgresql-core/test/clj_postgresql/t_types.clj b/clj-postgresql-core/test/clj_postgresql/t_types.clj new file mode 100644 index 0000000..05b98b1 --- /dev/null +++ b/clj-postgresql-core/test/clj_postgresql/t_types.clj @@ -0,0 +1,21 @@ +(ns clj-postgresql.t-types + (:require [clojure.test :refer :all] + [clj-postgresql.core :as pg] + [clj-postgresql.types] + [clojure.java.jdbc :as jdbc])) + +(def test-data + {"x" 42 "a" [4 3 2]}) + +(def test-table-name (str "test_json_" (rand-int 999))) + +(deftest types-test + (testing "Write and read json and jsonb" + (jdbc/with-db-transaction [tx (pg/spec)] + (jdbc/db-do-commands tx (str "CREATE TEMP TABLE " test-table-name " (json_field json, jsonb_field jsonb)")) + (is (= (first (jdbc/insert! tx test-table-name {:jsonb_field test-data :json_field test-data})) + {:jsonb_field test-data :json_field test-data})) + (let [row (first (jdbc/query tx (str "SELECT * FROM " test-table-name)))] + (is (= (:json_field row) test-data)) + (is (= (:jsonb_field row) test-data)))))) + diff --git a/clj-postgresql-gis/project.clj b/clj-postgresql-gis/project.clj new file mode 100644 index 0000000..eeae6c4 --- /dev/null +++ b/clj-postgresql-gis/project.clj @@ -0,0 +1,10 @@ +(defproject clj-postgresql/clj-postgresql-gis "1.0.1-SNAPSHOT" + :description "PostgreSQL helpers for Clojure projects. The PostGIS-parts." + :url "https://github.com/remodoy/clj-postgresql" + :license {:name "Two clause BSD license" + :url "http://github.com/remodoy/clj-postgresql/README.md"} + :dependencies [[org.clojure/clojure "1.12.0"] + [clj-postgresql/clj-postgresql-core "1.0.1-SNAPSHOT"] + [net.postgis/postgis-jdbc "2.5.0" :exclusions [postgresql org.postgresql/postgresql]] + [prismatic/schema "1.1.12"]] + :repl-options {:init-ns clj-postgresql.spatial}) diff --git a/src/clj_postgresql/coerce.clj b/clj-postgresql-gis/src/clj_postgresql/coerce.clj similarity index 100% rename from src/clj_postgresql/coerce.clj rename to clj-postgresql-gis/src/clj_postgresql/coerce.clj diff --git a/src/clj_postgresql/geojson.clj b/clj-postgresql-gis/src/clj_postgresql/geojson.clj similarity index 100% rename from src/clj_postgresql/geojson.clj rename to clj-postgresql-gis/src/clj_postgresql/geojson.clj diff --git a/clj-postgresql-gis/src/clj_postgresql/geometry.clj b/clj-postgresql-gis/src/clj_postgresql/geometry.clj new file mode 100644 index 0000000..0ecb5b5 --- /dev/null +++ b/clj-postgresql-gis/src/clj_postgresql/geometry.clj @@ -0,0 +1,37 @@ +(ns clj-postgresql.geometry + (:require [clojure.java.jdbc :as jdbc] + [clj-postgresql.types.maps :refer [map->parameter]] + [clj-postgresql.coerce :as coerce]) + (:import [org.postgis Geometry PGgeometry PGgeometryLW])) + +;;;; +;; +;; Data type conversion for SQL query parameters +;; +;;;; + +;; +;; Allow using PostsGIS Geometry objects as query parameters. +;; The Geometry will be made PGgeometryLW and passed to the backend with the efficient EWKB format. +;; +(extend-protocol jdbc/ISQLValue + Geometry + (sql-value [v] + (PGgeometryLW. v))) + +;; +;; Allow using geojson maps as parameters. +;; +(defmethod map->parameter :geometry + [m _] + (jdbc/sql-value (coerce/geojson->postgis m))) + +;; +;; Interpret query results. +;; PGgeometry typed results will be converted into geojson maps. +;; +(extend-protocol jdbc/IResultSetReadColumn + PGgeometry + (result-set-read-column [val _ _] + (coerce/postgis->geojson (.getGeometry val)))) + diff --git a/src/clj_postgresql/spatial.clj b/clj-postgresql-gis/src/clj_postgresql/spatial.clj similarity index 100% rename from src/clj_postgresql/spatial.clj rename to clj-postgresql-gis/src/clj_postgresql/spatial.clj diff --git a/clj-postgresql-pool/project.clj b/clj-postgresql-pool/project.clj new file mode 100644 index 0000000..add7fe1 --- /dev/null +++ b/clj-postgresql-pool/project.clj @@ -0,0 +1,9 @@ +(defproject clj-postgresql/clj-postgresql-pool "1.0.1-SNAPSHOT" + :description "FIXME: write description" + :url "http://example.com/FIXME" + :license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0" + :url "https://www.eclipse.org/legal/epl-2.0/"} + :dependencies [[org.clojure/clojure "1.12.0"] + [clj-postgresql/clj-postgresql-core "1.0.1-SNAPSHOT"] + [hikari-cp "2.13.0"]] + :repl-options {:init-ns clj-postgresql.pool}) diff --git a/src/clj_postgresql/pool.clj b/clj-postgresql-pool/src/clj_postgresql/pool.clj similarity index 71% rename from src/clj_postgresql/pool.clj rename to clj-postgresql-pool/src/clj_postgresql/pool.clj index ccd8230..b4868a1 100644 --- a/src/clj_postgresql/pool.clj +++ b/clj-postgresql-pool/src/clj_postgresql/pool.clj @@ -1,7 +1,7 @@ (ns clj-postgresql.pool "Hikari based connection pool" - (:require [hikari-cp.core :as hikari]) - (:import (java.util.concurrent TimeUnit))) + (:require [clj-postgresql.core :as core] + [hikari-cp.core :as hikari])) (defn db-spec->pool-config "Converts a db-spec with :host :port :dbname and :user to Hikari pool @@ -12,12 +12,13 @@ (let [host-part (when host (if port (format "%s:%s" host port) host))] (cond-> {:jdbc-url (format "jdbc:%s://%s/%s" dbtype (or host-part "") dbname) :username user} - hikari (merge hikari) - password (assoc :password password)))) + hikari (merge hikari) + password (assoc :password password)))) (defn pooled-db - [spec opts] - (let [config (merge (db-spec->pool-config spec) opts)] + [& {:as opts}] + (let [original-config (core/spec opts) + config (db-spec->pool-config original-config)] {:datasource (hikari/make-datasource config)})) (defn close-pooled-db! diff --git a/clj-postgresql-pool/test/clj_postgresql/t_pool.clj b/clj-postgresql-pool/test/clj_postgresql/t_pool.clj new file mode 100644 index 0000000..1aa0a51 --- /dev/null +++ b/clj-postgresql-pool/test/clj_postgresql/t_pool.clj @@ -0,0 +1,22 @@ +(ns clj-postgresql.t-pool + (:require [clojure.test :refer :all] + [clj-postgresql.pool :as pool] + [clojure.java.jdbc :as jdbc])) + +(defn query + [& args] + (jdbc/query (pool/pooled-db) args)) + +(defn query1 + [& args] + (let [result (apply query args)] + (first result))) + +(deftest pool-test + (testing "Pool query works" + (is (= (jdbc/execute! (pool/pooled-db) "CREATE TEMPORARY TABLE bob(id text)") [0])) + (is (= (query1 "SELECT true AS x") {:x true})) + (is (= (try + (jdbc/execute! (pool/pooled-db :hikari {:read-only true}) "CREATE TEMPORARY TABLE bob(id text)") + (catch org.postgresql.util.PSQLException e + (.getMessage e))) "ERROR: cannot execute CREATE TABLE in a read-only transaction")))) diff --git a/deps.edn b/deps.edn index 8366f55..783ef3b 100644 --- a/deps.edn +++ b/deps.edn @@ -1,5 +1,5 @@ {:deps - {org.clojure/clojure #:mvn{:version "1.10.0"}, + {org.clojure/clojure #:mvn{:version "1.12.0"}, hikari-cp #:mvn{:version "2.7.1"}, net.postgis/postgis-jdbc #:mvn{:version "2.3.0"}, clj-time #:mvn{:version "0.15.1"}, diff --git a/dev-resources/user.clj b/dev-resources/user.clj deleted file mode 100644 index 8ff782a..0000000 --- a/dev-resources/user.clj +++ /dev/null @@ -1,11 +0,0 @@ -;(ns user -; (:require [clj-postgresql.core :as pg] -; [clj-postgresql.spatial :as st] -; [clj-postgresql.geojson :as gj] -; [clj-postgresql.coerce :as coerce] -; [clojure.java.jdbc :as jdbc])) -; -;(def spec (delay (pg/spec))) -;(def pool (delay (pg/pool))) - - diff --git a/doc/intro.md b/doc/intro.md deleted file mode 100644 index ae7194d..0000000 --- a/doc/intro.md +++ /dev/null @@ -1,3 +0,0 @@ -# Introduction to postgresql - -TODO: write [great documentation](http://jacobian.org/writing/great-documentation/what-to-write/) diff --git a/project.clj b/project.clj index b3215ba..f72f11e 100644 --- a/project.clj +++ b/project.clj @@ -1,17 +1,15 @@ -(defproject clj-postgresql "0.7.0" +(defproject clj-postgresql "1.0.1-SNAPSHOT" :description "PostgreSQL helpers for Clojure projects" :url "https://github.com/remodoy/clj-postgresql" :license {:name "Two clause BSD license" :url "http://github.com/remodoy/clj-postgresql/README.md"} - :dependencies [[org.clojure/clojure "1.10.0"] - [org.postgresql/postgresql "42.2.5"] - [net.postgis/postgis-jdbc "2.3.0" :exclusions [postgresql org.postgresql/postgresql]] - [hikari-cp "2.7.1"] - [cheshire "5.8.1"] - [clj-time "0.15.1"] - [org.clojure/java.data "0.1.1"] - [org.clojure/java.jdbc "0.7.9"] - [prismatic/schema "1.1.10"]] - :profiles {:dev {:dependencies [[midje "1.9.8"]] - :plugins [[lein-midje "3.1.1"]]}}) + :plugins [[lein-sub "0.3.0"]] + :dependencies [[clj-postgresql/clj-postgresql-core "1.0.1-SNAPSHOT"] + [clj-postgresql/clj-postgresql-pool "1.0.1-SNAPSHOT"] + [clj-postgresql/clj-postgresql-gis "1.0.1-SNAPSHOT"] + [clj-postgresql/clj-postgresql-aws "1.0.1-SNAPSHOT"]] + :sub ["clj-postgresql-core" + "clj-postgresql-pool" + "clj-postgresql-gis" + "clj-postgresql-aws"]) diff --git a/src/clj_postgresql/core.clj b/src/clj_postgresql/core.clj deleted file mode 100644 index b2358e5..0000000 --- a/src/clj_postgresql/core.clj +++ /dev/null @@ -1,187 +0,0 @@ -(ns clj-postgresql.core - "Allow using PostgreSQL from Clojure as effortlessly as possible by reading connection parameter defaults from - PostgreSQL environment variables PGDATABASE, PGHOST, PGPORT, PGUSER and by reading password from ~/.pgpass if available." - (:require [clj-postgresql.types] - [clj-postgresql.pool :refer [pooled-db] :as pool] - [clj-postgresql.pgpass :as pgpass] - [clojure.java.jdbc :as jdbc]) - (:import org.postgresql.util.PGobject - org.postgresql.util.PGmoney - org.postgresql.util.PGInterval - org.postgresql.geometric.PGbox - org.postgresql.geometric.PGcircle - org.postgresql.geometric.PGline - org.postgresql.geometric.PGlseg - org.postgresql.geometric.PGpath - org.postgresql.geometric.PGpoint - org.postgresql.geometric.PGpolygon - org.postgis.PGgeometry)) - -(defn getenv->map - "Convert crazy non-map thingy which comes from (System/getenv) into a keywordized map. - If no argument given, fetch env with (System/getenv)." - ([x] - {:pre [(= (type x) java.util.Collections$UnmodifiableMap)] - :post [(map? %)]} - (zipmap - (map keyword (keys x)) - (vals x))) - ([] - (getenv->map (System/getenv)))) - -(defn default-spec - "Reasonable defaults as with the psql command line tool. - Use username for user and db. Don't use host." - [] - (let [username (java.lang.System/getProperty "user.name")] - {:dbtype "postgresql" - :user username - :dbname username})) - -(defn env-spec - "Get db spec by reading PG* variables from the environment." - [{:keys [PGDATABASE PGHOST PGPORT PGUSER] :as env}] - {:pre [(map? env)] - :post [(map? %)]} - (cond-> {} - PGDATABASE (assoc :dbname PGDATABASE) - PGHOST (assoc :host PGHOST) - PGPORT (assoc :port PGPORT) - PGUSER (assoc :user PGUSER))) - -(defn spec - "Create database spec for PostgreSQL. Uses PG* environment variables by default - and acceps options in the form: - (pg-spec :dbname ... :host ... :port ... :user ... :password ...)" - [& {:keys [password] :as opts}] - {:post [(contains? % :dbname) - (contains? % :user)]} - (let [spec-opts (select-keys opts [:dbname :host :port :user]) - extra-opts (dissoc opts :dbname :host :port :user :password) - db-spec (merge (default-spec) - (env-spec (getenv->map (System/getenv))) - spec-opts) - password (or password (pgpass/pgpass-lookup db-spec))] - (cond-> (merge extra-opts db-spec) - password (assoc :password password)))) - -(defn pool - [& rest] - (let [m (apply spec rest)] - (pooled-db m {}))) - -(defn close! - "Close db-spec if possible. Return true if the datasource was closeable and closed." - [{:keys [datasource]}] - (when (instance? java.io.Closeable datasource) - (.close ^java.io.Closeable datasource) - true)) - -(defn tables - [db] - (jdbc/with-db-metadata [md db] - (->> (doall (jdbc/metadata-result (.getTables md nil nil nil (into-array ["TABLE"])))) - (map :table_name) - (map keyword) - (set)))) - -;; -;; Types -;; - -(defn object - "Make a custom PGobject, e.g. (pg/object \"json\" \"{}\")" - [type value] - (doto (PGobject.) - (.setType (name type)) - (.setValue (str value)))) - -(defn interval - "Create a PGinterval. (pg/interval :hours 2)" - [& {:keys [years months days hours minutes seconds] - :or {years 0 months 0 days 0 hours 0 minutes 0 seconds 0.0}}] - (PGInterval. years months days hours minutes ^double seconds)) - -(defn money - "Create PGmoney object" - [amount] - (PGmoney. ^double amount)) - -(defn xml - "Make PostgreSQL XML object" - [s] - (object :xml (str s))) - -;; -;; Constructors for geometric Types -;; - -(defn point - "Create a PGpoint object" - ([x y] - (PGpoint. x y)) - ([obj] - (cond - (instance? PGpoint obj) obj - (coll? obj) (point (first obj) (second obj)) - :else (PGpoint. (str obj))))) - -(defn box - "Create a PGbox object" - ([p1 p2] - (PGbox. (point p1) (point p2))) - ([x1 y1 x2 y2] - (PGbox. x1 y1 x2 y2)) - ([obj] - (if (instance? PGbox obj) - obj - (PGbox. (str obj))))) - -(defn circle - "Create a PGcircle object" - ([x y r] - (PGcircle. x y r)) - ([center-point r] - (PGcircle. (point center-point) r)) - ([obj] - (if (instance? PGcircle obj) - obj - (PGcircle. (str obj))))) - -(defn line - "Create a PGline object" - ([x1 y1 x2 y2] - (PGline. x1 y1 x2 y2)) - ([p1 p2] - (PGline. (point p1) (point p2))) - ([obj] - (if (instance? PGline obj) - obj - (PGline. (str obj))))) - -(defn lseg - "Create a PGlseg object" - ([x1 y1 x2 y2] - (PGlseg. x1 y1 x2 y2)) - ([p1 p2] - (PGlseg. (point p1) (point p2))) - ([obj] - (if (instance? PGlseg obj) - obj - (PGlseg. (str obj))))) - -(defn path - "Create a PGpath object" - ([points open?] - (PGpath. (into-array PGpoint (map point points)) open?)) - ([obj] - (if (instance? PGpath obj) - obj - (PGpath. (str obj))))) - -(defn polygon - "Create a PGpolygon object" - [points-or-str] - (if (coll? points-or-str) - (PGpolygon. ^"[Lorg.postgresql.geometric.PGpoint;" (into-array PGpoint (map point points-or-str))) - (PGpolygon. ^String (str points-or-str)))) diff --git a/src/clj_postgresql/geometric/Point.clj b/src/clj_postgresql/geometric/Point.clj deleted file mode 100644 index 58378f3..0000000 --- a/src/clj_postgresql/geometric/Point.clj +++ /dev/null @@ -1,39 +0,0 @@ -(ns clj-postgresql.geometric.Point - "Example implementation of a PGobject that also implements - Clojure's ISeq and shows as a sequence of two numbers." - (:gen-class - :extends org.postgresql.geometric.PGpoint - :implements [clojure.lang.Counted clojure.lang.ISeq] - :main false)) - -(defn to-list - [^org.postgresql.geometric.PGpoint this] - (list (.-x this) (.-y this))) - -(defn -count - [this] - (.count (to-list this))) - -(defn -first - [this] - (.first (to-list this))) - -(defn -next - [this] - (.next (to-list this))) - -(defn -more - [this] - (.more (to-list this))) - -(defn -empty - [this] - (.empty (to-list this))) - -(defn -equiv - [this other] - (.equiv (to-list this) (to-list other))) - -(defn -seq - [this] - (.seq (to-list this))) diff --git a/src/clj_postgresql/protocol.clj b/src/clj_postgresql/protocol.clj deleted file mode 100644 index 468737d..0000000 --- a/src/clj_postgresql/protocol.clj +++ /dev/null @@ -1,277 +0,0 @@ -(ns clj-postgresql.protocol - (:import [java.net Socket] - [java.io InputStream OutputStream ObjectInputStream ObjectOutputStream] - [java.nio ByteBuffer] - [java.nio.charset Charset] - [java.nio.channels SocketChannel] - [java.net InetSocketAddress] - [java.security MessageDigest]) - (:require [clojure.pprint :as pp])) - -(def ^:constant ^Charset UTF-8 (Charset/forName "UTF-8")) - -(defn md5 - [& args] - (let [^MessageDigest md (MessageDigest/getInstance "MD5")] - (doseq [a args] - (if (string? a) - (.update md ^bytes (.getBytes ^String a ^Charset UTF-8)) - (.update md ^bytes a))) - (clojure.string/join (map (fn* [p1__418377#] (String/format "%02x" (into-array Object [p1__418377#]))) (.digest md))))) - -(defn int32 - [n] - (doto (ByteBuffer/allocate 4) - (.putInt n))) - -(defn int16 - [n] - (doto (ByteBuffer/allocate 2) - (.putShort n))) - -(defn byte1 - [n] - (doto (ByteBuffer/allocate 1) - (.put (byte n)))) - -(defn string - [s] - (let [bytes (.getBytes ^String (str s) ^Charset UTF-8)] - (doto (ByteBuffer/allocate (inc (alength bytes))) - (.put ^bytes bytes) - (.put (byte 0))))) - -(defn bytify-map - [m] - (mapcat (fn [[k v]] [(string (name k)) (string (str v))]) m)) - -(defn byte-count - [bytebufs] - (reduce + (map #(.capacity ^ByteBuffer %) bytebufs))) - -(defn startup-message - [m] - (let [msg (flatten [(int32 196608) - (bytify-map m) - (byte1 0)])] - (flatten [(int32 (+ (byte-count msg) 4)) - msg]))) - -(defn md5-auth - [auth-salt username password] - (let [a (md5 password username) - b (md5 a auth-salt)] - (str "md5" b))) - -(defn password-message - [content] - (let [bytes (string content)] - [(byte1 \p) - (int32 (+ (.capacity ^ByteBuffer bytes) 4)) - bytes])) - -(defn terminate-message - [] - [(byte1 \X) - (int32 4)]) - -(defn query-message - [query-string] - (let [buf (string query-string)] - [(byte1 \Q) - (int32 (+ (.capacity ^ByteBuffer buf) 4)) - buf])) - -(defmulti auth-req (fn [t bb] t)) -(defmethod auth-req 0 - [_ ^ByteBuffer bb] - :auth-ok) -(defmethod auth-req 5 - [_ ^ByteBuffer bb] - (let [salt (byte-array 4)] - (.get bb salt) - salt)) - -(defn get-string - [^ByteBuffer bb] - (loop [bytes []] - (let [x (.get bb)] - (if (zero? x) - (String. (byte-array bytes)) - (recur (conj bytes x)))))) - -(defmulti response (fn [response-type _ _] response-type)) -(defmethod response (int \R) - [_ len ^ByteBuffer bb] - (let [auth-type (.getInt bb)] - (auth-req auth-type bb))) -(defmethod response (int \E) - [_ len ^ByteBuffer bb] - :error) -(defmethod response (int \S) - [_ len ^ByteBuffer bb] - {:type :parameter-status - :name (get-string bb) - :value (get-string bb)}) -(defmethod response (int \K) - [_ len ^ByteBuffer bb] - {:type :backend-key-data - :pid (.getInt bb) - :key (.getInt bb)}) -(defmethod response (int \Z) - [_ len ^ByteBuffer bb] - {:type :ready-for-query - :backend-status (char (.get bb))}) -(defmethod response (int \T) - [_ len ^ByteBuffer bb] - (let [fields (.getShort bb)] - (println fields) - {:type :row-description - :field-count fields - :fields (for [i (range fields)] - {:name (get-string bb) - :table-oid (.getInt bb) - :column-attribute-num (.getShort bb) - :field-oid (.getInt bb) - :data-type-size (.getShort bb) - :type-modifier (.getInt bb) - :format-code (.getShort bb)})})) -(defmethod response (int \D) - [_ len ^ByteBuffer bb] - (let [cols (.getShort bb)] - {:type :data-row - :column-value-count cols - :values (for [i (range cols)] - (let [len (.getInt bb)] - {:len len - :val (when (pos? len) (let [bytea (byte-array len)] (.get bb bytea) bytea))}))})) -(defmethod response (int \C) - [_ len ^ByteBuffer bb] - {:type :command-complete - :tag (get-string bb)}) - -(defn send! - [^SocketChannel sc bufs] - (as-> bufs x - (map #(.rewind ^ByteBuffer %) x) - (into-array ByteBuffer x) - (.write sc ^"[Ljava.nio.ByteBuffer;" x))) - -(defn recv! - [^SocketChannel sc] - (let [bb (ByteBuffer/allocate 5)] - (.read sc bb) - (.rewind bb) - (let [type (.get bb) - len (.getInt bb) - content-bb (ByteBuffer/allocate (- len 4))] - (.read sc ^ByteBuffer content-bb) - (.rewind content-bb) - (response type len content-bb)))) - -(defn converse - [] - (with-open [sc (SocketChannel/open (InetSocketAddress. "127.0.0.1" 5432))] - (send! sc (startup-message {:user "postgres" :database "postgres"})) - (let [auth-salt (recv! sc)] - (send! sc (password-message (md5-auth auth-salt "postgres" "qRoJXpy")))) - (pp/pprint - (loop [preamble []] - (let [msg (recv! sc)] - (if (= (:type msg) :ready-for-query) - [msg preamble] - (recur (conj preamble msg)))))) - (send! sc (query-message "SELECT '{}'::json AS foo")) - (pp/pprint (recv! sc)) - (pp/pprint (recv! sc)) - (pp/pprint (recv! sc)) - (pp/pprint (recv! sc)) - (send! sc (terminate-message)))) - -;; startup -;; ReadyForQuery -;; dostuff -;; ReadyForQuery -;; ... -; -;; Simple query: -;; (Query_Q query-string) ->| -;; (CommandComplete_C command-tag) -;; (CopyInResponse_G overall-format num-colums column-formats) -;; (CopyOutResponse_H overall-format num-columns column-formats) -;; (RowDescription_T fields) -;; (DataRow_D values) -;; (EmptyQueryResponse_I) -- partner for command-complete -;; (ErrorReponse_E error-fields) -;; (ReadyForQuery_Z backend-status) -;; (NoticeResponse_N message) -;; -;; (Query_Q "SELECT 'test' AS foo") -;; (RowDescription [{name "foo", format 0, field-oid 705}]) -;; (DataRow ["test"]) -;; (CommandComplete :tag "SELECT 1") -;; (ReadyForQuery :backed-status \I) -;; -;; (Query_Q "") -;; (EmptyQueryResponse_I) -;; (ReadyForQuery_Z :backed-status \I) -;; -;; -;; -;; Extended query: -;; (Parse_P stmt-name query oids) -;; (ParseComplete_1) | (ErrorResponse_E) -;; (Bind_B portal-name stmt-name param-formats values result-formats) -;; (BindComplete_2) | (ErrorResponse_E) -;; (Describe_D portal-name) -;; (RowDescription_T fields) -;; (Execute_E portal-name row-limit) -;; -- no ReadyForQuery or RowDescription -;; -- ends with CommandComplete|EmptyQueryResponse|ErrorResponse|PortalSuspended -;; (Close_C stmt-name) -- can close stmt or portal, closing stmt closes portals too -;; (CloseComplete_3) -;; (Sync_S) -- will commit/rollback internal tx, won't touch BEGIN tx -;; (ReadyForQuery_Z status) -- I no tx, T in tx, E in failed tx -;; -;; -- can do multiple bind/describe/execute cycles -;; -;; Response meta from portal, as not automatic in extended query: -;; (Describe_D :portal portal-name) -- also can be :statement -;; -- RowDescription_T|NoData_n -;; (RowDescription_T fields) -;; -;; Parameter and respose meta from statement: -;; (Describe_D :statement statement-name) -;; (ParameterDescription_t oids) -;; (RowDescription_T fields)|(NoData_n) -- before bind field format is not known.. will be zeroes -;; -;; Hitting limit: -;; (PortalSuspended_s) -;; (Execute_E portal-name row-limit) -- to continue -;; -;; Async stuff: -;; (NoticeResponse_N msg) -;; (ParameterStatus_S param value) -- run-time param changed -;; (NotificationResponse_A pid channel payload) -;; -;; Cancel by opening new conn and using secret key - BackendKeyData in startup: -;; (CancelRequest pid key) -;; - - -;; -;; pg-async ? -;; -;; (def ch1 (parse "SELECT $1")) -;; (go -;; (let [[stmt field-info] parameter parameter-dispatch-fn) - -(defmethod map->parameter :geometry - [m _] - (jdbc/sql-value (coerce/geojson->postgis m))) - -(defn- to-pg-json [data json-type] - (doto (PGobject.) - (.setType (name json-type)) - (.setValue (json/generate-string data)))) - -(defmethod map->parameter :json - [m _] - (to-pg-json m :json)) - -(defmethod map->parameter :jsonb - [m _] - (to-pg-json m :jsonb)) -(extend-protocol jdbc/ISQLParameter - clojure.lang.IPersistentMap - (set-parameter [m ^PreparedStatement s ^long i] - (let [meta (.getParameterMetaData s)] - (if-let [type-name (keyword (.getParameterTypeName meta i))] - (.setObject s i (map->parameter m type-name)) - (.setObject s i m))))) - -;; -;; Convert clojure vectors to SQL parameter values -;; - -(defmulti vec->parameter parameter-dispatch-fn) - -(defmethod vec->parameter :json - [v _] - (to-pg-json v :json)) - -(defmethod vec->parameter :jsonb - [v _] - (to-pg-json v :jsonb)) - -(defmethod vec->parameter :inet - [v _] - (if (= (count v) 4) - (doto (PGobject.) (.setType "inet") (.setValue (clojure.string/join "." v))) - v)) - -(defmethod vec->parameter :default - [v _] - v) - -(extend-protocol jdbc/ISQLParameter - clojure.lang.IPersistentVector - (set-parameter [v ^PreparedStatement s ^long i] - (let [conn (.getConnection s) - meta (.getParameterMetaData s) - type-name (.getParameterTypeName meta i)] - (if-let [elem-type (when type-name (second (re-find #"^_(.*)" type-name)))] - (.setObject s i (.createArrayOf conn elem-type (to-array v))) - (.setObject s i (vec->parameter v type-name)))))) - -;; -;; Convert all sequables to SQL parameter values by handling them like vectors. -;; - -(extend-protocol jdbc/ISQLParameter - clojure.lang.Seqable - (set-parameter [seqable ^PreparedStatement s ^long i] - (jdbc/set-parameter (vec (seq seqable)) s i))) - -;; -;; Convert numbers to SQL parameter values. -;; Conversion is done for target types like timestamp -;; for which it makes sense to accept numeric values. -;; - -(defmulti num->parameter parameter-dispatch-fn) - -(defmethod num->parameter :timestamptz - [v _] - (java.sql.Timestamp. v)) - -(defmethod num->parameter :timestamp - [v _] - (java.sql.Timestamp. v)) - -(defmethod num->parameter :default - [v _] - v) - -(extend-protocol clojure.java.jdbc/ISQLParameter - java.lang.Number - (set-parameter [num ^java.sql.PreparedStatement s ^long i] - (let [meta (.getParameterMetaData s) - type-name (.getParameterTypeName meta i)] - (.setObject s i (num->parameter num type-name))))) - -;; Inet addresses -(extend-protocol clojure.java.jdbc/ISQLParameter - java.net.InetAddress - (set-parameter [^java.net.InetAddress inet-addr ^java.sql.PreparedStatement s ^long i] - (.setObject s i (doto (PGobject.) - (.setType "inet") - (.setValue (.getHostAddress inet-addr)))))) - -;;;; -;; -;; Data type conversions for query result set values. -;; -;;;; - - -;; -;; PGobject parsing magic -;; - -(defn read-pg-vector - "oidvector, int2vector, etc. are space separated lists" - [s] - (when (seq s) (clojure.string/split s #"\s+"))) - -(defn read-pg-array - "Arrays are of form {1,2,3}" - [s] - (when (seq s) (when-let [[_ content] (re-matches #"^\{(.+)\}$" s)] (if-not (empty? content) (clojure.string/split content #"\s*,\s*") [])))) - -(defmulti read-pgobject - "Convert returned PGobject to Clojure value." - #(keyword (when % (.getType ^org.postgresql.util.PGobject %)))) - -(defmethod read-pgobject :oidvector - [^org.postgresql.util.PGobject x] - (when-let [val (.getValue x)] - (mapv read-string (read-pg-vector val)))) - -(defmethod read-pgobject :int2vector - [^org.postgresql.util.PGobject x] - (when-let [val (.getValue x)] - (mapv read-string (read-pg-vector val)))) - -(defmethod read-pgobject :anyarray - [^org.postgresql.util.PGobject x] - (when-let [val (.getValue x)] - (vec (read-pg-array val)))) - -(defmethod read-pgobject :json - [^org.postgresql.util.PGobject x] - (when-let [val (.getValue x)] - (json/parse-string val))) - -(defmethod read-pgobject :jsonb - [^org.postgresql.util.PGobject x] - (when-let [val (.getValue x)] - (json/parse-string val))) - -(defmethod read-pgobject :default - [^org.postgresql.util.PGobject x] - (.getValue x)) - -;; -;; Extend clojure.java.jdbc's protocol for interpreting ResultSet column values. -;; -(extend-protocol jdbc/IResultSetReadColumn - - ;; Return the PostGIS geometry object instead of PGgeometry wrapper - org.postgis.PGgeometry - (result-set-read-column [val _ _] - (coerce/postgis->geojson (.getGeometry val))) - - ;; Parse SQLXML to a Clojure map representing the XML content - java.sql.SQLXML - (result-set-read-column [val _ _] - (xml/parse (.getBinaryStream val))) - - ;; Covert java.sql.Array to Clojure vector - java.sql.Array - (result-set-read-column [val _ _] - (vec (.getArray val))) - - ;; PGobjects have their own multimethod - org.postgresql.util.PGobject - (result-set-read-column [val _ _] - (read-pgobject val))) diff --git a/test/clj_postgresql/t_core.clj b/test/clj_postgresql/t_core.clj deleted file mode 100644 index 2be090b..0000000 --- a/test/clj_postgresql/t_core.clj +++ /dev/null @@ -1,39 +0,0 @@ -(ns clj-postgresql.t-core - (:use midje.sweet) - (:require [clj-postgresql.core :as pg] - [clojure.java.jdbc :as jdbc]) - (:import [java.net InetAddress])) - -(defn query - [& args] - (jdbc/query (pg/spec) args)) - -(defn query1 - [& args] - (let [result (apply query args)] - (first result))) - -(fact "Parsing data types works" - (query1 "SELECT true AS x") => {:x true} - (query1 "SELECT false AS x") => {:x false} - (query1 "SELECT false AS x, true AS y") => {:x false :y true} - (query1 "SELECT '1 2 3'::oidvector AS x") => {:x [1 2 3]} - (query1 "SELECT '{a,b}'::text[] AS x") => {:x ["a" "b"]} - (query1 "SELECT '{a,b}'::text[]::anyarray AS x") => {:x ["a" "b"]} - (query1 "SELECT '{\"foo\":1}'::json AS x") => {:x {"foo" 1}} - (query1 "SELECT '{\"foo\":1}'::jsonb AS x") => {:x {"foo" 1}} - (query1 "SELECT 'CaMeL' AS x") => {:x "CaMeL"}) - -(fact "Data type parameters work" - (query1 "SELECT true AS x WHERE true = ?" true) => {:x true} - (query1 "SELECT true AS x WHERE false = ?" false) => {:x true} - (query1 "SELECT true AS x WHERE 'a'::text = ?" "a") => {:x true} - (query1 "SELECT ?::json AS x" {"foo" {"bar" 1}}) => {:x {"foo" {"bar" 1}}} - (query1 "SELECT ?::json AS x" {:foo {:bar 1}}) => {:x {"foo" {"bar" 1}}} - (query1 "SELECT ?::jsonb AS x" {"foo" {"bar" 1}}) => {:x {"foo" {"bar" 1}}} - (query1 "SELECT ?::jsonb AS x" {:foo {:bar 1}}) => {:x {"foo" {"bar" 1}}} - (query1 "SELECT ?::int[] AS x" [1 2 7 6 5]) => {:x [1 2 7 6 5]} - (query1 "SELECT ?::text[] AS x" '("a" "b" "c" "d" "e")) => {:x ["a" "b" "c" "d" "e"]} - (query1 "SELECT ?::varchar[] AS x" ["a" 1 "B" 2.0]) => {:x ["a" "1" "B" "2.0"]} - (query1 "SELECT ?::inet AS x" (InetAddress/getByName "127.0.0.1")) => {:x "127.0.0.1"} - (query1 "SELECT ?::inet AS x" (InetAddress/getByName "::1")) => {:x "::1"}) diff --git a/test/clj_postgresql/t_pool.clj b/test/clj_postgresql/t_pool.clj deleted file mode 100644 index c574b2f..0000000 --- a/test/clj_postgresql/t_pool.clj +++ /dev/null @@ -1,21 +0,0 @@ -(ns clj-postgresql.t-pool - (:use midje.sweet) - (:require [clj-postgresql.core :as pg] - [clojure.java.jdbc :as jdbc])) - -(defn query - [& args] - (jdbc/query (pg/pool) args)) - -(defn query1 - [& args] - (let [result (apply query args)] - (first result))) - -(fact "Pool query works" - (jdbc/execute! (pg/pool) "CREATE TEMPORARY TABLE bob(id text)") => [0] - (query1 "SELECT true AS x") => {:x true} - (try - (jdbc/execute! (pg/pool :hikari {:read-only true}) "CREATE TEMPORARY TABLE bob(id text)") - (catch org.postgresql.util.PSQLException e - (.getMessage e))) => "ERROR: cannot execute CREATE TABLE in a read-only transaction") diff --git a/test/clj_postgresql/t_spatial.clj b/test/clj_postgresql/t_spatial.clj deleted file mode 100644 index bb3e48a..0000000 --- a/test/clj_postgresql/t_spatial.clj +++ /dev/null @@ -1,3 +0,0 @@ -(ns clj-postgresql.t-spatial - (:use [midje.sweet]) - (:require [clj-postgresql.spatial :as st])) diff --git a/test/clj_postgresql/t_types.clj b/test/clj_postgresql/t_types.clj deleted file mode 100644 index e779df7..0000000 --- a/test/clj_postgresql/t_types.clj +++ /dev/null @@ -1,21 +0,0 @@ -(ns clj-postgresql.t-types - (:use midje.sweet) - (:require [clj-postgresql.core :as pg] - [clj-postgresql.types] - [clojure.java.jdbc :as jdbc])) - -(def test-data - {"x" 42 "a" [4 3 2]}) - -(def test-table-name (str "test_json_" (rand-int 999))) - -(facts "Write and read json and jsonb" - (jdbc/with-db-transaction [tx (pg/spec)] - (jdbc/db-do-commands tx (str "CREATE TEMP TABLE " test-table-name " (json_field json, jsonb_field jsonb)")) - (fact (first (jdbc/insert! tx test-table-name {:jsonb_field test-data :json_field test-data})) - => {:jsonb_field test-data :json_field test-data}) - (let [row (first (jdbc/query tx (str "SELECT * FROM " test-table-name)))] - (fact row => truthy) - (fact (:json_field row) => test-data) - (fact (:jsonb_field row) => test-data)))) -