From d9ef34e7db610ec7c09507adad0ac8055ea05268 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Fri, 24 Mar 2023 21:38:06 -0700 Subject: [PATCH 01/67] Release all test connections. Fix create-database without arguments. --- deps.edn | 2 +- src/datahike/api.cljc | 7 +- src/datahike/connector.cljc | 30 +++--- src/datahike/experimental/versioning.cljc | 5 +- src/datahike/store.cljc | 11 ++- src/datahike/writer.cljc | 4 +- test/datahike/test/api_test.cljc | 57 +++++++---- .../datahike/test/attribute_refs/db_test.cljc | 3 +- .../test/attribute_refs/differences_test.cljc | 84 ++++++++++------ .../test/attribute_refs/pull_api_test.cljc | 3 +- .../test/attribute_refs/temporal_search.cljc | 3 +- .../test/attribute_refs/transact_test.cljc | 12 ++- test/datahike/test/cache_test.cljc | 3 +- test/datahike/test/entity_spec_test.cljc | 12 ++- test/datahike/test/gc_test.cljc | 48 ++++++--- test/datahike/test/insert.cljc | 6 +- test/datahike/test/listen_test.cljc | 3 +- test/datahike/test/middleware/query_test.cljc | 12 ++- test/datahike/test/migrate_test.clj | 20 +++- test/datahike/test/purge_test.cljc | 15 ++- test/datahike/test/query_rules_test.cljc | 3 +- test/datahike/test/query_test.cljc | 6 +- test/datahike/test/schema_test.cljc | 29 ++++-- test/datahike/test/store_test.cljc | 6 +- test/datahike/test/time_variance_test.cljc | 32 ++++-- test/datahike/test/transact_test.cljc | 99 ++++++++++++------- test/datahike/test/tuples_test.cljc | 30 ++++-- test/datahike/test/upsert_impl_test.cljc | 18 ++-- test/datahike/test/upsert_test.cljc | 24 +++-- test/datahike/test/versioning_test.cljc | 10 +- 30 files changed, 396 insertions(+), 201 deletions(-) diff --git a/deps.edn b/deps.edn index f30bdb4a4..6c9f37eeb 100644 --- a/deps.edn +++ b/deps.edn @@ -10,7 +10,7 @@ com.taoensso/timbre {:mvn/version "5.2.1"} io.replikativ/superv.async {:mvn/version "0.3.43"} io.replikativ/datalog-parser {:mvn/version "0.2.25"} - io.replikativ/zufall {:mvn/version "0.1.0"} + io.replikativ/zufall {:mvn/version "0.2.9"} junit/junit {:mvn/version "4.13.2"} medley/medley {:mvn/version "1.4.0"} metosin/spec-tools {:mvn/version "0.10.5"} diff --git a/src/datahike/api.cljc b/src/datahike/api.cljc index f17cf296a..dec2f50cc 100644 --- a/src/datahike/api.cljc +++ b/src/datahike/api.cljc @@ -115,7 +115,8 @@ (when-let [txs (:initial-tx config)] (let [conn (connect config)] (transact conn txs) - (release conn)))))) + (release conn))) + config))) (s/fdef delete-database @@ -256,8 +257,8 @@ release :args (s/cat :conn spec/SConnectionAtom) :ret nil?) -(def ^{:arglists '([conn]) - :doc "Releases a database connection"} +(def ^{:arglists '([conn] [conn release-all?]) + :doc "Releases a database connection. You need to release a connection as many times as you connected to it for it to be completely released. Set release-all? to true to force its release."} release dc/release) (s/fdef diff --git a/src/datahike/connector.cljc b/src/datahike/connector.cljc index 5027ce164..f73b2e920 100644 --- a/src/datahike/connector.cljc +++ b/src/datahike/connector.cljc @@ -44,6 +44,9 @@ (defn deref-conn [^Connection conn] (let [wrapped-atom (.-wrapped-atom conn)] + (when (= @wrapped-atom :released) + (throw (ex-info "Connection has been released." + {:type :connection-has-been-released}))) (if (not (w/streaming? (get @wrapped-atom :writer))) (let [store (:store @wrapped-atom) stored (k/get store (:branch (:config @wrapped-atom)) nil {:sync? true})] @@ -70,7 +73,7 @@ (swap! connections assoc conn-id {:conn conn :count 1})) (defn delete-connection! [conn-id] - (reset! (get-connection conn-id) :deleted) + (reset! (get-connection conn-id) :released) (swap! connections dissoc conn-id)) (defn version-check [{:keys [meta config] :as db}] @@ -189,14 +192,17 @@ ([config] (-connect config))) -(defn release [connection] - (let [db @(:wrapped-atom connection) - conn-id [(ds/store-identity (get-in db [:config :store])) - (get-in db [:config :branch])]] - (if-not (get @connections conn-id) - (log/info "Connection already released." conn-id) - (let [new-conns (swap! connections update-in [conn-id :count] dec)] - (when (zero? (get-in new-conns [conn-id :count])) - (delete-connection! conn-id) - (w/shutdown (:writer db)) - nil))))) +(defn release + ([connection] (release connection false)) + ([connection release-all?] + (when-not (= @(:wrapped-atom connection) :released) + (let [db @(:wrapped-atom connection) + conn-id [(ds/store-identity (get-in db [:config :store])) + (get-in db [:config :branch])]] + (if-not (get @connections conn-id) + (log/info "Connection already released." conn-id) + (let [new-conns (swap! connections update-in [conn-id :count] dec)] + (when (or release-all? (zero? (get-in new-conns [conn-id :count]))) + (delete-connection! conn-id) + (w/shutdown (:writer db)) + nil))))))) diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index 37d804e2e..b6eb3e08f 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -67,8 +67,8 @@ (k/update store :branches #(conj % new-branch) {:sync? true}))) (defn delete-branch! - "Removes this branch from set of known branches. The branch will still be - accessible until the next gc." + "Removes this branch from set of known branches. All of its connections need to + be released. The branch will still be accessible until the next gc." [conn branch] (when (= branch :db) (dt/raise "Cannot delete main :db branch. Delete database instead." @@ -78,7 +78,6 @@ (when-not (branches branch) (dt/raise "Branch does not exist." {:type :branch-does-not-exist :branch branch})) - (delete-connection! [(store-identity (get-in @conn [:config :store])) branch]) (k/update store :branches #(disj % branch) {:sync? true}))) (defn force-branch! diff --git a/src/datahike/store.cljc b/src/datahike/store.cljc index 145e5aaf9..c24b97402 100644 --- a/src/datahike/store.cljc +++ b/src/datahike/store.cljc @@ -7,7 +7,9 @@ [datahike.tools :as dt] [konserve.cache :as kc] [clojure.core.cache :as cache] - [taoensso.timbre :refer [info]])) + [taoensso.timbre :refer [info]] + [zufall.core :refer [rand-german-mammal]]) + #?(:clj (:import [java.nio.file Paths]))) (defn add-cache-and-handlers [raw-store config] (cond->> (kc/ensure-cache @@ -94,7 +96,7 @@ (defmethod default-config :mem [config] (merge - {:id (:datahike-store-id env "default")} + {:id (:datahike-store-id env (rand-german-mammal))} config)) (s/def :datahike.store.mem/backend #{:mem}) @@ -118,9 +120,12 @@ (defmethod connect-store :file [{:keys [path config]}] (fs/connect-fs-store path :opts {:sync? true} :config config)) +(defn- get-working-dir [] + (.toString (.toAbsolutePath (Paths/get "" (into-array String []))))) + (defmethod default-config :file [config] (merge - {:path (:datahike-store-path env "datahike-db") + {:path (:datahike-store-path env (str (get-working-dir) "/datahike-db-" (rand-german-mammal))) :scope (dt/get-hostname)} config)) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 94fb25f4e..2560abf9d 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -82,8 +82,8 @@ (defn streaming? [writer] (-streaming? writer)) -(defn backend-dispatch [config & _args] - (get-in config [:writer :backend] :self)) +(defn backend-dispatch [& args] + (get-in (first args) [:writer :backend] :self)) (defmulti create-database backend-dispatch) diff --git a/test/datahike/test/api_test.cljc b/test/datahike/test/api_test.cljc index a3528e696..918d1d75f 100644 --- a/test/datahike/test/api_test.cljc +++ b/test/datahike/test/api_test.cljc @@ -119,7 +119,8 @@ (is (thrown? clojure.lang.ExceptionInfo (d/transact conn nil))) (is (thrown? clojure.lang.ExceptionInfo (d/transact conn :foo))) (is (thrown? clojure.lang.ExceptionInfo (d/transact conn 1))) - (is (thrown? clojure.lang.ExceptionInfo (d/transact conn {:foo "bar"}))))) + (is (thrown? clojure.lang.ExceptionInfo (d/transact conn {:foo "bar"}))) + (d/release conn))) (deftest test-transact!-docs (let [cfg {:store {:backend :mem @@ -128,7 +129,8 @@ :schema-flexibility :read} conn (utils/setup-db cfg)] ;; add a single datom to an existing entity (1) - (is (d/transact! conn [[:db/add 1 :name "Ivan"]])))) + (is (d/transact! conn [[:db/add 1 :name "Ivan"]])) + (d/release conn))) ;; retract a single datom @@ -160,7 +162,8 @@ :name "Ivan" :likes [:pizza] :friends [{:db/id 2, :name "Oleg"}]} - (d/pull @conn '[:db/id :name :likes {:friends [:db/id :name]}] 1))))) + (d/pull @conn '[:db/id :name :likes {:friends [:db/id :name]}] 1))) + (d/release conn))) (deftest test-pull-many-docs (let [cfg {:store {:backend :mem @@ -172,7 +175,8 @@ conn (utils/setup-db cfg)] (is (= (d/pull-many @conn [:db/id :name] [1 2]) [{:db/id 1, :name "Ivan"} - {:db/id 2, :name "Oleg"}])))) + {:db/id 2, :name "Oleg"}])) + (d/release conn))) (deftest test-q-docs (let [cfg {:store {:backend :mem @@ -250,7 +254,8 @@ (is (= [{:db/id 1, :friend 296, :likes "pizza", :name "Ivan"}] (d/q '[:find [(pull ?e [*]) ...] :where [?e ?a ?v]] - @conn))))) + @conn))) + (d/release conn))) (deftest test-with-docs (let [cfg {:store {:backend :mem @@ -270,7 +275,8 @@ (is (= {:foo :bar} (dissoc (:tx-meta res) :db/txInstant))) (is (= '([1 :name "Ivan"]) - (map dvec (:tx-data res))))))) + (map dvec (:tx-data res))))) + (d/release conn))) ;; TODO testing properly on what? (deftest test-db-docs @@ -282,7 +288,8 @@ (is (= datahike.db.DB (type (d/db conn)))) (is (= datahike.db.DB - (type @conn))))) + (type @conn))) + (d/release conn))) (deftest test-history-docs (let [cfg {:store {:backend :mem @@ -313,7 +320,8 @@ (is (= #{["Alice" 25] ["Alice" 35] ["Bob" 30]} (d/q {:query '[:find ?n ?a :where [?e :name ?n] [?e :age ?a]] - :args [(d/history (d/db conn))]}))))) + :args [(d/history (d/db conn))]}))) + (d/release conn))) (deftest test-as-of-docs (let [cfg {:store {:backend :mem @@ -344,7 +352,8 @@ (is (= #{["Alice" 35] ["Bob" 30]} (d/q {:query '[:find ?n ?a :where [?e :name ?n] [?e :age ?a]] - :args [(d/db conn)]}))))) + :args [(d/db conn)]}))) + (d/release conn))) (deftest test-since-docs (let [cfg {:store {:backend :mem @@ -381,7 +390,8 @@ (is (= #{["Alice" 30] ["Bob" 30]} (d/q {:query '[:find ?n ?a :where [?e :name ?n] [?e :age ?a]] - :args [(d/db conn)]}))))) + :args [(d/db conn)]}))) + (d/release conn))) (deftest test-datoms-docs (let [cfg {:store {:backend :mem @@ -521,7 +531,8 @@ #_(is (= "fail" (->> (d/datoms @db {:index :avet :components [:name]}) (reverse) - (take 2)))))) + (take 2)))) + (d/release db))) (deftest test-seek-datoms-doc (let [cfg {:store {:backend :mem @@ -572,7 +583,8 @@ (is (= '([5 :likes "pie"] [5 :likes "pizza"]) - (map dvec (d/seek-datoms @db {:index :eavt :components [5 :likes "fish"]})))))) + (map dvec (d/seek-datoms @db {:index :eavt :components [5 :likes "fish"]})))) + (d/release db))) (deftest test-index-range-doc (let [cfg {:store {:backend :mem @@ -603,7 +615,8 @@ (map dvec (d/index-range @db {:attrid :likes :start "a" :end "zzzzzzzzz"})))) (is (= '([5 :likes "fries"] [9 :likes "pie"]) - (map dvec (d/index-range @db {:attrid :likes :start "egg" :end "pineapple"})))))) + (map dvec (d/index-range @db {:attrid :likes :start "egg" :end "pineapple"})))) + (d/release db))) (deftest test-database-hash (testing "Hashing without history" @@ -625,7 +638,8 @@ (let [_ (d/transact conn [[:db/retractEntity 1]]) hash-2 (hash @conn)] (is (not= hash-2 hash-1)) - (is (= hash-0 hash-2)))))))) + (is (= hash-0 hash-2)))))) + (d/release conn))) (testing "Hashing with history" (let [cfg {:store {:backend :mem :id "hashing-with-history"} @@ -645,7 +659,8 @@ (let [_ (d/transact conn [[:db/retractEntity 1]]) hash-2 (hash @conn)] (is (not= hash-1 hash-2)) - (is (not= hash-0 hash-2))))))))) + (is (not= hash-0 hash-2)))))) + (d/release conn)))) (deftest test-database-schema (letfn [(test-schema [cfg] @@ -697,7 +712,8 @@ :related-to related-to-schema} (coerced-schema @conn))) (is (= related-to-reverse-schema - (d/reverse-schema @conn)))))] + (d/reverse-schema @conn))) + (d/release conn)))] (let [base-cfg {:store {:backend :mem :id "api-db-schema-test"} :keep-history? false @@ -711,7 +727,8 @@ (is (= {} (d/schema @conn))) (is (= {} - (d/reverse-schema @conn))))) + (d/reverse-schema @conn))) + (d/release conn))) (testing "Empty database with write flexibility and no attribute refs" (test-schema base-cfg)) (testing "Empty database with write flexibility and attribute refs" @@ -727,7 +744,8 @@ :schema-flexibility :write} conn (utils/setup-db cfg)] (is (= #{:datahike/version :datahike/id :datahike/created-at :konserve/version :hitchhiker.tree/version :persistent.set/version :datahike/commit-id} - (-> @conn :meta keys set))))) + (-> @conn :meta keys set))) + (d/release conn))) (def ^:private metrics-base-cfg {:store {:backend :mem} :index :datahike.index/persistent-set @@ -837,7 +855,8 @@ :avet-count 0} schema-on-write? update-for-schema-on-write (dbi/-keep-history? @conn) (update-for-history schema-on-write? attribute-refs?) - (:attribute-refs? (.-config @conn)) update-for-attr-refs)))) + (:attribute-refs? (.-config @conn)) update-for-attr-refs)) + (d/release conn))) (deftest test-metrics-hht (test-metrics (assoc metrics-base-cfg :index :datahike.index/hitchhiker-tree diff --git a/test/datahike/test/attribute_refs/db_test.cljc b/test/datahike/test/attribute_refs/db_test.cljc index 8f6860f2f..c9da3db8f 100644 --- a/test/datahike/test/attribute_refs/db_test.cljc +++ b/test/datahike/test/attribute_refs/db_test.cljc @@ -38,4 +38,5 @@ (d/since @conn tx1)))) (is (= #{["Maria"]} (d/q '[:find ?v :where [?e :name ?v]] - (d/since @conn tx2))))))) + (d/since @conn tx2))))) + (d/release conn))) diff --git a/test/datahike/test/attribute_refs/differences_test.cljc b/test/datahike/test/attribute_refs/differences_test.cljc index dc8e3c552..0231d4de0 100644 --- a/test/datahike/test/attribute_refs/differences_test.cljc +++ b/test/datahike/test/attribute_refs/differences_test.cljc @@ -55,7 +55,8 @@ (d/datoms @conn :aevt)))) (testing "Empty AVET index for keyword DB" (is (= nil - (d/datoms @conn :avet))))) + (d/datoms @conn :avet)))) + (d/release conn)) (let [[no-ref-cfg ref-cfg] (init-cfgs) conn (setup-db ref-cfg)] @@ -70,7 +71,8 @@ indexed-refs #{(:db/ident ref) (:db/txInstant ref)}] (is (= (set (filter (fn [^Datom datom] (contains? indexed-refs (:a datom))) ref-datoms)) - (set (d/datoms @conn :avet)))))))) + (set (d/datoms @conn :avet)))))) + (d/release conn))) (deftest test-last-entity-id ;; TODO: What is the behavior wanted? (let [[no-ref-cfg ref-cfg] (init-cfgs) @@ -91,7 +93,8 @@ (testing "Last entity id for non-empty keyword DB" (d/transact conn simple-schema) (is (= (+ 1 const/e0) (:max-eid @conn))) - (is (= (+ 1 const/e0) (find-last-entity-id @conn))))) + (is (= (+ 1 const/e0) (find-last-entity-id @conn)))) + (d/release conn)) (let [conn (setup-db ref-cfg)] (testing "Last entity id for empty reference DB" @@ -101,7 +104,8 @@ (testing "Last entity id for non-empty reference DB" (d/transact conn simple-schema) (is (= (+ 1 const/ue0) (:max-eid @conn))) - (is (= (+ 1 const/ue0) (find-last-entity-id @conn))))))) + (is (= (+ 1 const/ue0) (find-last-entity-id @conn)))) + (d/release conn)))) (deftest test-transact-schema (testing "Schema for keyword DB" @@ -111,7 +115,8 @@ const/non-ref-implicit-schema)) (d/transact conn name-schema) (is (= (:schema @conn) - (merge const/non-ref-implicit-schema {:name (first name-schema)} {1 :name}))))) + (merge const/non-ref-implicit-schema {:name (first name-schema)} {1 :name}))) + (d/release conn))) (testing "Schema for reference DB" (let [[no-ref-cfg ref-cfg] (init-cfgs) conn (setup-db ref-cfg)] @@ -135,7 +140,8 @@ {(+ 1 const/ue0) :name}))) (is (contains? (-> (:rschema @conn2) :db/ident) :name)) (is (contains? (-> (:ident-ref-map @conn2) keys set) :name)) - (is (contains? (-> (:ref-ident-map @conn2) vals set) :name))))))) + (is (contains? (-> (:ref-ident-map @conn2) vals set) :name)) + (d/release conn2)))))) (deftest test-transact-tempid (testing "Tempid resolution for keyword DB" @@ -148,7 +154,8 @@ (is (= (:tempids (d/transact conn [[:db/add -2 :name "Petr"]])) {-2 (+ 3 const/e0), :db/current-tx (+ 3 const/tx0)})) (is (= (:tempids (d/transact conn [{:db/id "Serg" :name "Sergey"}])) - {"Serg" (+ 4 const/e0), :db/current-tx (+ 4 const/tx0)})))) + {"Serg" (+ 4 const/e0), :db/current-tx (+ 4 const/tx0)})) + (d/release conn))) (testing "Tempid resolution for reference DB" (let [[no-ref-cfg ref-cfg] (init-cfgs) @@ -160,7 +167,8 @@ (is (= (:tempids (d/transact conn [[:db/add -2 (get-in @conn [:ident-ref-map :name]) "Petr"]])) {-2 (+ 3 const/ue0), :db/current-tx (+ 3 const/tx0)})) (is (= (:tempids (d/transact conn [{:db/id "Serg" :name "Sergey"}])) - {"Serg" (+ 4 const/ue0), :db/current-tx (+ 4 const/utx0)}))))) + {"Serg" (+ 4 const/ue0), :db/current-tx (+ 4 const/utx0)})) + (d/release conn)))) (deftest test-system-attr-resolution (let [[no-ref-cfg ref-cfg] (init-cfgs) @@ -175,13 +183,15 @@ (let [conn (setup-db no-ref-cfg) tx-data (:tx-data (d/transact conn schema))] (is (= (keyword-attrs tx-data) - #{:db/ident :db/cardinality :db/valueType :db/txInstant})))) + #{:db/ident :db/cardinality :db/valueType :db/txInstant})) + (d/release conn))) (testing "Resolve attributes in reference DB" (let [conn (setup-db ref-cfg) tx-data (:tx-data (d/transact conn schema))] (is (= (keyword-attrs tx-data) - #{})))))) + #{})) + (d/release conn))))) (deftest test-system-enum-resolution (let [[no-ref-cfg ref-cfg] (init-cfgs) @@ -196,13 +206,15 @@ (let [conn (setup-db no-ref-cfg) tx-data (:tx-data (d/transact conn schema))] (is (= (unresolved-enums tx-data) - #{:name :db.cardinality/one :db.type/string})))) + #{:name :db.cardinality/one :db.type/string})) + (d/release conn))) (testing "Resolve enums in reference DB" (let [conn (setup-db ref-cfg) tx-data (:tx-data (d/transact conn schema))] (is (= (unresolved-enums tx-data) - #{:name})))))) + #{:name})) + (d/release conn))))) (deftest test-indexing (let [[no-ref-cfg ref-cfg] (init-cfgs) @@ -226,7 +238,8 @@ #{[:db/ident :age] [:db/ident :name]})) (d/transact conn tx1) (is (= (avet-a-v @conn) - #{[:db/ident :age] [:db/ident :name] [:name "Alice"]})))) + #{[:db/ident :age] [:db/ident :name] [:name "Alice"]})) + (d/release conn))) (testing "Entry in avet index only when indexing true for reference DB" (let [conn (setup-db ref-cfg) @@ -237,7 +250,8 @@ #{[1 :age] [1 :name]})) (d/transact conn tx1) (is (= (difference (avet-a-v @conn) initial-avet) - #{[(ref :name) "Alice"] [1 :age] [1 :name]})))))) + #{[(ref :name) "Alice"] [1 :age] [1 :name]})) + (d/release conn))))) (deftest test-transact-nested-data (let [[no-ref-cfg ref-cfg] (init-cfgs) @@ -258,21 +272,24 @@ (d/transact conn (vec (concat schema tx1))) (d/transact conn tx2) (is (= 1 (count (find-alices @conn)))) - (is (= 1 (count (find-bobs @conn)))))) + (is (= 1 (count (find-bobs @conn)))) + (d/release conn))) (testing "Resolve nesting in reference DB" (let [conn (setup-db ref-cfg)] (d/transact conn (vec (concat schema tx1))) (d/transact conn tx2) (is (= 1 (count (find-alices @conn)))) - (is (= 1 (count (find-bobs @conn)))))))) + (is (= 1 (count (find-bobs @conn)))) + (d/release conn))))) (deftest test-transact-data-with-keyword-attr (testing "Keyword transaction in keyword DB" (let [[no-ref-cfg ref-cfg] (init-cfgs) conn (setup-db no-ref-cfg) next-eid (inc (:max-eid @conn))] - (is (not (nil? (d/transact conn [[:db/add next-eid :db/ident :name]])))))) + (is (not (nil? (d/transact conn [[:db/add next-eid :db/ident :name]])))) + (d/release conn))) (testing "Keyword transaction in reference DB" (let [[no-ref-cfg ref-cfg] (init-cfgs) @@ -282,7 +299,8 @@ (re-pattern (str "Bad entity attribute :db/ident" " at \\[:db/add " next-eid " :db/ident :name\\]," " expected reference number")) - (d/transact conn [[:db/add next-eid :db/ident :name]])))))) + (d/transact conn [[:db/add next-eid :db/ident :name]]))) + (d/release conn)))) (deftest test-transact-data-with-reference-attr (testing "Reference transaction in keyword DB" @@ -293,13 +311,15 @@ (re-pattern (str "Bad entity attribute 1" " at \\[:db/add " next-eid " 1 :name\\]," " expected keyword or string")) - (d/transact conn [[:db/add next-eid 1 :name]]))))) + (d/transact conn [[:db/add next-eid 1 :name]]))) + (d/release conn))) (testing "Reference transaction in reference DB" (let [[no-ref-cfg ref-cfg] (init-cfgs) conn (setup-db ref-cfg) next-eid (inc (:max-eid @conn))] - (is (not (nil? (d/transact conn [[:db/add next-eid 1 :name]]))))))) + (is (not (nil? (d/transact conn [[:db/add next-eid 1 :name]])))) + (d/release conn)))) (deftest test-system-schema-protection (let [[no-ref-cfg ref-cfg] (init-cfgs) @@ -312,33 +332,38 @@ (testing "Transact system schema update as map" (is (thrown-with-msg? Throwable #"Entity with ID 1 is a system attribute :db/ident and cannot be changed" - (d/transact conn [{:db/id 1 :db/ident :name}])))))) + (d/transact conn [{:db/id 1 :db/ident :name}])))) + (d/release conn))) (deftest test-system-attribute-protection (testing "Use system keyword for schema in keyword DB" (let [[no-ref-cfg ref-cfg] (init-cfgs) conn (setup-db no-ref-cfg)] (is (thrown-with-msg? Throwable #"Using namespace 'db' for attribute identifiers is not allowed" - (d/transact conn [{:db/ident :db/unique}]))))) + (d/transact conn [{:db/ident :db/unique}]))) + (d/release conn))) (testing "Use system keyword for schema in keyword DB" (let [[no-ref-cfg ref-cfg] (init-cfgs) conn (setup-db ref-cfg)] (is (thrown-with-msg? Throwable #"Using namespace 'db' for attribute identifiers is not allowed" - (d/transact conn [{:db/ident :db/unique}])))))) + (d/transact conn [{:db/ident :db/unique}]))) + (d/release conn)))) (deftest test-system-enum-protection (testing "Use system keyword for schema in keyword DB" (let [[no-ref-cfg ref-cfg] (init-cfgs) conn (setup-db no-ref-cfg)] (is (thrown-with-msg? Throwable #"Using namespace 'db' for attribute identifiers is not allowed" - (d/transact conn [{:db/ident :db.cardinality/many}]))))) + (d/transact conn [{:db/ident :db.cardinality/many}]))) + (d/release conn))) (testing "Use system keyword for schema in keyword DB" (let [[no-ref-cfg ref-cfg] (init-cfgs) conn (setup-db ref-cfg)] (is (thrown-with-msg? Throwable #"Using namespace 'db' for attribute identifiers is not allowed" - (d/transact conn [{:db/ident :db.cardinality/many}])))))) + (d/transact conn [{:db/ident :db.cardinality/many}]))) + (d/release conn)))) (deftest test-read-schema (testing "No error in combination with schema-flexibility read for keyword DB" @@ -376,7 +401,8 @@ (is (= #{} (d/q '[:find ?n :in $ :where [_ ?a ?n] [?a :db/ident :name]] @conn))) (is (= #{["Alice"] ["Bob"]} - (d/q '[:find ?n :in $ ?a :where [_ ?a ?n]] @conn :name))))) + (d/q '[:find ?n :in $ ?a :where [_ ?a ?n]] @conn :name))) + (d/release conn))) (testing "Query keyword translation reference db" (let [[no-ref-cfg ref-cfg] (init-cfgs) @@ -397,7 +423,8 @@ (is (= #{["Alice"] ["Bob"]} (d/q '[:find ?n :in $ :where [_ ?a ?n] [?a :db/ident :name]] @conn))) (is (= #{["Alice"] ["Bob"]} - (d/q '[:find ?n :in $ ?a :where [_ ?a ?n]] @conn :name)))))) + (d/q '[:find ?n :in $ ?a :where [_ ?a ?n]] @conn :name))) + (d/release conn)))) (deftest test-pull-ref-db (let [[no-ref-cfg ref-cfg] (init-cfgs) @@ -432,4 +459,5 @@ (d/pull-many @conn '[:name] [ivan matthew]))) (is (= {:name "Ivan" :_father [{:db/id matthew}]} - (d/pull @conn '[:name :_father] ivan))))) + (d/pull @conn '[:name :_father] ivan))) + (d/release conn))) diff --git a/test/datahike/test/attribute_refs/pull_api_test.cljc b/test/datahike/test/attribute_refs/pull_api_test.cljc index b0ba82b1b..000bcef62 100644 --- a/test/datahike/test/attribute_refs/pull_api_test.cljc +++ b/test/datahike/test/attribute_refs/pull_api_test.cljc @@ -298,7 +298,8 @@ :part {:db/id (+ test-e0 3), :part {:db/id (+ test-e0 1), :spec {:db/id (+ test-e0 2)}, - :part {:db/id (+ test-e0 2)}}}}})))) + :part {:db/id (+ test-e0 2)}}}}})) + (d/release conn))) (deftest test-deep-recursion (let [start 100 diff --git a/test/datahike/test/attribute_refs/temporal_search.cljc b/test/datahike/test/attribute_refs/temporal_search.cljc index 84a21e1a7..ba82675d2 100644 --- a/test/datahike/test/attribute_refs/temporal_search.cljc +++ b/test/datahike/test/attribute_refs/temporal_search.cljc @@ -18,4 +18,5 @@ (is (= (d/q query @conn) (d/q query (d/history @conn)))) (is (< 1 - (count (d/q query (d/history @conn))))))) + (count (d/q query (d/history @conn))))) + (d/release conn))) diff --git a/test/datahike/test/attribute_refs/transact_test.cljc b/test/datahike/test/attribute_refs/transact_test.cljc index 3eb3b8df0..0b19f3250 100644 --- a/test/datahike/test/attribute_refs/transact_test.cljc +++ b/test/datahike/test/attribute_refs/transact_test.cljc @@ -131,7 +131,8 @@ (is (= (:weight (d/entity @conn (+ e0 1))) 400)) (is (thrown-with-msg? Throwable (re-pattern (str ":db\\.fn/cas failed on datom \\[" (+ ref-e0 1) " " weight-ref " 400\\], expected 200")) - (d/transact conn [[:db.fn/cas (+ ref-e0 1) weight-ref 200 210]])))) + (d/transact conn [[:db.fn/cas (+ ref-e0 1) weight-ref 200 210]]))) + (d/release conn)) (let [conn (setup-new-connection) e0 (:max-eid @conn) @@ -142,7 +143,8 @@ (is (= (:label (d/entity @conn (+ e0 1))) #{:x :y :z})) (is (thrown-with-msg? Throwable (re-pattern (str ":db\\.fn/cas failed on datom \\[" (+ ref-e0 1) " " label-ref " \\(:x :y :z\\)\\], expected :s")) - (d/transact conn [[:db.fn/cas (+ ref-e0 1) label-ref :s :t]])))) + (d/transact conn [[:db.fn/cas (+ ref-e0 1) label-ref :s :t]]))) + (d/release conn)) (let [conn (setup-new-connection) e0 (:max-eid @conn) @@ -153,7 +155,8 @@ (d/transact conn [[:db.fn/cas (+ e0 1) age-ref nil 42]]) (is (= (:age (d/entity @conn (+ e0 1))) 42)) (is (thrown-with-msg? Throwable (re-pattern (str ":db\\.fn/cas failed on datom \\[" (+ ref-e0 1) " " age-ref " 42\\], expected nil")) - (d/transact conn [[:db.fn/cas (+ ref-e0 1) age-ref nil 4711]]))))) + (d/transact conn [[:db.fn/cas (+ ref-e0 1) age-ref nil 4711]]))) + (d/release conn))) (deftest test-db-fn (let [conn (setup-new-connection) @@ -184,4 +187,5 @@ (let [{:keys [db-after]} (d/transact conn [[:db.fn/call inc-age "Petr"]]) e (d/entity db-after (+ e0 1))] (is (= (:age e) 32)) - (is (:had-birthday e))))) + (is (:had-birthday e))) + (d/release conn))) diff --git a/test/datahike/test/cache_test.cljc b/test/datahike/test/cache_test.cljc index 17fd2a423..00ea66d8c 100644 --- a/test/datahike/test/cache_test.cljc +++ b/test/datahike/test/cache_test.cljc @@ -33,5 +33,6 @@ (d/q {:query '[:find ?n ?op :where [?e :name ?n _ ?op]] - :args [(d/history @conn)]}))))) + :args [(d/history @conn)]}))) + (d/release conn))) diff --git a/test/datahike/test/entity_spec_test.cljc b/test/datahike/test/entity_spec_test.cljc index 079803c36..bff8efd68 100644 --- a/test/datahike/test/entity_spec_test.cljc +++ b/test/datahike/test/entity_spec_test.cljc @@ -51,7 +51,8 @@ (testing "assert empty entity" (is (thrown-with-msg? Throwable #"Entity 5 missing attributes #\{:account/balance :account/email\} of spec :account/guard" - (tx-with-ensure conn empty-account)))))) + (tx-with-ensure conn empty-account)))) + (d/release conn))) (testing "with read schema flexibility" (let [cfg (-> cfg-template (assoc :schema-flexibility :read) @@ -68,7 +69,8 @@ (testing "assert empty entity" (is (thrown-with-msg? Throwable #"Entity 5 missing attributes #\{:account/balance :account/email\} of spec :account/guard" - (tx-with-ensure conn empty-account))))))))) + (tx-with-ensure conn empty-account)))) + (d/release conn)))))) (defn is-email? [db eid] ;; email could not exist @@ -113,7 +115,8 @@ (testing "assert empty account" (is (thrown-with-msg? Throwable #"Entity 5 failed predicates #\{datahike.test.entity-spec-test/positive-balance\? datahike.test.entity-spec-test/is-email\?\} of spec :account/guard" - (tx-with-ensure empty-account))))))) + (tx-with-ensure empty-account))))) + (d/release conn))) (deftest test-attribute-and-predicate-assertion (let [schema (conj schema-template @@ -152,4 +155,5 @@ (testing "assert empty account with required attributes precidenting over predicates" (is (thrown-with-msg? Throwable #"Entity 5 missing attributes #\{:account/balance :account/email\} of spec :account/guard" - (tx-with-ensure empty-account))))))) + (tx-with-ensure empty-account))))) + (d/release conn))) diff --git a/test/datahike/test/gc_test.cljc b/test/datahike/test/gc_test.cljc index 8224efd21..06ee52f84 100644 --- a/test/datahike/test/gc_test.cljc +++ b/test/datahike/test/gc_test.cljc @@ -60,9 +60,12 @@ :age 5}]))))))) (testing "Check that we can still read the data." - (d/transact conn txs) - (> tx-data (mapv #(-> % rest vec)) @@ -199,7 +206,8 @@ (d/q '[:find ?a ?v ?t ?op :where [?e ?attr ?v ?t ?op] - [?attr :db/ident ?a]] @conn)))))) + [?attr :db/ident ?a]] @conn))) + (d/release conn)))) (deftest load-entities-history-test (testing "Migrate predefined set with historical data" @@ -244,7 +252,9 @@ (is (= (current-q source-conn) (current-q target-conn))) (is (= (history-q source-conn) - (history-q target-conn)))))) + (history-q target-conn))) + (d/release source-conn) + (d/release target-conn)))) (deftest test-binary-support (let [config {:store {:backend :mem @@ -261,4 +271,6 @@ (is (utils/all-true? (map #(or (= %1 %2) (utils/all-eq? (nth %1 2) (nth %2 2))) (d/datoms @conn :eavt) (filter #(< (datom/datom-tx %) (:max-tx @import-conn)) - (d/datoms @import-conn :eavt))))))) + (d/datoms @import-conn :eavt))))) + (d/release conn) + (d/release import-conn))) diff --git a/test/datahike/test/purge_test.cljc b/test/datahike/test/purge_test.cljc index 7dd332564..fcc6ac1e9 100644 --- a/test/datahike/test/purge_test.cljc +++ b/test/datahike/test/purge_test.cljc @@ -55,7 +55,8 @@ (d/transact conn [[:db/purge [:name name] :age 25]]) (are [x y] (= x y) nil (find-age @conn name) - nil (find-age (d/history @conn) name)))))) + nil (find-age (d/history @conn) name)))) + (d/release conn))) (deftest test-purge-attribute (let [conn (tu/setup-db (assoc-in cfg-template [:store :id] "test-purge-attribute"))] @@ -77,7 +78,8 @@ (d/transact conn [[:db.purge/entity [:name name] :age]]) (are [x y] (= x y) true (nil? (find-age @conn name)) - true (nil? (find-age (d/history @conn) name)))))))) + true (nil? (find-age (d/history @conn) name)))))) + (d/release conn))) (deftest test-purge-entity (let [conn (tu/setup-db (assoc-in cfg-template [:store :id] "test-purge-entity"))] @@ -99,14 +101,16 @@ (testing "purge something that is not present in the database" (is (thrown-with-msg? Throwable #"Can't find entity with ID \[:name \"Alice\"\] to be purged" - (d/transact conn [[:db.purge/entity [:name "Alice"]]])))))) + (d/transact conn [[:db.purge/entity [:name "Alice"]]])))) + (d/release conn))) (deftest test-purge-non-temporal-database (let [conn (tu/setup-db (-> (assoc-in cfg-template [:store :id] "purge-non-temporal") (assoc :keep-history? false)))] (testing "purge data in non temporal database" (is (thrown-with-msg? Throwable #"Purge entity is only available in temporal databases\." - (d/transact conn [[:db.purge/entity [:name "Alice"]]])))))) + (d/transact conn [[:db.purge/entity [:name "Alice"]]])))) + (d/release conn))) (defn find-ages [db name] (d/q '[:find ?a ?op @@ -136,4 +140,5 @@ (find-ages (d/history @conn) name))) (d/transact conn [[:db.history.purge/before (java.util.Date.)]]) (is (= #{[30 true]} - (find-ages (d/history @conn) name))))))) + (find-ages (d/history @conn) name))))) + (d/release conn))) diff --git a/test/datahike/test/query_rules_test.cljc b/test/datahike/test/query_rules_test.cljc index 063044589..59ea2bdd0 100644 --- a/test/datahike/test/query_rules_test.cljc +++ b/test/datahike/test/query_rules_test.cljc @@ -244,4 +244,5 @@ :where [[?child :name ?n] (parent-info ?child ["Alice"] ?age)]} - :args [@conn "Charlie" rules]}))))) + :args [@conn "Charlie" rules]}))) + (d/release conn))) diff --git a/test/datahike/test/query_test.cljc b/test/datahike/test/query_test.cljc index e4ca86cd6..788cb113e 100644 --- a/test/datahike/test/query_test.cljc +++ b/test/datahike/test/query_test.cljc @@ -406,7 +406,8 @@ (is (= 1 (count (d/q '[:find ?t :in $ :where [?t :version/id 0]] - @conn)))))) + @conn)))) + (d/release conn))) ;; https://github.com/replikativ/datahike/issues/471 (deftest keyword-keys-test @@ -454,7 +455,8 @@ keyword-result))) (testing "keyword equals symbol keys" (is (= symbol-result - keyword-result)))))) + keyword-result)))) + (d/release conn))) (deftest test-normalize-q-input (testing "query as vector" diff --git a/test/datahike/test/schema_test.cljc b/test/datahike/test/schema_test.cljc index 5d581a9b3..ac2f88dc2 100644 --- a/test/datahike/test/schema_test.cljc +++ b/test/datahike/test/schema_test.cljc @@ -102,7 +102,8 @@ ":db.type/long :db.type/valueType :db.type/symbol\\}")) (d/transact conn [{:db/ident :phone :db/cardinality :db.cardinality/one - :db/valueType :string}])))))) + :db/valueType :string}])))) + (d/release conn))) (deftest test-db-with-initial-schema (let [cfg "datahike:mem://test-db-with-initial-schema" @@ -161,7 +162,8 @@ 3 :age}) (dbi/-schema db))) (is (= #{[:name :db.type/string :db.cardinality/many] [:age :db.type/long :db.cardinality/one]} - (d/q find-schema-q db))))))) + (d/q find-schema-q db))))) + (d/release conn))) (defn testing-type [conn type-name tx-val tx-id wrong-val] (testing type-name @@ -230,7 +232,8 @@ (testing-type conn "long" (long 2) 20 :2) (testing-type conn "string" "one" 21 :one) (testing-type conn "symbol" 'one 22 :one) - (testing-type conn "uuid" (random-uuid) 23 1))) + (testing-type conn "uuid" (random-uuid) 23 1) + (d/release conn))) (deftest test-schema-cardinality (let [cfg "datahike:mem://test-schema-cardinality" @@ -277,7 +280,8 @@ (is (thrown-with-msg? Throwable #"Update not supported for these schema attributes" (d/transact conn [{:db/id [:db/ident :owner] - :db/cardinality :db.cardinality/many}])))))) + :db/cardinality :db.cardinality/many}])))) + (d/release conn))) (deftest test-schema-persistence (testing "test file persistence" @@ -295,7 +299,9 @@ (is (= #{[:name :db.type/string :db.cardinality/one]} (d/q find-schema-q (d/db conn))))) (testing "reconnect with db" (let [new-conn (d/connect cfg)] - (is (= #{[:name :db.type/string :db.cardinality/one]} (d/q find-schema-q (d/db new-conn)))))) + (is (= #{[:name :db.type/string :db.cardinality/one]} (d/q find-schema-q (d/db new-conn)))) + (d/release new-conn))) + (d/release conn) (d/delete-database cfg))) (testing "test mem persistence" (let [cfg "datahike:mem://test-schema-persistence" @@ -306,6 +312,7 @@ (testing "reconnect with db" (let [new-conn (d/connect cfg)] (is (= #{[:name :db.type/string :db.cardinality/one]} (d/q find-schema-q (d/db new-conn)))))) + (d/release conn) (d/delete-database cfg)))) (deftest test-schema-on-read-db @@ -321,7 +328,8 @@ (is (= #{[1 "Alice" 26]} (d/q '[:find ?e ?n ?a :where [?e :name ?n] [?e :age ?a]] (d/db conn)))) (is (= #{[2 "12" :bmw]} - (d/q '[:find ?e ?a ?c :where [?e :age ?a] [?e :car ?c]] (d/db conn)))))))) + (d/q '[:find ?e ?a ?c :where [?e :age ?a] [?e :car ?c]] (d/db conn))))) + (d/release conn)))) (deftest test-ident (testing "use db/ident as enum" @@ -340,7 +348,8 @@ (testing "insert data with enums" (d/transact conn [{:message "important" :tag :important} {:message "archive" :tag [:important :archive]}]) (is (= #{["important" :important] ["archive" :important] ["archive" :archive]} - (d/q '[:find ?m ?t :where [?e :message ?m] [?e :tag ?te] [?te :db/ident ?t]] (d/db conn)))))))) + (d/q '[:find ?m ?t :where [?e :message ?m] [?e :tag ?te] [?te :db/ident ?t]] (d/db conn)))) + (d/release conn))))) (deftest test-remove-schema (let [cfg "datahike:mem://test-remove-schema" @@ -353,7 +362,8 @@ #"Schema with attribute :name does not exist" (dbt/remove-schema db (da/datom 1 :db/ident :name))))) (testing "when upserting a non existing schema, it should not throw an exception" - (is (d/transact conn [name-schema]))))) + (is (d/transact conn [name-schema]))) + (d/release conn))) (deftest test-update-schema (let [cfg "datahike:mem://test-update-schema" @@ -395,4 +405,5 @@ (is (thrown-with-msg? Throwable #"Update not supported for these schema attributes" (d/transact conn {:tx-data [(assoc personal-id-schema :db/cardinality :db.cardinality/many)]})) - "It shouldn't be allowed to update :db/cardinality to :db.cardinality/many"))))) + "It shouldn't be allowed to update :db/cardinality to :db.cardinality/many"))) + (d/release conn))) diff --git a/test/datahike/test/store_test.cljc b/test/datahike/test/store_test.cljc index fb23e32a0..30d0e3803 100644 --- a/test/datahike/test/store_test.cljc +++ b/test/datahike/test/store_test.cljc @@ -45,7 +45,8 @@ (-> @conn :eavt type)))) (testing "upsert" (d/transact conn [{:db/id 1, :name "Paula"}]) - (is (= "Paula" (:name (d/entity @conn 1)))))))) + (is (= "Paula" (:name (d/entity @conn 1))))) + (d/release conn)))) (deftest test-binary-support (let [config {:store {:backend :mem @@ -64,4 +65,5 @@ [?e :payload ?arr] [?e :name ?n]] @conn - (byte-array [0 2 3]))))))) + (byte-array [0 2 3])))) + (d/release conn)))) diff --git a/test/datahike/test/time_variance_test.cljc b/test/datahike/test/time_variance_test.cljc index b3d75c09c..e06c247fc 100644 --- a/test/datahike/test/time_variance_test.cljc +++ b/test/datahike/test/time_variance_test.cljc @@ -66,7 +66,8 @@ [?e :age ?a ?t ?op] [?t :db/txInstant ?d]] (d/history @conn) - [:name "Alice"])))))) + [:name "Alice"])))) + (d/release conn))) (deftest test-historical-queries (let [cfg (assoc-in cfg-template [:store :id] "test-historical-queries") @@ -101,12 +102,14 @@ (d/q query-with-< history-db [:name "Alice"] date))))) (testing "print DB" (is (= "#datahike/HistoricalDB {:origin #datahike/DB {:max-tx 536870915 :max-eid 4}}" - (pr-str (d/history @conn))))))) + (pr-str (d/history @conn))))) + (d/release conn))) (deftest test-as-of-db (let [cfg (assoc-in cfg-template [:store :id] "test-as-of-db") conn (setup-db cfg) first-date (now) + _ (sleep 10) tx-id 536870914 query '[:find ?a :in $ ?e :where [?e :age ?a ?tx]]] (testing "get values at specific time" @@ -131,12 +134,14 @@ (d/transact conn [[:db/retractEntity [:name "Alice"]]]) (testing "after" (is (= #{} - (d/q find-alices-age (d/as-of @conn tx-id) "Alice")))))))) + (d/q find-alices-age (d/as-of @conn tx-id) "Alice")))))) + (d/release conn))) (deftest test-since-db (let [cfg (assoc-in cfg-template [:store :id] "test-since-db") conn (setup-db cfg) first-date (now) + _ (sleep 10) tx-id 536870914 query '[:find ?a :where [?e :age ?a]]] (testing "empty after first insertion" @@ -151,7 +156,8 @@ (d/q query (d/since @conn tx-id)))))) (testing "print DB" (is (= "#datahike/SinceDB {:origin #datahike/DB {:max-tx 536870914 :max-eid 4} :time-point 536870914}" - (pr-str (d/since @conn tx-id))))))) + (pr-str (d/since @conn tx-id))))) + (d/release conn))) (deftest test-no-history (let [initial-tx [{:db/ident :name @@ -178,7 +184,8 @@ (d/q query (d/history @conn))))) (testing "all other attributes are present in history" (is (= #{["Alice"] ["Bob"]} - (d/q '[:find ?n :where [?e :name ?n]] (d/history @conn))))))) + (d/q '[:find ?n :where [?e :name ?n]] (d/history @conn))))) + (d/release conn))) (deftest upsert-history (let [cfg {:store {:backend :mem @@ -354,7 +361,8 @@ (testing "since db attributes" (is (= db (:origin-db (d/since db current-tx)))) (is (= current-tx (:time-point (d/since db current-tx)))) - (is (= (:eavt db) (-> db (d/since current-tx) :origin-db :eavt)))))) + (is (= (:eavt db) (-> db (d/since current-tx) :origin-db :eavt)))) + (d/release conn))) (deftest test-filter-current-values-of-same-transaction (let [keyword-cfg {:store {:backend :mem} @@ -382,7 +390,8 @@ (is (= {:aka "Devil"} (d/pull as-of-db [:aka] michal))) (is (= nil - (d/pull @conn [:aka] michal))))) + (d/pull @conn [:aka] michal))) + (d/release conn))) (testing "cardinality many" (testing "keyword attributes" (let [schema [name-schema @@ -401,7 +410,8 @@ (is (= {:aka ["Devil"]} (d/pull as-of-db [:aka] michal))) (is (= nil - (d/pull @conn [:aka] michal))))) + (d/pull @conn [:aka] michal))) + (d/release conn))) (testing "reference attributes show all options" (let [schema [name-schema @@ -415,11 +425,13 @@ michal (:e (first (filter #(= "Michal" (:v %)) tx-data))) as-of-db (d/as-of @conn current-tx)] (is (= {:aka ["Devil" "Tupen"]} - (d/pull as-of-db [:aka] michal)))))))) + (d/pull as-of-db [:aka] michal))) + (d/release conn)))))) ;; https://github.com/replikativ/datahike/issues/572 (deftest as-of-should-fail-on-invalid-time-points (let [cfg (assoc-in cfg-template [:store :id] "as-of-invalid-time-points") conn (setup-db cfg)] (is (thrown-with-msg? Throwable #"Invalid transaction ID. Must be bigger than 536870912." - (d/as-of @conn 42))))) + (d/as-of @conn 42))) + (d/release conn))) diff --git a/test/datahike/test/transact_test.cljc b/test/datahike/test/transact_test.cljc index f38186295..83e6f86eb 100644 --- a/test/datahike/test/transact_test.cljc +++ b/test/datahike/test/transact_test.cljc @@ -193,7 +193,8 @@ #{["Petr"]})) (is (= (d/q '[:find ?v :where [1 :aka ?v]] @conn) - #{["Devil"] ["Tupen"]})))) + #{["Devil"] ["Tupen"]})) + (d/release conn))) (deftest test-db-fn-cas (let [conn (du/setup-db)] @@ -205,7 +206,8 @@ (d/transact conn {:tx-data [[:db/cas 1 :weight 300 400]]}) (is (= (:weight (d/entity @conn 1)) 400)) (is (thrown-with-msg? Throwable #":db.fn/cas failed on datom \[1 :weight 400\], expected 200" - (d/transact conn {:tx-data [[:db.fn/cas 1 :weight 200 210]]})))) + (d/transact conn {:tx-data [[:db.fn/cas 1 :weight 200 210]]}))) + (d/release conn)) (let [conn (du/setup-db {:initial-tx [{:db/ident :label :db/cardinality :db.cardinality/many}]})] (d/transact conn {:tx-data [[:db/add 1 :label :x]]}) @@ -213,19 +215,22 @@ (d/transact conn {:tx-data [[:db.fn/cas 1 :label :y :z]]}) (is (= (:label (d/entity @conn 1)) #{:x :y :z})) (is (thrown-with-msg? Throwable #":db.fn/cas failed on datom \[1 :label \(:x :y :z\)\], expected :s" - (d/transact conn {:tx-data [[:db.fn/cas 1 :label :s :t]]})))) + (d/transact conn {:tx-data [[:db.fn/cas 1 :label :s :t]]}))) + (d/release conn)) (let [conn (du/setup-db)] (d/transact conn {:tx-data [[:db/add 1 :name "Ivan"]]}) (d/transact conn {:tx-data [[:db.fn/cas 1 :age nil 42]]}) (is (= (:age (d/entity @conn 1)) 42)) (is (thrown-with-msg? Throwable #":db.fn/cas failed on datom \[1 :age 42\], expected nil" - (d/transact conn {:tx-data [[:db.fn/cas 1 :age nil 4711]]})))) + (d/transact conn {:tx-data [[:db.fn/cas 1 :age nil 4711]]}))) + (d/release conn)) (let [conn (du/setup-db)] (is (thrown-with-msg? Throwable #"Can't use tempid in '\[:db.fn/cas -1 :attr nil :val\]'. Tempids are allowed in :db/add only" (d/transact conn {:tx-data [[:db/add -1 :name "Ivan"] - [:db.fn/cas -1 :attr nil :val]]}))))) + [:db.fn/cas -1 :attr nil :val]]}))) + (d/release conn))) (deftest test-db-fn (let [conn (du/setup-db {:initial-tx [{:db/ident :aka :db/cardinality :db.cardinality/many}]}) @@ -253,7 +258,8 @@ (let [{:keys [db-after]} (d/transact conn {:tx-data [[:db.fn/call inc-age "Petr"]]}) e (d/entity db-after 1)] (is (= (:age e) 32)) - (is (:had-birthday e))))) + (is (:had-birthday e))) + (d/release conn))) #_(deftest test-db-ident-fn ;; TODO: check for :db/ident support within hhtree (let [conn (du/setup-db {:initial-tx [{:name {:db/unique :db.unique/identity}}]}) @@ -295,7 +301,8 @@ [3 "Sergey" 30 (+ const/tx0 2)]} (d/q '[:find ?e ?n ?a ?t :where [?e :name ?n ?t] - [?e :age ?a]] @conn))))) + [?e :age ?a]] @conn))) + (d/release conn))) (deftest test-resolve-eid-refs (let [conn (du/setup-db {:initial-tx [{:db/ident :friend @@ -317,46 +324,55 @@ (is (= (:tempids tx) {-1 3, -2 4, "B" 5, -3 6, :db/current-tx (+ const/tx0 2)})) (is (= (d/q {:query q :args [@conn "Sergey"]}) #{["Ivan"] ["Petr"]})) (is (= (d/q {:query q :args [@conn "Boris"]}) #{["Oleg"]})) - (is (= (d/q {:query q :args [@conn "Oleg"]}) #{["Boris"]}))) + (is (= (d/q {:query q :args [@conn "Oleg"]}) #{["Boris"]})) + (d/release conn)) (testing "Resolve eid for unique attributes with temporary reference value" - (let [conn (fn [] (du/setup-db {:initial-tx [{:db/ident :foo/match - :db/valueType :db.type/ref - :db/cardinality :db.cardinality/one - :db/unique :db.unique/identity}]})) + (let [cfg {:initial-tx [{:db/ident :foo/match + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity}]} query '[:find ?e ?a ?v :where [?e ?a ?v] [(= ?a :foo/match)]]] (testing "with maps" (testing "temp-eid first" - (let [report (d/transact (conn) [{:db/id 16 - :foo/match -1000001} - {:db/id -1000001 - :foo/match 16}]) + (let [conn (du/setup-db cfg) + report (d/transact conn [{:db/id 16 + :foo/match -1000001} + {:db/id -1000001 + :foo/match 16}]) id (get-in report [:tempids -1000001])] (is (= (d/q query (:db-after report)) - #{[16 :foo/match id] [id :foo/match 16]})))) + #{[16 :foo/match id] [id :foo/match 16]})) + (d/release conn))) (testing "temp-vid first" - (let [report (d/transact (conn) [{:db/id -1000001 - :foo/match 16} - {:db/id 16 - :foo/match -1000001}]) + (let [conn (du/setup-db cfg) + report (d/transact conn [{:db/id -1000001 + :foo/match 16} + {:db/id 16 + :foo/match -1000001}]) id (get-in report [:tempids -1000001])] (is (= (d/q query (:db-after report)) - #{[16 :foo/match id] [id :foo/match 16]}))))) + #{[16 :foo/match id] [id :foo/match 16]})) + (d/release conn)))) (testing "with vectors" (testing "temp-eid first" - (let [report (d/transact (conn) [[:db/add 16 :foo/match -1000001] - [:db/add -1000001 :foo/match 16]]) + (let [conn (du/setup-db cfg) + report (d/transact conn [[:db/add 16 :foo/match -1000001] + [:db/add -1000001 :foo/match 16]]) id (get-in report [:tempids -1000001])] (is (= (d/q query (:db-after report)) - #{[16 :foo/match id] [id :foo/match 16]})))) + #{[16 :foo/match id] [id :foo/match 16]})) + (d/release conn))) (testing "temp-vid first" - (let [report (d/transact (conn) [[:db/add -1000001 :foo/match 16] - [:db/add 16 :foo/match -1000001]]) + (let [conn (du/setup-db cfg) + report (d/transact conn [[:db/add -1000001 :foo/match 16] + [:db/add 16 :foo/match -1000001]]) id (get-in report [:tempids -1000001])] (is (= (d/q query (:db-after report)) - #{[16 :foo/match id] [id :foo/match 16]})))))))) + #{[16 :foo/match id] [id :foo/match 16]})) + (d/release conn))))))) (deftest test-resolve-current-tx (doseq [tx-tempid [:db/current-tx "datomic.tx" "datahike.tx"]] @@ -388,7 +404,8 @@ tx-id (get-in tx3 [:tempids tx-tempid])] (is (= tx-id (+ const/tx0 3))) (is (= (into {} (d/entity @conn tx-id)) - {:prop4 "prop4"}))))))) + {:prop4 "prop4"}))) + (d/release conn))))) (deftest test-tx-meta (testing "simple test" @@ -397,7 +414,8 @@ :age 5}] :tx-meta {:foo "bar"}})] (is (= (dissoc (:tx-meta tx) :db/txInstant :db/commitId) - {:foo "bar"})))) + {:foo "bar"})) + (d/release conn))) (testing "generative test" (let [conn (du/setup-db) Metadata (s/map-of keyword? (s/or :int int? @@ -410,7 +428,8 @@ :age 5}] :tx-meta generated})] (is (= (dissoc (:tx-meta tx-report) :db/txInstant :db/commitId) - generated)))) + generated)) + (d/release conn))) (testing "manual txInstant is the same as auto-generated" (let [conn (du/setup-db) date (tools/get-time) @@ -425,7 +444,8 @@ 536870913 true]] (mapv (comp #(into [] %) seq) - (d/datoms @conn :eavt)))))) + (d/datoms @conn :eavt)))) + (d/release conn))) (testing "missing schema definition" (let [schema [{:db/ident :name :db/cardinality :db.cardinality/one @@ -440,7 +460,8 @@ (is (thrown-with-msg? Throwable #"Bad entity attribute :foo at \[:db/add 536870914 :foo :bar 536870914\], not defined in current schema" (d/transact conn {:tx-data [{:name "Sergey" :age 5}] - :tx-meta {:foo :bar}}))))) + :tx-meta {:foo :bar}}))) + (d/release conn))) (testing "meta-data is available on the indices" (let [conn (du/setup-db) _ (d/transact conn {:tx-data [{:name "Sergey" @@ -449,7 +470,8 @@ (is (= #{[536870913 :bar]} (d/q '[:find ?e ?v :where [?e :foo ?v]] - @conn))))) + @conn))) + (d/release conn))) (testing "retracting metadata" (let [conn (du/setup-db) _ (d/transact conn {:tx-data [{:name "Sergey" @@ -459,7 +481,8 @@ (is (= #{} (d/q '[:find ?e ?v :where [?e :foo ?v]] - @conn))))) + @conn))) + (d/release conn))) (testing "overwrite metadata" (let [conn (du/setup-db) _ (d/transact conn {:tx-data [{:name "Sergey" @@ -469,7 +492,8 @@ (is (= #{[536870913 :baz]} (d/q '[:find ?e ?v :where [?e :foo ?v]] - @conn))))) + @conn))) + (d/release conn))) (testing "metadata has txInstant" (let [conn (du/setup-db) {:keys [tempids]} (d/transact conn {:tx-data [{:name "Sergey" @@ -479,4 +503,5 @@ '[:db/txInstant] (:db/current-tx tempids)) :db/txInstant - inst?))))) + inst?)) + (d/release conn)))) diff --git a/test/datahike/test/tuples_test.cljc b/test/datahike/test/tuples_test.cljc index 1397b7435..bf3e28535 100644 --- a/test/datahike/test/tuples_test.cljc +++ b/test/datahike/test/tuples_test.cljc @@ -46,7 +46,8 @@ (d/transact conn [{:prices ["a" "b" "fdsfdsf"]}]))))) (testing "of more than 8 values" (is (thrown-with-msg? ExceptionInfo #".*Cannot store more than 8 values .*" - (d/transact conn [{:prices [1 2 3 4 5 6 7 8 9]}])))))) + (d/transact conn [{:prices [1 2 3 4 5 6 7 8 9]}])))) + (d/release conn))) (testing "heterogeneous tuple" (let [conn (connect)] @@ -60,7 +61,8 @@ (d/transact conn [{:coord [100 :coord/west 9]}])))) (testing "with type mismatch" (is (thrown-with-msg? ExceptionInfo #".*Cannot store heterogeneous tuple: there is a mismatch between values.* and their types.*" - (d/transact conn [{:coord [100 9]}])))))) + (d/transact conn [{:coord [100 9]}])))) + (d/release conn))) (testing "composite tuple" (let [conn (connect) @@ -80,7 +82,8 @@ :db/cardinality :db.cardinality/one}])) (is (d/transact conn [{:reg/course "BIO-101" :reg/semester "2018-fall" - :reg/student "johndoe@university.edu"}]))))) + :reg/student "johndoe@university.edu"}])) + (d/release conn)))) (deftest test-transact-and-query-non-composite (testing "heterogeneous" @@ -93,7 +96,8 @@ (is (= #{[[100 :coord/west]]} (d/q '[:find ?v :where [_ :coord ?v]] - @conn))))) + @conn))) + (d/release conn))) (testing "homogeneous" (let [conn (connect)] (d/transact conn [{:db/ident :coord @@ -104,7 +108,8 @@ (is (= #{[[100 200 300]]} (d/q '[:find ?v :where [_ :coord ?v]] - @conn)))))) + @conn))) + (d/release conn)))) (deftest test-transact-and-query-composite (let [conn (connect)] @@ -127,7 +132,8 @@ (is (= #{[[123 nil nil]]} (d/q '[:find ?v :where [100 :a+b+c ?v]] - @conn))))) + @conn))) + (d/release conn))) (defn some-datoms [db es] @@ -215,7 +221,8 @@ [e :a+c+d [nil "C" "D"]]}) (is (thrown-with-msg? ExceptionInfo #"Can’t modify tuple attrs directly:.*" - (d/transact conn [{:db/id 100 :a+b ["A" "B"]}]))))) + (d/transact conn [{:db/id 100 :a+b ["A" "B"]}]))) + (d/release conn))) (deftest test-queries (let [conn (connect)] @@ -253,7 +260,8 @@ (is (= #{["A" "B"] ["A" "b"] ["a" "B"] ["a" "b"]} (d/q '[:find ?a ?b :where [?e :a+b ?a+b] - [(untuple ?a+b) [?a ?b]]] @conn))))) + [(untuple ?a+b) [?a ?b]]] @conn))) + (d/release conn))) (deftest test-lookup-refs (let [conn (connect)] @@ -322,7 +330,8 @@ :b "b" :a+b ["a" "b"] :c "c"} - (d/pull (d/db conn) '[*] [:a+b ["a" "b"]]))))) + (d/pull (d/db conn) '[*] [:a+b ["a" "b"]]))) + (d/release conn))) (deftest test-unique (let [conn (connect)] @@ -375,7 +384,8 @@ ;; adding entity with two tuple components in a single operation (d/transact conn [{:db/id 4 :a "a" :b "c"}]) (is (= {:db/id 4 :a "a" :b "c" :a+b ["a" "c"]} - (d/pull (d/db conn) '[*] 4)))))) + (d/pull (d/db conn) '[*] 4)))) + (d/release conn))) (deftest test-validation (let [db (db/empty-db {:a+b {:db/valueType :db.type/tuple diff --git a/test/datahike/test/upsert_impl_test.cljc b/test/datahike/test/upsert_impl_test.cljc index 7afb4cdb4..81cc7f4c5 100644 --- a/test/datahike/test/upsert_impl_test.cljc +++ b/test/datahike/test/upsert_impl_test.cljc @@ -97,8 +97,10 @@ (testing "IndexNode" (let [txs (vec (for [i (range 1000)] {:name (str "Peter" i) - :age i}))] - (is (d/transact (connect) txs)))) + :age i})) + conn (connect)] + (is (d/transact conn txs)) + (d/release conn))) (testing "simple upsert and history" (let [txs [[:db/add 199 :name "Peter"] @@ -106,7 +108,8 @@ conn (connect)] (is (d/transact conn txs)) (is (not (d/datoms @conn :eavt 199 :name "Peter"))) ;; no history - (is (d/datoms (d/history @conn) :eavt 199 :name "Peter")))) + (is (d/datoms (d/history @conn) :eavt 199 :name "Peter")) + (d/release conn))) (testing "transacting the same datoms twice should work with :avet" (let [dvec #(vector (:e %) (:a %) (:v %)) @@ -157,7 +160,8 @@ [10 (+ const/tx0 3) true] [10 (+ const/tx0 4) false] [1 (+ const/tx0 4) true]} - (d/q query (d/history @conn))))) + (d/q query (d/history @conn)))) + (d/release conn)) (testing "when only one transaction" (let [conn (setup-db cfg)] (d/transact conn [[:db/add [:name "Alice"] :age 20] @@ -171,7 +175,8 @@ [10 (+ const/tx0 2) true] [10 (+ const/tx0 2) false] [1 (+ const/tx0 2) true]} - (d/q query (d/history @conn)))))))) + (d/q query (d/history @conn)))) + (d/release conn))))) (deftest upsert-read-handlers (let [config {:store {:backend :file :path "/tmp/upsert-read-handlers"} @@ -194,6 +199,7 @@ (let [conn (d/connect config)] ;; Would fail if upsert read handlers are not present - (is (d/datoms @conn :eavt))) + (is (d/datoms @conn :eavt)) + (d/release conn)) (d/delete-database config))) diff --git a/test/datahike/test/upsert_test.cljc b/test/datahike/test/upsert_test.cljc index d9051d04c..6bc0df7bc 100644 --- a/test/datahike/test/upsert_test.cljc +++ b/test/datahike/test/upsert_test.cljc @@ -217,7 +217,8 @@ (testing "changing the datom value increases the history with 2 datoms: the retraction datom and the new value." (d/transact conn {:tx-data [{:db/id [:name "Alice"] :age 26}]}) - (is (= 3 (count (d/datoms (d/history @conn) :eavt [:name "Alice"] :age))))))) + (is (= 3 (count (d/datoms (d/history @conn) :eavt [:name "Alice"] :age))))) + (d/release conn))) (deftest temporal-history-mem (let [config {:store {:backend :mem :id "temp-hist-hht"} @@ -325,27 +326,36 @@ (testing "File upsert" (init-data file-conn) (let [cached-db @file-conn - fresh-db @(d/connect file-cfg) + fresh-conn (d/connect file-cfg) + fresh-db @fresh-conn actual-count (- initial-active-count inactive-count) cached-count (active-count cached-db) fresh-count (active-count fresh-db)] (is (= actual-count cached-count)) - (is (= cached-count fresh-count)))) + (is (= cached-count fresh-count)) + (d/release fresh-conn))) (testing "File pss upsert" (init-data file-pss-conn) (let [cached-db @file-pss-conn - fresh-db @(d/connect file-pss-cfg) + fresh-conn (d/connect file-pss-cfg) + fresh-db @fresh-conn actual-count (- initial-active-count inactive-count) cached-count (active-count cached-db) fresh-count (active-count fresh-db)] (is (= actual-count cached-count)) - (is (= cached-count fresh-count)))) + (is (= cached-count fresh-count)) + (d/release fresh-conn))) (testing "Mem upsert" (init-data mem-conn) (let [cached-db @mem-conn - fresh-db @(d/connect mem-cfg) + fresh-conn (d/connect mem-cfg) + fresh-db @fresh-conn actual-count (- initial-active-count inactive-count) cached-count (active-count cached-db) fresh-count (active-count fresh-db)] (is (= actual-count cached-count)) - (is (= cached-count fresh-count))))))) + (is (= cached-count fresh-count)) + (d/release fresh-conn)))) + (d/release file-conn) + (d/release file-pss-conn) + (d/release mem-conn))) diff --git a/test/datahike/test/versioning_test.cljc b/test/datahike/test/versioning_test.cljc index ed44ebe53..4b233301a 100644 --- a/test/datahike/test/versioning_test.cljc +++ b/test/datahike/test/versioning_test.cljc @@ -33,7 +33,8 @@ (let [foo-conn (d/connect (assoc cfg :branch :foo))] (d/transact foo-conn [{:age 42}]) ;; extracted data from foo and decide to merge it into :db - (merge! conn #{:foo} [{:age 42}])) + (merge! conn #{:foo} [{:age 42}]) + (d/release foo-conn)) (is (= 4 (count ( Date: Fri, 24 Mar 2023 22:07:35 -0700 Subject: [PATCH 02/67] Remove randomization from tests. --- test/datahike/test/attribute_refs/utils.cljc | 2 +- test/datahike/test/config_test.cljc | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/datahike/test/attribute_refs/utils.cljc b/test/datahike/test/attribute_refs/utils.cljc index 923f7322b..08c8b8d32 100644 --- a/test/datahike/test/attribute_refs/utils.cljc +++ b/test/datahike/test/attribute_refs/utils.cljc @@ -3,7 +3,7 @@ [datahike.db.interface :as dbi] [datahike.test.utils :refer [get-time]])) -(def ref-config {:store {:backend :mem} +(def ref-config {:store {:backend :mem :id "ref-config"} :writer {:backend :self} :attribute-refs? true :keep-history? false diff --git a/test/datahike/test/config_test.cljc b/test/datahike/test/config_test.cljc index d19bc25f4..dc7520904 100644 --- a/test/datahike/test/config_test.cljc +++ b/test/datahike/test/config_test.cljc @@ -57,8 +57,7 @@ (deftest load-config-test (testing "configuration defaults" (let [config (c/load-config)] - (is (= (merge {:store {:backend :mem - :id "default"} + (is (= (merge {:store {:backend :mem} :attribute-refs? c/*default-attribute-refs?* :keep-history? c/*default-keep-history?* :schema-flexibility c/*default-schema-flexibility* @@ -70,7 +69,7 @@ :store-cache-size c/*default-store-cache-size*} (when (seq (di/default-index-config c/*default-index*)) {:index-config (di/default-index-config c/*default-index*)})) - (-> config (dissoc :name))))))) + (update config :store dissoc :id)))))) (deftest core-config-test (testing "Schema on write in core empty database" From 54ac4724873feeefd526541ad1ad425e292d2fe2 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 25 Mar 2023 13:27:40 -0700 Subject: [PATCH 03/67] Release connections locally on deletions, but warn if they were not released. --- src/datahike/api.cljc | 2 +- src/datahike/connector.cljc | 18 +++--------------- src/datahike/experimental/versioning.cljc | 7 ++++--- src/datahike/writing.cljc | 13 +++++++++++-- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/datahike/api.cljc b/src/datahike/api.cljc index dec2f50cc..0cf785936 100644 --- a/src/datahike/api.cljc +++ b/src/datahike/api.cljc @@ -70,7 +70,7 @@ :temporal-index (s/? (s/cat :k (s/? (s/and #(= % :temporal-index))) :v boolean?)) :schema-on-read (s/? (s/cat :k (s/? (s/and #(= % :schema-on-read))) :v boolean?))) :nil (s/cat)) - :ret nil?) + :ret spec/SConfig) (declare transact release) diff --git a/src/datahike/connector.cljc b/src/datahike/connector.cljc index f73b2e920..69d85266a 100644 --- a/src/datahike/connector.cljc +++ b/src/datahike/connector.cljc @@ -1,5 +1,7 @@ (ns ^:no-doc datahike.connector - (:require [datahike.store :as ds] + (:require [datahike.connections :refer [get-connection add-connection! delete-connection! + connections]] + [datahike.store :as ds] [datahike.writing :as dsi] [datahike.config :as dc] [datahike.tools :as dt] @@ -62,20 +64,6 @@ (s/def ::connection #(and (instance? Connection %) (not= @(:wrapped-atom %) :deleted))) -(def connections (atom {})) - -(defn get-connection [conn-id] - (when-let [conn (get-in @connections [conn-id :conn])] - (swap! connections update-in [conn-id :count] inc) - conn)) - -(defn add-connection! [conn-id conn] - (swap! connections assoc conn-id {:conn conn :count 1})) - -(defn delete-connection! [conn-id] - (reset! (get-connection conn-id) :released) - (swap! connections dissoc conn-id)) - (defn version-check [{:keys [meta config] :as db}] (let [{dh-stored :datahike/version hh-stored :hitchhiker.tree/version diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index b6eb3e08f..19b867723 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -1,11 +1,11 @@ (ns datahike.experimental.versioning "Git-like versioning tools for Datahike." (:require [konserve.core :as k] + [datahike.connections :refer [delete-connection!]] [datahike.core :refer [transact]] [datahike.store :refer [store-identity]] [datahike.writing :refer [stored->db db->stored stored-db? update-and-flush-db create-commit-id]] - [datahike.connector :refer [delete-connection!]] [superv.async :refer [ Date: Sat, 25 Mar 2023 13:31:43 -0700 Subject: [PATCH 04/67] Add connections file... --- src/datahike/connections.cljc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 src/datahike/connections.cljc diff --git a/src/datahike/connections.cljc b/src/datahike/connections.cljc new file mode 100644 index 000000000..5026d3a3a --- /dev/null +++ b/src/datahike/connections.cljc @@ -0,0 +1,16 @@ +(ns ^:no-doc datahike.connections) + +(def connections (atom {})) + +(defn get-connection [conn-id] + (when-let [conn (get-in @connections [conn-id :conn])] + (swap! connections update-in [conn-id :count] inc) + conn)) + +(defn add-connection! [conn-id conn] + (swap! connections assoc conn-id {:conn conn :count 1})) + +(defn delete-connection! [conn-id] + (when-let [conn (get-connection conn-id)] + (reset! conn :released) + (swap! connections dissoc conn-id))) From b1baacb27d64e506304d0ceefad933224670b82d Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 25 Mar 2023 16:23:04 -0700 Subject: [PATCH 05/67] Fix create-database and connection spec. --- src/datahike/api.cljc | 20 ++++++++++---------- src/datahike/connector.cljc | 5 ++++- src/datahike/spec.cljc | 9 +++------ 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/datahike/api.cljc b/src/datahike/api.cljc index 0cf785936..6af91ce76 100644 --- a/src/datahike/api.cljc +++ b/src/datahike/api.cljc @@ -26,7 +26,7 @@ connect :args (s/alt :config (s/cat :config spec/SConfig) :nil (s/cat)) - :ret spec/SConnectionAtom) + :ret spec/SConnection) (def ^{:arglists '([] [config]) :doc "Connects to a datahike database via configuration map. For more information on the configuration refer to the [docs](https://github.com/replikativ/datahike/blob/master/doc/config.md). @@ -70,7 +70,7 @@ :temporal-index (s/? (s/cat :k (s/? (s/and #(= % :temporal-index))) :v boolean?)) :schema-on-read (s/? (s/cat :k (s/? (s/and #(= % :schema-on-read))) :v boolean?))) :nil (s/cat)) - :ret spec/SConfig) + :ret #(s/valid? spec/SConfig [%])) (declare transact release) @@ -132,7 +132,7 @@ (s/fdef transact! - :args (s/cat :conn spec/SConnectionAtom :txs spec/STransactions) + :args (s/cat :conn spec/SConnection :txs spec/STransactions) :ret #(s/valid? spec/STransactionReport @%)) (def ^{:arglists '([conn tx-data tx-meta]) :no-doc true} @@ -141,7 +141,7 @@ (s/fdef transact - :args (s/cat :conn spec/SConnectionAtom :txs spec/STransactions) + :args (s/cat :conn spec/SConnection :txs spec/STransactions) :ret spec/STransactionReport) (def ^{:arglists '([conn arg-map]) :doc "Applies transaction to the underlying database value and atomically updates the connection reference to point to the result of that transaction, the new db value. @@ -246,7 +246,7 @@ (s/fdef load-entities - :args (s/cat :conn spec/SConnectionAtom :txs spec/STransactions) + :args (s/cat :conn spec/SConnection :txs spec/STransactions) :ret #(s/valid? spec/STransactionReport @%)) ; This returns a throwable promise, so we have to dereference it.. (def ^{:arglists '([conn tx-data]) :doc "Load entities directly"} @@ -255,7 +255,7 @@ (s/fdef release - :args (s/cat :conn spec/SConnectionAtom) + :args (s/cat :conn spec/SConnection) :ret nil?) (def ^{:arglists '([conn] [conn release-all?]) :doc "Releases a database connection. You need to release a connection as many times as you connected to it for it to be completely released. Set release-all? to true to force its release."} @@ -706,7 +706,7 @@ (s/fdef db - :args (s/cat :conn spec/SConnectionAtom) + :args (s/cat :conn spec/SConnection) :ret spec/SDB) (defn db "Returns the underlying immutable database value from a connection. @@ -886,8 +886,8 @@ (s/fdef listen - :args (s/alt :no-key (s/cat :conn spec/SConnectionAtom :callback fn?) - :with-key (s/cat :conn spec/SConnectionAtom :key any? :callback fn?)) + :args (s/alt :no-key (s/cat :conn spec/SConnection :callback fn?) + :with-key (s/cat :conn spec/SConnection :key any? :callback fn?)) :ret any? :fn #(if (= :with-key (-> % :args first)) (= (:ret %) (-> % :args second :key)) @@ -904,7 +904,7 @@ (s/fdef unlisten - :args (s/cat :conn spec/SConnectionAtom :key any?) + :args (s/cat :conn spec/SConnection :key any?) :ret map?) (def ^{:arglists '([conn key]) :doc "Removes registered listener from connection. See also [[listen]]."} diff --git a/src/datahike/connector.cljc b/src/datahike/connector.cljc index 69d85266a..1b6551484 100644 --- a/src/datahike/connector.cljc +++ b/src/datahike/connector.cljc @@ -37,6 +37,9 @@ IMeta (meta [_] (meta wrapped-atom))) +(defn connection? [x] + (instance? Connection x)) + #?(:clj (defmethod print-method Connection [^Connection conn ^java.io.Writer w] @@ -62,7 +65,7 @@ (Connection. (atom db :meta {:listeners (atom {})}))) (s/def ::connection #(and (instance? Connection %) - (not= @(:wrapped-atom %) :deleted))) + (not= @(:wrapped-atom %) :released))) (defn version-check [{:keys [meta config] :as db}] (let [{dh-stored :datahike/version diff --git a/src/datahike/spec.cljc b/src/datahike/spec.cljc index e69ac4bbe..923d91616 100644 --- a/src/datahike/spec.cljc +++ b/src/datahike/spec.cljc @@ -1,5 +1,6 @@ (ns datahike.spec - (:require [datahike.datom :refer [datom?]] + (:require [datahike.connector :refer [connection?]] + [datahike.datom :refer [datom?]] [datahike.db.utils :as dbu] [spec-tools.data-spec :as ds] [clojure.spec.alpha :as s]) @@ -21,11 +22,7 @@ :deprecated/map :deprecated/config :deprecated/uri string?)) -(def SConnectionAtom - (fn [x] - (and - (instance? IAtom x) - (s/valid? SDB @x)))) +(def SConnection connection?) (def SEId (s/or :number number? :coll sequential? :keyword keyword?)) From 319345cdbe4b1f180e691b463b71dba196e7ed6b Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Mar 2023 23:09:37 -0700 Subject: [PATCH 06/67] First take on parallel operations for writer. --- src/datahike/index/persistent_set.cljc | 7 +-- src/datahike/writer.cljc | 4 +- src/datahike/writing.cljc | 60 +++++++++++++++----------- 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/src/datahike/index/persistent_set.cljc b/src/datahike/index/persistent_set.cljc index 33f887bd1..b0a7e0b73 100644 --- a/src/datahike/index/persistent_set.cljc +++ b/src/datahike/index/persistent_set.cljc @@ -147,13 +147,13 @@ (uuid (mapv (comp vec seq) (.keys node)))) (uuid))) -(defrecord CachedStorage [store config cache stats] +(defrecord CachedStorage [store config cache stats pending-writes] IStorage (store [_ node] (swap! stats update :writes inc) (let [address (gen-address node (:crypto-hash? config)) _ (trace "writing storage: " address " crypto: " (:crypto-hash? config))] - (k/assoc store address node {:sync? true}) + (swap! pending-writes conj (k/assoc store address node {:sync? false})) (wrapped/miss cache address node) address)) (accessed [_ address] @@ -176,7 +176,8 @@ (defn create-storage [store config] (CachedStorage. store config (atom (cache/lru-cache-factory {} :threshold (:store-cache-size config))) - (atom init-stats))) + (atom init-stats) + (atom []))) (def ^:const BRANCHING_FACTOR 512) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 2560abf9d..6aa124bf6 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -1,5 +1,5 @@ (ns ^:no-doc datahike.writer - (:require [superv.async :refer [S thread-try]] + (:require [superv.async :refer [S thread-try db - (assoc-in [:meta :datahike/commit-id] cid) - (assoc-in [:meta :datahike/parents] parents)) - db-to-store (db->stored db true)] - (k/assoc store cid db-to-store {:sync? true}) - (k/assoc store (:branch config) db-to-store {:sync? true}) - db)) + (go-try- + (let [parents (or parents #{(get config :branch)}) + parents (branch-heads-as-commits store parents) + cid (create-commit-id db) + db (-> db + (assoc-in [:meta :datahike/commit-id] cid) + (assoc-in [:meta :datahike/parents] parents)) + db-to-store (db->stored db true) + commit-log-op (k/assoc store cid db-to-store {:sync? false}) + branch-op (k/assoc store (:branch config) db-to-store {:sync? false})] + ( Date: Sun, 19 Mar 2023 23:10:32 -0700 Subject: [PATCH 07/67] Fix format. --- src/datahike/writing.cljc | 60 +++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index 622808ac5..c8752d82b 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -105,42 +105,42 @@ (defn commit! [store config db parents] (go-try- - (let [parents (or parents #{(get config :branch)}) - parents (branch-heads-as-commits store parents) - cid (create-commit-id db) - db (-> db - (assoc-in [:meta :datahike/commit-id] cid) - (assoc-in [:meta :datahike/parents] parents)) - db-to-store (db->stored db true) - commit-log-op (k/assoc store cid db-to-store {:sync? false}) - branch-op (k/assoc store (:branch config) db-to-store {:sync? false})] - ( db + (assoc-in [:meta :datahike/commit-id] cid) + (assoc-in [:meta :datahike/parents] parents)) + db-to-store (db->stored db true) + commit-log-op (k/assoc store cid db-to-store {:sync? false}) + branch-op (k/assoc store (:branch config) db-to-store {:sync? false})] + ( Date: Sat, 25 Mar 2023 04:03:57 -0700 Subject: [PATCH 08/67] Implement batching transactor. --- src/datahike/writer.cljc | 99 +++++++++++++++++++++++++++------------ src/datahike/writing.cljc | 42 +++++++++-------- 2 files changed, 92 insertions(+), 49 deletions(-) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 6aa124bf6..1fe91f5b5 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -3,8 +3,8 @@ [taoensso.timbre :as log] [datahike.core] [datahike.writing :as w] - [datahike.tools :as dt :refer [throwable-promise]] - [clojure.core.async :refer [chan close! promise-chan put! go go-loop ! poll!]]) (:import [clojure.lang ExceptionInfo])) (defprotocol PWriter @@ -12,7 +12,7 @@ (-shutdown [_] "Returns a channel that resolves when the writer has shut down.") (-streaming? [_] "Returns whether the transactor is streaming updates directly into the connection, so it does not need to fetch from store on read.")) -(defrecord LocalWriter [queue thread streaming?] +(defrecord LocalWriter [queue thread streaming? queue-size] PWriter (-dispatch! [_ arg-map] (let [p (promise-chan)] @@ -23,33 +23,70 @@ thread) (-streaming? [_] streaming?)) +(def ^:const default-queue-size 100000) + (defn create-thread "Creates new transaction thread" - [connection queue write-fn-map] + [connection queue write-fn-map queue-size] (thread-try S - (go-loop [] - (if-let [{:keys [op args callback] :as invocation} (! pending-txs-ch [res callback]))) + (recur)) + (do + (close! pending-txs-ch) + (log/debug "Writer thread gracefully closed")))) + ;; commit loop + (go-loop [tx (LocalWriter {:queue queue + :queue-size queue-size :thread thread :streaming? true}))) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index c8752d82b..b9ceea963 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -11,7 +11,8 @@ [konserve.core :as k] [taoensso.timbre :as log] [hasch.core :refer [uuid]] - [superv.async :refer [go-try- stored db true) + _ (when-let [pending-futures (:pending-writes (:storage store))] + (loop [[f & r] @pending-futures] + (when f ( Date: Sat, 25 Mar 2023 12:47:20 -0700 Subject: [PATCH 09/67] Expose buffer sizes and warn on back pressure. Reduce blocking => speedup. Rename get-time to get-date. --- src/datahike/db/transaction.cljc | 6 +- src/datahike/tools.cljc | 8 +- src/datahike/writer.cljc | 150 ++++++++++++++------------ src/datahike/writing.cljc | 59 +++++----- test/datahike/test/transact_test.cljc | 2 +- 5 files changed, 125 insertions(+), 100 deletions(-) diff --git a/src/datahike/db/transaction.cljc b/src/datahike/db/transaction.cljc index 2fd8738b1..5e08e475e 100644 --- a/src/datahike/db/transaction.cljc +++ b/src/datahike/db/transaction.cljc @@ -9,7 +9,7 @@ [datahike.db.search :as dbs] [datahike.db.utils :as dbu] [datahike.constants :refer [tx0]] - [datahike.tools :refer [get-time raise]] + [datahike.tools :refer [get-date raise]] [datahike.schema :as ds] [me.tonsky.persistent-sorted-set.arrays :as arrays]) #?(:cljs (:require-macros [datahike.datom :refer [datom]] @@ -488,7 +488,7 @@ (defn flush-tx-meta "Generates add-operations for transaction meta data." [{:keys [tx-meta db-before] :as report}] - (let [;; tx-meta (merge {:db/txInstant (get-time)} tx-meta) + (let [;; tx-meta (merge {:db/txInstant (get-date)} tx-meta) tid (current-tx report) {:keys [attribute-refs?]} (dbi/-config db-before)] (reduce-kv @@ -749,7 +749,7 @@ (interleave initial-es (repeat ::flush-tuples)) initial-es) initial-report (update initial-report :tx-meta - #(merge {:db/txInstant (get-time)} %)) + #(merge {:db/txInstant (get-date)} %)) meta-entities (flush-tx-meta initial-report)] (loop [report (update initial-report :db-after transient) es (if (dbi/-keep-history? db-before) diff --git a/src/datahike/tools.cljc b/src/datahike/tools.cljc index 38e0c8d80..f2c34b346 100644 --- a/src/datahike/tools.cljc +++ b/src/datahike/tools.cljc @@ -24,10 +24,14 @@ (defmacro case-tree [qs vs] (-case-tree qs vs))) -(defn ^:dynamic get-time [] - #?(:clj (java.util.Date.) +(defn ^:dynamic get-date [] + #?(:clj (Date.) :cljs (js/Date.))) +(defn ^:dynamic get-time-ms [] + #?(:clj (.getTime (Date.)) + :cljs (.getTime (js/Date.)))) + (defmacro raise "Logging an error and throwing an exception with message and structured data. Arguments: diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 1fe91f5b5..d6b0f7fc1 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -3,8 +3,8 @@ [taoensso.timbre :as log] [datahike.core] [datahike.writing :as w] - [datahike.tools :as dt :refer [throwable-promise get-time]] - [clojure.core.async :refer [chan close! promise-chan put! go go-loop ! poll!]]) + [datahike.tools :as dt :refer [throwable-promise get-time-ms]] + [clojure.core.async :refer [chan close! promise-chan put! go go-loop ! poll! buffer]]) (:import [clojure.lang ExceptionInfo])) (defprotocol PWriter @@ -12,39 +12,47 @@ (-shutdown [_] "Returns a channel that resolves when the writer has shut down.") (-streaming? [_] "Returns whether the transactor is streaming updates directly into the connection, so it does not need to fetch from store on read.")) -(defrecord LocalWriter [queue thread streaming? queue-size] +(defrecord LocalWriter [thread streaming? transaction-queue-size commit-queue-size + transaction-queue commit-queue] PWriter (-dispatch! [_ arg-map] (let [p (promise-chan)] - (put! queue (assoc arg-map :callback p)) + (put! transaction-queue (assoc arg-map :callback p)) p)) (-shutdown [_] - (close! queue) + (close! transaction-queue) thread) (-streaming? [_] streaming?)) -(def ^:const default-queue-size 100000) +(def ^:const default-queue-size 200000) (defn create-thread "Creates new transaction thread" - [connection queue write-fn-map queue-size] - (thread-try - S - (let [pending-txs-ch (chan queue-size) - store (:store @(:wrapped-atom connection))] - ;; processing loop - (go-loop [] - (if-let [{:keys [op args callback] :as invocation} ( (count transaction-queue-buffer) (/ transaction-queue-size 2)) + (log/warn "Transaction queue buffer more than 50% full, " + (count transaction-queue-buffer) "of" transaction-queue-size " filled." + "Reduce transaction frequency.")) + (let [op-fn (write-fn-map op) + res (try + (apply op-fn connection args) + ;; Only catch ExceptionInfo here (intentionally rejected transactions). + ;; Any other exceptions should crash the writer and signal the supervisor. + (catch Exception e + (log/errorf "Error during invocation" invocation e args) + ;; take a guess that a NPE was triggered by an invalid connection ;; short circuit on errors (put! callback (if (= (type e) NullPointerException) @@ -54,39 +62,44 @@ :connection connection :error e}) e)) - :error)))] - (when-not (= res :error) - (>! pending-txs-ch [res callback]))) - (recur)) - (do - (close! pending-txs-ch) - (log/debug "Writer thread gracefully closed")))) - ;; commit loop - (go-loop [tx ( (count commit-queue-buffer) (/ commit-queue-size 2)) + (log/warn "Commit queue buffer more than 50% full, " + (count commit-queue-buffer) "of" commit-queue-size " filled." + "Reduce transaction frequency.")))) + (recur)) + (do + (close! commit-queue) + (log/debug "Writer thread gracefully closed")))) + ;; commit loop + (go-loop [tx (LocalWriter - {:queue queue - :queue-size queue-size + {:transaction-queue transaction-queue + :transaction-queue-size transaction-queue-size + :commit-queu commit-queue + :commit-queue-size commit-queue-size :thread thread :streaming? true}))) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index b9ceea963..716bd5b1a 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -12,7 +12,7 @@ [taoensso.timbre :as log] [hasch.core :refer [uuid]] [superv.async :refer [go-try- db - (assoc-in [:meta :datahike/commit-id] cid) - (assoc-in [:meta :datahike/parents] parents)) - db-to-store (db->stored db true) - _ (when-let [pending-futures (:pending-writes (:storage store))] - (loop [[f & r] @pending-futures] - (when f ( db + (assoc-in [:meta :datahike/commit-id] cid) + (assoc-in [:meta :datahike/parents] parents)) + db-to-store (db->stored db true) + _ (when-let [pending-futures (:pending-writes (:storage store))] + (loop [[f & r] @pending-futures] + (when f ( Date: Sat, 25 Mar 2023 20:41:33 -0700 Subject: [PATCH 10/67] Provide synchronous flushing of pending writes. --- src/datahike/experimental/versioning.cljc | 4 +++- src/datahike/writing.cljc | 25 +++++++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index 19b867723..0605d6460 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -5,7 +5,8 @@ [datahike.core :refer [transact]] [datahike.store :refer [store-identity]] [datahike.writing :refer [stored->db db->stored stored-db? - update-and-flush-db create-commit-id]] + update-and-flush-db create-commit-id + flush-pending-writes]] [superv.async :refer [stored "Maps memory db to storage layout and flushes dirty indices." [db flush?] @@ -117,10 +137,7 @@ (assoc-in [:meta :datahike/commit-id] cid) (assoc-in [:meta :datahike/parents] parents)) db-to-store (db->stored db true) - _ (when-let [pending-futures (:pending-writes (:storage store))] - (loop [[f & r] @pending-futures] - (when f ( Date: Tue, 28 Mar 2023 01:36:42 -0700 Subject: [PATCH 11/67] Fix outstanding tests. --- src/datahike/experimental/versioning.cljc | 3 ++- src/datahike/writer.cljc | 31 +++++++++++++---------- src/datahike/writing.cljc | 6 ++++- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index 0605d6460..a0ce4adf3 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -141,4 +141,5 @@ ([conn parents tx-data tx-meta] (parent-check parents) (update-and-flush-db conn tx-data tx-meta transact - (conj parents (get-in @conn [:config :branch]))))) + (conj parents (get-in @conn [:config :branch])) + true))) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index d6b0f7fc1..7df7de49f 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -4,7 +4,7 @@ [datahike.core] [datahike.writing :as w] [datahike.tools :as dt :refer [throwable-promise get-time-ms]] - [clojure.core.async :refer [chan close! promise-chan put! go go-loop ! poll! buffer]]) + [clojure.core.async :refer [chan close! promise-chan put! go go-loop ! poll! buffer timeout]]) (:import [clojure.lang ExceptionInfo])) (defprotocol PWriter @@ -24,14 +24,14 @@ thread) (-streaming? [_] streaming?)) -(def ^:const default-queue-size 200000) +(def ^:const default-queue-size 20000) (defn create-thread "Creates new transaction thread" [connection write-fn-map transaction-queue-size commit-queue-size] - (let [transaction-queue-buffer (buffer transaction-queue-size) - transaction-queue (chan transaction-queue-buffer) - commit-queue-buffer (buffer commit-queue-size) + (let [transaction-queue-buffer (buffer transaction-queue-size) + transaction-queue (chan transaction-queue-buffer) + commit-queue-buffer (buffer commit-queue-size) commit-queue (chan commit-queue-buffer)] [transaction-queue commit-queue (thread-try @@ -41,8 +41,8 @@ (go-loop [] (if-let [{:keys [op args callback] :as invocation} ( (count transaction-queue-buffer) (/ transaction-queue-size 2)) - (log/warn "Transaction queue buffer more than 50% full, " + (when (> (count transaction-queue-buffer) (* 0.9 transaction-queue-size)) + (log/warn "Transaction queue buffer more than 90% full, " (count transaction-queue-buffer) "of" transaction-queue-size " filled." "Reduce transaction frequency.")) (let [op-fn (write-fn-map op) @@ -64,11 +64,12 @@ e)) :error))] (when-not (= res :error) - (put! commit-queue [res callback]) (when (> (count commit-queue-buffer) (/ commit-queue-size 2)) (log/warn "Commit queue buffer more than 50% full, " (count commit-queue-buffer) "of" commit-queue-size " filled." - "Reduce transaction frequency.")))) + "Throttling transaction processing. Reduce transaction frequency.") + ( res + (assoc-in [:tx-meta :db/commitId] commit-id) + (assoc-in [:db-after :meta :datahike/commit-id] commit-id))))) (catch Exception e (doseq [[_ callback] @txs] (put! callback e)) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index 55a8c66d2..a67521eb6 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -158,7 +158,11 @@ meta (assoc meta :datahike/updated-at txInstant) db (assoc db-after :meta meta) store (:store @(:wrapped-atom connection)) - db (if commit? (commit! store config db parents false) db)] + db (if commit? (commit! store config db parents true) db) + parents (or parents #{(get config :branch)}) + parents (branch-heads-as-commits store parents) + db (assoc-in db [:meta :datahike/parents] parents) + tx-report (assoc tx-report :db-after db)] (reset! connection db) tx-report (if commit? From ea4c1658e84d613c602bcecef0e0a90a6e458486 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 12 Nov 2023 19:45:21 -0800 Subject: [PATCH 12/67] Update Datahike API and writer --- src/datahike/api/specification.cljc | 2 +- src/datahike/writer.cljc | 2 +- src/datahike/writing.cljc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/datahike/api/specification.cljc b/src/datahike/api/specification.cljc index 7716acdea..c2767b897 100644 --- a/src/datahike/api/specification.cljc +++ b/src/datahike/api/specification.cljc @@ -136,7 +136,7 @@ Exists for Datomic API compatibility. Prefer using `@conn` directly if possible. :doc "Same as transact, but asynchronously returns a future." :supports-remote? false :referentially-transparent? false - :impl datahike.writing/transact!} + :impl datahike.writer/transact!} transact {:args (s/cat :conn spec/SConnection :txs spec/STransactions) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index ccf903c7c..880bba088 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -24,7 +24,7 @@ thread) (-streaming? [_] streaming?)) -(def ^:const default-queue-size 20000) +(def ^:const default-queue-size 100000) (defn create-thread "Creates new transaction thread" diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index 29206d654..5fec7d6ff 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -148,7 +148,7 @@ ([connection tx-data tx-meta update-fn] (update-and-commit! connection tx-data tx-meta update-fn nil)) ([connection tx-data tx-meta update-fn parents] - (update-and-commit! connection tx-data tx-meta update-fn parents (not (:datahike/noCommit tx-meta)))) + (update-and-commit! connection tx-data tx-meta update-fn parents false)) ([connection tx-data tx-meta update-fn parents commit?] (let [{:keys [db-after] {:keys [db/txInstant]} From b69466c2b78579ed22abeb8409cbb4bb26d71536 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 12 Nov 2023 22:44:14 -0800 Subject: [PATCH 13/67] Update persistent-sorted-set version and branching factor --- deps.edn | 2 +- src/datahike/index/persistent_set.cljc | 35 ++++++++++++++++++-------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/deps.edn b/deps.edn index 4ae527b1d..fba4da3f6 100644 --- a/deps.edn +++ b/deps.edn @@ -9,7 +9,7 @@ :exclusions [org.clojure/clojurescript]} io.replikativ/datalog-parser {:mvn/version "0.2.28"} io.replikativ/zufall {:mvn/version "0.2.9"} - persistent-sorted-set/persistent-sorted-set {:mvn/version "0.2.3"} + persistent-sorted-set/persistent-sorted-set {:mvn/version "0.3.0"} environ/environ {:mvn/version "1.2.0"} nrepl/bencode {:mvn/version "1.1.0"} com.taoensso/timbre {:mvn/version "6.3.1"} diff --git a/src/datahike/index/persistent_set.cljc b/src/datahike/index/persistent_set.cljc index eeec652db..d75e01c53 100644 --- a/src/datahike/index/persistent_set.cljc +++ b/src/datahike/index/persistent_set.cljc @@ -12,7 +12,7 @@ [taoensso.timbre :refer [trace]]) #?(:clj (:import [datahike.datom Datom] [org.fressian.handlers WriteHandler ReadHandler] - [me.tonsky.persistent_sorted_set PersistentSortedSet IStorage Leaf Branch ANode] + [me.tonsky.persistent_sorted_set PersistentSortedSet IStorage Leaf Branch ANode Settings] [java.util UUID List]))) (defn index-type->cmp @@ -180,17 +180,18 @@ (atom init-stats) (atom []))) -(def ^:const BRANCHING_FACTOR 512) +(def ^:const DEFAULT_BRANCHING_FACTOR 512) (defmethod di/empty-index :datahike.index/persistent-set [_index-name store index-type _] - (psset/set-branching-factor! BRANCHING_FACTOR) - (let [^PersistentSortedSet pset (psset/sorted-set-by (index-type->cmp-quick index-type false))] - (set! (.-_storage pset) (:storage store)) + (let [^PersistentSortedSet pset (psset/sorted-set* {:cmp (index-type->cmp-quick index-type false) + :storage (:storage store) + :branching-factor DEFAULT_BRANCHING_FACTOR})] + #_(set! (.-_storage pset) (:storage store)) (with-meta pset {:index-type index-type}))) (defmethod di/init-index :datahike.index/persistent-set [_index-name store datoms index-type _ {:keys [indexed]}] - (psset/set-branching-factor! BRANCHING_FACTOR) + #_(psset/set-branching-factor! BRANCHING_FACTOR) (let [arr (if (= index-type :avet) (->> datoms (filter #(contains? indexed (.-a ^Datom %))) @@ -199,14 +200,26 @@ (not (arrays/array? datoms)) (arrays/into-array))) _ (arrays/asort arr (index-type->cmp-quick index-type false)) - ^PersistentSortedSet pset (psset/from-sorted-array (index-type->cmp-quick index-type false) arr)] + ^PersistentSortedSet pset (psset/from-sorted-array (index-type->cmp-quick index-type false) + arr + (alength arr) + {:branching-factor DEFAULT_BRANCHING_FACTOR})] (set! (.-_storage pset) (:storage store)) (with-meta pset {:index-type index-type}))) +;; temporary import from psset until public +(defn- map->settings ^Settings [m] + (Settings. + (int (or (:branching-factor m) 0)) + nil ;; weak ref default + )) + + (defmethod di/add-konserve-handlers :datahike.index/persistent-set [config store] ;; deal with circular reference between storage and store - (let [storage (atom nil) + (let [settings (map->settings {:branching-factor DEFAULT_BRANCHING_FACTOR}) + storage (atom nil) store (assoc store :serializers {:FressianSerializer (fressian-serializer @@ -218,17 +231,17 @@ ;; The following fields are reset as they cannot be accessed from outside: ;; - 'edit' is set to false, i.e. the set is assumed to be persistent, not transient ;; - 'version' is set back to 0 - (PersistentSortedSet. meta cmp address @storage nil count nil 0)))) + (PersistentSortedSet. meta cmp address @storage nil count settings 0)))) "datahike.index.PersistentSortedSet.Leaf" (reify ReadHandler (read [_ reader _tag _component-count] (let [{:keys [keys level]} (.readObject reader)] - (Leaf. keys)))) + (Leaf. keys settings)))) "datahike.index.PersistentSortedSet.Branch" (reify ReadHandler (read [_ reader _tag _component-count] (let [{:keys [keys level addresses]} (.readObject reader)] - (Branch. (int level) ^List keys ^List (seq addresses))))) + (Branch. (int level) ^List keys ^List (seq addresses) settings)))) "datahike.datom.Datom" (reify ReadHandler (read [_ reader _tag _component-count] From 5cebfafe634367d7188cdc1ef33279f16ea88810 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 12 Nov 2023 22:45:49 -0800 Subject: [PATCH 14/67] Fix formatting and whitespace in versioning and persistent_set files --- src/datahike/experimental/versioning.cljc | 2 +- src/datahike/index/persistent_set.cljc | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index 7ba77d772..aa478b294 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -142,4 +142,4 @@ (parent-check parents) (update-and-commit! conn tx-data tx-meta #(with @%1 %2 %3) (conj parents (get-in @conn [:config :branch]))) - true)) + true)) diff --git a/src/datahike/index/persistent_set.cljc b/src/datahike/index/persistent_set.cljc index d75e01c53..898055f0f 100644 --- a/src/datahike/index/persistent_set.cljc +++ b/src/datahike/index/persistent_set.cljc @@ -183,7 +183,7 @@ (def ^:const DEFAULT_BRANCHING_FACTOR 512) (defmethod di/empty-index :datahike.index/persistent-set [_index-name store index-type _] - (let [^PersistentSortedSet pset (psset/sorted-set* {:cmp (index-type->cmp-quick index-type false) + (let [^PersistentSortedSet pset (psset/sorted-set* {:cmp (index-type->cmp-quick index-type false) :storage (:storage store) :branching-factor DEFAULT_BRANCHING_FACTOR})] #_(set! (.-_storage pset) (:storage store)) @@ -200,8 +200,8 @@ (not (arrays/array? datoms)) (arrays/into-array))) _ (arrays/asort arr (index-type->cmp-quick index-type false)) - ^PersistentSortedSet pset (psset/from-sorted-array (index-type->cmp-quick index-type false) - arr + ^PersistentSortedSet pset (psset/from-sorted-array (index-type->cmp-quick index-type false) + arr (alength arr) {:branching-factor DEFAULT_BRANCHING_FACTOR})] (set! (.-_storage pset) (:storage store)) @@ -211,11 +211,9 @@ ;; temporary import from psset until public (defn- map->settings ^Settings [m] (Settings. - (int (or (:branching-factor m) 0)) - nil ;; weak ref default + (int (or (:branching-factor m) 0)) + nil ;; weak ref default )) - - (defmethod di/add-konserve-handlers :datahike.index/persistent-set [config store] ;; deal with circular reference between storage and store (let [settings (map->settings {:branching-factor DEFAULT_BRANCHING_FACTOR}) From 8edbb24127ed3f597e057fa819b7597e4c88f662 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 12 Nov 2023 23:13:56 -0800 Subject: [PATCH 15/67] Add Leaf type hint for native compilation. --- src/datahike/index/persistent_set.cljc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/datahike/index/persistent_set.cljc b/src/datahike/index/persistent_set.cljc index 898055f0f..49669e956 100644 --- a/src/datahike/index/persistent_set.cljc +++ b/src/datahike/index/persistent_set.cljc @@ -183,7 +183,7 @@ (def ^:const DEFAULT_BRANCHING_FACTOR 512) (defmethod di/empty-index :datahike.index/persistent-set [_index-name store index-type _] - (let [^PersistentSortedSet pset (psset/sorted-set* {:cmp (index-type->cmp-quick index-type false) + (let [^PersistentSortedSet pset (psset/sorted-set* {:cmp (index-type->cmp-quick index-type false) :storage (:storage store) :branching-factor DEFAULT_BRANCHING_FACTOR})] #_(set! (.-_storage pset) (:storage store)) @@ -200,8 +200,8 @@ (not (arrays/array? datoms)) (arrays/into-array))) _ (arrays/asort arr (index-type->cmp-quick index-type false)) - ^PersistentSortedSet pset (psset/from-sorted-array (index-type->cmp-quick index-type false) - arr + ^PersistentSortedSet pset (psset/from-sorted-array (index-type->cmp-quick index-type false) + arr (alength arr) {:branching-factor DEFAULT_BRANCHING_FACTOR})] (set! (.-_storage pset) (:storage store)) @@ -211,12 +211,14 @@ ;; temporary import from psset until public (defn- map->settings ^Settings [m] (Settings. - (int (or (:branching-factor m) 0)) - nil ;; weak ref default + (int (or (:branching-factor m) 0)) + nil ;; weak ref default )) + + (defmethod di/add-konserve-handlers :datahike.index/persistent-set [config store] ;; deal with circular reference between storage and store - (let [settings (map->settings {:branching-factor DEFAULT_BRANCHING_FACTOR}) + (let [settings ^Settings (map->settings {:branching-factor DEFAULT_BRANCHING_FACTOR}) storage (atom nil) store (assoc store From fc89bd313c406bfff962a0dcf6bf51f9e1c9fa19 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 12 Nov 2023 23:16:44 -0800 Subject: [PATCH 16/67] Fix indentation for persistent set --- src/datahike/index/persistent_set.cljc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/datahike/index/persistent_set.cljc b/src/datahike/index/persistent_set.cljc index 49669e956..2b7ee0305 100644 --- a/src/datahike/index/persistent_set.cljc +++ b/src/datahike/index/persistent_set.cljc @@ -183,7 +183,7 @@ (def ^:const DEFAULT_BRANCHING_FACTOR 512) (defmethod di/empty-index :datahike.index/persistent-set [_index-name store index-type _] - (let [^PersistentSortedSet pset (psset/sorted-set* {:cmp (index-type->cmp-quick index-type false) + (let [^PersistentSortedSet pset (psset/sorted-set* {:cmp (index-type->cmp-quick index-type false) :storage (:storage store) :branching-factor DEFAULT_BRANCHING_FACTOR})] #_(set! (.-_storage pset) (:storage store)) @@ -200,8 +200,8 @@ (not (arrays/array? datoms)) (arrays/into-array))) _ (arrays/asort arr (index-type->cmp-quick index-type false)) - ^PersistentSortedSet pset (psset/from-sorted-array (index-type->cmp-quick index-type false) - arr + ^PersistentSortedSet pset (psset/from-sorted-array (index-type->cmp-quick index-type false) + arr (alength arr) {:branching-factor DEFAULT_BRANCHING_FACTOR})] (set! (.-_storage pset) (:storage store)) @@ -211,11 +211,9 @@ ;; temporary import from psset until public (defn- map->settings ^Settings [m] (Settings. - (int (or (:branching-factor m) 0)) - nil ;; weak ref default + (int (or (:branching-factor m) 0)) + nil ;; weak ref default )) - - (defmethod di/add-konserve-handlers :datahike.index/persistent-set [config store] ;; deal with circular reference between storage and store (let [settings ^Settings (map->settings {:branching-factor DEFAULT_BRANCHING_FACTOR}) From 0e588288773a78c258a1ac09f8c74a26f9f5982b Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 12 Nov 2023 23:30:57 -0800 Subject: [PATCH 17/67] Add boolean argument to update-and-commit function --- src/datahike/experimental/versioning.cljc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index aa478b294..eb866b41e 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -141,5 +141,6 @@ ([conn parents tx-data tx-meta] (parent-check parents) (update-and-commit! conn tx-data tx-meta #(with @%1 %2 %3) - (conj parents (get-in @conn [:config :branch]))) + (conj parents (get-in @conn [:config :branch])) + true) true)) From 2c00059cccde3551005c46733d4d4039389c7391 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Mon, 13 Nov 2023 00:55:44 -0800 Subject: [PATCH 18/67] Deref transact! function call in server.clj --- http-server/datahike/http/server.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http-server/datahike/http/server.clj b/http-server/datahike/http/server.clj index 2fc6ff6f8..cb9e00bb0 100644 --- a/http-server/datahike/http/server.clj +++ b/http-server/datahike/http/server.clj @@ -148,7 +148,7 @@ :summary "Internal endpoint. DO NOT USE!" :no-doc true :handler (fn [{{:keys [body]} :parameters}] - (let [res (apply datahike.writing/transact! body)] + (let [res @(apply datahike.writing/transact! body)] {:status 200 :body res})) :operationId "transact"}, From bcc1201a9d9a37c304784f4450de415e7f102e45 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Mon, 13 Nov 2023 01:05:56 -0800 Subject: [PATCH 19/67] Fix transact! function to handle updates correctly --- http-server/datahike/http/server.clj | 2 +- src/datahike/writing.cljc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/http-server/datahike/http/server.clj b/http-server/datahike/http/server.clj index cb9e00bb0..35d9e09fc 100644 --- a/http-server/datahike/http/server.clj +++ b/http-server/datahike/http/server.clj @@ -148,7 +148,7 @@ :summary "Internal endpoint. DO NOT USE!" :no-doc true :handler (fn [{{:keys [body]} :parameters}] - (let [res @(apply datahike.writing/transact! body)] + (let [res (apply datahike.writing/transact! body)] {:status 200 :body res})) :operationId "transact"}, diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index 5fec7d6ff..86559b500 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -276,7 +276,7 @@ (defn transact! [connection {:keys [tx-data tx-meta]}] (log/debug "Transacting" (count tx-data) " objects with meta: " tx-meta) (log/trace "Transaction data" tx-data) - (update-and-commit! connection tx-data tx-meta #(core/with @%1 %2 %3))) + (update-and-commit! connection tx-data tx-meta #(core/with @%1 %2 %3) true)) (defn load-entities [connection entities] (log/debug "Loading" (count entities) " entities.") From b96bc8942da5aef9621886aa39c1f9c5c24b86fa Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Mon, 13 Nov 2023 15:18:10 -0800 Subject: [PATCH 20/67] Add Leaf type hint for native compilation --- src/datahike/index/persistent_set.cljc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datahike/index/persistent_set.cljc b/src/datahike/index/persistent_set.cljc index 2b7ee0305..33be32534 100644 --- a/src/datahike/index/persistent_set.cljc +++ b/src/datahike/index/persistent_set.cljc @@ -216,7 +216,7 @@ )) (defmethod di/add-konserve-handlers :datahike.index/persistent-set [config store] ;; deal with circular reference between storage and store - (let [settings ^Settings (map->settings {:branching-factor DEFAULT_BRANCHING_FACTOR}) + (let [settings (map->settings {:branching-factor DEFAULT_BRANCHING_FACTOR}) storage (atom nil) store (assoc store @@ -234,7 +234,7 @@ (reify ReadHandler (read [_ reader _tag _component-count] (let [{:keys [keys level]} (.readObject reader)] - (Leaf. keys settings)))) + (Leaf. ^List keys settings)))) "datahike.index.PersistentSortedSet.Branch" (reify ReadHandler (read [_ reader _tag _component-count] From f3cb0aa623b1dc7f4405e19b2bac3bb6a8cf9839 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Tue, 14 Nov 2023 11:06:25 -0800 Subject: [PATCH 21/67] Minor cleanups. --- http-server/datahike/http/server.clj | 17 +++++-------- src/datahike/http/writer.clj | 7 ++--- src/datahike/readers.cljc | 2 +- src/datahike/writer.cljc | 6 ++--- src/datahike/writing.cljc | 2 +- test/datahike/test/http/writer_test.clj | 34 ++++++++++++------------- 6 files changed, 32 insertions(+), 36 deletions(-) diff --git a/http-server/datahike/http/server.clj b/http-server/datahike/http/server.clj index 35d9e09fc..534291d96 100644 --- a/http-server/datahike/http/server.clj +++ b/http-server/datahike/http/server.clj @@ -11,7 +11,8 @@ [datahike.transit :as transit] [datahike.json :as json] [datahike.api :refer :all :as api] - [datahike.writing :as datahike.writing] + [datahike.writing] + [datahike.writer] [reitit.ring :as ring] [reitit.coercion.spec] [reitit.swagger :as swagger] @@ -36,7 +37,7 @@ (fn [request] (let [{{body :body} :parameters :keys [headers params method]} request - _ (log/trace "req-body" f body (= f #'api/create-database)) + _ (log/trace "request body" f body) ;; TODO move this to client ret-body (cond (= f #'api/create-database) @@ -50,7 +51,7 @@ :else (apply f body))] - (log/trace "ret-body" ret-body) + (log/trace "return body" ret-body) (merge {:status 200 :body @@ -100,11 +101,7 @@ {:handlers transit/write-handlers})))) (defn default-route-opts [muuntaja-with-opts] - {;; :reitit.middleware/transform dev/print-request-diffs ;; pretty diffs - ;; :validate spec/validate ;; enable spec validation for route data - ;; :reitit.spec/wrap spell/closed ;; strict top-level validation - ;; :exception pretty/exception - :data {:coercion reitit.coercion.spec/coercion + {:data {:coercion reitit.coercion.spec/coercion :muuntaja muuntaja-with-opts :middleware [swagger/swagger-feature parameters/parameters-middleware @@ -138,7 +135,7 @@ :handler (fn [{{:keys [body]} :parameters}] {:status 200 :body (apply datahike.writing/create-database - (dissoc (first body) :remote-peer) + (dissoc (first body) :remote-peer :writer) (rest body))}) :operationId "create-database"}, :swagger {:tags ["Internal"]}}] @@ -148,7 +145,7 @@ :summary "Internal endpoint. DO NOT USE!" :no-doc true :handler (fn [{{:keys [body]} :parameters}] - (let [res (apply datahike.writing/transact! body)] + (let [res @(apply datahike.writer/transact! body)] {:status 200 :body res})) :operationId "transact"}, diff --git a/src/datahike/http/writer.clj b/src/datahike/http/writer.clj index 23fd5708e..c7d4edd22 100644 --- a/src/datahike/http/writer.clj +++ b/src/datahike/http/writer.clj @@ -5,8 +5,7 @@ [datahike.http.client :refer [request-transit] :as client] [datahike.tools :as dt :refer [throwable-promise]] [taoensso.timbre :as log] - [clojure.core.async :refer [promise-chan put!]]) - (:import [java.io ByteArrayOutputStream])) + [clojure.core.async :refer [promise-chan put!]])) (defrecord DatahikeServerWriter [remote-peer conn] PWriter @@ -40,7 +39,9 @@ (request-transit :post "create-database-writer" writer - (vec (concat [(assoc config :remote-peer writer)] (rest args)))) + (vec (concat [(-> config + (assoc :remote-peer writer) + (dissoc :writer))] (rest args)))) (dissoc :remote-peer))) p)) diff --git a/src/datahike/readers.cljc b/src/datahike/readers.cljc index 538885fc0..c2737ef5e 100644 --- a/src/datahike/readers.cljc +++ b/src/datahike/readers.cljc @@ -17,7 +17,7 @@ (defn db-from-reader [{:keys [schema datoms store-id commit-id] :as raw-db}] (if (and store-id commit-id) #?(:cljs (throw (ex-info "Reader not supported." {:type :reader-not-supported - :raw-db db})) + :raw-db raw-db})) :clj (if-let [conn (get-connection store-id)] (let [store (:store @conn)] diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 880bba088..67399f98c 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -108,9 +108,7 @@ ;; public API -(declare transact load-entities) - -(def default-write-fn-map {'transact w/transact! +(def default-write-fn-map {'transact! w/transact! 'load-entities w/load-entities}) (defmulti create-writer @@ -167,7 +165,7 @@ writer (:writer @(:wrapped-atom connection))] (go (let [tx-report ( Date: Sat, 18 Nov 2023 02:28:01 -0800 Subject: [PATCH 22/67] Fix commit-db variable assignment --- src/datahike/writer.cljc | 3 ++- src/datahike/writing.cljc | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 67399f98c..11168a441 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -89,9 +89,10 @@ (try (let [start-ts (get-time-ms) {{:keys [datahike/commit-id]} :meta - :as _commit-db} ( (:listeners (meta connection)) (deref))] - (callback tx-report)) + (when commit? + (reset! connection db) + (doseq [[_ callback] (some-> (:listeners (meta connection)) (deref))] + (callback tx-report))) tx-report))) (defprotocol PDatabaseManager From 73cf977863fd2d661372f44e885afc2c9870bada Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 18 Nov 2023 02:31:52 -0800 Subject: [PATCH 23/67] Fix callback bug in commit function --- src/datahike/writing.cljc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index ffcf8d022..cc8cf057c 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -165,9 +165,9 @@ (assoc-in tx-report [:tx-meta :db/commitId] (get-in db [:meta :datahike/commit-id])))] (when commit? - (reset! connection db) - (doseq [[_ callback] (some-> (:listeners (meta connection)) (deref))] - (callback tx-report))) + (reset! connection db)) + (doseq [[_ callback] (some-> (:listeners (meta connection)) (deref))] + (callback tx-report)) tx-report))) (defprotocol PDatabaseManager From d616ce84c96941e8c5ac1dc47c4ae39671fb2e0a Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 18 Nov 2023 03:47:13 -0800 Subject: [PATCH 24/67] Refactor connection handling and fix writer test --- src/datahike/connections.cljc | 10 +++++----- src/datahike/connector.cljc | 6 +++--- src/datahike/readers.cljc | 4 ++-- test/datahike/test/http/writer_test.clj | 5 +++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/datahike/connections.cljc b/src/datahike/connections.cljc index 5026d3a3a..a29939bd5 100644 --- a/src/datahike/connections.cljc +++ b/src/datahike/connections.cljc @@ -1,16 +1,16 @@ (ns ^:no-doc datahike.connections) -(def connections (atom {})) +(def ^:dynamic *connections* (atom {})) (defn get-connection [conn-id] - (when-let [conn (get-in @connections [conn-id :conn])] - (swap! connections update-in [conn-id :count] inc) + (when-let [conn (get-in @*connections* [conn-id :conn])] + (swap! *connections* update-in [conn-id :count] inc) conn)) (defn add-connection! [conn-id conn] - (swap! connections assoc conn-id {:conn conn :count 1})) + (swap! *connections* assoc conn-id {:conn conn :count 1})) (defn delete-connection! [conn-id] (when-let [conn (get-connection conn-id)] (reset! conn :released) - (swap! connections dissoc conn-id))) + (swap! *connections* dissoc conn-id))) diff --git a/src/datahike/connector.cljc b/src/datahike/connector.cljc index f6d276b60..a8a3f767c 100644 --- a/src/datahike/connector.cljc +++ b/src/datahike/connector.cljc @@ -1,6 +1,6 @@ (ns ^:no-doc datahike.connector (:require [datahike.connections :refer [get-connection add-connection! delete-connection! - connections]] + *connections*]] [datahike.readers] [datahike.store :as ds] [datahike.writing :as dsi] @@ -201,9 +201,9 @@ (let [db @(:wrapped-atom connection) conn-id [(ds/store-identity (get-in db [:config :store])) (get-in db [:config :branch])]] - (if-not (get @connections conn-id) + (if-not (get @*connections* conn-id) (log/info "Connection already released." conn-id) - (let [new-conns (swap! connections update-in [conn-id :count] dec)] + (let [new-conns (swap! *connections* update-in [conn-id :count] dec)] (when (or release-all? (zero? (get-in new-conns [conn-id :count]))) (delete-connection! conn-id) (w/shutdown (:writer db)) diff --git a/src/datahike/readers.cljc b/src/datahike/readers.cljc index c2737ef5e..80344baa5 100644 --- a/src/datahike/readers.cljc +++ b/src/datahike/readers.cljc @@ -1,5 +1,5 @@ (ns datahike.readers - (:require [datahike.connections :refer [get-connection connections]] + (:require [datahike.connections :refer [get-connection *connections*]] [datahike.writing :as dw] [datahike.datom :refer [datom] :as dd] [datahike.impl.entity :as de] @@ -38,7 +38,7 @@ (SinceDB. origin time-point)) (defn connection-from-reader [conn-id] - (:conn (@connections conn-id))) + (:conn (@*connections* conn-id))) (defn entity-from-reader [{:keys [db eid]}] (de/entity db eid)) diff --git a/test/datahike/test/http/writer_test.clj b/test/datahike/test/http/writer_test.clj index 376065be9..84242a0d4 100644 --- a/test/datahike/test/http/writer_test.clj +++ b/test/datahike/test/http/writer_test.clj @@ -5,15 +5,16 @@ [datahike.http.writer] [datahike.api :as d])) + (deftest test-http-writer (testing "Testing distributed datahike.http.writer implementation." - (let [port 31283 + (let [port 31283 server (start-server {:port port :join? false :dev-mode false :token "securerandompassword"})] (try - (let [cfg {:store {:backend :mem :id "distributed_writer"} + (let [cfg {:store {:backend :file :path "/tmp/distributed_writer"} :keep-history? true :schema-flexibility :read :writer {:backend :datahike-server From 386f3071a18c418d1f975d0536af5151cd7b5180 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 18 Nov 2023 03:50:44 -0800 Subject: [PATCH 25/67] Add missing files --- http-server/datahike/http/server.clj | 20 ++++++++++++-------- src/datahike/http/writer.clj | 5 +++-- src/datahike/writing.cljc | 4 ++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/http-server/datahike/http/server.clj b/http-server/datahike/http/server.clj index 534291d96..15e5cfb4e 100644 --- a/http-server/datahike/http/server.clj +++ b/http-server/datahike/http/server.clj @@ -5,6 +5,7 @@ (:require [clojure.string :as str] [clojure.edn :as edn] + [datahike.connections :refer [*connections*]] [datahike.api.specification :refer [api-specification ->url]] [datahike.http.middleware :as middleware] [datahike.readers :refer [edn-readers]] @@ -116,7 +117,8 @@ multipart/multipart-middleware middleware/patch-swagger-json]}}) -(def internal-writer-routes + +(defn internal-writer-routes [server-connections] [["/delete-database-writer" {:post {:parameters {:body (st/spec {:spec any? :name "delete-database-writer"})}, @@ -139,19 +141,21 @@ (rest body))}) :operationId "create-database"}, :swagger {:tags ["Internal"]}}] - ["/transact-writer" + ["/transact!-writer" {:post {:parameters {:body (st/spec {:spec any? :name "transact-writer"})}, :summary "Internal endpoint. DO NOT USE!" :no-doc true :handler (fn [{{:keys [body]} :parameters}] - (let [res @(apply datahike.writer/transact! body)] - {:status 200 - :body res})) + (binding [*connections* server-connections] + (let [conn (api/connect (dissoc (first body) :remote-peer :writer)) ;; TODO maybe release? + res @(apply datahike.writer/transact! conn (rest body))] + {:status 200 + :body res}))) :operationId "transact"}, :swagger {:tags ["Internal"]}}]]) -(defn app [config route-opts] +(defn app [config route-opts server-connections] (-> (ring/ring-handler (ring/router (concat @@ -166,7 +170,7 @@ [(partial middleware/token-auth config) (partial middleware/auth config)]))) (concat (create-routes config) - internal-writer-routes))) route-opts) + (internal-writer-routes server-connections)))) route-opts) (ring/routes (swagger-ui/create-swagger-ui-handler {:path "/" @@ -178,7 +182,7 @@ :access-control-allow-methods [:get :put :post :delete]))) (defn start-server [config] - (run-jetty (app config (default-route-opts muuntaja-with-opts)) config)) + (run-jetty (app config (default-route-opts muuntaja-with-opts) (atom {})) config)) (defn stop-server [^org.eclipse.jetty.server.Server server] (.stop server)) diff --git a/src/datahike/http/writer.clj b/src/datahike/http/writer.clj index c7d4edd22..634faba8f 100644 --- a/src/datahike/http/writer.clj +++ b/src/datahike/http/writer.clj @@ -11,13 +11,14 @@ PWriter (-dispatch! [_ arg-map] (let [{:keys [op args]} arg-map - p (promise-chan)] + p (promise-chan) + config (:config @(:wrapped-atom conn))] (log/debug "Sending operation to datahike-server:" op) (log/trace "Arguments:" arg-map) (put! p (try (request-transit :post - (str op "-writer") remote-peer (vec (concat [conn] args)) + (str op "-writer") remote-peer (vec (concat [config] args)) read-handlers write-handlers) (catch Exception e e))) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index cc8cf057c..b34c455ff 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -1,6 +1,6 @@ (ns datahike.writing "Manage all state changes and access to state of durable store." - (:require [datahike.connections :refer [delete-connection! connections]] + (:require [datahike.connections :refer [delete-connection! *connections*]] [datahike.db :as db] [datahike.db.utils :as dbu] [datahike.index :as di] @@ -244,7 +244,7 @@ config-store-id (ds/store-identity (:store config)) active-conns (filter (fn [[store-id _branch]] (= store-id config-store-id)) - (keys @connections))] + (keys @*connections*))] (doseq [conn active-conns] (log/warn "Deleting database without releasing all connections first: " conn "." "All connections will be released now, but this cannot be ensured for remote readers.") From 1f7a42861bbcefa9ad025be1261cd7abccef84e6 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 18 Nov 2023 03:53:51 -0800 Subject: [PATCH 26/67] Fix format --- http-server/datahike/http/server.clj | 1 - test/datahike/test/http/writer_test.clj | 1 - 2 files changed, 2 deletions(-) diff --git a/http-server/datahike/http/server.clj b/http-server/datahike/http/server.clj index 15e5cfb4e..eb99953b3 100644 --- a/http-server/datahike/http/server.clj +++ b/http-server/datahike/http/server.clj @@ -117,7 +117,6 @@ multipart/multipart-middleware middleware/patch-swagger-json]}}) - (defn internal-writer-routes [server-connections] [["/delete-database-writer" {:post {:parameters {:body (st/spec {:spec any? diff --git a/test/datahike/test/http/writer_test.clj b/test/datahike/test/http/writer_test.clj index 84242a0d4..1263874b5 100644 --- a/test/datahike/test/http/writer_test.clj +++ b/test/datahike/test/http/writer_test.clj @@ -5,7 +5,6 @@ [datahike.http.writer] [datahike.api :as d])) - (deftest test-http-writer (testing "Testing distributed datahike.http.writer implementation." (let [port 31283 From 64226de50efcd29117dd03c7e8088d3cc8c2785f Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 18 Nov 2023 04:16:00 -0800 Subject: [PATCH 27/67] Ensure atomic extraction of pending write operations --- src/datahike/writing.cljc | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index b34c455ff..48fef8601 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -26,23 +26,22 @@ (count keys-to-check)))) (defn flush-pending-writes [store sync?] - (if sync? - (when-let [pending-futures (:pending-writes (:storage store))] - ;; spin lock - (loop [pfs @pending-futures] - (let [f (first pfs)] - (when f - (let [fv (poll! f)] - (if fv - (recur (rest pfs)) - (do (Thread/sleep 1) - (recur pfs))))))) - (reset! (:pending-writes (:storage store)) [])) - (go-try- - (when-let [pending-futures (:pending-writes (:storage store))] - (loop [[f & r] @pending-futures] - (when f (stored "Maps memory db to storage layout and flushes dirty indices." From b3b96e0a62ff95c8137f78488a2cb0850256c4d0 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 18 Nov 2023 04:22:23 -0800 Subject: [PATCH 28/67] Refactor flush-pending-writes function --- src/datahike/writing.cljc | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index 48fef8601..0ec21967e 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -26,22 +26,23 @@ (count keys-to-check)))) (defn flush-pending-writes [store sync?] - (when-let [pending-futures (:pending-writes (:storage store))] - (let [current-futures (atom nil)] + (let [pending-futures (:pending-writes (:storage store)) + current-futures (atom nil)] ;; atomic extraction and reset - (swap! pending-futures (fn [old] (reset! current-futures old) [])) - (if sync? - (loop [pfs @current-futures] - (let [f (first pfs)] - (when f - (let [fv (poll! f)] - (if fv - (recur (rest pfs)) - (do (Thread/sleep 1) - (recur pfs))))))) - (go-try- - (loop [[f & r] @current-futures] - (when f (stored "Maps memory db to storage layout and flushes dirty indices." From c8583f4700fb8f09045302cc2815ce0390ed1938 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 18 Nov 2023 04:49:28 -0800 Subject: [PATCH 29/67] Refactor config normalization in connector --- src/datahike/connector.cljc | 9 +++++++-- src/datahike/writer.cljc | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/datahike/connector.cljc b/src/datahike/connector.cljc index a8a3f767c..ed5bc9078 100644 --- a/src/datahike/connector.cljc +++ b/src/datahike/connector.cljc @@ -133,6 +133,11 @@ :stored-config stored-config :diff (diff config stored-config)})))) +(defn- normalize-config [cfg] + (-> cfg + (update :store ds/store-identity) + (dissoc :writer))) + (extend-protocol PConnector String (-connect [uri] @@ -148,8 +153,8 @@ (if-let [conn (get-connection conn-id)] (let [conn-config (:config @(:wrapped-atom conn)) ;; replace store config with its identity - cfg (update config :store ds/store-identity) - conn-cfg (update conn-config :store ds/store-identity)] + cfg (normalize-config config) + conn-cfg (normalize-config conn-config)] (when-not (= cfg conn-cfg) (dt/raise "Configuration does not match existing connections." {:type :config-does-not-match-existing-connections diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 11168a441..8ee3b6477 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -51,7 +51,7 @@ ;; Only catch ExceptionInfo here (intentionally rejected transactions). ;; Any other exceptions should crash the writer and signal the supervisor. (catch Exception e - (log/errorf "Error during invocation" invocation e args) + (log/error "Error during invocation" invocation e args) ;; take a guess that a NPE was triggered by an invalid connection ;; short circuit on errors (put! callback From 40ad4f38fe6b5583de344001dc60adc805e36127 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 18 Nov 2023 05:03:30 -0800 Subject: [PATCH 30/67] Format. --- src/datahike/connector.cljc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/datahike/connector.cljc b/src/datahike/connector.cljc index ed5bc9078..03b4a9ca4 100644 --- a/src/datahike/connector.cljc +++ b/src/datahike/connector.cljc @@ -133,9 +133,9 @@ :stored-config stored-config :diff (diff config stored-config)})))) -(defn- normalize-config [cfg] - (-> cfg - (update :store ds/store-identity) +(defn- normalize-config [cfg] + (-> cfg + (update :store ds/store-identity) (dissoc :writer))) (extend-protocol PConnector From f8c14b78f90bfcaac1b11556d774bb045bb26b30 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 18 Nov 2023 15:18:24 -0800 Subject: [PATCH 31/67] Only update commit-id of connection in writer. --- src/datahike/writer.cljc | 2 +- src/datahike/writing.cljc | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 8ee3b6477..7c12855ce 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -92,7 +92,7 @@ :as commit-db} ( (:listeners (meta connection)) (deref))] (callback tx-report)) tx-report))) From 6febe0a287b85a20f3f9eec85e6506b8d2aab0ed Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 14:42:52 -0800 Subject: [PATCH 32/67] Separate update logic from commiting. --- src/datahike/connector.cljc | 2 +- src/datahike/experimental/versioning.cljc | 8 ++--- src/datahike/index/persistent_set.cljc | 1 + src/datahike/writer.cljc | 2 ++ src/datahike/writing.cljc | 43 +++++++++-------------- 5 files changed, 25 insertions(+), 31 deletions(-) diff --git a/src/datahike/connector.cljc b/src/datahike/connector.cljc index 03b4a9ca4..7b9d8c5dc 100644 --- a/src/datahike/connector.cljc +++ b/src/datahike/connector.cljc @@ -38,7 +38,7 @@ IMeta (meta [_] (meta wrapped-atom)) - IRef + IRef ;; TODO This is unoffically supported, it triggers watches on each update, not on commits. For proper listeners use the API. (addWatch [_ key f] (add-watch wrapped-atom key f)) (removeWatch [_ key] (remove-watch wrapped-atom key))) diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index eb866b41e..dca134fd8 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -5,7 +5,7 @@ [datahike.core :refer [with]] [datahike.store :refer [store-identity]] [datahike.writing :refer [stored->db db->stored stored-db? - update-and-commit! create-commit-id + update-connection! commit! create-commit-id flush-pending-writes]] [superv.async :refer [ (:listeners (meta connection)) (deref))] + (callback tx-report)) (deliver p tx-report))) p)) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index 2d695e3b3..742075af8 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -144,30 +144,21 @@ ( (:listeners (meta connection)) (deref))] - (callback tx-report)) - tx-report))) +(defn update-connection! [connection tx-data tx-meta update-fn] + (let [ret-atom (atom nil)] + (swap! connection + (fn [old] + (let [{:keys [writer]} old + {:keys [db-after] + {:keys [db/txInstant]} + :tx-meta + :as tx-report} (update-fn old tx-data tx-meta) + new-meta (assoc (:meta db-after) :datahike/updated-at txInstant) + db (assoc db-after :meta new-meta :writer writer) + tx-report (assoc tx-report :db-after db)] + (reset! ret-atom tx-report) + db))) + @ret-atom)) (defprotocol PDatabaseManager (-create-database [config opts]) @@ -276,8 +267,8 @@ (defn transact! [connection {:keys [tx-data tx-meta]}] (log/debug "Transacting" (count tx-data) " objects with meta: " tx-meta) (log/trace "Transaction data" tx-data) - (update-and-commit! connection tx-data tx-meta #(core/with @%1 %2 %3))) + (update-connection! connection tx-data tx-meta #(core/with %1 %2 %3))) (defn load-entities [connection entities] (log/debug "Loading" (count entities) " entities.") - (update-and-commit! connection entities nil #(core/load-entities-with @%1 %2 %3))) + (update-connection! connection entities nil #(core/load-entities-with %1 %2 %3))) From a642251106111e7a4b01c9bb17d977f5ef07527c Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 15:05:12 -0800 Subject: [PATCH 33/67] Fix commit! function arguments --- src/datahike/experimental/versioning.cljc | 6 +++--- src/datahike/writer.cljc | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index dca134fd8..1cc81bba7 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -141,6 +141,6 @@ ([conn parents tx-data tx-meta] (parent-check parents) (let [db (:db-after (update-connection! conn tx-data tx-meta #(with %1 %2 %3))) - {:keys [config store]} @(:wrapped-atom conn)] - (commit! store config db parents true)) - true)) + {:keys [store]} @(:wrapped-atom conn)] + (commit! store (:config db) db parents) + true))) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 1b0f20e17..089c063bc 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -89,7 +89,7 @@ (try (let [start-ts (get-time-ms) {{:keys [datahike/commit-id]} :meta - :as commit-db} ( Date: Sun, 19 Nov 2023 15:19:19 -0800 Subject: [PATCH 34/67] Add parents to connection metadata --- src/datahike/experimental/versioning.cljc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index 1cc81bba7..796a56c0a 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -143,4 +143,5 @@ (let [db (:db-after (update-connection! conn tx-data tx-meta #(with %1 %2 %3))) {:keys [store]} @(:wrapped-atom conn)] (commit! store (:config db) db parents) + (swap! conn assoc-in [:meta :datahike/parents] parents) true))) From 648a2d2644a4b05938193838edf8fcc2b9519dc2 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 15:23:50 -0800 Subject: [PATCH 35/67] Fix parents assignment in writer.cljc --- src/datahike/writer.cljc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 089c063bc..7d4297ef9 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -88,11 +88,13 @@ (let [db (:db-after (first (last @txs)))] (try (let [start-ts (get-time-ms) - {{:keys [datahike/commit-id]} :meta + {{:keys [datahike/commit-id datahike/parents]} :meta :as _commit-db} ( % + (assoc-in [:meta :datahike/commit-id] commit-id) + (assoc-in [:meta :datahike/parents] parents))) ;; notify all processes that transaction is complete (doseq [[res callback] @txs] (put! callback From 3f3467badc67576929b0a0104d0f123481e190c3 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 15:30:23 -0800 Subject: [PATCH 36/67] Fix commit metadata in writer.cljc --- src/datahike/writer.cljc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 7d4297ef9..b889ea50c 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -92,7 +92,7 @@ :as _commit-db} ( % + (swap! connection #(-> % (assoc-in [:meta :datahike/commit-id] commit-id) (assoc-in [:meta :datahike/parents] parents))) ;; notify all processes that transaction is complete @@ -100,7 +100,8 @@ (put! callback (-> res (assoc-in [:tx-meta :db/commitId] commit-id) - (assoc-in [:db-after :meta :datahike/commit-id] commit-id))))) + (assoc-in [:db-after :meta :datahike/commit-id] commit-id) + (assoc-in [:db-after :meta :datahike/parents] parents))))) (catch Exception e (doseq [[_ callback] @txs] (put! callback e)) From e1807fe1550ae97f26cb820cef23e432608cd7bc Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 16:22:15 -0800 Subject: [PATCH 37/67] Update commit-id in merge! --- src/datahike/experimental/versioning.cljc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index 796a56c0a..dd8396dc1 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -141,7 +141,9 @@ ([conn parents tx-data tx-meta] (parent-check parents) (let [db (:db-after (update-connection! conn tx-data tx-meta #(with %1 %2 %3))) - {:keys [store]} @(:wrapped-atom conn)] - (commit! store (:config db) db parents) - (swap! conn assoc-in [:meta :datahike/parents] parents) + {:keys [store]} @(:wrapped-atom conn) + {{:keys [datahike/commit-id]} :meta} (commit! store (:config db) db parents)] + (swap! conn #(-> % + (assoc-in [:meta :datahike/parents] parents) + (assoc-in [:meta :datahike/commit-id] commit-id))) true))) From d7c85ac8c8f352ef67eeb913376021f476ca3d95 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 17:07:28 -0800 Subject: [PATCH 38/67] Add add-commit-meta function --- src/datahike/experimental/versioning.cljc | 11 ++++------- src/datahike/writer.cljc | 6 ++---- src/datahike/writing.cljc | 6 ++++++ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index dd8396dc1..295f36297 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -5,8 +5,8 @@ [datahike.core :refer [with]] [datahike.store :refer [store-identity]] [datahike.writing :refer [stored->db db->stored stored-db? - update-connection! commit! create-commit-id - flush-pending-writes]] + update-connection! commit! add-commit-meta! + create-commit-id flush-pending-writes]] [superv.async :refer [ % - (assoc-in [:meta :datahike/parents] parents) - (assoc-in [:meta :datahike/commit-id] commit-id))) + {:keys [store]} @(:wrapped-atom conn)] + (add-commit-meta! conn (commit! store (:config db) db parents)) true))) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index b889ea50c..30d38db26 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -89,12 +89,10 @@ (try (let [start-ts (get-time-ms) {{:keys [datahike/commit-id datahike/parents]} :meta - :as _commit-db} ( % - (assoc-in [:meta :datahike/commit-id] commit-id) - (assoc-in [:meta :datahike/parents] parents))) + (w/add-commit-meta! connection commit-db) ;; notify all processes that transaction is complete (doseq [[res callback] @txs] (put! callback diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index 742075af8..7b7bb3cd4 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -144,6 +144,12 @@ ( % + (assoc-in [:meta :datahike/parents] parents) + (assoc-in [:meta :datahike/commit-id] commit-id))))) + (defn update-connection! [connection tx-data tx-meta update-fn] (let [ret-atom (atom nil)] (swap! connection From c8526cfb20d6c534fb898b656fc4feb8d70ebd73 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 17:23:45 -0800 Subject: [PATCH 39/67] Fix commit-queue input channel in writer.cljc --- src/datahike/writer.cljc | 2 +- src/datahike/writing.cljc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 30d38db26..3b0cafbf6 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -106,7 +106,7 @@ (log/error "Writer thread shutting down because of commit error " e) (close! commit-queue) (close! transaction-queue))) - (recur ( % (assoc-in [:meta :datahike/parents] parents) (assoc-in [:meta :datahike/commit-id] commit-id))))) From 633d3fbe38fd09b3500d93198ca5583806eac32e Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 17:33:27 -0800 Subject: [PATCH 40/67] Try to simplify commit! --- src/datahike/writing.cljc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index dc61add1f..f46104b2e 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -130,7 +130,8 @@ ([store config db parents sync?] (async+sync sync? *default-sync-translation* (go-try- - (let [parents (or parents #{(get config :branch)}) + (let [{:keys [store config]} db + parents (or parents #{(get config :branch)}) parents (branch-heads-as-commits store parents) cid (create-commit-id db) db (-> db From eb5b13206ea2407c1d36caeeb89455cf7fac4f18 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 17:37:39 -0800 Subject: [PATCH 41/67] Simplify commit! --- src/datahike/experimental/versioning.cljc | 5 ++--- src/datahike/writer.cljc | 2 +- src/datahike/writing.cljc | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index 295f36297..f2bc687a4 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -140,7 +140,6 @@ (merge! conn parents tx-data nil)) ([conn parents tx-data tx-meta] (parent-check parents) - (let [db (:db-after (update-connection! conn tx-data tx-meta #(with %1 %2 %3))) - {:keys [store]} @(:wrapped-atom conn)] - (add-commit-meta! conn (commit! store (:config db) db parents)) + (let [db (:db-after (update-connection! conn tx-data tx-meta #(with %1 %2 %3)))] + (add-commit-meta! conn (commit! db parents)) true))) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 3b0cafbf6..40c9cc1c4 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -89,7 +89,7 @@ (try (let [start-ts (get-time-ms) {{:keys [datahike/commit-id datahike/parents]} :meta - :as commit-db} ( Date: Sun, 19 Nov 2023 18:05:21 -0800 Subject: [PATCH 42/67] Add current branch head again. --- src/datahike/experimental/versioning.cljc | 3 ++- src/datahike/writer.cljc | 2 +- src/datahike/writing.cljc | 18 ++++++++++-------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/datahike/experimental/versioning.cljc b/src/datahike/experimental/versioning.cljc index f2bc687a4..e6e83ea05 100644 --- a/src/datahike/experimental/versioning.cljc +++ b/src/datahike/experimental/versioning.cljc @@ -140,6 +140,7 @@ (merge! conn parents tx-data nil)) ([conn parents tx-data tx-meta] (parent-check parents) - (let [db (:db-after (update-connection! conn tx-data tx-meta #(with %1 %2 %3)))] + (let [db (:db-after (update-connection! conn tx-data tx-meta #(with %1 %2 %3))) + parents (conj parents (get-in @conn [:config :branch]))] (add-commit-meta! conn (commit! db parents)) true))) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 40c9cc1c4..2b30d59e9 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -67,7 +67,7 @@ (when (> (count commit-queue-buffer) (/ commit-queue-size 2)) (log/warn "Commit queue buffer more than 50% full, " (count commit-queue-buffer) "of" commit-queue-size " filled." - "Throttling transaction processing. Reduce transaction frequency.") + "Throttling transaction processing. Reduce transaction frequency and check your storage throughput.") ( Date: Sun, 19 Nov 2023 19:11:09 -0800 Subject: [PATCH 43/67] Send committed db as db-after to ensure data structure integrity. --- src/datahike/writer.cljc | 3 +-- src/datahike/writing.cljc | 13 +++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 2b30d59e9..8ee17c125 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -98,8 +98,7 @@ (put! callback (-> res (assoc-in [:tx-meta :db/commitId] commit-id) - (assoc-in [:db-after :meta :datahike/commit-id] commit-id) - (assoc-in [:db-after :meta :datahike/parents] parents))))) + (assoc :db-after commit-db))))) (catch Exception e (doseq [[_ callback] @txs] (put! callback e)) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index 641a761cc..92029e0bb 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -26,13 +26,14 @@ (count keys-to-check)))) (defn flush-pending-writes [store sync?] - (let [pending-futures (:pending-writes (:storage store)) - current-futures (atom nil)] + (let [pending-writes (:pending-writes (:storage store)) + current-writes (atom nil)] ;; atomic extraction and reset - (when pending-futures - (swap! pending-futures (fn [old] (reset! current-futures old) []))) + (when pending-writes + ;; at least as new as the current commit, maybe some more writes are included + (swap! pending-writes (fn [old] (reset! current-writes old) []))) (if sync? - (loop [pfs @current-futures] + (loop [pfs @current-writes] (let [f (first pfs)] (when f (let [fv (poll! f)] @@ -41,7 +42,7 @@ (do (Thread/sleep 1) (recur pfs))))))) (go-try- - (loop [[f & r] @current-futures] + (loop [[f & r] @current-writes] (when f (stored From 0eabf43a9a978e8832ac5e4197d7a85902b772fd Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 20:27:56 -0800 Subject: [PATCH 44/67] Improve error handling and use JSON for writer. --- http-server/datahike/http/server.clj | 86 +++++++++++++++++----------- src/datahike/http/client.clj | 1 - src/datahike/http/writer.clj | 45 +++++++++------ src/datahike/writer.cljc | 24 +++++--- 4 files changed, 96 insertions(+), 60 deletions(-) diff --git a/http-server/datahike/http/server.clj b/http-server/datahike/http/server.clj index eb99953b3..347d2d620 100644 --- a/http-server/datahike/http/server.clj +++ b/http-server/datahike/http/server.clj @@ -36,33 +36,38 @@ (defn generic-handler [config f] (fn [request] - (let [{{body :body} :parameters - :keys [headers params method]} request - _ (log/trace "request body" f body) + (try + (let [{{body :body} :parameters + :keys [headers params method]} request + _ (log/trace "request body" f body) ;; TODO move this to client - ret-body - (cond (= f #'api/create-database) + ret-body + (cond (= f #'api/create-database) ;; remove remote-peer and re-add - (assoc - (apply f (dissoc (first body) :remote-peer) (rest body)) - :remote-peer (:remote-peer (first body))) + (assoc + (apply f (dissoc (first body) :remote-peer) (rest body)) + :remote-peer (:remote-peer (first body))) - (= f #'api/delete-database) - (apply f (dissoc (first body) :remote-peer) (rest body)) + (= f #'api/delete-database) + (apply f (dissoc (first body) :remote-peer) (rest body)) - :else - (apply f body))] - (log/trace "return body" ret-body) - (merge - {:status 200 - :body - (when-not (headers "no-return-value") - ret-body)} - (when (and (= method :get) - (get params "args-id") - (get-in config [:cache :get :max-age])) - {:headers {"Cache-Control" (str (when-not (:token config) "public, ") - "max-age=" (get-in config [:cache :get :max-age]))}}))))) + :else + (apply f body))] + (log/trace "return body" ret-body) + (merge + {:status 200 + :body + (when-not (headers "no-return-value") + ret-body)} + (when (and (= method :get) + (get params "args-id") + (get-in config [:cache :get :max-age])) + {:headers {"Cache-Control" (str (when-not (:token config) "public, ") + "max-age=" (get-in config [:cache :get :max-age]))}}))) + (catch Exception e + {:status 500 + :body {:msg (ex-message e) + :ex-data (ex-data e)}}))))) (declare create-routes) @@ -124,8 +129,13 @@ :summary "Internal endpoint. DO NOT USE!" :no-doc true :handler (fn [{{:keys [body]} :parameters}] - {:status 200 - :body (apply datahike.writing/delete-database body)}) + (try + {:status 200 + :body (apply datahike.writing/delete-database body)} + (catch Exception e + {:status 500 + :body {:msg (ex-message e) + :ex-data (ex-data e)}}))) :operationId "delete-database"}, :swagger {:tags ["Internal"]}}] ["/create-database-writer" @@ -134,10 +144,15 @@ :summary "Internal endpoint. DO NOT USE!" :no-doc true :handler (fn [{{:keys [body]} :parameters}] - {:status 200 - :body (apply datahike.writing/create-database - (dissoc (first body) :remote-peer :writer) - (rest body))}) + (try + {:status 200 + :body (apply datahike.writing/create-database + (dissoc (first body) :remote-peer :writer) + (rest body))} + (catch Exception e + {:status 500 + :body {:msg (ex-message e) + :ex-data (ex-data e)}}))) :operationId "create-database"}, :swagger {:tags ["Internal"]}}] ["/transact!-writer" @@ -147,10 +162,15 @@ :no-doc true :handler (fn [{{:keys [body]} :parameters}] (binding [*connections* server-connections] - (let [conn (api/connect (dissoc (first body) :remote-peer :writer)) ;; TODO maybe release? - res @(apply datahike.writer/transact! conn (rest body))] - {:status 200 - :body res}))) + (try + (let [conn (api/connect (dissoc (first body) :remote-peer :writer)) ;; TODO maybe release? + res @(apply datahike.writer/transact! conn (rest body))] + {:status 200 + :body res}) + (catch Exception e + {:status 500 + :body {:msg (ex-message e) + :ex-data (ex-data e)}})))) :operationId "transact"}, :swagger {:tags ["Internal"]}}]]) diff --git a/src/datahike/http/client.clj b/src/datahike/http/client.clj index b42c61170..49de9b8c3 100644 --- a/src/datahike/http/client.clj +++ b/src/datahike/http/client.clj @@ -113,7 +113,6 @@ ;; read exception (let [msg (ex-message e) data (ex-data e) - _ (prn msg data) new-data (update data :body #(when % diff --git a/src/datahike/http/writer.clj b/src/datahike/http/writer.clj index 634faba8f..8ad0c7b1d 100644 --- a/src/datahike/http/writer.clj +++ b/src/datahike/http/writer.clj @@ -1,8 +1,7 @@ (ns datahike.http.writer "Remote writer implementation for datahike.http.server through datahike.http.client." (:require [datahike.writer :refer [PWriter create-writer create-database delete-database]] - [datahike.transit :refer [read-handlers write-handlers]] - [datahike.http.client :refer [request-transit] :as client] + [datahike.http.client :refer [request-json] :as client] [datahike.tools :as dt :refer [throwable-promise]] [taoensso.timbre :as log] [clojure.core.async :refer [promise-chan put!]])) @@ -17,9 +16,10 @@ (log/trace "Arguments:" arg-map) (put! p (try - (request-transit :post - (str op "-writer") remote-peer (vec (concat [config] args)) - read-handlers write-handlers) + (request-json :post + (str op "-writer") + remote-peer + (vec (concat [config] args))) (catch Exception e e))) p)) @@ -36,14 +36,17 @@ (let [p (throwable-promise) {:keys [writer] :as config} (first args)] ;; redirect call to remote-peer as writer config - (deliver p (-> - (request-transit :post - "create-database-writer" - writer - (vec (concat [(-> config - (assoc :remote-peer writer) - (dissoc :writer))] (rest args)))) - (dissoc :remote-peer))) + (deliver p (try (-> + (request-json :post + "create-database-writer" + writer + (vec (concat [(-> config + (assoc :remote-peer writer) + (dissoc :writer))] + (rest args)))) + (dissoc :remote-peer)) + (catch Exception e + e))) p)) (defmethod delete-database :datahike-server @@ -51,9 +54,15 @@ (let [p (throwable-promise) {:keys [writer] :as config} (first args)] ;; redirect call to remote-peer as writer config - (deliver p (-> (request-transit :post - "delete-database-writer" - writer - (vec (concat [(assoc config :remote-peer writer)] (rest args)))) - (dissoc :remote-peer))) + (deliver p (try + (-> (request-json :post + "delete-database-writer" + writer + (vec (concat [(-> config + (assoc :remote-peer writer) + (dissoc :writer))] + (rest args)))) + (dissoc :remote-peer)) + (catch Exception e + e))) p)) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 8ee17c125..cb426e9d5 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -24,11 +24,15 @@ thread) (-streaming? [_] streaming?)) -(def ^:const default-queue-size 100000) +(def ^:const DEFAULT_QUEUE_SIZE 100000) + +;; minimum wait time between commits in ms +;; this reduces write pressure on the storage +(def ^:const DEFAULT_COMMIT_WAIT_TIME 10) ;; in ms (defn create-thread "Creates new transaction thread" - [connection write-fn-map transaction-queue-size commit-queue-size] + [connection write-fn-map transaction-queue-size commit-queue-size commit-wait-time] (let [transaction-queue-buffer (buffer transaction-queue-size) transaction-queue (chan transaction-queue-buffer) commit-queue-buffer (buffer commit-queue-size) @@ -105,6 +109,7 @@ (log/error "Writer thread shutting down because of commit error " e) (close! commit-queue) (close! transaction-queue))) + (LocalWriter {:transaction-queue transaction-queue :transaction-queue-size transaction-queue-size @@ -168,8 +175,9 @@ (let [tx-report ( (:listeners (meta connection)) (deref))] - (callback tx-report)) + (when (map? tx-report) ;; not error + (doseq [[_ callback] (some-> (:listeners (meta connection)) (deref))] + (callback tx-report))) (deliver p tx-report))) p)) From 6a364daf8cd289e4b2ee17417d7213e6a45e6c55 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 20:31:54 -0800 Subject: [PATCH 45/67] Add missing paren. --- http-server/datahike/http/server.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http-server/datahike/http/server.clj b/http-server/datahike/http/server.clj index 347d2d620..35131672c 100644 --- a/http-server/datahike/http/server.clj +++ b/http-server/datahike/http/server.clj @@ -67,7 +67,7 @@ (catch Exception e {:status 500 :body {:msg (ex-message e) - :ex-data (ex-data e)}}))))) + :ex-data (ex-data e)}})))) (declare create-routes) From efdd8daf8536b90fd8d8725808b6789ee24b7cf1 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 21:22:06 -0800 Subject: [PATCH 46/67] Improve error handling. --- http-server/datahike/http/server.clj | 38 ++++++++++++++++------------ src/datahike/http/client.clj | 5 ++-- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/http-server/datahike/http/server.clj b/http-server/datahike/http/server.clj index 35131672c..8e19c375b 100644 --- a/http-server/datahike/http/server.clj +++ b/http-server/datahike/http/server.clj @@ -129,13 +129,18 @@ :summary "Internal endpoint. DO NOT USE!" :no-doc true :handler (fn [{{:keys [body]} :parameters}] - (try - {:status 200 - :body (apply datahike.writing/delete-database body)} - (catch Exception e - {:status 500 - :body {:msg (ex-message e) - :ex-data (ex-data e)}}))) + (let [cfg (dissoc (first body) :remote-peer :writer)] + (try + ;; force release to avoid throwing the release warning + (try + (api/release (api/connect cfg) true) + (catch Exception _)) + {:status 200 + :body (apply datahike.writing/delete-database cfg (rest body))} + (catch Exception e + {:status 500 + :body {:msg (ex-message e) + :ex-data (ex-data e)}})))) :operationId "delete-database"}, :swagger {:tags ["Internal"]}}] ["/create-database-writer" @@ -144,15 +149,16 @@ :summary "Internal endpoint. DO NOT USE!" :no-doc true :handler (fn [{{:keys [body]} :parameters}] - (try - {:status 200 - :body (apply datahike.writing/create-database - (dissoc (first body) :remote-peer :writer) - (rest body))} - (catch Exception e - {:status 500 - :body {:msg (ex-message e) - :ex-data (ex-data e)}}))) + (let [cfg (dissoc (first body) :remote-peer :writer)] + (try + {:status 200 + :body (apply datahike.writing/create-database + cfg + (rest body))} + (catch Exception e + {:status 500 + :body {:msg (ex-message e) + :ex-data (ex-data e)}})))) :operationId "create-database"}, :swagger {:tags ["Internal"]}}] ["/transact!-writer" diff --git a/src/datahike/http/client.clj b/src/datahike/http/client.clj index 49de9b8c3..8cab0570e 100644 --- a/src/datahike/http/client.clj +++ b/src/datahike/http/client.clj @@ -111,13 +111,14 @@ {:query-params {"args-id" (uuid data)}}))) (catch Exception e ;; read exception - (let [msg (ex-message e) + (let [msg (if-let [m (ex-message e)] m "Nothing returned. Is the server reachable?") data (ex-data e) new-data (update data :body #(when % (j/read-value % remote/json-mapper)))] - (throw (ex-info msg new-data))))) + (throw (ex-info (or (:msg (:body new-data)) msg) + (or (:ex-data (:body new-data)) new-data)))))) response (:body response) response (j/read-value response remote/json-mapper)] (log/trace "response" response) From f7faf37b6bdead712c08f5586ccccf759e7eea4e Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 21:23:10 -0800 Subject: [PATCH 47/67] Fix format. --- src/datahike/http/writer.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datahike/http/writer.clj b/src/datahike/http/writer.clj index 8ad0c7b1d..f15c7c282 100644 --- a/src/datahike/http/writer.clj +++ b/src/datahike/http/writer.clj @@ -17,8 +17,8 @@ (put! p (try (request-json :post - (str op "-writer") - remote-peer + (str op "-writer") + remote-peer (vec (concat [config] args))) (catch Exception e e))) From 7a1c9e0c3e0aaf533eeb94c58ecb1ad7246632b1 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 19 Nov 2023 22:07:06 -0800 Subject: [PATCH 48/67] Fix server connections. --- http-server/datahike/http/server.clj | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/http-server/datahike/http/server.clj b/http-server/datahike/http/server.clj index 8e19c375b..ca9dc1f2d 100644 --- a/http-server/datahike/http/server.clj +++ b/http-server/datahike/http/server.clj @@ -129,18 +129,18 @@ :summary "Internal endpoint. DO NOT USE!" :no-doc true :handler (fn [{{:keys [body]} :parameters}] - (let [cfg (dissoc (first body) :remote-peer :writer)] - (try - ;; force release to avoid throwing the release warning + (binding [*connections* server-connections] + (let [cfg (dissoc (first body) :remote-peer :writer)] (try - (api/release (api/connect cfg) true) - (catch Exception _)) - {:status 200 - :body (apply datahike.writing/delete-database cfg (rest body))} - (catch Exception e - {:status 500 - :body {:msg (ex-message e) - :ex-data (ex-data e)}})))) + (try + (api/release (api/connect cfg) true) + (catch Exception _)) + {:status 200 + :body (apply datahike.writing/delete-database cfg (rest body))} + (catch Exception e + {:status 500 + :body {:msg (ex-message e) + :ex-data (ex-data e)}}))))) :operationId "delete-database"}, :swagger {:tags ["Internal"]}}] ["/create-database-writer" From 1e841b99064bf5fd5bd3febfb52c42d3dd22f28f Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Mon, 20 Nov 2023 00:34:56 -0800 Subject: [PATCH 49/67] Update DEFAULT_COMMIT_WAIT_TIME to 0 --- src/datahike/writer.cljc | 2 +- src/datahike/writing.cljc | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index cb426e9d5..5215e8bdc 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -28,7 +28,7 @@ ;; minimum wait time between commits in ms ;; this reduces write pressure on the storage -(def ^:const DEFAULT_COMMIT_WAIT_TIME 10) ;; in ms +(def ^:const DEFAULT_COMMIT_WAIT_TIME 0) ;; in ms (defn create-thread "Creates new transaction thread" diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index 92029e0bb..a728896cf 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -121,11 +121,8 @@ commit-id))))))) (defn create-commit-id [db] - (let [{:keys [hash max-tx max-eid config] - {:keys [parents]} :meta} db] - (if (:crypto-hash? config) - (uuid [hash max-tx max-eid parents]) - (uuid)))) + (let [{:keys [hash max-tx max-eid meta]} db] + (uuid [hash max-tx max-eid meta]))) (defn commit! ([db parents] From 457b483a36a0fa5795b29139374913c8a05cbf50 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Mon, 20 Nov 2023 22:36:23 -0800 Subject: [PATCH 50/67] Add resources to native compilation, fixes libdatahike build. --- config.edn | 1 + 1 file changed, 1 insertion(+) diff --git a/config.edn b/config.edn index c24742a96..dfa2bceb6 100644 --- a/config.edn +++ b/config.edn @@ -31,6 +31,7 @@ :native {;; jvm build :src-dirs ["src" "libdatahike/src"] :java-src-dirs ["java"] + :resource-dir "resources" :target-dir "target" :class-dir "target/classes" :deps-file "deps.edn" From 3c38e14c34b0eff8b1f9527650cfb80548cb3ddd Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Tue, 21 Nov 2023 01:20:56 -0800 Subject: [PATCH 51/67] Use konserve git version konserve upstream filestore fix. --- deps.edn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index fba4da3f6..6adf75755 100644 --- a/deps.edn +++ b/deps.edn @@ -3,7 +3,9 @@ :exclusions [org.clojure/clojurescript]} io.replikativ/hitchhiker-tree {:mvn/version "0.2.222" :exclusions [org.clojure/clojurescript]} - io.replikativ/konserve {:mvn/version "0.7.311" + io.replikativ/konserve {;:mvn/version "0.7.311" + :git/url "https://github.com/replikativ/konserve" + :git/sha "5ca5560383b03c245c92616ce2720facf4640dfd" :exclusions [org.clojure/clojurescript]} io.replikativ/superv.async {:mvn/version "0.3.46" :exclusions [org.clojure/clojurescript]} From 302c822b430c69d91a15b6eb7e2ebd0123f2f1c4 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Tue, 21 Nov 2023 23:52:10 -0800 Subject: [PATCH 52/67] Instrumented konserve release --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 6adf75755..ac13b988d 100644 --- a/deps.edn +++ b/deps.edn @@ -5,7 +5,7 @@ :exclusions [org.clojure/clojurescript]} io.replikativ/konserve {;:mvn/version "0.7.311" :git/url "https://github.com/replikativ/konserve" - :git/sha "5ca5560383b03c245c92616ce2720facf4640dfd" + :git/sha "75f434ccecb5e722e4ae07dfb8215ca46818a648" :exclusions [org.clojure/clojurescript]} io.replikativ/superv.async {:mvn/version "0.3.46" :exclusions [org.clojure/clojurescript]} From 77886ed90f6d8a2ae606e89e22b1aec975e99011 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Wed, 22 Nov 2023 00:10:46 -0800 Subject: [PATCH 53/67] Use new konserve version. --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index ac13b988d..f74bf5db6 100644 --- a/deps.edn +++ b/deps.edn @@ -5,7 +5,7 @@ :exclusions [org.clojure/clojurescript]} io.replikativ/konserve {;:mvn/version "0.7.311" :git/url "https://github.com/replikativ/konserve" - :git/sha "75f434ccecb5e722e4ae07dfb8215ca46818a648" + :git/sha "127545aadb1adfb44b8df1cc5e0558b9a6987b49" :exclusions [org.clojure/clojurescript]} io.replikativ/superv.async {:mvn/version "0.3.46" :exclusions [org.clojure/clojurescript]} From 75bcce21dfa4d2d28b1f6fee4c4d8d64a0729c8d Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Wed, 22 Nov 2023 08:22:52 -0800 Subject: [PATCH 54/67] Update git/sha for io.replikativ/konserve --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index f74bf5db6..c7848d7aa 100644 --- a/deps.edn +++ b/deps.edn @@ -5,7 +5,7 @@ :exclusions [org.clojure/clojurescript]} io.replikativ/konserve {;:mvn/version "0.7.311" :git/url "https://github.com/replikativ/konserve" - :git/sha "127545aadb1adfb44b8df1cc5e0558b9a6987b49" + :git/sha "c9642bd5ed07d8391cb8d61354eb9cbbababc788" :exclusions [org.clojure/clojurescript]} io.replikativ/superv.async {:mvn/version "0.3.46" :exclusions [org.clojure/clojurescript]} From bfb2e97ce7a664b92d4d04d9ee430d1c78b691f4 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Wed, 22 Nov 2023 09:34:36 -0800 Subject: [PATCH 55/67] Update git/sha for io.replikativ/konserve --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index c7848d7aa..e8d6148da 100644 --- a/deps.edn +++ b/deps.edn @@ -5,7 +5,7 @@ :exclusions [org.clojure/clojurescript]} io.replikativ/konserve {;:mvn/version "0.7.311" :git/url "https://github.com/replikativ/konserve" - :git/sha "c9642bd5ed07d8391cb8d61354eb9cbbababc788" + :git/sha "133cb6e773b092cd2646baf1f3d004b3dcda61e2" :exclusions [org.clojure/clojurescript]} io.replikativ/superv.async {:mvn/version "0.3.46" :exclusions [org.clojure/clojurescript]} From f347d69ee18bb637a6fd0cc3a614f9880e8d10fb Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Wed, 22 Nov 2023 21:46:17 -0800 Subject: [PATCH 56/67] Add simple stress test for file store --- test/datahike/test/stress_test.cljc | 54 +++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 test/datahike/test/stress_test.cljc diff --git a/test/datahike/test/stress_test.cljc b/test/datahike/test/stress_test.cljc new file mode 100644 index 000000000..8d69271ca --- /dev/null +++ b/test/datahike/test/stress_test.cljc @@ -0,0 +1,54 @@ +(ns datahike.test.stress-test + (:require [datahike.api :as d] + #?(:cljs [cljs.test :as t :refer-macros [is deftest testing]] + :clj [clojure.test :as t :refer [is deftest testing]]))) + + +(deftest stress-test + (testing "Test lots of parallel reads and writes." + (let [avet? true + num-writes 10000 + num-reads 1000 + + schema [{:db/ident :name + :db/cardinality :db.cardinality/one + :db/index true + :db/unique :db.unique/identity + :db/valueType :db.type/string} + {:db/ident :sibling + :db/cardinality :db.cardinality/many + :db/valueType :db.type/ref} + {:db/ident :age + :db/cardinality :db.cardinality/one + :db/index avet? + :db/valueType :db.type/long}] + + cfg {:store {:backend :file :path "/tmp/dh-stress" + :config {:sync-blob? true :in-place? false}} + :keep-history? false + :schema-flexibility :read + :initial-tx []} + + _ (d/delete-database cfg) + _ (d/create-database cfg) + conn (d/connect cfg) + + _ (d/transact conn schema) + + last-transact + (last + (for [i (shuffle (range num-writes))] + (do #_(prn "write") + (d/transact! conn {:tx-data [[:db/add (inc i) :age i]]}))))] + + (dotimes [_ num-reads] + #_(prn "read") + (d/q '[:find ?e :where [?e :age ?a]] + @conn)) + + @last-transact + (is (= num-writes + (d/q '[:find (count ?e) . + :where + [?e :age ?a]] + @conn)))))) From c80cf14266c907f196f1c4f267bad34e797f43a5 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Wed, 22 Nov 2023 21:48:51 -0800 Subject: [PATCH 57/67] Fix format --- test/datahike/test/stress_test.cljc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/datahike/test/stress_test.cljc b/test/datahike/test/stress_test.cljc index 8d69271ca..3627ada72 100644 --- a/test/datahike/test/stress_test.cljc +++ b/test/datahike/test/stress_test.cljc @@ -3,7 +3,6 @@ #?(:cljs [cljs.test :as t :refer-macros [is deftest testing]] :clj [clojure.test :as t :refer [is deftest testing]]))) - (deftest stress-test (testing "Test lots of parallel reads and writes." (let [avet? true @@ -39,7 +38,7 @@ (last (for [i (shuffle (range num-writes))] (do #_(prn "write") - (d/transact! conn {:tx-data [[:db/add (inc i) :age i]]}))))] + (d/transact! conn {:tx-data [[:db/add (inc i) :age i]]}))))] (dotimes [_ num-reads] #_(prn "read") From 14b2cd27c11e8e27c6feb96d919c0f431bd11abd Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Thu, 23 Nov 2023 00:32:38 -0800 Subject: [PATCH 58/67] Update git/sha for io.replikativ/konserve --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index e8d6148da..585a320f2 100644 --- a/deps.edn +++ b/deps.edn @@ -5,7 +5,7 @@ :exclusions [org.clojure/clojurescript]} io.replikativ/konserve {;:mvn/version "0.7.311" :git/url "https://github.com/replikativ/konserve" - :git/sha "133cb6e773b092cd2646baf1f3d004b3dcda61e2" + :git/sha "8d9253895f7952c056d9a44ac84cb32e3ed404be" :exclusions [org.clojure/clojurescript]} io.replikativ/superv.async {:mvn/version "0.3.46" :exclusions [org.clojure/clojurescript]} From 2f0fb13ad88842dbf52d769762a49f1061ea4dac Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Thu, 23 Nov 2023 01:16:04 -0800 Subject: [PATCH 59/67] Don't run spec tests on stress-test --- test/datahike/test/stress_test.cljc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/datahike/test/stress_test.cljc b/test/datahike/test/stress_test.cljc index 3627ada72..df094b8d6 100644 --- a/test/datahike/test/stress_test.cljc +++ b/test/datahike/test/stress_test.cljc @@ -3,7 +3,7 @@ #?(:cljs [cljs.test :as t :refer-macros [is deftest testing]] :clj [clojure.test :as t :refer [is deftest testing]]))) -(deftest stress-test +(deftest ^:no-spec stress-test (testing "Test lots of parallel reads and writes." (let [avet? true num-writes 10000 @@ -34,12 +34,14 @@ _ (d/transact conn schema) + ;; write in parallel and force the transactor to keep flusing last-transact (last (for [i (shuffle (range num-writes))] (do #_(prn "write") (d/transact! conn {:tx-data [[:db/add (inc i) :age i]]}))))] + ;; read while we are writing (dotimes [_ num-reads] #_(prn "read") (d/q '[:find ?e :where [?e :age ?a]] @@ -50,4 +52,5 @@ (d/q '[:find (count ?e) . :where [?e :age ?a]] - @conn)))))) + @conn))) + (d/release conn true)))) \ No newline at end of file From f667a315978db3818532584863671dd0b599f406 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Fri, 24 Nov 2023 02:00:49 -0800 Subject: [PATCH 60/67] Fix JSON handlers in writer. Small fixes. --- src/datahike/http/client.clj | 71 +++++++++++++++++++----------------- src/datahike/http/writer.clj | 4 +- src/datahike/writer.cljc | 12 +++--- 3 files changed, 46 insertions(+), 41 deletions(-) diff --git a/src/datahike/http/client.clj b/src/datahike/http/client.clj index 8cab0570e..c9982332f 100644 --- a/src/datahike/http/client.clj +++ b/src/datahike/http/client.clj @@ -88,41 +88,44 @@ (log/trace "response" response) response))) -(defn request-json [method end-point remote-peer data] - (let [{:keys [url token]} - remote-peer - fmt "application/json" - url (str url "/" end-point) - out (j/write-value-as-bytes data remote/json-mapper) - _ (log/trace "request" url end-point token data out) - response - (try - (http/request (merge - {:method method - :uri url - :headers - (merge {:content-type fmt - :accept fmt} - (when token - {:authorization (str "token " token)})) - :as :stream - :body out} - (when (= method :get) - {:query-params {"args-id" (uuid data)}}))) - (catch Exception e +(defn request-json + ([method end-point remote-peer data] + (request-json method end-point remote-peer data remote/json-mapper)) + ([method end-point remote-peer data mapper] + (let [{:keys [url token]} + remote-peer + fmt "application/json" + url (str url "/" end-point) + out (j/write-value-as-bytes data mapper) + _ (log/trace "request" url end-point token data out) + response + (try + (http/request (merge + {:method method + :uri url + :headers + (merge {:content-type fmt + :accept fmt} + (when token + {:authorization (str "token " token)})) + :as :stream + :body out} + (when (= method :get) + {:query-params {"args-id" (uuid data)}}))) + (catch Exception e ;; read exception - (let [msg (if-let [m (ex-message e)] m "Nothing returned. Is the server reachable?") - data (ex-data e) - new-data - (update data :body - #(when % - (j/read-value % remote/json-mapper)))] - (throw (ex-info (or (:msg (:body new-data)) msg) - (or (:ex-data (:body new-data)) new-data)))))) - response (:body response) - response (j/read-value response remote/json-mapper)] - (log/trace "response" response) - response)) + (let [msg (if-let [m (ex-message e)] m "Nothing returned. Is the server reachable?") + data (ex-data e) + new-data + (update data :body + #(when % + (j/read-value % mapper)))] + (throw (ex-info (or (:msg (:body new-data)) msg) + (or (:ex-data (:body new-data)) new-data)))))) + response (:body response) + response (j/read-value response mapper)] + (log/trace "response" response) + response))) (defn request-json-raw [method end-point remote-peer data] (let [{:keys [url token]} diff --git a/src/datahike/http/writer.clj b/src/datahike/http/writer.clj index f15c7c282..7af9822ad 100644 --- a/src/datahike/http/writer.clj +++ b/src/datahike/http/writer.clj @@ -2,6 +2,7 @@ "Remote writer implementation for datahike.http.server through datahike.http.client." (:require [datahike.writer :refer [PWriter create-writer create-database delete-database]] [datahike.http.client :refer [request-json] :as client] + [datahike.json :as json] [datahike.tools :as dt :refer [throwable-promise]] [taoensso.timbre :as log] [clojure.core.async :refer [promise-chan put!]])) @@ -19,7 +20,8 @@ (request-json :post (str op "-writer") remote-peer - (vec (concat [config] args))) + (vec (concat [config] args)) + json/mapper) (catch Exception e e))) p)) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 5215e8bdc..836fa5979 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -24,7 +24,7 @@ thread) (-streaming? [_] streaming?)) -(def ^:const DEFAULT_QUEUE_SIZE 100000) +(def ^:const DEFAULT_QUEUE_SIZE 120000) ;; minimum wait time between commits in ms ;; this reduces write pressure on the storage @@ -99,10 +99,10 @@ (w/add-commit-meta! connection commit-db) ;; notify all processes that transaction is complete (doseq [[res callback] @txs] - (put! callback - (-> res - (assoc-in [:tx-meta :db/commitId] commit-id) - (assoc :db-after commit-db))))) + (let [res (-> res + (assoc-in [:tx-meta :db/commitId] commit-id) + (assoc :db-after commit-db))] + (put! callback res)))) (catch Exception e (doseq [[_ callback] @txs] (put! callback e)) @@ -136,7 +136,7 @@ (map->LocalWriter {:transaction-queue transaction-queue :transaction-queue-size transaction-queue-size - :commit-queu commit-queue + :commit-queue commit-queue :commit-queue-size commit-queue-size :thread thread :streaming? true}))) From 05b91b78ebcff86d239d66bbb357ad2cdc0e4c31 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Fri, 24 Nov 2023 11:03:03 -0800 Subject: [PATCH 61/67] Test with old konserve version --- deps.edn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deps.edn b/deps.edn index 585a320f2..b31148a8d 100644 --- a/deps.edn +++ b/deps.edn @@ -3,9 +3,9 @@ :exclusions [org.clojure/clojurescript]} io.replikativ/hitchhiker-tree {:mvn/version "0.2.222" :exclusions [org.clojure/clojurescript]} - io.replikativ/konserve {;:mvn/version "0.7.311" - :git/url "https://github.com/replikativ/konserve" - :git/sha "8d9253895f7952c056d9a44ac84cb32e3ed404be" + io.replikativ/konserve {:mvn/version "0.7.311" + ;:git/url "https://github.com/replikativ/konserve" + ;:git/sha "8d9253895f7952c056d9a44ac84cb32e3ed404be" :exclusions [org.clojure/clojurescript]} io.replikativ/superv.async {:mvn/version "0.3.46" :exclusions [org.clojure/clojurescript]} From a3167fffffc08eecf32dcb78ac9aaaafe7bae607 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 25 Nov 2023 10:43:08 -0800 Subject: [PATCH 62/67] Use konserve release and remove commented code --- deps.edn | 4 +--- src/datahike/index/persistent_set.cljc | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/deps.edn b/deps.edn index b31148a8d..8b21c24d3 100644 --- a/deps.edn +++ b/deps.edn @@ -3,9 +3,7 @@ :exclusions [org.clojure/clojurescript]} io.replikativ/hitchhiker-tree {:mvn/version "0.2.222" :exclusions [org.clojure/clojurescript]} - io.replikativ/konserve {:mvn/version "0.7.311" - ;:git/url "https://github.com/replikativ/konserve" - ;:git/sha "8d9253895f7952c056d9a44ac84cb32e3ed404be" + io.replikativ/konserve {:mvn/version "0.7.314" :exclusions [org.clojure/clojurescript]} io.replikativ/superv.async {:mvn/version "0.3.46" :exclusions [org.clojure/clojurescript]} diff --git a/src/datahike/index/persistent_set.cljc b/src/datahike/index/persistent_set.cljc index d384d63ab..9a177fd1b 100644 --- a/src/datahike/index/persistent_set.cljc +++ b/src/datahike/index/persistent_set.cljc @@ -187,12 +187,10 @@ (let [^PersistentSortedSet pset (psset/sorted-set* {:cmp (index-type->cmp-quick index-type false) :storage (:storage store) :branching-factor DEFAULT_BRANCHING_FACTOR})] - #_(set! (.-_storage pset) (:storage store)) (with-meta pset {:index-type index-type}))) (defmethod di/init-index :datahike.index/persistent-set [_index-name store datoms index-type _ {:keys [indexed]}] - #_(psset/set-branching-factor! BRANCHING_FACTOR) (let [arr (if (= index-type :avet) (->> datoms (filter #(contains? indexed (.-a ^Datom %))) From 4458d5eff5c8fbe82b6686bdc39574aa2a7f59c3 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 25 Nov 2023 11:04:58 -0800 Subject: [PATCH 63/67] Update datahike logo --- resources/datahike-logo.txt | 65 ++++++++++++++----------------------- 1 file changed, 25 insertions(+), 40 deletions(-) diff --git a/resources/datahike-logo.txt b/resources/datahike-logo.txt index 95b8ac6a9..538884280 100644 --- a/resources/datahike-logo.txt +++ b/resources/datahike-logo.txt @@ -1,40 +1,25 @@ - :; - .==. - ::-= .. - :: -= ==: - ::. -;. .=:=. - .= ==: =. - .= :=: =. .. - .;. .= - =: ==: - :=: .= :; .=::=: - .=-:; =` :: ;:- -=. - .= -=. =- :;._=- -:; - .;` -=. =- ==: -=. - =- ===: .=` -:+: - =- == :=: - =- -- - :: - :: - .::. - .== - =: - - - - - . . .: . - .== =: ==. =: - .== . =: - =: - .== .=: =: =: - .== .=: =: =: - .::...== _::.. _;=::. .;:_. =: .::. .. =: ._ .._.. - .=====:== .======: .======. .======. =::=;==: .=. =: :=- :=====. -.==:---=== ==- -== -:=:-- :=: -== ==: :=. .=. =: :=- .=- :=. -:=: ==; :-. ==. .=; :: ==. =; =: .=. =: :=- =: =: -;=. .=; ... ==. .+: ....:=. =: =: .=. =:.=- =: . .;; -==. .== .=======. :=: .=======. =: =: .=. =:==. .========= -==. .== .==----==. .=: .==----:=. =: =: .=. =: =: =:- - - - -:=: :=; :=: ;=. .+: .=: :=. =: =: .=. =: -=: =: -.==. .==; :=: .==. .+: .=: .==. =: =: .=. =: -=. := =: - :==;;===; .==:::===. .===:. +==.:===. =: =: .=. =: :=. =:_..== - ====- == -;===-:=. -===. -====-.=. =: =: .=. =: == -:===: + :: + ::. . + ; .:. .=. + : ::: -: + : = -. .. + .= :` -: .;;. + ;-:. .- -: :- -:. + ; :. .- .=: -: + : -=: - -=. + : - - + :` + :- + =: + + .. .. . + =: :. ; = + =. .=. :. = + .. =. ... :=.. .._ :. .. . = . .. + =====. :=:== :===; .==:= :::-=: = = .; .=-:= +:= .=..:` =: .= =- =: :: = = = :; :` .; +=: =. ._.=: .= ._.=: :. = = =:: =....= +=. ;. ==--=: .= =:--=: :. = = =.=. =----- +:: =;.=. =: .= .= =: :. = = = -: = . +.=:.==..=:.;=: =:. =:.;=: :. = = = :; -:..:: + -:=-- -:=--` -=: -:=--- -` - - - - -::- From 559f7b88de023871c2bcbfbb86bb3bbaf176ab6d Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Thu, 30 Nov 2023 23:18:04 -0800 Subject: [PATCH 64/67] Add missing flush statement on db creation. --- src/datahike/writing.cljc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index a728896cf..67fe77d73 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -230,6 +230,7 @@ {:temporal-eavt-key (di/-flush temporal-eavt backend) :temporal-aevt-key (di/-flush temporal-aevt backend) :temporal-avet-key (di/-flush temporal-avet backend)}))] + (flush-pending-writes store true) (k/assoc store :branches #{:db} {:sync? true}) (k/assoc store cid db-to-store {:sync? true}) (k/assoc store :db db-to-store {:sync? true}) From c2e4d5e712f666b17986f0ff57522bf9eadd55c3 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 3 Dec 2023 13:25:09 -0800 Subject: [PATCH 65/67] Use Exception for node not found. last -> peek. --- src/datahike/index/persistent_set.cljc | 5 ++++- src/datahike/writer.cljc | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/datahike/index/persistent_set.cljc b/src/datahike/index/persistent_set.cljc index 9a177fd1b..57bb4c5da 100644 --- a/src/datahike/index/persistent_set.cljc +++ b/src/datahike/index/persistent_set.cljc @@ -166,7 +166,10 @@ (if-let [cached (wrapped/lookup cache address)] cached (let [node (k/get store address nil {:sync? true})] - (assert (not (nil? node)) "Node not found in storage.") + (when (not (nil? node)) + (throw (ex-info "Node not found in storage." {:type :node-not-found + :address address + :store store}))) (swap! stats update :reads inc) (wrapped/miss cache address node) node)))) diff --git a/src/datahike/writer.cljc b/src/datahike/writer.cljc index 836fa5979..78fd0af06 100644 --- a/src/datahike/writer.cljc +++ b/src/datahike/writer.cljc @@ -89,7 +89,7 @@ (recur (poll! commit-queue)))) (log/trace "Batched transaction count: " (count @txs)) ;; commit latest tx to disk - (let [db (:db-after (first (last @txs)))] + (let [db (:db-after (first (peek @txs)))] (try (let [start-ts (get-time-ms) {{:keys [datahike/commit-id datahike/parents]} :meta From 1faaefb532b02ec95659ce12b1b3e7a37edb95c5 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 3 Dec 2023 13:28:59 -0800 Subject: [PATCH 66/67] Fix condition for node exception. --- src/datahike/index/persistent_set.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datahike/index/persistent_set.cljc b/src/datahike/index/persistent_set.cljc index 57bb4c5da..8d21debb6 100644 --- a/src/datahike/index/persistent_set.cljc +++ b/src/datahike/index/persistent_set.cljc @@ -166,7 +166,7 @@ (if-let [cached (wrapped/lookup cache address)] cached (let [node (k/get store address nil {:sync? true})] - (when (not (nil? node)) + (when (nil? node) (throw (ex-info "Node not found in storage." {:type :node-not-found :address address :store store}))) From d21e65ae6041bb1468c0cc1fcd7d9200f3d53501 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Mon, 4 Dec 2023 11:42:39 -0800 Subject: [PATCH 67/67] Use dt/raise instead of throw. --- src/datahike/index/persistent_set.cljc | 12 +++++++----- src/datahike/writing.cljc | 4 +++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/datahike/index/persistent_set.cljc b/src/datahike/index/persistent_set.cljc index 8d21debb6..3ee125a81 100644 --- a/src/datahike/index/persistent_set.cljc +++ b/src/datahike/index/persistent_set.cljc @@ -6,6 +6,7 @@ [datahike.datom :as dd] [datahike.constants :refer [tx0 txmax]] [datahike.index.interface :as di :refer [IIndex]] + [datahike.tools :as dt] [konserve.core :as k] [konserve.serializers :refer [fressian-serializer]] [hasch.core :refer [uuid]] @@ -167,9 +168,9 @@ cached (let [node (k/get store address nil {:sync? true})] (when (nil? node) - (throw (ex-info "Node not found in storage." {:type :node-not-found - :address address - :store store}))) + (dt/raise "Node not found in storage." {:type :node-not-found + :address address + :store store})) (swap! stats update :reads inc) (wrapped/miss cache address node) node)))) @@ -250,8 +251,9 @@ {"datahike.index.PersistentSortedSet" (reify WriteHandler (write [_ writer pset] - (assert (not (nil? (.-_address ^PersistentSortedSet pset))) - "Must be flushed.") + (when (nil? (.-_address ^PersistentSortedSet pset)) + (dt/raise "Must be flushed." {:type :must-be-flushed + :pset pset})) (.writeTag writer "datahike.index.PersistentSortedSet" 1) (.writeObject writer {:meta (meta pset) :address (.-_address ^PersistentSortedSet pset) diff --git a/src/datahike/writing.cljc b/src/datahike/writing.cljc index 67fe77d73..0c3c60cee 100644 --- a/src/datahike/writing.cljc +++ b/src/datahike/writing.cljc @@ -110,7 +110,9 @@ (defn branch-heads-as-commits [store parents] (set (doall (for [p parents] (do - (assert (not (nil? p)) "Parent cannot be nil.") + (when (nil? p) + (dt/raise "Parent cannot be nil." {:type :parent-cannot-be-nil + :parent p})) (if-not (keyword? p) p (let [{{:keys [datahike/commit-id]} :meta :as old-db} (k/get store p nil {:sync? true})]