diff --git a/.gitignore b/.gitignore index dbb2e57..193f853 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ pom.xml ## OSx .DS_Store +.cake diff --git a/README.md b/README.md index edf01c8..024353f 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ -Cache dot clj +clj-cache ============= - A Clojure library that caches the results of impure functions. This library provides 3 internal caching strategies and can also cache externally or persistently using the java [ehcache](http://github.com/alienscience/cache-dot-clj/blob/master/ehcache/README.md) package. + A Clojure library that caches the results of impure functions. This library provides 3 internal caching strategies and can also cache externally or persistently using the java [ehcache](http://github.com/alienscience/clj-cache/blob/master/ehcache/README.md) package. I have found this useful for caching the results of database calls and for holding HTML snippets. -This library is available at [clojars.org](http://clojars.org/uk.org.alienscience/cache-dot-clj) for use with Leiningen, Cake or Maven. -/ - :dependencies [[uk.org.alienscience/cache-dot-clj "0.0.3"]] +This library is available at [clojars.org](http://clojars.org/clj-cache) for use with Leiningen, Cake or Maven. + + :dependencies [[clj-cache "0.0.4"]] The internal caching functions consist of small modifications to the memoization functions described in these two excellent blog posts, [the rule of three](http://kotka.de/blog/2010/03/The_Rule_of_Three.html) and [memoize done right](http://kotka.de/blog/2010/03/memoize_done_right.html). I'd recommend these posts to Clojure programmers as they discuss flexible apis and concurrency in real world detail. @@ -17,9 +17,9 @@ Example ------- (ns an-example - (:use cache-dot-clj.cache)) + (:use clj-cache.cache)) - (defn-cached get-user-from-db + (defn-cached get-user-from-db (lru-cache-strategy 1000) "Gets a user details from a database. Caches the last 1000 users read in i.e support serving a 1000 concurrent users @@ -30,7 +30,7 @@ Example ;; First read of the user is slow (get-user-from-db "fred") - + ;; Second is fast (get-user-from-db "fred") @@ -49,10 +49,10 @@ Internal Algorithms ;; Least Recently Used (lru-cache-strategy cache-size) - ;; Least Recently Used + ;; Least Recently Used ;; (faster LRU removal, slower under multiple threads) (mutable-lru-cache-strategy cache-size) - + ;; Time to live (ttl-cache-strategy time-to-live-millisecs) @@ -66,7 +66,7 @@ External Algorithms Please see the READMEs in each subdirectory: -- An interface to [ehcache](http://github.com/alienscience/cache-dot-clj/blob/master/ehcache/README.md). Ehcache provides persistent caches that survive application restarts and caches distributed over many machines. +- An interface to [ehcache](http://github.com/alienscience/clj-cache/blob/master/ehcache/README.md). Ehcache provides persistent caches that survive application restarts and caches distributed over many machines. Available Functions diff --git a/ehcache/README.md b/ehcache/README.md index d9a7e84..61a1b08 100644 --- a/ehcache/README.md +++ b/ehcache/README.md @@ -1,18 +1,18 @@ -# Ehcache support for cache-dot-clj +# Ehcache support for clj-cache -Cache-dot-clj can be used with the java [Ehcache](http://ehcache.org/) package to provide distributed caching and persistence. +This plugin allows clj-cache to be used with the java [Ehcache](http://ehcache.org/) library to provide distributed caching and persistence. A PDF user guide for Ehcache is available [here](http://ehcache.org/documentation/EhcacheUserGuide-1.7.1.pdf). -Ehcache is flexible and powerful but comes with a large API and may require XML configuration. The `cache-dot-clj.ehcache` namespace does its best to isolate casual users from the complexity without limiting access to the underlying java objects and features. However, feedback and feature requests are very welcome. +Ehcache is flexible and powerful but comes with a large API and may require XML configuration. The `clj-cache.ehcache` namespace does its best to isolate casual users from the complexity without limiting access to the underlying java objects and features. However, feedback and feature requests are very welcome. ## Example using a default Ehcache configuration (ns an-example - (:use cache-dot-clj.cache) - (:require [cache-dot-clj.ehcache :as ehcache])) + (:use clj-cache.cache) + (:require [clj-cache.ehcache :as ehcache])) (defn-cached get-user-from-db (ehcache/strategy) @@ -34,30 +34,30 @@ Ehcache is flexible and powerful but comes with a large API and may require XML ## Dependencies -To use Ehcache and cache-dot-clj pull in the following dependency using Leiningen, Cake or Maven: +To use Ehcache and clj-cache pull in the following dependency using Leiningen, Cake or Maven: + + [clj-cache-ehcache "0.0.4"] - [uk.org.alienscience/ehcache-dot-clj "0.0.3"] - Ehcache uses slf4j to do logging and a slf4j plugin must be included as a dependency. To log to stderr you can use: - [org.sljf4j/slf4j-simple "1.5.11"] + [org.sljf4j/slf4j-simple "1.6.1"] Or if logging is not required: - [org.sljf4j/slf4j-nop "1.5.11"] + [org.sljf4j/slf4j-nop "1.6.1"] ## Limitations -This package assumes all keys and values in the cache are java.io.Serializable. This covers most clojure datastructures but means that different versions of clojure (e.g 1.1 and 1.2) shouldn't share the same distributed cache. +This package assumes all keys and values in the cache are `java.io.Serializable`. This covers most clojure datastructures but means that different versions of clojure (e.g 1.1 and 1.2) shouldn't share the same distributed cache. -Internally, cache-dot-clj uses features found in clojure to limit the number of calls to slow functions on a cache miss. Ehcache can also do this using locking. Locking in Ehcache is relatively new, requires an additional package and is not well documented. Because of this cache-dot-clj does not yet support locking with Ehcache. However, if there is interest, locking can be added at a later date. +Internally, clj-cache uses features found in clojure to limit the number of calls to slow functions on a cache miss. Ehcache can also do this using locking. Locking in Ehcache is relatively new, requires an additional package and is not well documented. Because of this clj-cache does not yet support locking with Ehcache. However, if there is interest, locking can be added at a later date. # API ## strategy [] [config] [manager config] -Returns a strategy for use with cache-dot-clj.cache using the +Returns a strategy for use with clj-cache.cache using the default configuration or the given cache configuration. The config can be a object of class [net.sf.ehcache.config.CacheConfiguration](http://ehcache.org/apidocs/net/sf/ehcache/config/CacheConfiguration.html) or a clojure map containing keys that correspond to the setters of the Cache configuration. The keys are converted to camelCase internally @@ -66,16 +66,16 @@ of the Cache configuration. The keys are converted to camelCase internally {:max-elements-in-memory 100} calls setMaxElementsInMemory(100) A CacheManager can also be passed in as the first argument, without this the singleton CacheManager is used (which should be fine for most uses). - + ## new-manager [] [config] Creates a new cache manager. The config can be a filename string, URL object or an InputStream containing an XML configuration. To set the configuration without using an external XML file, a clojure [prxml](http://richhickey.github.com/clojure-contrib/prxml-api.html#clojure.contrib.prxml/prxml) style datastructure can be used. ### example - (new-manager + (new-manager [:ehcache - [:disk-store + [:disk-store {:path "java.io.tmpdir/mycaches"}] [:default-cache {:max-elements-in-memory 100 @@ -93,29 +93,29 @@ Creates a new cache manager. The config can be a filename string, URL object or - lookup [cache k] Looks up an item in the given cache. Returns a vector [element-exists? value]. - -- invalidate [cache k] - + +- invalidate [cache k] + Invalidates the cache entry with the given key. -- create-config [] +- create-config [] Creates a CacheConfiguration object. The functions below can also be called with a CacheManager as the first argument. If a a CacheManager is not passed in then the singleton CacheManager is used. -- create-cache [cache-name config] +- create-cache [cache-name config] Returns an ehcache Cache object with the given name and config. - -- cache-seq [] + +- cache-seq [] Returns a sequence containing the names of the currently used caches within a cache manager. - -- remove-cache [cache-name] + +- remove-cache [cache-name] Removes the cache with the given name. -- shutdown [] +- shutdown [] Shuts down a cache manager. - + diff --git a/ehcache/project.clj b/ehcache/project.clj index 49369cb..d4ae6b4 100755 --- a/ehcache/project.clj +++ b/ehcache/project.clj @@ -1,7 +1,7 @@ -(defproject uk.org.alienscience/ehcache-dot-clj "0.0.4-SNAPSHOT" - :description "Ehcache support for cache-dot-clj." +(defproject clj-cache-ehcache "0.0.4-SNAPSHOT" + :description "Ehcache support for clj-cache" :dependencies [[net.sf.ehcache/ehcache-core "2.4.2"] - [uk.org.alienscience/cache-dot-clj "0.0.4-SNAPSHOT"]] + [clj-cache "0.0.4-SNAPSHOT"]] :dev-dependencies [[org.clojure/clojure "1.2.0"] [org.clojure/clojure-contrib "1.2.0"] [swank-clojure "1.2.1"] diff --git a/ehcache/src/cache_dot_clj/bean.clj b/ehcache/src/clj_cache/bean.clj similarity index 98% rename from ehcache/src/cache_dot_clj/bean.clj rename to ehcache/src/clj_cache/bean.clj index adb230e..db8aa7a 100644 --- a/ehcache/src/cache_dot_clj/bean.clj +++ b/ehcache/src/clj_cache/bean.clj @@ -17,7 +17,7 @@ (ns #^{:author "Justin Balthrop" :doc "Modify bean attributes in clojure."} - cache-dot-clj.bean + clj-cache.bean (:import [java.beans Introspector])) (defn- property-key [property] diff --git a/ehcache/src/cache_dot_clj/ehcache.clj b/ehcache/src/clj_cache/ehcache.clj similarity index 96% rename from ehcache/src/cache_dot_clj/ehcache.clj rename to ehcache/src/clj_cache/ehcache.clj index 3d4e163..2a208c9 100644 --- a/ehcache/src/cache_dot_clj/ehcache.clj +++ b/ehcache/src/clj_cache/ehcache.clj @@ -1,4 +1,4 @@ -(ns cache-dot-clj.ehcache +(ns clj-cache.ehcache (:import [net.sf.ehcache CacheManager Cache Element Ehcache] net.sf.ehcache.config.CacheConfiguration net.sf.ehcache.constructs.blocking.BlockingCache @@ -6,7 +6,7 @@ javax.management.MBeanServer java.lang.management.ManagementFactory java.io.Serializable) - (:require [cache-dot-clj.bean :as bean-utils]) + (:require [clj-cache.bean :as bean-utils]) (:require [clojure.contrib.string :as str]) (:use clojure.contrib.prxml)) @@ -118,7 +118,10 @@ ;; By default the key (args of the fn) would be a clojure.lang.ArraySeq, and for some reason ;; seemily identical versions (i.e. = would be true) ehcache would have misses (only) after ;; persisted to disk. By converting the ArraySeq's over then the keys match within ehcache. -(def cache-key vec) +(defn cache-key [key] + (if (string? key) + key + (vec key))) (defn add "Adds an item to the given cache and returns the value added" @@ -139,7 +142,7 @@ (.remove cache ^Serializable (cache-key k))) (defn- make-strategy - "Create a strategy map for use with cache-dot-clj.cache" + "Create a strategy map for use with clj-cache.cache" [init-fn] {:init init-fn :lookup lookup @@ -159,7 +162,7 @@ [create-cache config])) (defn strategy - "Returns a strategy for use with cache-dot-clj.cache using the + "Returns a strategy for use with clj-cache.cache using the default configuration or the given cache configuration. The config can be a object of class net.sf.ehcache.config.CacheConfiguration diff --git a/ehcache/test/cache_dot_clj/test/ehcache.clj b/ehcache/test/clj_cache/test/ehcache.clj similarity index 97% rename from ehcache/test/cache_dot_clj/test/ehcache.clj rename to ehcache/test/clj_cache/test/ehcache.clj index b2a8bcf..beaac51 100644 --- a/ehcache/test/cache_dot_clj/test/ehcache.clj +++ b/ehcache/test/clj_cache/test/ehcache.clj @@ -1,14 +1,14 @@ -(ns cache-dot-clj.test.ehcache +(ns clj-cache.test.ehcache "Ehcache tests" (:use clojure.test) - (:use cache-dot-clj.cache) + (:use clj-cache.cache) (:use [clojure.set :only [union]]) (:use [clj-file-utils.core :only [rm-rf mkdir-p exists?]]) - (:require [cache-dot-clj.ehcache :as ehcache] + (:require [clj-cache.ehcache :as ehcache] [clojure.java.io :as io] [clojure.contrib.jmx :as jmx])) -;;--- Copy and paste of cache-dot-clj.test.cache (different src tree) +;;--- Copy and paste of clj-cache.test.cache (different src tree) (defn slow [a] (Thread/sleep a) a) diff --git a/jedis/project.clj b/jedis/project.clj deleted file mode 100755 index 2accd77..0000000 --- a/jedis/project.clj +++ /dev/null @@ -1,12 +0,0 @@ -(defproject uk.org.alienscience/jedis-dot-clj "0.0.3" - :description "Redis support for cache-dot-clj." - :dependencies [[redis.clients/jedis "1.5.1"] - [uk.org.alienscience/cache-dot-clj "0.0.3"]] - :dev-dependencies [[org.clojure/clojure "1.2.0"] - [org.clojure/clojure-contrib "1.2.0"] - [swank-clojure "1.2.1"]] - :license {:name "Eclipse Public License - v 1.0" - :url "http://www.eclipse.org/legal/epl-v10.html" - :distribution :repo - :comments "same as Clojure"}) - diff --git a/masai/README.markdown b/masai/README.markdown new file mode 100644 index 0000000..17eb44c --- /dev/null +++ b/masai/README.markdown @@ -0,0 +1,21 @@ +# Masai support for cache-dot-clj + +Masai is a common interface to several key-value stores. Right now, Masai supports both Redis and Tokyo Cabinet, and will support more in the future. This adds Masai support to cache-dot-clj, giving you the ability to cache to any of the databases that Masai supports. + +## Example using Masai's redis backend + + (ns an-example + (:use cache-dot-clj.cache) + (:require [cache-dot-clj.masai :as masai])) + + (defn-cached get-user-from-db + (masai/strategy) + [username] + ;; Slow database read goes here + ) + +## Dependencies + +To use Masai and cache-dot-clj pull in the following dependency using Leiningen, Cake or Maven: + + [uk.org.alienscience/masai-dot-clj "0.0.4-SNAPSHOT"] diff --git a/masai/project.clj b/masai/project.clj new file mode 100644 index 0000000..c939c40 --- /dev/null +++ b/masai/project.clj @@ -0,0 +1,12 @@ +(defproject clj-cache-masai "0.0.4-SNAPSHOT" + :description "Masai support for cache-dot-clj." + :dependencies [[org.clojars.raynes/masai "0.5.1-SNAPSHOT"] + [clj-cache "0.0.4-SNAPSHOT"] + [cereal "0.1.1"]] + :dev-dependencies [[org.clojure/clojure "1.2.0"] + [org.clojars.raynes/jedis "2.0.0-SNAPSHOT"] + [tokyocabinet "1.24.1-SNAPSHOT"]] + :license {:name "Eclipse Public License - v 1.0" + :url "http://www.eclipse.org/legal/epl-v10.html" + :distribution :repos + :comments "same as Clojure"}) \ No newline at end of file diff --git a/masai/src/clj_cache/masai.clj b/masai/src/clj_cache/masai.clj new file mode 100644 index 0000000..b2722b1 --- /dev/null +++ b/masai/src/clj_cache/masai.clj @@ -0,0 +1,67 @@ +(ns clj-cache.masai + (:use cereal.format) + (require [masai.db :as db] + [cereal.java :as j])) + +(def form (j/make)) + +(defn add + "Add an item to the given cache and return the value added." + [^DB cache k v] + (db/put! cache (str k) (encode form v)) + v) + +(defn lookup + "Looks up an item in the given cache. Returns a vector: + [element-exists? value]" + [^DB cache k] + (let [record (db/get cache (str k))] + [(-> record nil? not) (and record (decode form record))])) + +(defn invalidate + "Removes an item from the cache." + [^DB cache k] + (db/delete! cache (str k))) + +(defn- make-strategy + "Create a strategy map for use with cache-dot-clj.cache" + [init-fn] + {:init init-fn + :lookup lookup + :miss! add + :invalidate! invalidate + :description "Masai backend" + :plugs-into :external-memoize}) + +(defn- prefix [f-name s] (str f-name "/" s)) + +(defn- key-format [f-name] + (fn [^String s] (bytes (.getBytes (prefix f-name s))))) + +(defn- open [db] + (db/open db) + db) + +(defmacro init [type opts] + (require + (case type + :redis 'masai.redis + :tokyo 'masai.tokyo)) + `(fn [x#] + (open + (~(case type + :redis 'masai.redis/make + :tokyo 'masai.tokyo/make) + (assoc ~opts :key-format (key-format x#)))))) + +(defn strategy + "Returns a strategy for use with cache-dot-clj.cache. Given + no arguments, uses Redis as the backend with default configuration. + If passed an argument, that argument is expected to be a keyword + naming the Masai backend to use. Possible keywords are :tokyo and + :redis. If given two arguments, the second argument is expected to + be a map of options to pass to whatever backend your using as options." + ([] (make-strategy (init :redis nil))) + ([back opts] (case back + :redis (make-strategy (init :redis opts)) + :tokyo (make-strategy (init :tokyo opts))))) \ No newline at end of file diff --git a/masai/test/clj_cache/masai_test.clj b/masai/test/clj_cache/masai_test.clj new file mode 100644 index 0000000..a15bfbd --- /dev/null +++ b/masai/test/clj_cache/masai_test.clj @@ -0,0 +1,52 @@ +(ns clj-cache.masai-test + (:use clojure.test + clj-cache.cache) + (:require [clj-cache.masai :as masai])) + +(defn slow [a] (Thread/sleep a) a) + +(def fast-default (cached slow (masai/strategy))) + +(defmacro how-long [expr] + `(let [start# (. System (nanoTime)) + ret# ~expr + msecs# (/ (double (- (. System (nanoTime)) start#)) + 1000000.0)] + {:msecs msecs# :ret ret#})) + +(defn expect [situation f check t should] + (let [{:keys [msecs ret]} (how-long (f t))] + (is (check msecs t) (str situation " (expected time:" t ", actual time: " msecs ") " should)) + (is (= ret t) (str situation " returns correct value")))) + +(defn is-caching [f t] + (invalidate-cache f t) + (expect "First call" f > t "hits function" ) + (expect "Second call" f < t "is cached") + (expect "Third call" f < t "is cached")) + +(deftest is-caching-ehcache (is-caching fast-default 100)) + +(defn invalidating [f t1 t2 t3] + (invalidate-cache f t1) + (invalidate-cache f t2) + (invalidate-cache f t3) + (expect "First call" f > t1 "hits function") + (expect "First call" f > t2 "hits function") + (expect "First call" f > t3 "hits function") + (invalidate-cache f t1) + (expect "Invalidated entry" f > t1 "hits function") + (expect "Second call" f < t2 "is cached") + (expect "Second call" f < t3 "is cached") + (expect "Third call" f < t1 "is cached")) + +(deftest invalidating-ehcache (invalidating fast-default 50 51 52)) + +(defn-cached cached-fn + (masai/strategy) + "A cached function definition" + [t] + (Thread/sleep t) + t) + +(deftest is-caching-def (is-caching cached-fn 100)) diff --git a/project.clj b/project.clj index 8258c15..7575aad 100755 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject uk.org.alienscience/cache-dot-clj "0.0.4-SNAPSHOT" +(defproject clj-cache "0.0.4-SNAPSHOT" :description "A resettable memoize." :dependencies [] :dev-dependencies [[org.clojure/clojure "1.2.0"] diff --git a/src/cache_dot_clj/outside.clj b/src/cache_dot_clj/outside.clj deleted file mode 100644 index 5a99785..0000000 --- a/src/cache_dot_clj/outside.clj +++ /dev/null @@ -1,25 +0,0 @@ - -(ns cache-dot-clj.outside - "Functions to help with caches outside the current VM" - (import [java.security MessageDigest]) - (import [sun.misc BASE64Encoder])) - -(defn serialise - "Serialises the given data structure. If the datastructure contains - unprintable types they need to be handled by adding methods to - the print-dup multimethod. - The result can be deserialised using read-string" - [d] - (binding [*print-dup* true] - (with-out-str (pr d)))) - -(defn digest-data - "Returns the BASE64 encoded MD5 of the serialised form of the - given data structure" - [d] - (let [md5 (MessageDigest/getInstance "MD5") - d-bytes (.getBytes (serialise d) "UTF-8") - md (.digest md5 d-bytes) - b64 (new BASE64Encoder)] - (.encode b64 md))) - diff --git a/src/cache_dot_clj/cache.clj b/src/clj_cache/cache.clj similarity index 99% rename from src/cache_dot_clj/cache.clj rename to src/clj_cache/cache.clj index 243d713..d413d1e 100644 --- a/src/cache_dot_clj/cache.clj +++ b/src/clj_cache/cache.clj @@ -1,6 +1,6 @@ -(ns cache-dot-clj.cache +(ns clj-cache.cache "Resettable memoize" - (:require [cache-dot-clj.datastructures :as ds])) + (:require [clj-cache.datastructures :as ds])) (declare naive-strategy) diff --git a/src/cache_dot_clj/datastructures.clj b/src/clj_cache/datastructures.clj similarity index 80% rename from src/cache_dot_clj/datastructures.clj rename to src/clj_cache/datastructures.clj index 055b65a..2a44ab3 100644 --- a/src/cache_dot_clj/datastructures.clj +++ b/src/clj_cache/datastructures.clj @@ -1,6 +1,6 @@ -(ns cache-dot-clj.datastructures - "Datastructures for use with cache-dot-clj" +(ns clj-cache.datastructures + "Datastructures for use with clj-cache" (:import [java.util LinkedHashMap Collections])) diff --git a/test/cache_dot_clj/test/cache.clj b/test/clj_cache/test/cache.clj similarity index 99% rename from test/cache_dot_clj/test/cache.clj rename to test/clj_cache/test/cache.clj index 8084d6d..8571b25 100644 --- a/test/cache_dot_clj/test/cache.clj +++ b/test/clj_cache/test/cache.clj @@ -1,7 +1,7 @@ -(ns cache-dot-clj.test.cache +(ns clj-cache.test.cache "Resettable memoize tests" (:use clojure.test) - (:use cache-dot-clj.cache)) + (:use clj-cache.cache)) (defn slow [a] (Thread/sleep a) a) (def fast-naive (cached slow naive-strategy))