Handle range requests

cacheproxy/cmd/cacheproxy/main.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/gorilla/handlers"
+	"github.com/uovobw/httpcache/cacheproxy"
+)
+
+var (
+	bindTo = flag.String("bind", "0.0.0.0:8080", "address to bind to")
+	debug  = flag.Bool("debug", false, "enable debugging")
+	target = flag.String("target", "", "base url to cache")
+)
+
+func init() {
+	flag.Parse()
+	flag.VisitAll(func(f *flag.Flag) {
+		log.Printf("%s=%v", f.Name, f.Value)
+	})
+	if *target == "" {
+		log.Fatalln("you must specify a target url")
+	}
+	if *debug {
+		logger := log.New(os.Stdout, "", 0)
+		cacheproxy.SetLogger(logger)
+	}
+}
+
+func main() {
+	URL, err := url.Parse(*target)
+	if err != nil {
+		log.Fatal(err)
+	}
+	proxy := cacheproxy.NewSingleHostReverseProxy(URL)
+	log.Fatal(http.ListenAndServe(*bindTo, handlers.CombinedLoggingHandler(os.Stdout, proxy)))
+}

cacheproxy/proxy.go

@@ -0,0 +1,32 @@
+package cacheproxy
+
+import (
+	"github.com/uovobw/httpcache"
+	"log"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+)
+
+var (
+	memoryCache = httpcache.NewMemoryCache()
+	transport   = httpcache.NewTransport(memoryCache)
+)
+
+// NewSingleHostReverseProxy wraps net/http/httputil.NewSingleHostReverseProxy
+// and sets the Host header based on the target URL.
+func NewSingleHostReverseProxy(url *url.URL) *httputil.ReverseProxy {
+	proxy := httputil.NewSingleHostReverseProxy(url)
+	oldDirector := proxy.Director
+	proxy.Director = func(r *http.Request) {
+		oldDirector(r)
+		r.Host = url.Host
+	}
+	proxy.Transport = transport
+	return proxy
+}
+
+// SetLogger wraps httpcache.SetLogger
+func SetLogger(l *log.Logger) {
+	transport.SetLogger(l)
+}

diskcache/diskcache_test.go

@@ -5,20 +5,12 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
-
-	. "gopkg.in/check.v1"
 )
 
-func Test(t *testing.T) { TestingT(t) }
-
-type S struct{}
-
-var _ = Suite(&S{})
-
-func (s *S) Test(c *C) {
+func TestDiskCache(t *testing.T) {
 	tempDir, err := ioutil.TempDir("", "httpcache")
 	if err != nil {
-		c.Fatalf("TempDir,: %v", err)
+		t.Fatalf("TempDir,: %v", err)
 	}
 	defer os.RemoveAll(tempDir)
 
@@ -27,17 +19,25 @@ func (s *S) Test(c *C) {
 	key := "testKey"
 	_, ok := cache.Get(key)
 
-	c.Assert(ok, Equals, false)
+	if ok != false {
+		t.Fatal("Get() without Add()")
+	}
 
 	val := []byte("some bytes")
 	cache.Set(key, val)
 
 	retVal, ok := cache.Get(key)
-	c.Assert(ok, Equals, true)
-	c.Assert(bytes.Equal(retVal, val), Equals, true)
+	if ok != true {
+		t.Fatal("did not retrieve the key i just set")
+	}
+	if bytes.Equal(retVal, val) != true {
+		t.Fatal("retrieved value not equal to the stored one")
+	}
 
 	cache.Delete(key)
 
 	_, ok = cache.Get(key)
-	c.Assert(ok, Equals, false)
+	if ok != false {
+		t.Fatal("Delete() key still present")
+	}
 }

httpcache.go

@@ -11,8 +11,12 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"log"
 	"net/http"
 	"net/http/httputil"
+	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -23,9 +27,21 @@ const (
 	fresh
 	transparent
 	// XFromCache is the header added to responses that are returned from the cache
-	XFromCache = "X-From-Cache"
+	XFromCache         = "X-From-Cache"
+	rangeSeparator     = "-"
+	rangeTypeSeparator = "="
 )
 
+var (
+	logger           *log.Logger
+	bytesRangeRegexp *regexp.Regexp
+)
+
+func init() {
+	logger = log.New(ioutil.Discard, "httpcache", 0)
+	bytesRangeRegexp = regexp.MustCompile("bytes=([0-9]*)-([0-9]*)")
+}
+
 // A Cache interface is used by the Transport to store and retrieve responses.
 type Cache interface {
 	// Get returns the []byte representation of a cached response and a bool
@@ -51,7 +67,142 @@ func CachedResponse(c Cache, req *http.Request) (resp *http.Response, err error)
 	}
 
 	b := bytes.NewBuffer(cachedVal)
-	return http.ReadResponse(bufio.NewReader(b), req)
+	returnResponse, err := http.ReadResponse(bufio.NewReader(b), req)
+	if err != nil {
+		return nil, fmt.Errorf("error loading response from cache: %s\n", err.Error())
+	}
+
+	if req.Header.Get("range") != "" {
+		strContentLength := returnResponse.Header.Get("content-length")
+		contentLength, err := strconv.ParseInt(strContentLength, 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("response loaded from cache has null or malformed content-length: %d", contentLength)
+		}
+		rangeRequestStart, rangeRequestEnd, err := findRanges(req, contentLength)
+		if err != nil {
+			return nil, err
+		}
+		if !validateRanges(rangeRequestStart, rangeRequestEnd, returnResponse) {
+			return nil, nil
+		}
+
+		body, err := ioutil.ReadAll(returnResponse.Body)
+		if err != nil {
+			logger.Printf("error reading cached response body: %s", err.Error())
+			return returnResponse, nil
+		}
+		returnResponse.Body.Close()
+
+		returnResponse.Body = ioutil.NopCloser(bytes.NewReader(body[rangeRequestStart:rangeRequestEnd]))
+		returnResponse.Header.Set("content-range", fmt.Sprintf("bytes %d-%d/%d", rangeRequestStart, rangeRequestEnd, contentLength))
+	}
+	return returnResponse, nil
+}
+
+// findRanges parses the range header value and the content length and returns (start, end, error)
+// it is not exported since it does not implement multiple ranges and only accepts bytes= ranges
+func findRanges(r *http.Request, totalLength int64) (start, end int64, err error) {
+	rawRange := r.Header.Get("range")
+	if rawRange == "" {
+		return -1, -1, fmt.Errorf("not a ranged request")
+	}
+	if !strings.HasPrefix(rawRange, "bytes=") {
+		return -1, -1, fmt.Errorf("non-bytes request %s range type unsupported", rawRange)
+	}
+	if strings.Contains(rawRange, ",") {
+		return -1, -1, fmt.Errorf("unsupported multiple ranges: %s", rawRange)
+	}
+	matchedValues := bytesRangeRegexp.FindStringSubmatch(rawRange)[1:]
+	strStart := matchedValues[0]
+	strEnd := matchedValues[1]
+	// range in the form STRSTART-
+	if strEnd == "" {
+		end = totalLength
+		start, err = strconv.ParseInt(strStart, 10, 64)
+		if err != nil {
+			return -1, -1, err
+		}
+		// range in the form -STREND
+	} else if strStart == "" {
+		end = totalLength
+		start, err = strconv.ParseInt(strEnd, 10, 64)
+		if err != nil {
+			return -1, -1, err
+		}
+		start = totalLength - start
+		// range in the form STRSTART-STREND
+	} else {
+		start, err = strconv.ParseInt(strStart, 10, 64)
+		if err != nil {
+			return -1, -1, err
+		}
+		end, err = strconv.ParseInt(strEnd, 10, 64)
+		if err != nil {
+			return -1, -1, err
+		}
+	}
+	if start >= end {
+		return -1, -1, fmt.Errorf("invalid start %d >= end %d", start, end)
+	}
+	return start, end, nil
+}
+
+// validateRanges checks that a cached request for a given response is within data that has been already loaded
+func validateRanges(start, end int64, resp *http.Response) (ok bool) {
+	// if the response cites partial content we need to compare the partial content we have stored
+	// with the ranges we require and, if not compatbile, fetch it again
+	// http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html section 14.16
+	if resp.StatusCode == http.StatusPartialContent {
+		rawContentRange := resp.Header.Get("content-range")
+		if rawContentRange == "" {
+			logger.Printf("request for %s is stored as 206-partial-content but has no content-range header!", resp.Request.URL.String())
+			return false
+		}
+		if !strings.Contains(rawContentRange, "bytes") {
+			logger.Printf("non-satisfiable range type in %s", rawContentRange)
+			return false
+		}
+		// the format is START-END/TOTAL or START-END/* if TOTAL is unknown
+		// if we find * we re-fetch the request as most probably the content was ephemereal
+		// or is highly probable iot has changed
+		if strings.Contains(rawContentRange, "*") {
+			return false
+		}
+		re := regexp.MustCompile("bytes ([0-9]+)-([0-9]+)/([0-9]+)")
+		// the first element is always the full match, skip it
+		matchedValues := re.FindStringSubmatch(rawContentRange)[1:]
+		currentStart, err := strconv.ParseInt(matchedValues[0], 10, 64)
+		if err != nil {
+			logger.Printf("cached response has malformed content-range header %s", rawContentRange)
+			return false
+		}
+		currentEnd, err := strconv.ParseInt(matchedValues[1], 10, 64)
+		if err != nil {
+			logger.Printf("cached response has malformed content-range header %s", rawContentRange)
+			return false
+		}
+		total, err := strconv.ParseInt(matchedValues[2], 10, 64)
+		if err != nil {
+			logger.Printf("cached response has malformed content-range header %s", rawContentRange)
+			return false
+		}
+		// validate the request ranges against the response headers
+		if start < currentStart || end > currentEnd || end > total {
+			logger.Printf("start: %d, currentStart: %d, end: %d, currentEnd: %d, total: %d", start, currentStart, end, currentEnd, total)
+			return false
+		}
+		return true
+		// the response is full content, use the content-length header to verify ranges
+	}
+	contentLength, err := strconv.ParseInt(resp.Header.Get("content-length"), 10, 64)
+	if err != nil {
+		logger.Printf("stored response has malformed or invalid content length %d", contentLength)
+		return false
+	}
+	if end > contentLength {
+		return false
+	}
+	return true
 }
 
 // MemoryCache is an implemtation of Cache that stores responses in an in-memory map.
@@ -106,6 +257,12 @@ func NewTransport(c Cache) *Transport {
 	return &Transport{Cache: c, MarkCachedResponses: true}
 }
 
+// SetLogger takes a *log.Logger and replaces the current one that discards all messages
+// this method is not thread safe
+func (t *Transport) SetLogger(l *log.Logger) {
+	logger = l
+}
+
 // Client returns an *http.Client that caches responses.
 func (t *Transport) Client() *http.Client {
 	return &http.Client{Transport: t}
@@ -138,6 +295,9 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 	var cachedResp *http.Response
 	if cacheableMethod {
 		cachedResp, err = CachedResponse(t.Cache, req)
+		if err != nil {
+			fmt.Print(err)
+		}
 	} else {
 		// Need to invalidate an existing value
 		t.Cache.Delete(cacheKey)
@@ -148,7 +308,11 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 		transport = http.DefaultTransport
 	}
 
-	if cachedResp != nil && err == nil && cacheableMethod && req.Header.Get("range") == "" {
+	if !cacheableMethod {
+		return transport.RoundTrip(req)
+	}
+
+	if cachedResp != nil && err == nil && cacheableMethod {
 		if t.MarkCachedResponses {
 			cachedResp.Header.Set(XFromCache, "1")
 		}
@@ -362,7 +526,7 @@ func getEndToEndHeaders(respHeaders http.Header) []string {
 		}
 	}
 	endToEndHeaders := []string{}
-	for respHeader, _ := range respHeaders {
+	for respHeader := range respHeaders {
 		if _, ok := hopByHopHeaders[respHeader]; !ok {
 			endToEndHeaders = append(endToEndHeaders, respHeader)
 		}