From 0135dc9ce09de5810f9f1605a44b83aae4693f07 Mon Sep 17 00:00:00 2001
From: kaspernj <k@spernj.org>
Date: Sat, 6 Dec 2014 16:14:43 +0100
Subject: [PATCH 1/3] Initial work on implementing streaming.

---
 Gemfile.lock                       |  2 ++
 http2.gemspec                      |  2 +-
 include/response_reader.rb         |  8 +++++++-
 lib/http2.rb                       |  4 ++--
 spec/http2/json_streaming_spec.rb  | 12 ++++++++++++
 spec/spec_root/json_streaming.json |  6 ++++++
 6 files changed, 30 insertions(+), 4 deletions(-)
 create mode 100644 spec/http2/json_streaming_spec.rb
 create mode 100644 spec/spec_root/json_streaming.json

diff --git a/Gemfile.lock b/Gemfile.lock
index 50df2fc..41ccd23 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -62,6 +62,7 @@ GEM
     tpool (0.0.4)
     tsafe (0.0.11)
     wref (0.0.6)
+    yajl-ruby (1.2.1)
 
 PLATFORMS
   ruby
@@ -75,3 +76,4 @@ DEPENDENCIES
   rdoc (~> 3.12)
   rspec (~> 2.8.0)
   sqlite3
+  yajl-ruby
diff --git a/http2.gemspec b/http2.gemspec
index f6518bb..91cf9a4 100644
--- a/http2.gemspec
+++ b/http2.gemspec
@@ -31,5 +31,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency("hayabusa", ">= 0.0.25")
   s.add_development_dependency("sqlite3")
   s.add_development_dependency("codeclimate-test-reporter")
+  s.add_development_dependency("yajl-ruby")
 end
-
diff --git a/include/response_reader.rb b/include/response_reader.rb
index 84460f9..7d01776 100644
--- a/include/response_reader.rb
+++ b/include/response_reader.rb
@@ -225,7 +225,7 @@ def parse_body(line)
       return parse_body_chunked(line)
     else
       puts "Http2: Adding #{line.to_s.bytesize} to the body." if @debug
-      @response.body << line
+      @response.body << line unless skip_body?
       @http2.on_content_call(@args, line)
       return :break if @response.content_length && @response.body.length >= @response.content_length
     end
@@ -252,4 +252,10 @@ def parse_body_chunked(line)
 
     raise "Should have read newline but didnt: '#{nl}'." if nl != @nl
   end
+
+private
+
+  def skip_body?
+    @args[:skip_body]
+  end
 end
diff --git a/lib/http2.rb b/lib/http2.rb
index 9e1cfa3..05673b5 100644
--- a/lib/http2.rb
+++ b/lib/http2.rb
@@ -15,13 +15,13 @@
 #  print "#{res.headers}"
 # end
 class Http2
-  #Autoloader for subclasses.
+  # Autoloader for subclasses.
   def self.const_missing(name)
     require "#{File.dirname(__FILE__)}/../include/#{::StringCases.camel_to_snake(name)}.rb"
     return Http2.const_get(name)
   end
 
-  #Converts a URL to "is.gd"-short-URL.
+  # Converts a URL to "is.gd"-short-URL.
   def self.isgdlink(url)
     Http2.new(host: "is.gd") do |http|
       resp = http.get("/api.php?longurl=#{url}")
diff --git a/spec/http2/json_streaming_spec.rb b/spec/http2/json_streaming_spec.rb
new file mode 100644
index 0000000..d53393f
--- /dev/null
+++ b/spec/http2/json_streaming_spec.rb
@@ -0,0 +1,12 @@
+require "spec_helper"
+
+describe Http2 do
+  it "should stream json results" do
+    with_http do |http|
+      result = http.get(url: "json_streaming.json", stream_json: true, skip_body: true)
+      result.json_streaming_results do |result|
+        puts "Result: #{result}"
+      end
+    end
+  end
+end
diff --git a/spec/spec_root/json_streaming.json b/spec/spec_root/json_streaming.json
new file mode 100644
index 0000000..11dc55e
--- /dev/null
+++ b/spec/spec_root/json_streaming.json
@@ -0,0 +1,6 @@
+{
+  "Results:" [
+    {"id": 1, "name":"Kasper"},
+    {"id": 2, "name":"Christina"}
+  ]
+}
\ No newline at end of file

From b62e23de2d109329e06b8e2fd462b706ea5bfd61 Mon Sep 17 00:00:00 2001
From: kaspernj <k@spernj.org>
Date: Sun, 7 Dec 2014 17:19:33 +0100
Subject: [PATCH 2/3] Made text-streaming work.

---
 Gemfile.lock                      |  3 +
 http2.gemspec                     |  1 +
 include/connection.rb             | 14 ++---
 include/get_request.rb            |  8 +--
 include/post_multipart_request.rb |  2 +-
 include/post_request.rb           |  8 +--
 include/response.rb               | 98 ++++++++++++++++++++++++++++---
 include/response_reader.rb        | 54 +++++++++++------
 lib/http2.rb                      | 35 +++++++----
 spec/http2/json_streaming_spec.rb | 12 ----
 spec/http2/streaming_spec.rb      | 30 ++++++++++
 11 files changed, 198 insertions(+), 67 deletions(-)
 delete mode 100644 spec/http2/json_streaming_spec.rb
 create mode 100644 spec/http2/streaming_spec.rb

diff --git a/Gemfile.lock b/Gemfile.lock
index 41ccd23..35e7647 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -3,6 +3,7 @@ PATH
   specs:
     http2 (0.0.29)
       string-cases
+      thread_queues (>= 0.0.2)
 
 GEM
   remote: http://rubygems.org/
@@ -59,6 +60,8 @@ GEM
     sqlite3 (1.3.10)
     string-cases (0.0.0)
     string-strtr (0.0.3)
+    thread_queues (0.0.2)
+      string-cases
     tpool (0.0.4)
     tsafe (0.0.11)
     wref (0.0.6)
diff --git a/http2.gemspec b/http2.gemspec
index 91cf9a4..e85995c 100644
--- a/http2.gemspec
+++ b/http2.gemspec
@@ -24,6 +24,7 @@ Gem::Specification.new do |s|
   s.summary = "A lightweight framework for doing http-connections in Ruby. Supports cookies, keep-alive, compressing and much more."
 
   s.add_runtime_dependency("string-cases", ">= 0")
+  s.add_runtime_dependency("thread_queues", ">= 0.0.2")
   s.add_development_dependency("rake")
   s.add_development_dependency("rspec", "~> 2.8.0")
   s.add_development_dependency("rdoc", "~> 3.12")
diff --git a/include/connection.rb b/include/connection.rb
index 12d5e6e..794048a 100644
--- a/include/connection.rb
+++ b/include/connection.rb
@@ -1,6 +1,6 @@
 class Http2::Connection
   def initialize(http2)
-    @http2, @debug, @args = http2, http2.debug, http2.args
+    @http2, @debug, @args = http2, http2.debug?, http2.args
     reconnect
   end
 
@@ -59,7 +59,7 @@ def write(str)
 
   # Reconnects to the host.
   def reconnect
-    puts "Http2: Reconnect." if @debug
+    @http2.debug "Reconnect." if @debug
 
     #Open connection.
     if @args[:proxy] && @args[:ssl]
@@ -67,7 +67,7 @@ def reconnect
     elsif @args[:proxy]
       connect_proxy
     else
-      puts "Http2: Opening socket connection to '#{@http2.host}:#{@http2.port}'." if @debug
+      @http2.debug "Opening socket connection to '#{@http2.host}:#{@http2.port}'." if @debug
       @sock_plain = TCPSocket.new(@http2.host, @http2.port)
     end
 
@@ -87,7 +87,7 @@ def socket_working?
     if @keepalive_timeout && @request_last
       between = Time.now.to_i - @request_last.to_i
       if between >= @keepalive_timeout
-        puts "Http2: We are over the keepalive-wait - returning false for socket_working?." if @debug
+        @http2.debug "We are over the keepalive-wait - returning false for socket_working?." if @debug
         return false
       end
     end
@@ -103,7 +103,7 @@ def close
   end
 
   def connect_proxy_ssl
-    puts "Http2: Initializing proxy stuff." if @debug
+    @http2.debug "Initializing proxy stuff." if @debug
     @sock_plain = TCPSocket.new(@args[:proxy][:host], @args[:proxy][:port])
 
     @sock_plain.write("CONNECT #{@args[:host]}:#{@args[:port]} HTTP/1.0#{@nl}")
@@ -122,12 +122,12 @@ def connect_proxy_ssl
   end
 
   def connect_proxy
-    puts "Http2: Opening socket connection to '#{@args[:host]}:#{@args[:port]}' through proxy '#{@args[:proxy][:host]}:#{@args[:proxy][:port]}'." if @debug
+    @http2.debug "Opening socket connection to '#{@args[:host]}:#{@args[:port]}' through proxy '#{@args[:proxy][:host]}:#{@args[:proxy][:port]}'." if @debug
     @sock_plain = TCPSocket.new(@args[:proxy][:host], @args[:proxy][:port].to_i)
   end
 
   def apply_ssl
-    puts "Http2: Initializing SSL." if @debug
+    @http2.debug "Initializing SSL." if @debug
     require "openssl" unless ::Kernel.const_defined?(:OpenSSL)
 
     ssl_context = OpenSSL::SSL::SSLContext.new
diff --git a/include/get_request.rb b/include/get_request.rb
index db68665..f991d34 100644
--- a/include/get_request.rb
+++ b/include/get_request.rb
@@ -1,17 +1,17 @@
 class Http2::GetRequest
   def initialize(http2, args)
-    @http2, @args, @debug, @nl = http2, http2.parse_args(args), http2.debug, http2.nl
+    @http2, @args, @debug, @nl = http2, http2.parse_args(args), http2.debug?, http2.nl
   end
 
   def execute
     @http2.mutex.synchronize do
-      puts "Http2: Writing headers: #{header_string}" if @debug
+      @http2.debug "Writing headers: #{header_string}" if @debug
       @http2.connection.write(header_string)
 
-      puts "Http2: Reading response." if @debug
+      @http2.debug "Reading response." if @debug
       resp = @http2.read_response(@args)
 
-      puts "Http2: Done with get request." if @debug
+      @http2.debug "Done with get request." if @debug
       return resp
     end
   end
diff --git a/include/post_multipart_request.rb b/include/post_multipart_request.rb
index 2dbe25d..d78dc26 100644
--- a/include/post_multipart_request.rb
+++ b/include/post_multipart_request.rb
@@ -11,7 +11,7 @@ def initialize(http2, *args)
 
   def execute
     generate_raw(@phash) do |helper, praw|
-      puts "Http2: Header string: #{header_string}" if @debug
+      @http2.debug "Header string: #{header_string}" if @debug
 
       @http2.mutex.synchronize do
         @conn.write(header_string(praw))
diff --git a/include/post_request.rb b/include/post_request.rb
index a668170..fa00e92 100644
--- a/include/post_request.rb
+++ b/include/post_request.rb
@@ -1,12 +1,12 @@
 class Http2::PostRequest
-  VALID_ARGUMENTS_POST = [:post, :url, :default_headers, :headers, :json, :method, :cookies, :on_content, :content_type]
+  VALID_ARGUMENTS_POST = [:body_as, :post, :url, :default_headers, :headers, :json, :method, :cookies, :on_content, :content_type]
 
   def initialize(http2, args)
     args.each do |key, val|
       raise "Invalid key: '#{key}'." unless VALID_ARGUMENTS_POST.include?(key)
     end
 
-    @http2, @args, @debug, @nl = http2, http2.parse_args(args), http2.debug, http2.nl
+    @http2, @args, @debug, @nl = http2, http2.parse_args(args), http2.debug?, http2.nl
     @conn = @http2.connection
   end
 
@@ -14,8 +14,8 @@ def execute
     @data = raw_data
 
     @http2.mutex.synchronize do
-      puts "Http2: Doing post." if @debug
-      puts "Http2: Header str: #{header_str}" if @debug
+      @http2.debug "Doing post." if @debug
+      @http2.debug "Header str: #{header_str}" if @debug
 
       @conn.write(header_string)
       return @http2.read_response(@args)
diff --git a/include/response.rb b/include/response.rb
index 60b9d8f..e8e5867 100644
--- a/include/response.rb
+++ b/include/response.rb
@@ -1,15 +1,64 @@
 #This object will be returned as the response for each request.
 class Http2::Response
   #All the data the response contains. Headers, body, cookies, requested URL and more.
-  attr_reader :args
+  attr_reader :args, :tempfile, :buffer
   attr_accessor :body, :charset, :code, :content_type, :http_version
 
   #This method should not be called manually.
   def initialize(args = {})
     @args = args
     @args[:headers] = {} unless @args.key?(:headers)
-    @body = args[:body] || ""
     @debug = @args[:debug]
+
+    if args[:request_args][:args][:body_as] == :tempfile
+      @tempfile = Tempfile.new
+    elsif args[:request_args][:args][:body_as] == :buffer
+      require "thread_queues"
+      @queue = ThreadQueues::BufferedQueue.new(50)
+      @buffer = ThreadQueues::StringBuffer.new(@queue)
+    else
+      @body = ""
+    end
+  end
+
+  def buffer
+    if @buffer
+      if block_given?
+        begin
+          yield @buffer
+        ensure
+          @queue.close
+        end
+      else
+        return @buffer
+      end
+    else
+      raise "Not in buffer-mode"
+    end
+  end
+
+  def tempfile
+    if @tempfile
+      return @tempfile
+    else
+      raise "Not in tempfile-mode"
+    end
+  end
+
+  def body
+    if body?
+      return @body
+    else
+      raise "Not in body-mode"
+    end
+  end
+
+  def body?
+    return true if @body
+  end
+
+  def finish
+    @queue.close if @queue
   end
 
   #Returns headers given from the host for the result.
@@ -45,6 +94,16 @@ def content_length
     end
   end
 
+  def content_length?
+    if header?("content-length")
+      return true
+    elsif @body
+      return true
+    end
+
+    return false
+  end
+
   #Returns the requested URL as a string.
   #===Examples
   # res.requested_url #=> "?show=status&action=getstatus"
@@ -55,25 +114,46 @@ def requested_url
 
   # Checks the data that has been sat on the object and raises various exceptions, if it does not validate somehow.
   def validate!
-    puts "Http2: Validating response length." if @debug
+    @http2.debug "Validating response length." if @debug
     validate_body_versus_content_length!
   end
 
+  def add_to_body(obj)
+    if @tempfile
+      @tempfile.write(obj)
+    elsif @queue
+      @queue.push(obj)
+    elsif @body
+      @body << obj
+    else
+      raise "Don't know how to add to body?"
+    end
+  end
+
 private
 
   # Checks that the length of the body is the same as the given content-length if given.
   def validate_body_versus_content_length!
     unless self.header?("content-length")
-      puts "Http2: No content length given - skipping length validation." if @debug
+      @http2.debug "No content length given - skipping length validation." if @debug
       return nil
     end
 
-    content_length = header("content-length").to_i
-    body_length = @body.bytesize
+    if @body
+      body_length = @body.bytesize
+    elsif @tempfile
+      body_length File.size(@tempfile.path)
+    end
+
+    if body_length
+      content_length = header("content-length").to_i
 
-    puts "Http2: Body length: #{body_length}" if @debug
-    puts "Http2: Content length: #{content_length}" if @debug
+      @http2.debug "Body length: #{body_length}" if @debug
+      @http2.debug "Content length: #{content_length}" if @debug
 
-    raise "Body does not match the given content-length: '#{body_length}', '#{content_length}'." if body_length != content_length
+      if body_length != content_length
+        raise "Body does not match the given content-length: '#{body_length}', '#{content_length}'."
+      end
+    end
   end
 end
diff --git a/include/response_reader.rb b/include/response_reader.rb
index 7d01776..f683375 100644
--- a/include/response_reader.rb
+++ b/include/response_reader.rb
@@ -6,22 +6,35 @@ def initialize(args)
     @transfer_encoding = nil
     @response = Http2::Response.new(request_args: args, debug: @debug)
     @rec_count = 0
-    @args, @debug, @http2, @sock = args[:args], args[:http2].debug, args[:http2], args[:sock]
+    @content_read_length = 0
+    @args, @debug, @http2, @sock = args[:args], args[:http2].debug?, args[:http2], args[:sock]
     @nl = @http2.nl
     @conn = @http2.connection
+  end
 
+  def read
     read_headers
-    read_body if @length == nil || @length > 0
+    read_body
     finish
   end
 
+  def wait_for_headers
+    sleep 0.05 while @mode == "headers"
+  end
+
+  def wait_for_body
+    sleep 0.05 until @mode == "finished"
+  end
+
+private
+
   def read_headers
     loop do
       line = @conn.gets
       check_line_read(line)
 
       if line == "\n" || line == "\r\n" || line == @nl
-        puts "Http2: Changing mode to body!" if @debug
+        @http2.debug "Changing mode to body!" if @debug
         raise "No headers was given at all? Possibly corrupt state after last request?" if @response.headers.empty?
         @mode = "body"
         @http2.on_content_call(@args, @nl)
@@ -33,6 +46,8 @@ def read_headers
   end
 
   def read_body
+    return if @length == 0
+
     loop do
       if @length
         line = @conn.read(@length)
@@ -44,7 +59,7 @@ def read_body
       check_line_read(line)
       stat = parse_body(line)
       break if stat == :break
-      next if stat == :next
+      # next if stat == :next
     end
   end
 
@@ -55,14 +70,17 @@ def finish
     end
 
     # Validate that the response is as it should be.
-    puts "Http2: Validating response." if @debug
+    @http2.debug "Validating response." if @debug
 
-    if !@response.code
+    unless @response.code
       raise "No status-code was received from the server. Headers: '#{@response.headers}' Body: '#{resp.body}'."
     end
 
+    @mode = "finished"
+    @response.finish
     @response.validate!
     check_and_decode
+
     @http2.autostate_register(@response) if @http2.args[:autostate]
     handle_errors
 
@@ -71,8 +89,6 @@ def finish
     end
   end
 
-private
-
   def check_and_follow_redirect
     if (@response.code == "302" || @response.code == "303" || @response.code == "307") && @response.header?("location") && @http2.args[:follow_redirects]
       url, args = url_and_args_from_location
@@ -100,7 +116,7 @@ def url_and_args_from_location
   def check_and_decode
     # Check if the content is gzip-encoded - if so: decode it!
     if @encoding == "gzip"
-      puts "Http2: Decoding GZip." if @debug
+      @http2.debug "Decoding GZip." if @debug
       require "zlib"
       require "stringio"
       io = StringIO.new(@response.body)
@@ -110,7 +126,7 @@ def check_and_decode
       begin
         valid_string = ic.encode("UTF-8")
       rescue
-        valid_string = untrusted_str.force_encoding("UTF-8").encode("UTF-8", :invalid => :replace, :replace => "").encode("UTF-8")
+        valid_string = untrusted_str.force_encoding("UTF-8").encode("UTF-8", invalid: :replace, replace: "").encode("UTF-8")
       end
 
       @response.body = valid_string
@@ -153,10 +169,10 @@ def parse_cookie(cookie_line)
   def parse_keep_alive(keep_alive_line)
     keep_alive_line.scan(/([a-z]+)=(\d+)/) do |match|
       if match[0] == "timeout"
-        puts "Http2: Keepalive-max set to: '#{@keepalive_max}'." if @debug
+        @http2.debug "Keepalive-max set to: '#{@keepalive_max}'." if @debug
         @http2.keepalive_timeout = match[1].to_i
       elsif match[0] == "max"
-        puts "Http2: Keepalive-timeout set to: '#{@keepalive_timeout}'." if @debug
+        @http2.debug "Keepalive-timeout set to: '#{@keepalive_timeout}'." if @debug
         @http2.keepalive_max = match[1].to_i
       end
     end
@@ -198,7 +214,7 @@ def set_header_special_values(key, value)
       @connection = value.downcase
     elsif key == "content-encoding"
       @encoding = value.downcase
-      puts "Http2: Setting encoding to #{@encoding}" if @debug
+      @http2.debug "Setting encoding to #{@encoding}" if @debug
     elsif key == "content-length"
       @length = value.to_i
     elsif key == "transfer-encoding"
@@ -207,7 +223,7 @@ def set_header_special_values(key, value)
   end
 
   def parse_normal_header(line, key, orig_key, value)
-    puts "Http2: Parsed header: #{orig_key}: #{value}" if @debug
+    @http2.debug "Parsed header: #{orig_key}: #{value}" if @debug
     @response.headers[key] = [] unless @response.headers.key?(key)
     @response.headers[key] << value
 
@@ -221,13 +237,15 @@ def parse_normal_header(line, key, orig_key, value)
   def parse_body(line)
     return :break if @length == 0
 
+    @content_read_length += line.length
+
     if @transfer_encoding == "chunked"
       return parse_body_chunked(line)
     else
-      puts "Http2: Adding #{line.to_s.bytesize} to the body." if @debug
-      @response.body << line unless skip_body?
+      @http2.debug "Adding #{line.to_s.bytesize} to the body." if @debug
+      @response.add_to_body(line)
       @http2.on_content_call(@args, line)
-      return :break if @response.content_length && @response.body.length >= @response.content_length
+      return :break if @response.content_length? && @content_read_length >= @response.content_length
     end
   end
 
@@ -237,7 +255,7 @@ def parse_body_chunked(line)
     if len > 0
       read = @conn.read(len)
       return :break if read == "" || read == "\n" || read == "\r\n"
-      @response.body << read
+      @response.add_to_body(read)
       @http2.on_content_call(@args, read)
     end
 
diff --git a/lib/http2.rb b/lib/http2.rb
index 05673b5..cd70775 100644
--- a/lib/http2.rb
+++ b/lib/http2.rb
@@ -29,7 +29,7 @@ def self.isgdlink(url)
     end
   end
 
-  attr_reader :autostate, :connection, :cookies, :args, :debug, :mutex, :resp, :raise_errors, :nl
+  attr_reader :autostate, :connection, :cookies, :args, :mutex, :resp, :raise_errors, :nl
   attr_accessor :keepalive_max, :keepalive_timeout
 
   VALID_ARGUMENTS_INITIALIZE = [:host, :port, :skip_port_in_host_header, :ssl, :nl, :user_agent, :raise_errors, :follow_redirects, :debug, :encoding_gzip, :autostate, :basic_auth, :extra_headers, :proxy]
@@ -82,18 +82,18 @@ def change(args)
   #===Examples
   # http.destroy
   def destroy
-    @args = nil
-    @cookies = nil
-    @debug = nil
-    @mutex = nil
-    @uagent = nil
-    @keepalive_timeout = nil
-    @request_last = nil
-
     @connection.destroy
     @connection = nil
   end
 
+  def debug(message)
+    print "Http2: #{message}\n" if @debug
+  end
+
+  def debug?
+    return @debug
+  end
+
   #Forces various stuff into arguments-hash like URL from original arguments and enables single-string-shortcuts and more.
   def parse_args(*args)
     if args.length == 1 && args.first.is_a?(String)
@@ -217,7 +217,18 @@ def on_content_call(args, str)
   #===Examples
   # res = http.read_response
   def read_response(args = {})
-    ::Http2::ResponseReader.new(http2: self, sock: @sock, args: args).response
+    response_reader = ::Http2::ResponseReader.new(http2: self, sock: @sock, args: args)
+
+    if args[:async]
+      Thread.new do
+        Thread.current.abort_on_exception = true
+        response_reader.read
+      end
+    else
+      response_reader.read
+    end
+
+    return response_reader.response
   end
 
   def to_s
@@ -232,7 +243,7 @@ def inspect
 
   #Registers the states from a result.
   def autostate_register(res)
-    puts "Http2: Running autostate-register on result." if @debug
+    debug "Running autostate-register on result." if @debug
     @autostate_values.clear
 
     res.body.to_s.scan(/<input type="hidden" name="__(EVENTTARGET|EVENTARGUMENT|VIEWSTATE|LASTFOCUS)" id="(.*?)" value="(.*?)" \/>/) do |match|
@@ -240,7 +251,7 @@ def autostate_register(res)
       id = match[1]
       value = match[2]
 
-      puts "Http2: Registered autostate-value with name '#{name}' and value '#{value}'." if @debug
+      debug "Registered autostate-value with name '#{name}' and value '#{value}'." if @debug
       @autostate_values[name] = Http2::Utils.urldec(value)
     end
 
diff --git a/spec/http2/json_streaming_spec.rb b/spec/http2/json_streaming_spec.rb
deleted file mode 100644
index d53393f..0000000
--- a/spec/http2/json_streaming_spec.rb
+++ /dev/null
@@ -1,12 +0,0 @@
-require "spec_helper"
-
-describe Http2 do
-  it "should stream json results" do
-    with_http do |http|
-      result = http.get(url: "json_streaming.json", stream_json: true, skip_body: true)
-      result.json_streaming_results do |result|
-        puts "Result: #{result}"
-      end
-    end
-  end
-end
diff --git a/spec/http2/streaming_spec.rb b/spec/http2/streaming_spec.rb
new file mode 100644
index 0000000..aec19b6
--- /dev/null
+++ b/spec/http2/streaming_spec.rb
@@ -0,0 +1,30 @@
+require "spec_helper"
+
+describe Http2 do
+  it "should stream to queue" do
+    with_http do |http|
+      result = http.get(url: "json_streaming.json", async: true, body_as: :buffer)
+
+      result.buffer do |buffer|
+        buffer.gets.should eq "{\n"
+        buffer.gets.should eq "  \"Results:\" [\n"
+        buffer.gets.should eq '    {"id": 1, "name":"Kasper"},' + "\n"
+        buffer.gets.should eq '    {"id": 2, "name":"Christina"}' + "\n"
+        buffer.gets.should eq "  ]\n"
+        buffer.gets.should eq "}"
+      end
+    end
+  end
+
+  it "should stream json results" do
+    with_http do |http|
+      result = http.get(url: "json_streaming.json", async: true)
+
+      puts "Running free - wee!"
+
+      result.json_streaming_results do |result|
+        puts "Result: #{result}"
+      end
+    end
+  end
+end

From fb934f70f5ff7f06dee2352b89ba6afba45a09ea Mon Sep 17 00:00:00 2001
From: kaspernj <k@spernj.org>
Date: Fri, 20 Mar 2015 12:22:22 +0100
Subject: [PATCH 3/3] Latest work on streaming.

---
 Gemfile                            |  3 +++
 Gemfile.lock                       |  9 +++++++--
 http2.gemspec                      |  2 +-
 include/response_reader.rb         |  6 ------
 spec/http2/streaming_spec.rb       | 21 +++++++++++++++------
 spec/http2_spec.rb                 |  3 +++
 spec/spec_root/json_streaming.json |  2 +-
 7 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/Gemfile b/Gemfile
index c80ee36..7193932 100644
--- a/Gemfile
+++ b/Gemfile
@@ -1,3 +1,6 @@
 source "http://rubygems.org"
 
 gemspec
+
+gem "thread_queues", path: "/home/kaspernj/Dev/Ruby/thread_queues"
+gem "json_streamer", path: "/home/kaspernj/Dev/Ruby/json_streamer"
diff --git a/Gemfile.lock b/Gemfile.lock
index 35e7647..36f81e1 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -5,6 +5,12 @@ PATH
       string-cases
       thread_queues (>= 0.0.2)
 
+PATH
+  remote: /home/kaspernj/Dev/Ruby/thread_queues
+  specs:
+    thread_queues (0.0.3)
+      string-cases
+
 GEM
   remote: http://rubygems.org/
   specs:
@@ -60,8 +66,6 @@ GEM
     sqlite3 (1.3.10)
     string-cases (0.0.0)
     string-strtr (0.0.3)
-    thread_queues (0.0.2)
-      string-cases
     tpool (0.0.4)
     tsafe (0.0.11)
     wref (0.0.6)
@@ -79,4 +83,5 @@ DEPENDENCIES
   rdoc (~> 3.12)
   rspec (~> 2.8.0)
   sqlite3
+  thread_queues!
   yajl-ruby
diff --git a/http2.gemspec b/http2.gemspec
index e85995c..88176f8 100644
--- a/http2.gemspec
+++ b/http2.gemspec
@@ -32,5 +32,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency("hayabusa", ">= 0.0.25")
   s.add_development_dependency("sqlite3")
   s.add_development_dependency("codeclimate-test-reporter")
-  s.add_development_dependency("yajl-ruby")
+  s.add_development_dependency("json_streamer")
 end
diff --git a/include/response_reader.rb b/include/response_reader.rb
index f683375..85b7aaf 100644
--- a/include/response_reader.rb
+++ b/include/response_reader.rb
@@ -270,10 +270,4 @@ def parse_body_chunked(line)
 
     raise "Should have read newline but didnt: '#{nl}'." if nl != @nl
   end
-
-private
-
-  def skip_body?
-    @args[:skip_body]
-  end
 end
diff --git a/spec/http2/streaming_spec.rb b/spec/http2/streaming_spec.rb
index aec19b6..3a17700 100644
--- a/spec/http2/streaming_spec.rb
+++ b/spec/http2/streaming_spec.rb
@@ -7,7 +7,7 @@
 
       result.buffer do |buffer|
         buffer.gets.should eq "{\n"
-        buffer.gets.should eq "  \"Results:\" [\n"
+        buffer.gets.should eq '  "Results": ' + "[\n"
         buffer.gets.should eq '    {"id": 1, "name":"Kasper"},' + "\n"
         buffer.gets.should eq '    {"id": 2, "name":"Christina"}' + "\n"
         buffer.gets.should eq "  ]\n"
@@ -17,13 +17,22 @@
   end
 
   it "should stream json results" do
-    with_http do |http|
-      result = http.get(url: "json_streaming.json", async: true)
+    with_http(debug: true) do |http|
+      result = http.get(url: "json_streaming.json", async: true, body_as: :buffer)
+      result.buffer do |buffer|
+        require "json_streamer"
+
+        parser = Yajl::Parser.new(symbolize_keys: true)
+        parser.on_parse_complete = lambda { |obj|
+          puts "ObjectParsed: #{obj}"
+        }
 
-      puts "Running free - wee!"
+        buffer.each_line do |line|
+          puts "Giving line to parser: #{line}"
+          parser << line
+        end
 
-      result.json_streaming_results do |result|
-        puts "Result: #{result}"
+        # hash = Yajl::Parser.parse(buffer)
       end
     end
   end
diff --git a/spec/http2_spec.rb b/spec/http2_spec.rb
index c491ef1..e098ee2 100644
--- a/spec/http2_spec.rb
+++ b/spec/http2_spec.rb
@@ -120,6 +120,9 @@
       # Hack JSON data from Hayabusa.
       json_data = JSON.parse(data["_POST"].keys.first)
       json_data["testkey"].should eq "testvalue"
+
+      res.json?.should eq true
+      res.json["testkey"].should eq "testvalue"
     end
   end
 
diff --git a/spec/spec_root/json_streaming.json b/spec/spec_root/json_streaming.json
index 11dc55e..625f982 100644
--- a/spec/spec_root/json_streaming.json
+++ b/spec/spec_root/json_streaming.json
@@ -1,5 +1,5 @@
 {
-  "Results:" [
+  "Results": [
     {"id": 1, "name":"Kasper"},
     {"id": 2, "name":"Christina"}
   ]