From d308c287c9b90147d0bb5fab66c80041f166d47f Mon Sep 17 00:00:00 2001 From: Rahul Thomas Date: Thu, 18 Oct 2018 22:54:17 -0700 Subject: [PATCH 1/3] moved output adapters to separate package --- {octopus => adapters}/outputadapter.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) rename {octopus => adapters}/outputadapter.go (82%) diff --git a/octopus/outputadapter.go b/adapters/outputadapter.go similarity index 82% rename from octopus/outputadapter.go rename to adapters/outputadapter.go index 32980bc..1fd6e6b 100644 --- a/octopus/outputadapter.go +++ b/adapters/outputadapter.go @@ -1,10 +1,12 @@ -package octopus +package adapters import ( "fmt" "io" "log" "os" + + oct "github.com/rapidclock/web-octopus/octopus" ) // OutputAdapter is the interface for the Adapter that is used to handle @@ -15,14 +17,14 @@ import ( // Implementers of the interface should listen on this channel for output from // the crawler. type OutputAdapter interface { - Consume(quitCh <-chan bool) chan<- CrawlOutput + Consume(quitCh <-chan bool) chan<- oct.CrawlOutput } // StdOpAdapter is an output adapter that just prints the output onto the screen. type StdOpAdapter struct{} -func (s *StdOpAdapter) Consume(quitCh <-chan bool) chan<- CrawlOutput { - listenCh := make(chan CrawlOutput) +func (s *StdOpAdapter) Consume(quitCh <-chan bool) chan<- oct.CrawlOutput { + listenCh := make(chan oct.CrawlOutput) go func() { for { select { @@ -41,13 +43,13 @@ type FileWriterAdapter struct { FilePath string } -func (fw *FileWriterAdapter) Consume(quitCh <-chan bool) chan<- CrawlOutput { - listenCh := make(chan CrawlOutput) +func (fw *FileWriterAdapter) Consume(quitCh <-chan bool) chan<- oct.CrawlOutput { + listenCh := make(chan oct.CrawlOutput) fw.writeToFile(quitCh, listenCh) return listenCh } -func (fw *FileWriterAdapter) writeToFile(quitCh <-chan bool, ch <-chan CrawlOutput) { +func (fw *FileWriterAdapter) writeToFile(quitCh <-chan bool, ch <-chan oct.CrawlOutput) { fp, err := fw.getFilePointer() if err != nil { fp.Close() From d8af331c0e999a5fddc4d1a8301912ffa90fae0b Mon Sep 17 00:00:00 2001 From: Rahul Thomas Date: Thu, 18 Oct 2018 23:07:56 -0700 Subject: [PATCH 2/3] fixed adapter reference issue --- {adapters => adapter}/outputadapter.go | 2 +- octopus/models.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) rename {adapters => adapter}/outputadapter.go (99%) diff --git a/adapters/outputadapter.go b/adapter/outputadapter.go similarity index 99% rename from adapters/outputadapter.go rename to adapter/outputadapter.go index 1fd6e6b..002df6d 100644 --- a/adapters/outputadapter.go +++ b/adapter/outputadapter.go @@ -1,4 +1,4 @@ -package adapters +package adapter import ( "fmt" diff --git a/octopus/models.go b/octopus/models.go index 6103e4d..5a64998 100644 --- a/octopus/models.go +++ b/octopus/models.go @@ -3,6 +3,8 @@ package octopus import ( "io" "time" + + "github.com/rapidclock/web-octopus/adapter" ) // Node is used to represent each crawled link and its associated depth of crawl. @@ -39,7 +41,7 @@ type CrawlOptions struct { CrawlRate time.Duration RespectRobots bool IncludeBody bool - OpAdapter OutputAdapter + OpAdapter adapter.OutputAdapter } type CrawlOutput struct { From 34afa26a6dcc1b688b1480fb4d5c79874f5c2e94 Mon Sep 17 00:00:00 2001 From: Rahul Thomas Date: Thu, 18 Oct 2018 23:21:31 -0700 Subject: [PATCH 3/3] moved interface outputadapter to octopus package --- adapter/{outputadapter.go => basicadapters.go} | 11 ----------- octopus/models.go | 15 ++++++++++++--- 2 files changed, 12 insertions(+), 14 deletions(-) rename adapter/{outputadapter.go => basicadapters.go} (74%) diff --git a/adapter/outputadapter.go b/adapter/basicadapters.go similarity index 74% rename from adapter/outputadapter.go rename to adapter/basicadapters.go index 002df6d..598c542 100644 --- a/adapter/outputadapter.go +++ b/adapter/basicadapters.go @@ -9,17 +9,6 @@ import ( oct "github.com/rapidclock/web-octopus/octopus" ) -// OutputAdapter is the interface for the Adapter that is used to handle -// output from the Octopus Crawler. -// The contract stipulates that the crawler provides the channel -// to listen for a quit command. -// The crawler pumps its output onto the returned channel of the Consume method. -// Implementers of the interface should listen on this channel for output from -// the crawler. -type OutputAdapter interface { - Consume(quitCh <-chan bool) chan<- oct.CrawlOutput -} - // StdOpAdapter is an output adapter that just prints the output onto the screen. type StdOpAdapter struct{} diff --git a/octopus/models.go b/octopus/models.go index 5a64998..b1e4f50 100644 --- a/octopus/models.go +++ b/octopus/models.go @@ -3,8 +3,6 @@ package octopus import ( "io" "time" - - "github.com/rapidclock/web-octopus/adapter" ) // Node is used to represent each crawled link and its associated depth of crawl. @@ -41,10 +39,21 @@ type CrawlOptions struct { CrawlRate time.Duration RespectRobots bool IncludeBody bool - OpAdapter adapter.OutputAdapter + OpAdapter OutputAdapter } type CrawlOutput struct { Node Body io.ReadCloser } + +// OutputAdapter is the interface for the Adapter that is used to handle +// output from the Octopus Crawler. +// The contract stipulates that the crawler provides the channel +// to listen for a quit command. +// The crawler pumps its output onto the returned channel of the Consume method. +// Implementers of the interface should listen on this channel for output from +// the crawler. +type OutputAdapter interface { + Consume(quitCh <-chan bool) chan<- CrawlOutput +}