diff --git a/LinkCrawler/LinkCrawler.Tests/LinkCrawler.Tests.csproj b/LinkCrawler/LinkCrawler.Tests/LinkCrawler.Tests.csproj index 2f7ce1b..2a7d599 100644 --- a/LinkCrawler/LinkCrawler.Tests/LinkCrawler.Tests.csproj +++ b/LinkCrawler/LinkCrawler.Tests/LinkCrawler.Tests.csproj @@ -60,7 +60,9 @@ - + + + {db53303b-f9fb-4d77-b656-d05db0420e6a} diff --git a/LinkCrawler/LinkCrawler/LinkCrawler.cs b/LinkCrawler/LinkCrawler/LinkCrawler.cs index 04ffad1..c8f7d99 100644 --- a/LinkCrawler/LinkCrawler/LinkCrawler.cs +++ b/LinkCrawler/LinkCrawler/LinkCrawler.cs @@ -7,6 +7,8 @@ using RestSharp; using System; using System.Collections.Generic; +using System.Threading; +using System.Diagnostics; namespace LinkCrawler { @@ -20,6 +22,7 @@ public class LinkCrawler public bool OnlyReportBrokenLinksToOutput { get; set; } public static List VisitedUrlList { get; set; } private ISettings _settings; + private int counter = 0; public LinkCrawler(IEnumerable outputs, IValidUrlParser validUrlParser, ISettings settings) { @@ -35,7 +38,18 @@ public LinkCrawler(IEnumerable outputs, IValidUrlParser validUrlParser, public void Start() { + Stopwatch sw = new Stopwatch(); + sw.Start(); SendRequest(BaseUrl); + while (counter > 0) + { + Thread.Sleep(100); + } + sw.Stop(); + foreach (var output in Outputs) + { + output.WriteInfo(String.Format("Elapsed {0:c}", sw.Elapsed)); + } } public void SendRequest(string crawlUrl, string referrerUrl = "") @@ -43,6 +57,8 @@ public void SendRequest(string crawlUrl, string referrerUrl = "") var requestModel = new RequestModel(crawlUrl, referrerUrl, BaseUrl); var restClient = new RestClient(new Uri(crawlUrl)) { FollowRedirects = false }; + Interlocked.Increment(ref counter); + restClient.ExecuteAsync(RestRequest, response => { if (response == null) @@ -59,6 +75,8 @@ public void ProcessResponse(IResponseModel responseModel) if (responseModel.ShouldCrawl) CrawlForLinksInResponse(responseModel); + + Interlocked.Decrement(ref counter); } public void CrawlForLinksInResponse(IResponseModel responseModel) diff --git a/LinkCrawler/LinkCrawler/Utils/Outputs/ConsoleOutput.cs b/LinkCrawler/LinkCrawler/Utils/Outputs/ConsoleOutput.cs index 1301c28..90b9da6 100644 --- a/LinkCrawler/LinkCrawler/Utils/Outputs/ConsoleOutput.cs +++ b/LinkCrawler/LinkCrawler/Utils/Outputs/ConsoleOutput.cs @@ -15,5 +15,11 @@ public void WriteInfo(IResponseModel responseModel) { Console.WriteLine(responseModel.ToString()); } + + public void WriteInfo(string info) + { + Console.WriteLine(info); + } + } } diff --git a/LinkCrawler/LinkCrawler/Utils/Outputs/CsvOutput.cs b/LinkCrawler/LinkCrawler/Utils/Outputs/CsvOutput.cs index 0b6ed79..95a9368 100644 --- a/LinkCrawler/LinkCrawler/Utils/Outputs/CsvOutput.cs +++ b/LinkCrawler/LinkCrawler/Utils/Outputs/CsvOutput.cs @@ -40,6 +40,11 @@ public void WriteInfo(IResponseModel responseModel) Write(responseModel); } + public void WriteInfo(string info) + { + // Write nothing to csv + } + private void Write(IResponseModel responseModel) { _writer?.WriteLine("{1}{0}{2}{0}{3}{0}{4}", diff --git a/LinkCrawler/LinkCrawler/Utils/Outputs/IOutput.cs b/LinkCrawler/LinkCrawler/Utils/Outputs/IOutput.cs index c924c13..c1183d7 100644 --- a/LinkCrawler/LinkCrawler/Utils/Outputs/IOutput.cs +++ b/LinkCrawler/LinkCrawler/Utils/Outputs/IOutput.cs @@ -6,5 +6,6 @@ public interface IOutput { void WriteError(IResponseModel responseModel); void WriteInfo(IResponseModel responseModel); + void WriteInfo(string info); } } diff --git a/LinkCrawler/LinkCrawler/Utils/Outputs/SlackOutput.cs b/LinkCrawler/LinkCrawler/Utils/Outputs/SlackOutput.cs index 9454a69..a2cf0b0 100644 --- a/LinkCrawler/LinkCrawler/Utils/Outputs/SlackOutput.cs +++ b/LinkCrawler/LinkCrawler/Utils/Outputs/SlackOutput.cs @@ -21,5 +21,10 @@ public void WriteInfo(IResponseModel responseModel) { // Write nothing to Slack } + + public void WriteInfo(string info) + { + // Write nothing to Slack + } } }