From 9bb95ebec592f8cd45362aeec5141b5240f04779 Mon Sep 17 00:00:00 2001
From: romans91 <to.ru@Live.com>
Date: Thu, 16 Nov 2017 23:43:01 +1300
Subject: [PATCH 1/2] Added SqlLite output

---
 LinkCrawler/LinkCrawler/App.config            | 22 +++++--
 LinkCrawler/LinkCrawler/LinkCrawler.csproj    | 23 +++++++
 .../Utils/Outputs/SqlLiteOutput.cs            | 60 +++++++++++++++++++
 .../LinkCrawler/Utils/Settings/Constants.cs   |  1 +
 .../LinkCrawler/Utils/Settings/ISettings.cs   |  2 +
 .../Utils/Settings/MockSettings.cs            |  2 +
 .../LinkCrawler/Utils/Settings/Settings.cs    |  3 +
 LinkCrawler/LinkCrawler/packages.config       |  3 +
 8 files changed, 111 insertions(+), 5 deletions(-)
 create mode 100644 LinkCrawler/LinkCrawler/Utils/Outputs/SqlLiteOutput.cs

diff --git a/LinkCrawler/LinkCrawler/App.config b/LinkCrawler/LinkCrawler/App.config
index a87ddf3..b3d2382 100644
--- a/LinkCrawler/LinkCrawler/App.config
+++ b/LinkCrawler/LinkCrawler/App.config
@@ -1,7 +1,9 @@
-<?xml version="1.0" encoding="utf-8" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <configSections>
     <section name="outputProviders" type="System.Configuration.DictionarySectionHandler" />
+    <!-- For more information on Entity Framework configuration, visit http://go.microsoft.com/fwlink/?LinkID=237468 -->
+    <section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false" />
   </configSections>
   <startup>
     <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
@@ -20,14 +22,24 @@
     <add key="Slack.WebHook.Url" value="https://hooks.slack.com/services/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" />
     <add key="Slack.WebHook.Bot.Name" value="Homer Bot" />
     <add key="Slack.WebHook.Bot.IconEmoji" value=":homer:" />
-    <add key="Slack.WebHook.Bot.MessageFormat" value="*Doh! There is a link not working*
-Url: {0}
-Statuscode: {1}
-The link is placed on this page: {2}" />
+    <add key="Slack.WebHook.Bot.MessageFormat" value="*Doh! There is a link not working* Url: {0} Statuscode: {1} The link is placed on this page: {2}" />
   </appSettings>
   <outputProviders>
     <add key="Console" value="LinkCrawler.Utils.Outputs.ConsoleOutput" />
     <!--<add key="Csv" value="LinkCrawler.Utils.Outputs.CsvOutput" />-->
     <!--<add key="Slack" value="LinkCrawler.Utils.Outputs.SlackOutput" />-->
   </outputProviders>
+  <entityFramework>
+    <defaultConnectionFactory type="System.Data.Entity.Infrastructure.SqlConnectionFactory, EntityFramework" />
+    <providers>
+      <provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer" />
+      <provider invariantName="System.Data.SQLite.EF6" type="System.Data.SQLite.EF6.SQLiteProviderServices, System.Data.SQLite.EF6" />
+    </providers>
+  </entityFramework>
+  <system.data>
+    <DbProviderFactories>
+      <remove invariant="System.Data.SQLite.EF6" />
+      <add name="SQLite Data Provider (Entity Framework 6)" invariant="System.Data.SQLite.EF6" description=".NET Framework Data Provider for SQLite (Entity Framework 6)" type="System.Data.SQLite.EF6.SQLiteProviderFactory, System.Data.SQLite.EF6" />
+    <remove invariant="System.Data.SQLite" /><add name="SQLite Data Provider" invariant="System.Data.SQLite" description=".NET Framework Data Provider for SQLite" type="System.Data.SQLite.SQLiteFactory, System.Data.SQLite" /></DbProviderFactories>
+  </system.data>
 </configuration>
\ No newline at end of file
diff --git a/LinkCrawler/LinkCrawler/LinkCrawler.csproj b/LinkCrawler/LinkCrawler/LinkCrawler.csproj
index 210c55a..d8fac9a 100644
--- a/LinkCrawler/LinkCrawler/LinkCrawler.csproj
+++ b/LinkCrawler/LinkCrawler/LinkCrawler.csproj
@@ -12,6 +12,8 @@
     <TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
+    <NuGetPackageImportStamp>
+    </NuGetPackageImportStamp>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <PlatformTarget>AnyCPU</PlatformTarget>
@@ -45,6 +47,14 @@
       <HintPath>..\packages\CommonServiceLocator.StructureMapAdapter.Unofficial.3.0.4.125\lib\Net40\CommonServiceLocator.StructureMapAdapter.Unofficial.dll</HintPath>
       <Private>True</Private>
     </Reference>
+    <Reference Include="EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
+      <HintPath>..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="EntityFramework.SqlServer, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
+      <HintPath>..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.SqlServer.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
     <Reference Include="HtmlAgilityPack, Version=1.4.9.0, Culture=neutral, PublicKeyToken=bd319b19eaf3b43a, processorArchitecture=MSIL">
       <HintPath>..\packages\HtmlAgilityPack.1.4.9\lib\Net45\HtmlAgilityPack.dll</HintPath>
       <Private>True</Private>
@@ -66,8 +76,13 @@
       <Private>True</Private>
     </Reference>
     <Reference Include="System" />
+    <Reference Include="System.ComponentModel.DataAnnotations" />
     <Reference Include="System.Configuration" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Data.SQLite, Version=1.0.106.0, Culture=neutral, PublicKeyToken=db937bc2d44ff139, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Data.SQLite.Core.1.0.106.0\lib\net451\System.Data.SQLite.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
     <Reference Include="Microsoft.CSharp" />
@@ -85,6 +100,7 @@
     <Compile Include="Utils\Outputs\CsvOutput.cs" />
     <Compile Include="Utils\Outputs\IOutput.cs" />
     <Compile Include="Utils\Outputs\SlackOutput.cs" />
+    <Compile Include="Utils\Outputs\SqlLiteOutput.cs" />
     <Compile Include="Utils\Settings\MockSettings.cs" />
     <Compile Include="Utils\StructureMapRegistry.cs" />
     <Compile Include="Utils\Extensions\RegexExtensions.cs" />
@@ -113,6 +129,13 @@
   </ItemGroup>
   <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="..\packages\System.Data.SQLite.Core.1.0.106.0\build\net451\System.Data.SQLite.Core.targets" Condition="Exists('..\packages\System.Data.SQLite.Core.1.0.106.0\build\net451\System.Data.SQLite.Core.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('..\packages\System.Data.SQLite.Core.1.0.106.0\build\net451\System.Data.SQLite.Core.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\System.Data.SQLite.Core.1.0.106.0\build\net451\System.Data.SQLite.Core.targets'))" />
+  </Target>
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.
   <Target Name="BeforeBuild">
diff --git a/LinkCrawler/LinkCrawler/Utils/Outputs/SqlLiteOutput.cs b/LinkCrawler/LinkCrawler/Utils/Outputs/SqlLiteOutput.cs
new file mode 100644
index 0000000..60e58ff
--- /dev/null
+++ b/LinkCrawler/LinkCrawler/Utils/Outputs/SqlLiteOutput.cs
@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Data.SQLite;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using LinkCrawler.Models;
+using LinkCrawler.Utils.Settings;
+
+namespace LinkCrawler.Utils.Outputs
+{
+    class SqlLiteOutput : IOutput, IDisposable
+    {
+        private readonly ISettings _settings;
+        private SQLiteConnection _sqLiteConnection;
+
+        public SqlLiteOutput(ISettings settings)
+        {
+            _settings = settings;
+            SetUp();
+        }
+
+        private void SetUp()
+        {
+            SQLiteConnection.CreateFile(_settings.SqlLiteFilePath);
+
+            _sqLiteConnection = new SQLiteConnection(String.Format("Data Source={0}Version=3;", _settings.SqlLiteFilePath));
+            string sql = "CREATE TABLE links (code SMALLINT, status VARCHAR(27), url TEXT NOT NULL UNIQUE, referrer TEXT)";
+            SQLiteCommand command = new SQLiteCommand(sql, _sqLiteConnection);
+            command.ExecuteNonQuery();
+        }
+
+        public void WriteError(IResponseModel responseModel)
+        {
+            Write(responseModel);
+        }
+
+        public void WriteInfo(IResponseModel responseModel)
+        {
+            Write(responseModel);
+        }
+
+        public void WriteInfo(string[] Info)
+        {
+
+        }
+
+        public void Write(IResponseModel responseModel)
+        {
+            string sql = String.Format("insert into links (code, status, url, referrer) values ({0}, '{1}', '{2}', '{3}')", responseModel.StatusCodeNumber, responseModel.StatusCode, responseModel.RequestedUrl, responseModel.ReferrerUrl);
+            SQLiteCommand command = new SQLiteCommand(sql, _sqLiteConnection);
+            command.ExecuteNonQuery();
+        }
+
+        public void Dispose()
+        {
+            _sqLiteConnection.Close();
+        }
+    }
+}
diff --git a/LinkCrawler/LinkCrawler/Utils/Settings/Constants.cs b/LinkCrawler/LinkCrawler/Utils/Settings/Constants.cs
index 434ad5b..4aa1c18 100644
--- a/LinkCrawler/LinkCrawler/Utils/Settings/Constants.cs
+++ b/LinkCrawler/LinkCrawler/Utils/Settings/Constants.cs
@@ -15,6 +15,7 @@ public static class AppSettings
             public const string CsvFilePath = "Csv.FilePath";
             public const string CsvOverwrite = "Csv.Overwrite";
             public const string CsvDelimiter = "Csv.Delimiter";
+            public const string SqlLiteFilePath = "SqlLite.Filepath";
             public const string SuccessHttpStatusCodes = "SuccessHttpStatusCodes";
             public const string OutputProviders = "outputProviders";
             public const string PrintSummary = "PrintSummary";
diff --git a/LinkCrawler/LinkCrawler/Utils/Settings/ISettings.cs b/LinkCrawler/LinkCrawler/Utils/Settings/ISettings.cs
index cc03c8f..92a2ee3 100644
--- a/LinkCrawler/LinkCrawler/Utils/Settings/ISettings.cs
+++ b/LinkCrawler/LinkCrawler/Utils/Settings/ISettings.cs
@@ -26,6 +26,8 @@ public interface ISettings
 
         string CsvDelimiter { get; }
 
+        string SqlLiteFilePath { get; }
+
         bool IsSuccess(HttpStatusCode statusCode);
 
         bool PrintSummary { get; }
diff --git a/LinkCrawler/LinkCrawler/Utils/Settings/MockSettings.cs b/LinkCrawler/LinkCrawler/Utils/Settings/MockSettings.cs
index 3f576d8..024cbd5 100644
--- a/LinkCrawler/LinkCrawler/Utils/Settings/MockSettings.cs
+++ b/LinkCrawler/LinkCrawler/Utils/Settings/MockSettings.cs
@@ -14,6 +14,8 @@ public class MockSettings : ISettings {
 
         public bool CsvOverwrite => true;
 
+        public string SqlLiteFilePath => @"C:\tmp\output.sqlite";
+
         public bool OnlyReportBrokenLinksToOutput => false;
 
         public string SlackWebHookBotIconEmoji => ":homer:";
diff --git a/LinkCrawler/LinkCrawler/Utils/Settings/Settings.cs b/LinkCrawler/LinkCrawler/Utils/Settings/Settings.cs
index a560270..b14b96a 100644
--- a/LinkCrawler/LinkCrawler/Utils/Settings/Settings.cs
+++ b/LinkCrawler/LinkCrawler/Utils/Settings/Settings.cs
@@ -39,6 +39,9 @@ public class Settings : ISettings
         public string CsvDelimiter =>
             ConfigurationManager.AppSettings[Constants.AppSettings.CsvDelimiter];
 
+        public string SqlLiteFilePath =>
+            ConfigurationManager.AppSettings[Constants.AppSettings.SqlLiteFilePath];
+
         public bool PrintSummary =>
             ConfigurationManager.AppSettings[Constants.AppSettings.PrintSummary].ToBool();
 
diff --git a/LinkCrawler/LinkCrawler/packages.config b/LinkCrawler/LinkCrawler/packages.config
index 43bf28e..da401bb 100644
--- a/LinkCrawler/LinkCrawler/packages.config
+++ b/LinkCrawler/LinkCrawler/packages.config
@@ -4,7 +4,10 @@
   <package id="Bootstrapper.StructureMap" version="2.0.3.13" targetFramework="net452" />
   <package id="CommonServiceLocator" version="1.3" targetFramework="net452" />
   <package id="CommonServiceLocator.StructureMapAdapter.Unofficial" version="3.0.4.125" targetFramework="net452" />
+  <package id="EntityFramework" version="6.0.0" targetFramework="net452" />
   <package id="HtmlAgilityPack" version="1.4.9" targetFramework="net452" />
   <package id="RestSharp" version="105.2.3" targetFramework="net452" />
   <package id="structuremap" version="3.0.4.125" targetFramework="net452" />
+  <package id="System.Data.SQLite" version="1.0.106.0" targetFramework="net452" />
+  <package id="System.Data.SQLite.Core" version="1.0.106.0" targetFramework="net452" />
 </packages>
\ No newline at end of file

From 0a34e5df6e4903f7d86a679a70e58e2b3bda2c81 Mon Sep 17 00:00:00 2001
From: romans91 <to.ru@Live.com>
Date: Sat, 18 Nov 2017 01:52:01 +1300
Subject: [PATCH 2/2] Normalized database

---
 LinkCrawler/LinkCrawler/LinkCrawler.csproj    |  8 -----
 .../Utils/Outputs/SqlLiteOutput.cs            | 35 +++++++++++++++----
 2 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/LinkCrawler/LinkCrawler/LinkCrawler.csproj b/LinkCrawler/LinkCrawler/LinkCrawler.csproj
index d8fac9a..8e63e16 100644
--- a/LinkCrawler/LinkCrawler/LinkCrawler.csproj
+++ b/LinkCrawler/LinkCrawler/LinkCrawler.csproj
@@ -47,14 +47,6 @@
       <HintPath>..\packages\CommonServiceLocator.StructureMapAdapter.Unofficial.3.0.4.125\lib\Net40\CommonServiceLocator.StructureMapAdapter.Unofficial.dll</HintPath>
       <Private>True</Private>
     </Reference>
-    <Reference Include="EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
-      <HintPath>..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="EntityFramework.SqlServer, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
-      <HintPath>..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.SqlServer.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
     <Reference Include="HtmlAgilityPack, Version=1.4.9.0, Culture=neutral, PublicKeyToken=bd319b19eaf3b43a, processorArchitecture=MSIL">
       <HintPath>..\packages\HtmlAgilityPack.1.4.9\lib\Net45\HtmlAgilityPack.dll</HintPath>
       <Private>True</Private>
diff --git a/LinkCrawler/LinkCrawler/Utils/Outputs/SqlLiteOutput.cs b/LinkCrawler/LinkCrawler/Utils/Outputs/SqlLiteOutput.cs
index 60e58ff..2747703 100644
--- a/LinkCrawler/LinkCrawler/Utils/Outputs/SqlLiteOutput.cs
+++ b/LinkCrawler/LinkCrawler/Utils/Outputs/SqlLiteOutput.cs
@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Data.SQLite;
 using System.Linq;
+using System.Net;
 using System.Text;
 using System.Threading.Tasks;
 using LinkCrawler.Models;
@@ -25,9 +26,18 @@ private void SetUp()
             SQLiteConnection.CreateFile(_settings.SqlLiteFilePath);
 
             _sqLiteConnection = new SQLiteConnection(String.Format("Data Source={0}Version=3;", _settings.SqlLiteFilePath));
-            string sql = "CREATE TABLE links (code SMALLINT, status VARCHAR(27), url TEXT NOT NULL UNIQUE, referrer TEXT)";
-            SQLiteCommand command = new SQLiteCommand(sql, _sqLiteConnection);
-            command.ExecuteNonQuery();
+
+            string createStatusCodesTable = String.Format("CREATE TABLE status_codes (code SMALLINT NOT NULL PRIMARY KEY, status VARCHAR({0}))", GetMaxStatusCodeWordLength());
+            new SQLiteCommand(createStatusCodesTable, _sqLiteConnection).ExecuteNonQuery();
+
+            foreach (var code in Enum.GetValues(typeof(HttpStatusCode)))
+            {
+                string insertStatusCode = String.Format("INSERT OR REPLACE INTO status_codes (code, status) VALUES({0}, '{1}')", (int)code, code);
+                new SQLiteCommand(insertStatusCode, _sqLiteConnection).ExecuteNonQuery();
+            }
+
+            string createLinksTable = "CREATE TABLE links (code SMALLINT, url TEXT NOT NULL PRIMARY KEY, referrer TEXT)";
+            new SQLiteCommand(createLinksTable, _sqLiteConnection).ExecuteNonQuery();
         }
 
         public void WriteError(IResponseModel responseModel)
@@ -47,14 +57,27 @@ public void WriteInfo(string[] Info)
 
         public void Write(IResponseModel responseModel)
         {
-            string sql = String.Format("insert into links (code, status, url, referrer) values ({0}, '{1}', '{2}', '{3}')", responseModel.StatusCodeNumber, responseModel.StatusCode, responseModel.RequestedUrl, responseModel.ReferrerUrl);
-            SQLiteCommand command = new SQLiteCommand(sql, _sqLiteConnection);
-            command.ExecuteNonQuery();
+            string insertLink = String.Format("INSERT OR UPDATE INTO links (code, url, referrer) values ({0}, '{1}', '{2}')", responseModel.StatusCodeNumber, responseModel.RequestedUrl, responseModel.ReferrerUrl);
+            new SQLiteCommand(insertLink, _sqLiteConnection).ExecuteNonQuery();
         }
 
         public void Dispose()
         {
             _sqLiteConnection.Close();
         }
+
+        private static int GetMaxStatusCodeWordLength()
+        {
+            int result = 0;
+            foreach (var code in Enum.GetValues(typeof(HttpStatusCode)))
+            {
+                if (result < code.ToString().Length)
+                {
+                    result = code.ToString().Length;
+                }
+
+            }
+            return result;
+        }
     }
 }