Shader gen produce shader data

Engine/source/core/util/hashFunction.cpp

@@ -31,6 +31,8 @@
 
 #include "core/util/hashFunction.h"
 
+#include "core/util/endian.h"
+
 namespace Torque
 {
 
@@ -268,4 +270,21 @@ U64 hash64( const U8 *k, U32 length, U64 initval )
   return c;
 }
 
+// Generate a single 64bit hash from the input string.
+//
+// Don't get paranoid!  This has 1 in 18446744073709551616
+// chance for collision... it won't happen in this lifetime.
+//
+String getStringHash64(const String& in)
+{
+   String cacheKey = in;
+   cacheKey.replace("\n", " ");
+   U64 hash = hash64((const U8*)cacheKey.c_str(), cacheKey.length(), 0);
+   hash = convertHostToLEndian(hash);
+   U32 high = (U32)(hash >> 32);
+   U32 low = (U32)(hash & 0x00000000FFFFFFFF);
+   cacheKey = String::ToString("%x%x", high, low);
+   return cacheKey;
+}
+
 } // namespace

Engine/source/core/util/hashFunction.h

@@ -34,6 +34,8 @@ extern U32 hash(const U8 *k, U32 length, U32 initval);
 
 extern U64 hash64(const U8 *k, U32 length, U64 initval);
 
+extern String getStringHash64(const String& in);
+
 }
 
 #endif // _HASHFUNCTION_H_

Engine/source/gfx/D3D11/gfxD3D11Shader.cpp

@@ -681,6 +681,24 @@ bool GFXD3D11Shader::_init()
    return true;
 }
 
+static String buildMacroHash(const D3D_SHADER_MACRO* defines)
+{
+   String combined;
+
+   if (!defines)
+      return "";
+
+   for (const D3D_SHADER_MACRO* m = defines; m->Name != nullptr; ++m)
+   {
+      combined += m->Name;
+      combined += "=";
+      combined += m->Definition ? m->Definition : "";
+      combined += ";";
+   }
+
+   return Torque::getStringHash64(combined);
+}
+
 bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
                                     GFXShaderStage shaderStage,
                                     const D3D_SHADER_MACRO *defines)
@@ -706,8 +724,40 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
    Con::printf( "Compiling Shader: '%s'", filePath.getFullPath().c_str() );
 #endif
 
+   String macroHash = buildMacroHash(defines);
+
+   Torque::Path cachePath = filePath;
+   cachePath.setExtension("tso");
+   cachePath.setFileName(cachePath.getFileName() + "_" + macroHash);
+
+   if (Torque::FS::IsFile(cachePath))
+   {
+      Torque::FS::FileNodeRef rawFile = Torque::FS::GetFileNode(filePath);
+      Torque::FS::FileNodeRef cachedFile = Torque::FS::GetFileNode(cachePath);
+
+      if (rawFile != NULL && cachedFile != NULL)
+      {
+         if (cachedFile->getModifiedTime() >= rawFile->getModifiedTime())
+         {
+
+            FileStream fs;
+            if (fs.open(cachePath, Torque::FS::File::Read))
+            {
+               U32 size = fs.getStreamSize();
+
+               D3DCreateBlob(size, &code);
+               fs.read(size, code->GetBufferPointer());
+
+               res = 1;
+            }
+         }
+      }
+   }
+
+   bool loadedFromCache = (code != NULL);
+
    // Is it an HLSL shader?
-   if(filePath.getExtension().equal("hlsl", String::NoCase))
+   if(filePath.getExtension().equal("hlsl", String::NoCase) && !loadedFromCache)
    {
       // Set this so that the D3DInclude::Open will have this
       // information for relative paths.
@@ -788,6 +838,20 @@ bool GFXD3D11Shader::_compileShader( const Torque::Path &filePath,
 
    AssertISV(SUCCEEDED(res), "Unable to compile shader!");
 
+   // succeeded write out a cache
+   if (!loadedFromCache)
+   {
+      if (SUCCEEDED(res) && code)
+      {
+         // Save cache
+         FileStream out;
+         if (out.open(cachePath, Torque::FS::File::Write))
+         {
+            out.write(code->GetBufferSize(), code->GetBufferPointer());
+         }
+      }
+   }
+
    if(code != NULL)
    {
       switch (shaderStage)

Engine/source/gfx/gfxAPI.cpp

@@ -180,3 +180,37 @@ ImplementEnumType( GFXBlendOp,
    { GFXBlendOpMax, "GFXBlendOpMax" }
 
 EndImplementEnumType;
+
+ImplementEnumType(GFXShaderConstType,
+   "The shader const types.\n"
+   "@ingroup GFX")
+
+   { GFXSCT_Uknown, "GFXSCT_Uknown" },
+   { GFXSCT_ConstBuffer, "GFXSCT_ConstBuffer" },
+   { GFXSCT_Float, "GFXSCT_Float" },
+   { GFXSCT_Float2, "GFXSCT_Float2" },
+   { GFXSCT_Float3, "GFXSCT_Float3" },
+   { GFXSCT_Float4, "GFXSCT_Float4" },
+   { GFXSCT_Float2x2, "GFXSCT_Float2x2" },
+   { GFXSCT_Float3x3, "GFXSCT_Float3x3" },
+   { GFXSCT_Float3x4, "GFXSCT_Float3x4" },
+   { GFXSCT_Float4x3, "GFXSCT_Float4x3" },
+   { GFXSCT_Float4x4, "GFXSCT_Float4x4" },
+   { GFXSCT_Int, "GFXSCT_Int" },
+   { GFXSCT_Int2, "GFXSCT_Int2" },
+   { GFXSCT_Int3, "GFXSCT_Int3" },
+   { GFXSCT_Int4, "GFXSCT_Int4" },
+   { GFXSCT_UInt, "GFXSCT_UInt" },
+   { GFXSCT_UInt2, "GFXSCT_UInt2" },
+   { GFXSCT_UInt3, "GFXSCT_UInt3" },
+   { GFXSCT_UInt4, "GFXSCT_UInt4" },
+   { GFXSCT_Bool, "GFXSCT_Bool" },
+   { GFXSCT_Bool2, "GFXSCT_Bool2" },
+   { GFXSCT_Bool3, "GFXSCT_Bool3" },
+   { GFXSCT_Bool4, "GFXSCT_Bool4" },
+   { GFXSCT_Sampler, "GFXSCT_Sampler" },
+   { GFXSCT_SamplerCube, "GFXSCT_SamplerCube" },
+   { GFXSCT_SamplerCubeArray, "GFXSCT_SamplerCubeArray" },
+   { GFXSCT_SamplerTextureArray, "GFXSCT_SamplerTextureArray" }
+
+EndImplementEnumType;

Engine/source/gfx/gfxAPI.h

@@ -44,6 +44,7 @@ DefineEnumType( GFXTextureFilterType );
 DefineEnumType( GFXCullMode );
 DefineEnumType( GFXStencilOp );
 DefineEnumType( GFXBlendOp );
+DefineEnumType(GFXShaderConstType);
 DefineEnumType( GFXAdapterType );
 
 DECLARE_STRUCT( GFXVideoMode );
@@ -57,5 +58,6 @@ DefineConsoleType( TypeGFXTextureFilterType, GFXTextureFilterType );
 DefineConsoleType( TypeGFXCullMode, GFXCullMode );
 DefineConsoleType( TypeGFXStencilOp, GFXStencilOp );
 DefineConsoleType( TypeGFXBlendOp, GFXBlendOp );
+DefineConsoleType( TypeGFXShaderConstType, GFXShaderConstType);
 
 #endif // !_GFXAPI_H_

Engine/source/gfx/gfxShader.cpp

@@ -42,8 +42,12 @@ GFXShader::GFXShader()
 
 GFXShader::~GFXShader()
 {
-   Torque::FS::RemoveChangeNotification( mVertexFile, this, &GFXShader::_onFileChanged );
-   Torque::FS::RemoveChangeNotification( mPixelFile, this, &GFXShader::_onFileChanged );
+   if (!mVertexFile.isEmpty())
+      Torque::FS::RemoveChangeNotification( mVertexFile, this, &GFXShader::_onFileChanged );
+   if (!mPixelFile.isEmpty())
+      Torque::FS::RemoveChangeNotification( mPixelFile, this, &GFXShader::_onFileChanged );
+   if (!mGeometryFile.isEmpty())
+      Torque::FS::RemoveChangeNotification(mGeometryFile, this, &GFXShader::_onFileChanged);
 
    SAFE_DELETE(mInstancingFormat);
 }

Engine/source/gfx/gfxShader.h

@@ -77,7 +77,9 @@ enum GFXShaderStage
    GEOMETRY_SHADER   = BIT(2),
    DOMAIN_SHADER     = BIT(3),
    HULL_SHADER       = BIT(4),
-   COMPUTE_SHADER    = BIT(5)
+   COMPUTE_SHADER    = BIT(5),
+   ALL_STAGES = VERTEX_SHADER | PIXEL_SHADER | GEOMETRY_SHADER |
+   DOMAIN_SHADER | HULL_SHADER | COMPUTE_SHADER
 };
 
 /// Instances of this struct are returned GFXShaderConstBuffer

Engine/source/materials/shaderData.cpp

@@ -78,6 +78,8 @@ ShaderData::ShaderData()
    mOGLVertexShaderName = StringTable->EmptyString();
    mOGLPixelShaderName = StringTable->EmptyString();
    mOGLGeometryShaderName = StringTable->EmptyString();
+
+   mInstancingFormat = NULL;
 }
 
 void ShaderData::initPersistFields()
@@ -204,7 +206,7 @@ const Vector<GFXShaderMacro>& ShaderData::_getMacros()
    return mShaderMacros;
 }
 
-GFXShader* ShaderData::getShader( const Vector<GFXShaderMacro> &macros )
+GFXShader* ShaderData::getShader( const Vector<GFXShaderMacro> &macros)
 {
    PROFILE_SCOPE( ShaderData_GetShader );
 
@@ -217,14 +219,16 @@ GFXShader* ShaderData::getShader( const Vector<GFXShaderMacro> &macros )
    String cacheKey;
    GFXShaderMacro::stringize( macros, &cacheKey );
 
+   cacheKey = Torque::getStringHash64(cacheKey);
+
    // Lookup the shader for this instance.
    ShaderCache::Iterator iter = mShaders.find( cacheKey );
    if ( iter != mShaders.end() )
       return iter->value;
 
    // Create the shader instance... if it fails then
    // bail out and return nothing to the caller.
-   GFXShader *shader = _createShader( finalMacros );
+   GFXShader *shader = _createShader( finalMacros);
    if ( !shader )
       return NULL;
 
@@ -235,7 +239,7 @@ GFXShader* ShaderData::getShader( const Vector<GFXShaderMacro> &macros )
    return shader;
 }
 
-GFXShader* ShaderData::_createShader( const Vector<GFXShaderMacro> &macros )
+GFXShader* ShaderData::_createShader( const Vector<GFXShaderMacro> &macros)
 {
    F32 pixver = mPixVersion;
    if ( mUseDevicePixVersion )
@@ -257,30 +261,32 @@ GFXShader* ShaderData::_createShader( const Vector<GFXShaderMacro> &macros )
    {
       case Direct3D11:
       {
-         if (mDXVertexShaderName != String::EmptyString)
+         if (mDXVertexShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, mDXVertexShaderName);
-         if (mDXPixelShaderName != String::EmptyString)
+         if (mDXPixelShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, mDXPixelShaderName);
-         if (mDXGeometryShaderName != String::EmptyString)
+         if (mDXGeometryShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::GEOMETRY_SHADER, mDXGeometryShaderName);
          success = shader->init( pixver,
                                  macros,
-                                 samplers);
+                                 samplers,
+                                 mInstancingFormat);
          break;
       }
 
       case OpenGL:
       {
-         if(mOGLVertexShaderName != String::EmptyString)
+         if(mOGLVertexShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::VERTEX_SHADER, mOGLVertexShaderName);
-         if (mOGLPixelShaderName != String::EmptyString)
+         if (mOGLPixelShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::PIXEL_SHADER, mOGLPixelShaderName);
-         if (mOGLGeometryShaderName != String::EmptyString)
+         if (mOGLGeometryShaderName != StringTable->EmptyString())
             shader->setShaderStageFile(GFXShaderStage::GEOMETRY_SHADER, mOGLGeometryShaderName);
 
          success = shader->init( pixver,
                                  macros,
-                                 samplers);
+                                 samplers,
+                                 mInstancingFormat);
          break;
       }
 
@@ -348,6 +354,33 @@ void ShaderData::_onLMActivate( const char *lm, bool activate )
    reloadAllShaders();
 }
 
+void ShaderData::setShaderStageFile(GFXShaderStage stage, String fileName)
+{
+   const bool isGL = GFX->getAdapterType() == GFXAdapterType::OpenGL;
+   switch (stage)
+   {
+   case VERTEX_SHADER:
+      isGL ? mOGLVertexShaderName = StringTable->insert(fileName) : mDXVertexShaderName = StringTable->insert(fileName);
+      break;
+   case PIXEL_SHADER:
+      isGL ? mOGLPixelShaderName = StringTable->insert(fileName) : mDXPixelShaderName = StringTable->insert(fileName);
+      break;
+   case GEOMETRY_SHADER:
+      isGL ? mOGLGeometryShaderName = StringTable->insert(fileName) : mDXGeometryShaderName = StringTable->insert(fileName);
+      break;
+   case DOMAIN_SHADER:
+      break;
+   case HULL_SHADER:
+      break;
+   case COMPUTE_SHADER:
+      break;
+   case ALL_STAGES:
+      break;
+   default:
+      break;
+   }
+}
+
 bool ShaderData::hasSamplerDef(const String &_samplerName, int &pos) const
 {
    String samplerName = _samplerName.startsWith("$") ? _samplerName : "$"+_samplerName;

Engine/source/materials/shaderData.h

@@ -80,10 +80,8 @@ class ShaderData : public SimObject
    /// them if the content has changed.
    const Vector<GFXShaderMacro>& _getMacros();
 
-   /// Helper for converting an array of macros
-   /// into a formatted string.
-   void _stringizeMacros(const Vector<GFXShaderMacro>& macros,
-      String* outString);
+   // the instancing format.
+   GFXVertexFormat* mInstancingFormat;
 
    /// Creates a new shader returning NULL on error.
    GFXShader* _createShader(const Vector<GFXShaderMacro>& macros);
@@ -106,8 +104,11 @@ class ShaderData : public SimObject
    void setSamplerName(const String& name, int idx) { mSamplerNames[idx] = name; }
    String getSamplerName(int idx) const { return mSamplerNames[idx]; }
 
+   void setShaderStageFile(GFXShaderStage stage, String fileName);
+
    bool hasSamplerDef(const String& samplerName, int& pos) const;
    bool hasRTParamsDef(const int pos) const { return mRTParams[pos]; }
+   void setInstancingFormat(GFXVertexFormat* instancingFormat) { mInstancingFormat = instancingFormat; }
 
    ShaderData();
 
@@ -122,6 +123,7 @@ class ShaderData : public SimObject
    /// all loaded ShaderData objects in the system.
    static void reloadAllShaders();
 
+   void setPixVersion(F32 pixVersion) { mPixVersion = pixVersion; }
    /// Returns the required pixel shader version for this shader.
    F32 getPixVersion() const { return mPixVersion; }