From bc12ebf06719302b2efbff38d7f899ae16b8b243 Mon Sep 17 00:00:00 2001 From: RaphaelIT7 Date: Sun, 22 Dec 2024 05:59:49 +0100 Subject: [PATCH] [filesystem] optimize file search by splitting it into two passes Pass 1. We first go thru all search paths and try to open it using CWin32ReadOnlyFile. In most cases the file can be opened using it, so we can skip CStdioFile in this pass which should be a good speedup. Pass 2. We failed to open it using CWin32ReadOnlyFile, now were going thru everything again but this time were going to use CStdioFile. If it now fails, the file doesn't exist. --- filesystem/basefilesystem.cpp | 73 +++++++++++++++++++++++++++------ filesystem/basefilesystem.h | 8 ++-- filesystem/filesystem_stdio.cpp | 17 ++++---- filesystem/packfile.cpp | 4 +- 4 files changed, 77 insertions(+), 25 deletions(-) diff --git a/filesystem/basefilesystem.cpp b/filesystem/basefilesystem.cpp index fa69b0ff9..0c71f4237 100644 --- a/filesystem/basefilesystem.cpp +++ b/filesystem/basefilesystem.cpp @@ -550,15 +550,20 @@ void CBaseFileSystem::LogAccessToFile( char const *accesstype, char const *fullp // *options - // Output : FILE //----------------------------------------------------------------------------- -FILE *CBaseFileSystem::Trace_FOpen( const char *filenameT, const char *options, unsigned flags, int64 *size ) +FILE *CBaseFileSystem::Trace_FOpen( const char *filenameT, const char *options, unsigned flags, int64 *size, bool bNative ) { +#ifndef _WIN32 + if ( bNative ) + return NULL; +#endif + AUTOBLOCKREPORTER_FN( Trace_FOpen, this, true, filenameT, FILESYSTEM_BLOCKING_SYNCHRONOUS, FileBlockingItem::FB_ACCESS_OPEN ); char filename[MAX_PATH]; FixUpPath ( filenameT, filename, sizeof( filename ) ); - FILE *fp = FS_fopen( filename, options, flags, size ); + FILE *fp = FS_fopen( filename, options, flags, size, bNative ); if ( fp ) { @@ -880,7 +885,10 @@ bool CBaseFileSystem::AddPackFileFromPath( const char *pPath, const char *pakfil return false; CPackFile *pf = new CZipPackFile( this ); - pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, NULL ); + pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, NULL, true ); + if ( !pf->m_hPackFileHandleFS ) + pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, NULL, false ); + if ( !pf->m_hPackFileHandleFS ) { delete pf; @@ -1029,7 +1037,9 @@ void CBaseFileSystem::AddPackFiles( const char *pPath, const CUtlSymbol &pathID, sp->SetPackFile( pf ); pf->m_lPackFileTime = GetFileTime( fullpath ); - pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, NULL ); + pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, NULL, true ); + if ( !pf->m_hPackFileHandleFS ) + pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, NULL, false ); if ( pf->m_hPackFileHandleFS ) { @@ -1155,7 +1165,10 @@ void CBaseFileSystem::AddMapPackFile( const char *pPath, const char *pPathID, Se } { - FILE *fp = Trace_FOpen( fullpath, "rb", 0, NULL ); + FILE *fp = Trace_FOpen( fullpath, "rb", 0, NULL, true ); + if ( !fp ) + fp = Trace_FOpen( fullpath, "rb", 0, NULL, false ); + if ( !fp ) { // Couldn't open it @@ -1252,7 +1265,9 @@ void CBaseFileSystem::BeginMapAccess() #endif { // Try opening the file as a regular file - pPackFile->m_hPackFileHandleFS = Trace_FOpen( pPackFile->m_ZipName, "rb", 0, NULL ); + pPackFile->m_hPackFileHandleFS = Trace_FOpen( pPackFile->m_ZipName, "rb", 0, NULL, true ); + if ( !pPackFile->m_hPackFileHandleFS ) + pPackFile->m_hPackFileHandleFS = Trace_FOpen( pPackFile->m_ZipName, "rb", 0, NULL, false ); // !NOTE! Pack files inside of VPK not supported //#if defined( SUPPORT_PACKED_STORE ) @@ -2141,12 +2156,12 @@ class CFileOpenInfo }; -void CBaseFileSystem::HandleOpenRegularFile( CFileOpenInfo &openInfo, bool bIsAbsolutePath ) +void CBaseFileSystem::HandleOpenRegularFile( CFileOpenInfo &openInfo, bool bIsAbsolutePath, bool bNative ) { openInfo.m_pFileHandle = NULL; int64 size; - FILE *fp = Trace_FOpen( openInfo.m_AbsolutePath, openInfo.m_pOptions, openInfo.m_Flags, &size ); + FILE *fp = Trace_FOpen( openInfo.m_AbsolutePath, openInfo.m_pOptions, openInfo.m_Flags, &size, bNative ); if ( fp ) { if ( m_pLogFile ) @@ -2186,7 +2201,7 @@ void CBaseFileSystem::HandleOpenRegularFile( CFileOpenInfo &openInfo, bool bIsAb // *filetime - // Output : FileHandle_t //----------------------------------------------------------------------------- -FileHandle_t CBaseFileSystem::FindFileInSearchPath( CFileOpenInfo &openInfo ) +FileHandle_t CBaseFileSystem::FindFileInSearchPath( CFileOpenInfo &openInfo, bool bNative ) { VPROF( "CBaseFileSystem::FindFile" ); @@ -2240,7 +2255,7 @@ FileHandle_t CBaseFileSystem::FindFileInSearchPath( CFileOpenInfo &openInfo ) openInfo.SetAbsolutePath( "%s%s", openInfo.m_pSearchPath->GetPathString(), szLowercaseFilename ); // now have an absolute name - HandleOpenRegularFile( openInfo, false ); + HandleOpenRegularFile( openInfo, false, bNative ); return (FileHandle_t)openInfo.m_pFileHandle; } @@ -2367,7 +2382,9 @@ FileHandle_t CBaseFileSystem::OpenForRead( const char *pFileNameT, const char *p } // Otherwise, it must be a regular file, specified by absolute filename - HandleOpenRegularFile( openInfo, true ); + HandleOpenRegularFile( openInfo, true, true ); + if ( !openInfo.m_pFileHandle ) + HandleOpenRegularFile( openInfo, true, false ); // !FIXME! We probably need to deal with CRC tracking, right? @@ -2393,7 +2410,37 @@ FileHandle_t CBaseFileSystem::OpenForRead( const char *pFileNameT, const char *p CSearchPathsIterator iter( this, &pFileName, pathID, pathFilter ); for ( openInfo.m_pSearchPath = iter.GetFirst(); openInfo.m_pSearchPath != NULL; openInfo.m_pSearchPath = iter.GetNext() ) { - FileHandle_t filehandle = FindFileInSearchPath( openInfo ); + FileHandle_t filehandle = FindFileInSearchPath( openInfo, true ); + if ( filehandle ) + { + // Check if search path is excluded due to pure server white list, + // then we should make a note of this fact, and keep searching + if ( !openInfo.m_pSearchPath->m_bIsTrustedForPureServer && openInfo.m_ePureFileClass == ePureServerFileClass_AnyTrusted ) + { + #ifdef PURE_SERVER_DEBUG_SPEW + Msg( "Ignoring %s from %s for pure server operation\n", openInfo.m_pFileName, openInfo.m_pSearchPath->GetDebugString() ); + #endif + + m_FileTracker2.NoteFileIgnoredForPureServer( openInfo.m_pFileName, pathID, openInfo.m_pSearchPath->m_storeId ); + Close( filehandle ); + openInfo.m_pFileHandle = NULL; + if ( ppszResolvedFilename && *ppszResolvedFilename ) + { + free( *ppszResolvedFilename ); + *ppszResolvedFilename = NULL; + } + continue; + } + + // + openInfo.HandleFileCRCTracking( openInfo.m_pFileName ); + return filehandle; + } + } + + for ( openInfo.m_pSearchPath = iter.GetFirst(); openInfo.m_pSearchPath != NULL; openInfo.m_pSearchPath = iter.GetNext() ) + { + FileHandle_t filehandle = FindFileInSearchPath( openInfo, false ); if ( filehandle ) { // Check if search path is excluded due to pure server white list, @@ -2454,7 +2501,7 @@ FileHandle_t CBaseFileSystem::OpenForWrite( const char *pFileName, const char *p } int64 size; - FILE *fp = Trace_FOpen( pTmpFileName, pOptions, 0, &size ); + FILE *fp = Trace_FOpen( pTmpFileName, pOptions, 0, &size, false ); // No Native since were writing not reading. if ( !fp ) { return ( FileHandle_t )0; diff --git a/filesystem/basefilesystem.h b/filesystem/basefilesystem.h index 97a2a7f94..1450b63d6 100644 --- a/filesystem/basefilesystem.h +++ b/filesystem/basefilesystem.h @@ -692,7 +692,7 @@ abstract_class CBaseFileSystem : public CTier1AppSystem< IFileSystem > //---------------------------------------------------------------------------- // Purpose: Functions implementing basic file system behavior. //---------------------------------------------------------------------------- - virtual FILE *FS_fopen( const char *filename, const char *options, unsigned flags, int64 *size ) = 0; + virtual FILE *FS_fopen( const char *filename, const char *options, unsigned flags, int64 *size, bool bNative ) = 0; virtual void FS_setbufsize( FILE *fp, unsigned nBytes ) = 0; virtual void FS_fclose( FILE *fp ) = 0; virtual void FS_fseek( FILE *fp, int64 pos, int seekType ) = 0; @@ -760,7 +760,7 @@ abstract_class CBaseFileSystem : public CTier1AppSystem< IFileSystem > FileWarningLevel_t m_fwLevel; void (*m_pfnWarning)( PRINTF_FORMAT_STRING const char *fmt, ... ); - FILE *Trace_FOpen( const char *filename, const char *options, unsigned flags, int64 *size ); + FILE *Trace_FOpen( const char *filename, const char *options, unsigned flags, int64 *size, bool bNative ); void Trace_FClose( FILE *fp ); void Trace_FRead( int size, FILE* file ); void Trace_FWrite( int size, FILE* file ); @@ -784,9 +784,9 @@ abstract_class CBaseFileSystem : public CTier1AppSystem< IFileSystem > void AddVPKFile( const char *pPath, const char *pPathID, SearchPathAdd_t addType ); bool RemoveVPKFile( const char *pPath, const char *pPathID ); - void HandleOpenRegularFile( CFileOpenInfo &openInfo, bool bIsAbsolutePath ); + void HandleOpenRegularFile( CFileOpenInfo &openInfo, bool bIsAbsolutePath, bool bNative ); - FileHandle_t FindFileInSearchPath( CFileOpenInfo &openInfo ); + FileHandle_t FindFileInSearchPath( CFileOpenInfo &openInfo, bool bNative ); long FastFileTime( const CSearchPath *path, const char *pFileName ); const char *GetWritePath( const char *pFilename, const char *pathID ); diff --git a/filesystem/filesystem_stdio.cpp b/filesystem/filesystem_stdio.cpp index 9de5421ab..2700442f7 100644 --- a/filesystem/filesystem_stdio.cpp +++ b/filesystem/filesystem_stdio.cpp @@ -66,7 +66,7 @@ class CFileSystem_Stdio : public CBaseFileSystem protected: // implementation of CBaseFileSystem virtual functions - virtual FILE *FS_fopen( const char *filename, const char *options, unsigned flags, int64 *size ); + virtual FILE *FS_fopen( const char *filename, const char *options, unsigned flags, int64 *size, bool bNative ); virtual void FS_setbufsize( FILE *fp, unsigned nBytes ); virtual void FS_fclose( FILE *fp ); virtual void FS_fseek( FILE *fp, int64 pos, int seekType ); @@ -415,7 +415,7 @@ void CFileSystem_Stdio::FreeOptimalReadBuffer( void *p ) //----------------------------------------------------------------------------- // Purpose: low-level filesystem wrapper //----------------------------------------------------------------------------- -FILE *CFileSystem_Stdio::FS_fopen( const char *filenameT, const char *options, unsigned flags, int64 *size ) +FILE *CFileSystem_Stdio::FS_fopen( const char *filenameT, const char *options, unsigned flags, int64 *size, bool bNative ) { CStdFilesystemFile *pFile = NULL; @@ -424,14 +424,16 @@ FILE *CFileSystem_Stdio::FS_fopen( const char *filenameT, const char *options, u CBaseFileSystem::FixUpPath ( filenameT, filename, sizeof( filename ) ); #ifdef _WIN32 - if ( CWin32ReadOnlyFile::CanOpen( filename, options ) ) + if ( bNative && CWin32ReadOnlyFile::CanOpen( filename, options ) ) { pFile = CWin32ReadOnlyFile::FS_fopen( filename, options, size ); - if ( pFile ) - { - return (FILE *)pFile; - } + + // If you have filesystem_native 1 checking if a file exists can take twice as long. There are some cases where CWin32[...]::FS_fopen fails but CStdioFile::FS_fopen works + return (FILE *)pFile; // We do two passes, one with bNative and one without. This should improve performance since most of the time CWin32ReadOnlyFile::FS_fopen will work. } +#else + if ( bNative ) // The Native pass should fail for any other platform, as the second pass should use the normal CStdioFile. + return (FILE *)pFile; #endif pFile = CStdioFile::FS_fopen( filename, options, size ); @@ -750,6 +752,7 @@ int CFileSystem_Stdio::HintResourceNeed( const char *hintlist, int forgetEveryth //----------------------------------------------------------------------------- CStdioFile *CStdioFile::FS_fopen( const char *filenameT, const char *options, int64 *size ) { + VPROF_BUDGET( "CStdioFile::FS_fopen", VPROF_BUDGETGROUP_OTHER_FILESYSTEM ); FILE *pFile = NULL; char *p = NULL; char filename[MAX_PATH]; diff --git a/filesystem/packfile.cpp b/filesystem/packfile.cpp index 586e2cfec..1281223c9 100644 --- a/filesystem/packfile.cpp +++ b/filesystem/packfile.cpp @@ -129,7 +129,9 @@ CFileHandle *CZipPackFile::OpenFile( const char *pFileName, const char *pOptions #endif { // Try to open it as a regular file first - m_hPackFileHandleFS = m_fs->Trace_FOpen( m_ZipName, "rb", 0, NULL ); + m_hPackFileHandleFS = m_fs->Trace_FOpen( m_ZipName, "rb", 0, NULL, true ); + if ( !m_hPackFileHandleFS ) + m_hPackFileHandleFS = m_fs->Trace_FOpen( m_ZipName, "rb", 0, NULL, false ); // !NOTE! Pack files inside of VPK not supported }