Skip to content

[filesystem] optimize file search by splitting it into two passes #113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
RaphaelIT7 wants to merge 2 commits into
base: master
Choose a base branch
from
Open

[filesystem] optimize file search by splitting it into two passes #113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bc12ebf
[filesystem] optimize file search by splitting it into two passes
RaphaelIT7 Dec 22, 2024
594ecd8
Merge branch 'master' into patch-9
RaphaelIT7 May 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 60 additions & 13 deletions filesystem/basefilesystem.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -530,14 +530,19 @@ void CBaseFileSystem::LogAccessToFile( char const *accesstype, char const *fullp
// *options -
// Output : FILE
//-----------------------------------------------------------------------------
FILE *CBaseFileSystem::Trace_FOpen( const char *filenameT, const char *options, unsigned flags, int64 *size )
FILE *CBaseFileSystem::Trace_FOpen( const char *filenameT, const char *options, unsigned flags, int64 *size, bool bNative )
{
#ifndef _WIN32
if ( bNative )
return NULL;
#endif

AUTOBLOCKREPORTER_FN( Trace_FOpen, this, true, filenameT, FILESYSTEM_BLOCKING_SYNCHRONOUS, FileBlockingItem::FB_ACCESS_OPEN );

char filename[MAX_PATH];
FixUpPath ( filenameT, filename );

FILE *fp = FS_fopen( filename, options, flags, size );
FILE *fp = FS_fopen( filename, options, flags, size, bNative );
if ( fp )
{
if ( options[0] == 'r' )
Expand Down Expand Up @@ -858,7 +863,10 @@ bool CBaseFileSystem::AddPackFileFromPath( const char *pPath, const char *pakfil
return false;

CPackFile *pf = new CZipPackFile( this );
pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, nullptr );
pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, nullptr, true );
if ( !pf->m_hPackFileHandleFS )
pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, nullptr, false );

if ( !pf->m_hPackFileHandleFS )
{
delete pf;
Expand Down Expand Up @@ -1007,7 +1015,9 @@ void CBaseFileSystem::AddPackFiles( const char *pPath, const CUtlSymbol &pathID,
sp->SetPackFile( pf );
pf->m_lPackFileTime = GetFileTime( fullpath );

pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, nullptr );
pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, nullptr, true );
if ( !pf->m_hPackFileHandleFS )
pf->m_hPackFileHandleFS = Trace_FOpen( fullpath, "rb", 0, nullptr, false );

if ( pf->m_hPackFileHandleFS )
{
Expand Down Expand Up @@ -1133,7 +1143,10 @@ void CBaseFileSystem::AddMapPackFile( const char *pPath, const char *pPathID, Se
}

{
FILE *fp = Trace_FOpen( fullpath, "rb", 0, nullptr );
FILE *fp = Trace_FOpen( fullpath, "rb", 0, nullptr, true );
if ( !fp )
fp = Trace_FOpen( fullpath, "rb", 0, nullptr, false );

if ( !fp )
{
// Couldn't open it
Expand Down Expand Up @@ -1230,7 +1243,9 @@ void CBaseFileSystem::BeginMapAccess()
#endif
{
// Try opening the file as a regular file
pPackFile->m_hPackFileHandleFS = Trace_FOpen( pPackFile->m_ZipName, "rb", 0, nullptr );
pPackFile->m_hPackFileHandleFS = Trace_FOpen( pPackFile->m_ZipName, "rb", 0, nullptr, true );
if ( !pPackFile->m_hPackFileHandleFS )
pPackFile->m_hPackFileHandleFS = Trace_FOpen( pPackFile->m_ZipName, "rb", 0, nullptr, false );

// !NOTE! Pack files inside of VPK not supported
//#if defined( SUPPORT_PACKED_STORE )
Expand Down Expand Up @@ -2111,12 +2126,12 @@ class CFileOpenInfo
};


void CBaseFileSystem::HandleOpenRegularFile( CFileOpenInfo &openInfo, bool bIsAbsolutePath )
void CBaseFileSystem::HandleOpenRegularFile( CFileOpenInfo &openInfo, bool bIsAbsolutePath, bool bNative )
{
openInfo.m_pFileHandle = nullptr;

int64 size;
FILE *fp = Trace_FOpen( openInfo.m_AbsolutePath, openInfo.m_pOptions, openInfo.m_Flags, &size );
FILE *fp = Trace_FOpen( openInfo.m_AbsolutePath, openInfo.m_pOptions, openInfo.m_Flags, &size, bNative );
if ( fp )
{
if ( m_pLogFile )
Expand Down Expand Up @@ -2156,7 +2171,7 @@ void CBaseFileSystem::HandleOpenRegularFile( CFileOpenInfo &openInfo, bool bIsAb
// *filetime -
// Output : FileHandle_t
//-----------------------------------------------------------------------------
FileHandle_t CBaseFileSystem::FindFileInSearchPath( CFileOpenInfo &openInfo )
FileHandle_t CBaseFileSystem::FindFileInSearchPath( CFileOpenInfo &openInfo, bool bNative )
{
VPROF( "CBaseFileSystem::FindFile" );

Expand Down Expand Up @@ -2210,7 +2225,7 @@ FileHandle_t CBaseFileSystem::FindFileInSearchPath( CFileOpenInfo &openInfo )
openInfo.SetAbsolutePath( "%s%s", openInfo.m_pSearchPath->GetPathString(), szLowercaseFilename );

// now have an absolute name
HandleOpenRegularFile( openInfo, false );
HandleOpenRegularFile( openInfo, false, bNative );
return (FileHandle_t)openInfo.m_pFileHandle;
}

Expand Down Expand Up @@ -2337,7 +2352,9 @@ FileHandle_t CBaseFileSystem::OpenForRead( const char *pFileNameT, const char *p
}

// Otherwise, it must be a regular file, specified by absolute filename
HandleOpenRegularFile( openInfo, true );
HandleOpenRegularFile( openInfo, true, true );
if ( !openInfo.m_pFileHandle )
HandleOpenRegularFile( openInfo, true, false );

// !FIXME! We probably need to deal with CRC tracking, right?

Expand All @@ -2350,7 +2367,37 @@ FileHandle_t CBaseFileSystem::OpenForRead( const char *pFileNameT, const char *p
CSearchPathsIterator iter( this, &pFileName, pathID, pathFilter );
for ( openInfo.m_pSearchPath = iter.GetFirst(); openInfo.m_pSearchPath != nullptr; openInfo.m_pSearchPath = iter.GetNext() )
{
FileHandle_t filehandle = FindFileInSearchPath( openInfo );
FileHandle_t filehandle = FindFileInSearchPath( openInfo, true );
if ( filehandle )
{
// Check if search path is excluded due to pure server white list,
// then we should make a note of this fact, and keep searching
if ( !openInfo.m_pSearchPath->m_bIsTrustedForPureServer && openInfo.m_ePureFileClass == ePureServerFileClass_AnyTrusted )
{
#ifdef PURE_SERVER_DEBUG_SPEW
Msg( "Ignoring %s from %s for pure server operation\n", openInfo.m_pFileName, openInfo.m_pSearchPath->GetDebugString() );
#endif

m_FileTracker2.NoteFileIgnoredForPureServer( openInfo.m_pFileName, pathID, openInfo.m_pSearchPath->m_storeId );
Close( filehandle );
openInfo.m_pFileHandle = NULL;
if ( ppszResolvedFilename && *ppszResolvedFilename )
{
free( *ppszResolvedFilename );
*ppszResolvedFilename = NULL;
}
continue;
}

//
openInfo.HandleFileCRCTracking( openInfo.m_pFileName );
return filehandle;
}
}

for ( openInfo.m_pSearchPath = iter.GetFirst(); openInfo.m_pSearchPath != NULL; openInfo.m_pSearchPath = iter.GetNext() )
{
FileHandle_t filehandle = FindFileInSearchPath( openInfo, false );
if ( filehandle )
{
// Check if search path is excluded due to pure server white list,
Expand Down Expand Up @@ -2411,7 +2458,7 @@ FileHandle_t CBaseFileSystem::OpenForWrite( const char *pFileName, const char *p
}

int64 size;
FILE *fp = Trace_FOpen( pTmpFileName, pOptions, 0, &size );
FILE *fp = Trace_FOpen( pTmpFileName, pOptions, 0, &size, false ); // No Native since were writing not reading.
if ( !fp )
{
return nullptr;
Expand Down
8 changes: 4 additions & 4 deletions filesystem/basefilesystem.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,7 +695,7 @@ abstract_class CBaseFileSystem : public CTier1AppSystem< IFileSystem >
//----------------------------------------------------------------------------
// Purpose: Functions implementing basic file system behavior.
//----------------------------------------------------------------------------
virtual FILE *FS_fopen( const char *filename, const char *options, unsigned flags, int64 *size ) = 0;
virtual FILE *FS_fopen( const char *filename, const char *options, unsigned flags, int64 *size, bool bNative ) = 0;
virtual void FS_setbufsize( FILE *fp, unsigned nBytes ) = 0;
virtual void FS_fclose( FILE *fp ) = 0;
virtual void FS_fseek( FILE *fp, int64 pos, int seekType ) = 0;
Expand Down Expand Up @@ -763,7 +763,7 @@ abstract_class CBaseFileSystem : public CTier1AppSystem< IFileSystem >
FileWarningLevel_t m_fwLevel;
void (*m_pfnWarning)( PRINTF_FORMAT_STRING const char *fmt, ... );

FILE *Trace_FOpen( const char *filename, const char *options, unsigned flags, int64 *size );
FILE *Trace_FOpen( const char *filename, const char *options, unsigned flags, int64 *size, bool bNative );
void Trace_FClose( FILE *fp );
void Trace_FRead( int size, FILE* file );
void Trace_FWrite( int size, FILE* file );
Expand All @@ -787,9 +787,9 @@ abstract_class CBaseFileSystem : public CTier1AppSystem< IFileSystem >
void AddVPKFile( const char *pPath, const char *pPathID, SearchPathAdd_t addType );
bool RemoveVPKFile( const char *pPath, const char *pPathID );

void HandleOpenRegularFile( CFileOpenInfo &openInfo, bool bIsAbsolutePath );
void HandleOpenRegularFile( CFileOpenInfo &openInfo, bool bIsAbsolutePath, bool bNative );

FileHandle_t FindFileInSearchPath( CFileOpenInfo &openInfo );
FileHandle_t FindFileInSearchPath( CFileOpenInfo &openInfo, bool bNative );
time_t FastFileTime( const CSearchPath *path, const char *pFileName );

const char *GetWritePath( const char *pFilename, const char *pathID );
Expand Down
17 changes: 10 additions & 7 deletions filesystem/filesystem_stdio.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ class CFileSystem_Stdio : public CBaseFileSystem

protected:
// implementation of CBaseFileSystem virtual functions
FILE *FS_fopen( const char *filename, const char *options, unsigned flags, int64 *size ) override;
FILE *FS_fopen( const char *filename, const char *options, unsigned flags, int64 *size, bool bNative ) override;
void FS_setbufsize( FILE *fp, unsigned nBytes ) override;
void FS_fclose( FILE *fp ) override;
void FS_fseek( FILE *fp, int64 pos, int seekType ) override;
Expand Down Expand Up @@ -391,22 +391,24 @@ void CFileSystem_Stdio::FreeOptimalReadBuffer( void *p )
//-----------------------------------------------------------------------------
// Purpose: low-level filesystem wrapper
//-----------------------------------------------------------------------------
FILE *CFileSystem_Stdio::FS_fopen( const char *filenameT, const char *options, unsigned flags, int64 *size )
FILE *CFileSystem_Stdio::FS_fopen( const char *filenameT, const char *options, unsigned flags, int64 *size, bool bNative )
{
char filename[ MAX_PATH ];
CBaseFileSystem::FixUpPath ( filenameT, filename );

alignas(FILE *) CStdFilesystemFile *pFile = nullptr;

#ifdef _WIN32
if ( CWin32ReadOnlyFile::CanOpen( filename, options ) )
if ( bNative && CWin32ReadOnlyFile::CanOpen( filename, options ) )
{
pFile = CWin32ReadOnlyFile::FS_fopen( filename, options, size );
if ( pFile )
{
return reinterpret_cast<FILE *>(pFile);
}

// If you have filesystem_native 1 checking if a file exists can take twice as long. There are some cases where CWin32[...]::FS_fopen fails but CStdioFile::FS_fopen works
return reinterpret_cast<FILE *>(pFile); // We do two passes, one with bNative and one without. This should improve performance since most of the time CWin32ReadOnlyFile::FS_fopen will work.
}
#else
if ( bNative ) // The Native pass should fail for any other platform, as the second pass should use the normal CStdioFile.
return reinterpret_cast<FILE *>(pFile);
#endif

pFile = CStdioFile::FS_fopen( filename, options, size );
Expand Down Expand Up @@ -723,6 +725,7 @@ int CFileSystem_Stdio::HintResourceNeed( const char *hintlist, int forgetEveryth
//-----------------------------------------------------------------------------
CStdioFile *CStdioFile::FS_fopen( const char *filenameT, const char *options, int64 *size )
{
VPROF_BUDGET( "CStdioFile::FS_fopen", VPROF_BUDGETGROUP_OTHER_FILESYSTEM );
char filename[MAX_PATH];
V_strcpy_safe( filename, filenameT );

Expand Down
4 changes: 3 additions & 1 deletion filesystem/packfile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ CFileHandle *CZipPackFile::OpenFile( const char *pFileName, const char *pOptions
#endif
{
// Try to open it as a regular file first
m_hPackFileHandleFS = m_fs->Trace_FOpen( m_ZipName, "rb", 0, NULL );
m_hPackFileHandleFS = m_fs->Trace_FOpen( m_ZipName, "rb", 0, NULL, true );
if ( !m_hPackFileHandleFS )
m_hPackFileHandleFS = m_fs->Trace_FOpen( m_ZipName, "rb", 0, NULL, false );

// !NOTE! Pack files inside of VPK not supported
}
Expand Down
Loading