From 948a7a1c1214664189026c2a50fabec258e54c0a Mon Sep 17 00:00:00 2001 From: Grr1337 Date: Sun, 28 Dec 2025 16:05:01 +0100 Subject: [PATCH 1/4] Update HashlistUtils.class.php Outsourced algorithm to SQL for better performance. --- src/inc/utils/HashlistUtils.class.php | 149 ++++++++++++++++---------- 1 file changed, 93 insertions(+), 56 deletions(-) diff --git a/src/inc/utils/HashlistUtils.class.php b/src/inc/utils/HashlistUtils.class.php index 58265158f..980efaf74 100644 --- a/src/inc/utils/HashlistUtils.class.php +++ b/src/inc/utils/HashlistUtils.class.php @@ -770,7 +770,7 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, $hashtype = intval($hashtype); $accessGroup = Factory::getAccessGroupFactory()->get($accessGroupId); $brainFeatures = intval($brainFeatures); - + if ($format < DHashlistFormat::PLAIN || $format > DHashlistFormat::BINARY) { throw new HTException("Invalid hashlist format!"); } @@ -792,11 +792,11 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, else if ($brainId && $brainFeatures < 1 || $brainFeatures > 3) { throw new HTException("Invalid brain features selected!"); } - + Factory::getAgentFactory()->getDB()->beginTransaction(); $hashlist = new Hashlist(null, $name, $format, $hashtype, 0, $separator, 0, $secret, $hexsalted, $salted, $accessGroup->getId(), '', $brainId, $brainFeatures, 0); $hashlist = Factory::getHashlistFactory()->save($hashlist); - + $dataSource = ""; switch ($source) { case "paste": @@ -821,7 +821,6 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, Factory::getAgentFactory()->getDB()->rollback(); throw new HTException("Required file does not exist!"); } - // replace countLines with fileLineCount? Seems like a better option, not OS-dependent else if (Util::countLines($tmpfile) > SConfig::getInstance()->getVal(DConfig::MAX_HASHLIST_SIZE)) { Factory::getAgentFactory()->getDB()->rollback(); throw new HTException("Hashlist has too many lines!"); @@ -833,11 +832,10 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, Factory::getAgentFactory()->getDB()->commit(); $added = 0; $preFound = 0; - + switch ($format) { case DHashlistFormat::PLAIN: if ($salted) { - // find out if the first line contains field separator rewind($file); $bufline = stream_get_line($file, 1024); if (strpos($bufline, $saltSeparator) === false) { @@ -847,70 +845,109 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, else { $saltSeparator = ""; } - rewind($file); + + + // -- Start of new codeblock -- Factory::getAgentFactory()->getDB()->beginTransaction(); - $values = array(); + rewind($file); + + $db = Factory::getAgentFactory()->getDB(); + $db->query("CREATE TEMPORARY TABLE tmp_hashes ( + hash VARCHAR(255) PRIMARY KEY + ) ENGINE=InnoDB"); + + $batchSize = 1000; + $batch = []; + rewind($file); + while (($line = fgets($file)) !== false) { + $line = trim($line); + if ($line === '') continue; + + if ($saltSeparator !== '') { + $parts = explode($saltSeparator, $line, 2); + $hash = $parts[0]; + } else { + $hash = $line; + } + $batch[] = $db->quote($hash); + + if (count($batch) >= $batchSize) { + + $db->query("INSERT IGNORE INTO tmp_hashes (hash) VALUES (" . implode("),(", $batch) . ")"); + $batch = []; + } + } + if (count($batch) > 0) { + $db->query("INSERT IGNORE INTO tmp_hashes (hash) VALUES (" . implode("),(", $batch) . ")"); + } + + $foundHashes = []; + if (SConfig::getInstance()->getVal(DConfig::HASHLIST_IMPORT_CHECK)) { + $sql = " + SELECT * + FROM Hash h + INNER JOIN tmp_hashes t ON h.hash = t.hash + WHERE h.isCracked = 1 + "; + $stmt = $db->query($sql); + $rows = $stmt->fetchAll(PDO::FETCH_ASSOC); + foreach ($rows as $row) { + $foundHashes[$row['hash']] = Factory::getHashFactory()->createObjectFromDict($row['hashId'], $row); + } + } + // -- End of changed code block -- + + $values = []; $bufferCount = 0; - while (!feof($file)) { - $line = trim(fgets($file)); - if (strlen($line) == 0) { - continue; - } - $hash = $line; - $salt = ""; - if ($saltSeparator != "") { - $pos = strpos($line, $saltSeparator); - if ($pos !== false) { - $hash = substr($line, 0, $pos); - $salt = substr($line, $pos + 1); + rewind($file); + while (($line = fgets($file)) !== false) { + $line = trim($line); + if ($line === '') continue; + + if ($saltSeparator !== '') { + $parts = explode($saltSeparator, $line, 2); + $hash = $parts[0]; + $salt = $parts[1] ?? ""; + } else { + $hash = $line; + $salt = ""; } - } - if (strlen($hash) == 0) { - continue; - } - //TODO: check hash length here - - // if selected check if it is cracked - $found = null; - if (SConfig::getInstance()->getVal(DConfig::HASHLIST_IMPORT_CHECK)) { - $qF = new QueryFilter(Hash::HASH, $hash, "="); - $check = Factory::getHashFactory()->filter([Factory::FILTER => $qF]); - foreach ($check as $c) { - if ($c->getIsCracked()) { - $found = $c; - break; - } + + $found = $foundHashes[$hash] ?? null; + + if ($found == null) { + $values[] = new Hash(null, $hashlist->getId(), $hash, $salt, "", 0, null, 0, 0); + } else { + $values[] = new Hash(null, $hashlist->getId(), $hash, $salt, $found->getPlaintext(), time(), null, 1, 0); + $preFound++; + } + + $bufferCount++; + if ($bufferCount >= 10000) { + $result = Factory::getHashFactory()->massSave($values); + $added += $result->rowCount(); + Factory::getAgentFactory()->getDB()->commit(); + Factory::getAgentFactory()->getDB()->beginTransaction(); + $values = []; + $bufferCount = 0; } - } - if ($found == null) { - $values[] = new Hash(null, $hashlist->getId(), $hash, $salt, "", 0, null, 0, 0); - } - else { - $values[] = new Hash(null, $hashlist->getId(), $hash, $salt, $found->getPlaintext(), time(), null, 1, 0); - $preFound++; - } - $bufferCount++; - if ($bufferCount >= 10000) { - $result = Factory::getHashFactory()->massSave($values); - $added += $result->rowCount(); - Factory::getAgentFactory()->getDB()->commit(); - Factory::getAgentFactory()->getDB()->beginTransaction(); - $values = array(); - $bufferCount = 0; - } } + if (sizeof($values) > 0) { - $result = Factory::getHashFactory()->massSave($values); - $added += $result->rowCount(); + $result = Factory::getHashFactory()->massSave($values); + $added += $result->rowCount(); } + fclose($file); unlink($tmpfile); + Factory::getHashlistFactory()->mset($hashlist, [Hashlist::HASH_COUNT => $added, Hashlist::CRACKED => $preFound]); Factory::getAgentFactory()->getDB()->commit(); Util::createLogEntry("User", $user->getId(), DLogEntry::INFO, "New Hashlist created: " . $hashlist->getHashlistName()); - NotificationHandler::checkNotifications(DNotificationType::NEW_HASHLIST, new DataSet(array(DPayloadKeys::HASHLIST => $hashlist))); + break; + case DHashlistFormat::WPA: $added = 0; $values = []; From 5013098ebd260ebb2ec7003a1d68f9ba6a297e82 Mon Sep 17 00:00:00 2001 From: Grr1337 Date: Sun, 28 Dec 2025 18:08:11 +0100 Subject: [PATCH 2/4] Update HashlistUtils.class.php implemented prepared queries for input sanitization --- src/inc/utils/HashlistUtils.class.php | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/inc/utils/HashlistUtils.class.php b/src/inc/utils/HashlistUtils.class.php index 980efaf74..c6fd57de3 100644 --- a/src/inc/utils/HashlistUtils.class.php +++ b/src/inc/utils/HashlistUtils.class.php @@ -853,9 +853,15 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, $db = Factory::getAgentFactory()->getDB(); $db->query("CREATE TEMPORARY TABLE tmp_hashes ( - hash VARCHAR(255) PRIMARY KEY + hash VARCHAR(255) ) ENGINE=InnoDB"); + + $statementInsertInTempDb = $db->prepare( + "INSERT INTO tmp_hashes (hash) VALUES (:hash)" + ); + + $batchSize = 1000; $batch = []; rewind($file); @@ -869,16 +875,20 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, } else { $hash = $line; } - $batch[] = $db->quote($hash); + $batch[] = $hash; if (count($batch) >= $batchSize) { - - $db->query("INSERT IGNORE INTO tmp_hashes (hash) VALUES (" . implode("),(", $batch) . ")"); + foreach($batch as $h) { + $statementInsertInTempDb->execute([':hash' => $h]); + } $batch = []; + } } if (count($batch) > 0) { - $db->query("INSERT IGNORE INTO tmp_hashes (hash) VALUES (" . implode("),(", $batch) . ")"); + foreach($batch as $h){ + $statementInsertInTempDb->execute([':hash' => $h]); + } } $foundHashes = []; @@ -895,7 +905,8 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, $foundHashes[$row['hash']] = Factory::getHashFactory()->createObjectFromDict($row['hashId'], $row); } } - // -- End of changed code block -- + +// -- End of changed code block -- $values = []; $bufferCount = 0; @@ -932,7 +943,6 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, $bufferCount = 0; } } - if (sizeof($values) > 0) { $result = Factory::getHashFactory()->massSave($values); $added += $result->rowCount(); From 28e7a7753bd883fcf7ac8edca6f5535523ae4458 Mon Sep 17 00:00:00 2001 From: Grr1337 Date: Sun, 28 Dec 2025 18:19:09 +0100 Subject: [PATCH 3/4] Update HashlistUtils.class.php Made Code formatting more pretty --- src/inc/utils/HashlistUtils.class.php | 82 ++++++++++++++------------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/src/inc/utils/HashlistUtils.class.php b/src/inc/utils/HashlistUtils.class.php index c6fd57de3..4e56712c4 100644 --- a/src/inc/utils/HashlistUtils.class.php +++ b/src/inc/utils/HashlistUtils.class.php @@ -846,67 +846,71 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, $saltSeparator = ""; } - - // -- Start of new codeblock -- - Factory::getAgentFactory()->getDB()->beginTransaction(); + Factory::getAgentFactory()->getDB()->beginTransaction(); rewind($file); $db = Factory::getAgentFactory()->getDB(); - $db->query("CREATE TEMPORARY TABLE tmp_hashes ( + $db->query( + "CREATE TEMPORARY TABLE tmp_hashes ( hash VARCHAR(255) - ) ENGINE=InnoDB"); - + ) ENGINE=InnoDB" + ); $statementInsertInTempDb = $db->prepare( - "INSERT INTO tmp_hashes (hash) VALUES (:hash)" - ); + "INSERT INTO tmp_hashes (hash) VALUES (:hash)" + ); - $batchSize = 1000; $batch = []; rewind($file); + while (($line = fgets($file)) !== false) { - $line = trim($line); - if ($line === '') continue; + $line = trim($line); + if ($line === '') { + continue; + } - if ($saltSeparator !== '') { - $parts = explode($saltSeparator, $line, 2); - $hash = $parts[0]; - } else { - $hash = $line; - } - $batch[] = $hash; + if ($saltSeparator !== '') { + $parts = explode($saltSeparator, $line, 2); + $hash = $parts[0]; + } else { + $hash = $line; + } - if (count($batch) >= $batchSize) { - foreach($batch as $h) { - $statementInsertInTempDb->execute([':hash' => $h]); - } - $batch = []; - + $batch[] = $hash; + + if (count($batch) >= $batchSize) { + foreach ($batch as $h) { + $statementInsertInTempDb->execute([':hash' => $h]); } + $batch = []; + } } + if (count($batch) > 0) { - foreach($batch as $h){ + foreach ($batch as $h) { $statementInsertInTempDb->execute([':hash' => $h]); - } + } } $foundHashes = []; if (SConfig::getInstance()->getVal(DConfig::HASHLIST_IMPORT_CHECK)) { - $sql = " - SELECT * - FROM Hash h - INNER JOIN tmp_hashes t ON h.hash = t.hash - WHERE h.isCracked = 1 - "; - $stmt = $db->query($sql); - $rows = $stmt->fetchAll(PDO::FETCH_ASSOC); - foreach ($rows as $row) { - $foundHashes[$row['hash']] = Factory::getHashFactory()->createObjectFromDict($row['hashId'], $row); - } + $sql = " + SELECT * + FROM Hash h + INNER JOIN tmp_hashes t ON h.hash = t.hash + WHERE h.isCracked = 1 + "; + $stmt = $db->query($sql); + $rows = $stmt->fetchAll(PDO::FETCH_ASSOC); + + foreach ($rows as $row) { + $foundHashes[$row['hash']] = Factory::getHashFactory()->createObjectFromDict( + $row['hashId'], + $row + ); + } } - -// -- End of changed code block -- $values = []; $bufferCount = 0; From 5ddf35c2edda4cf1750d483f45599f505f9b6dc9 Mon Sep 17 00:00:00 2001 From: Grr1337 Date: Mon, 29 Dec 2025 20:18:23 +0100 Subject: [PATCH 4/4] Update HashlistUtils.class.php bugfix - updated varchar length to 1024 --- src/inc/utils/HashlistUtils.class.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/utils/HashlistUtils.class.php b/src/inc/utils/HashlistUtils.class.php index 4e56712c4..e3ea65006 100644 --- a/src/inc/utils/HashlistUtils.class.php +++ b/src/inc/utils/HashlistUtils.class.php @@ -852,7 +852,7 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, $db = Factory::getAgentFactory()->getDB(); $db->query( "CREATE TEMPORARY TABLE tmp_hashes ( - hash VARCHAR(255) + hash VARCHAR(1024) ) ENGINE=InnoDB" );