Add supertask runtime estimation

Show estimated total runtime of a supertask, allowing the user to interactively size a supertask by adding or removing subtasks
* Allow adding and removing subtasks to supertask
* Parse hashcat command to compute runtime per subtask. Gratefully uses the code submitted by shivanraptor in hashtopolis#672
* Save line count of dictionary files and rule files in database
* Compute runtime per subtask of supertask, depending on benchmark values entered by the user
* Works for -a0 and -a3 hashcat attacks. Adds 'Unknown' to runtime estimate when not every subtask runtime could be estimated
* Feature includes custom charsets for masks in -a3 attacks

src/dba/models/File.class.php

@@ -9,14 +9,16 @@ class File extends AbstractModel {
   private $isSecret;
   private $fileType;
   private $accessGroupId;
+  private $lineCount;
 
-  function __construct($fileId, $filename, $size, $isSecret, $fileType, $accessGroupId) {
+  function __construct($fileId, $filename, $size, $isSecret, $fileType, $accessGroupId, $lineCount) {
     $this->fileId = $fileId;
     $this->filename = $filename;
     $this->size = $size;
     $this->isSecret = $isSecret;
     $this->fileType = $fileType;
     $this->accessGroupId = $accessGroupId;
+    $this->lineCount = $lineCount;
   }
 
   function getKeyValueDict() {
@@ -27,6 +29,7 @@ function getKeyValueDict() {
     $dict['isSecret'] = $this->isSecret;
     $dict['fileType'] = $this->fileType;
     $dict['accessGroupId'] = $this->accessGroupId;
+    $dict['lineCount'] = $this->lineCount;
 
     return $dict;
   }
@@ -94,11 +97,21 @@ function getAccessGroupId() {
   function setAccessGroupId($accessGroupId) {
     $this->accessGroupId = $accessGroupId;
   }
+
+  function getLineCount() {
+    return $this->lineCount;
+  }
+
+  function setLineCount($lineCount) {
+    $this->lineCount = $lineCount;
+  }
+
 
   const FILE_ID = "fileId";
   const FILENAME = "filename";
   const SIZE = "size";
   const IS_SECRET = "isSecret";
   const FILE_TYPE = "fileType";
   const ACCESS_GROUP_ID = "accessGroupId";
+  const LINE_COUNT = "lineCount";
 }

src/dba/models/FileFactory.class.php

@@ -23,7 +23,7 @@ function getCacheValidTime() {
    * @return File
    */
   function getNullObject() {
-    $o = new File(-1, null, null, null, null, null);
+    $o = new File(-1, null, null, null, null, null, null);
     return $o;
   }
 
@@ -33,7 +33,7 @@ function getNullObject() {
    * @return File
    */
   function createObjectFromDict($pk, $dict) {
-    $o = new File($dict['fileId'], $dict['filename'], $dict['size'], $dict['isSecret'], $dict['fileType'], $dict['accessGroupId']);
+    $o = new File($dict['fileId'], $dict['filename'], $dict['size'], $dict['isSecret'], $dict['fileType'], $dict['accessGroupId'], $dict['lineCount']);
     return $o;
   }
 

src/inc/Util.class.php

@@ -359,8 +359,12 @@ public static function insertFile($path, $name, $type, $accessGroupId) {
     // check if there is an old deletion request for the same filename
     $qF = new QueryFilter(FileDelete::FILENAME, $name, "=");
     Factory::getFileDeleteFactory()->massDeletion([Factory::FILTER => $qF]);
-
-    $file = new File(null, $name, Util::filesize($path), 1, $fileType, $accessGroupId);
+    if ($fileType == 1) {
+      $file = new File(null, $name, Util::filesize($path), 1, $fileType, $accessGroupId, Util::rulefileLineCount($path));
+    }
+    else {
+      $file = new File(null, $name, Util::filesize($path), 1, $fileType, $accessGroupId, Util::fileLineCount($path));
+    }
     $file = Factory::getFileFactory()->save($file);
     if ($file == null) {
       return false;
@@ -647,7 +651,48 @@ public static function filesize($file) {
 
     return $pos;
   }
+
+  /**
+   * This counts the number of lines in a given file
+   * @param $file string Filepath you want to get the size from
+   * @return int -1 if the file doesn't exist, else filesize
+   */
+  public static function fileLineCount($file) {
+    if (!file_exists($file)) {
+      return -1;
+    }
+    // TODO: find out what a prettier solution for this would be, as opposed to setting the max execution time to an arbitrary two hours
+    ini_set('max_execution_time', '7200');
+    $file = new \SplFileObject($file, 'r');
+    $file->seek(PHP_INT_MAX);
+
+    return $file->key();
+  }
 
+  /**
+   * This counts the number of lines in a rule file, excluding lines starting with # and empty lines
+   * @param $file string Filepath you want to get the size from
+   * @return int -1 if the file doesn't exist, else filesize
+   */
+  public static function rulefileLineCount($file) {
+    if (!file_exists($file)) {
+      return -1;
+    }
+    // TODO: find out what a prettier solution for this would be, as opposed to setting the max execution time to an arbitrary two hours
+    ini_set('max_execution_time', '7200');
+    $lineCount = 0;
+    $handle = fopen($file, "r");
+    while(!feof($handle)){
+      $line = fgets($handle);
+      if (!(Util::startsWith($line, '#') or trim($line) == "")) {
+        $lineCount = $lineCount + 1;
+      }
+    }
+
+    fclose($handle);
+    return $lineCount;
+  }
+
   /**
    * Refreshes the page with the current url, also includes the query string.
    */
@@ -1348,7 +1393,9 @@ public static function arrayOfIds($array) {
     }
     return $arr;
   }
-
+
+  // new function added: fileLineCount(). This function is independent of OS.
+  // TODO check whether we can remove one of these functions
   public static function countLines($tmpfile) {
     if (stripos(PHP_OS, "WIN") === 0) {
       // windows line count

src/inc/defines/files.php

@@ -19,4 +19,7 @@ class DFileAction {
 
   const EDIT_FILE      = "editFile";
   const EDIT_FILE_PERM = DAccessControl::MANAGE_FILE_ACCESS;
+
+  const COUNT_FILE_LINES      = "countFileLines";
+  const COUNT_FILE_LINES_PERM = DAccessControl::MANAGE_FILE_ACCESS;
 }

src/inc/defines/tasks.php

@@ -55,6 +55,12 @@ class DSupertaskAction {
 
   const BULK_SUPERTASK      = "bulkSupertaskCreation";
   const BULK_SUPERTASK_PERM = DAccessControl::CREATE_SUPERTASK_ACCESS;
+
+  const REMOVE_PRETASK_FROM_SUPERTASK      = "removePretaskFromSupertask";
+  const REMOVE_PRETASK_FROM_SUPERTASK_PERM = DAccessControl::CREATE_SUPERTASK_ACCESS;
+
+  const ADD_PRETASK_TO_SUPERTASK      = "addPretaskToSupertask";
+  const ADD_PRETASK_TO_SUPERTASK_PERM = DAccessControl::CREATE_SUPERTASK_ACCESS;
 }
 
 class DTaskAction {

src/inc/handlers/FileHandler.class.php

@@ -27,6 +27,11 @@ public function handle($action) {
           FileUtils::saveChanges($_POST['fileId'], $_POST['filename'], $_POST['accessGroupId'], AccessControl::getInstance()->getUser());
           FileUtils::setFileType($_POST['fileId'], $_POST['filetype'], AccessControl::getInstance()->getUser());
           break;
+        case DFileAction::COUNT_FILE_LINES:
+          AccessControl::getInstance()->checkPermission(DFileAction::COUNT_FILE_LINES_PERM);
+          FileUtils::fileCountLines($_POST['file']);
+          UI::addMessage(UI::SUCCESS, "Line count has been successfully calculated!");
+          break;
         default:
           UI::addMessage(UI::ERROR, "Invalid action!");
           break;

src/inc/handlers/SupertaskHandler.class.php

@@ -29,6 +29,14 @@ public function handle($action) {
           AccessControl::getInstance()->checkPermission(DSupertaskAction::BULK_SUPERTASK_PERM);
           SupertaskUtils::bulkSupertask($_POST['name'], $_POST['command'], $_POST['isCpu'], $_POST['isSmall'], $_POST['crackerBinaryTypeId'], $_POST['benchtype'], @$_POST['basefile'], @$_POST['iterfile'], Login::getInstance()->getUser());
           break;
+        case DSupertaskAction::REMOVE_PRETASK_FROM_SUPERTASK:
+          AccessControl::getInstance()->checkPermission(DSupertaskAction::REMOVE_PRETASK_FROM_SUPERTASK_PERM);
+          SupertaskUtils::removePretaskFromSupertask($_POST['supertaskId'], $_POST['pretaskId']);
+          break;
+        case DSupertaskAction::ADD_PRETASK_TO_SUPERTASK:
+          AccessControl::getInstance()->checkPermission(DSupertaskAction::ADD_PRETASK_TO_SUPERTASK_PERM);
+          SupertaskUtils::addPretaskToSupertask($_POST['supertaskId'], $_POST['pretaskId']);
+          break;
         default:
           UI::addMessage(UI::ERROR, "Invalid action!");
           break;

src/inc/utils/FileUtils.class.php

@@ -338,4 +338,30 @@ public static function getFile($fileId, $user) {
     }
     return $file;
   }
+
+  /**
+   * @param $fileId
+   * @throws HTException
+   */
+  public static function fileCountLines($fileId) {
+    $file = Factory::getFileFactory()->get($fileId);
+    $fileName = $file->getFilename();
+    $filePath = dirname(__FILE__) . "/../../files/" . $fileName;
+    if (!file_exists($filePath)) {
+      throw new HTException("File not found!");
+    }
+    if ($file->getFileType() == 1) {
+      $count = Util::rulefileLineCount($filePath);
+    }
+    else {
+      $count = Util::fileLineCount($filePath);
+    }
+
+    if ($count == -1) {
+      throw new HTException("Could not determine line count.");
+    }
+    else {
+      Factory::getFileFactory()->set($file, File::LINE_COUNT, $count);
+    }
+  }
 }

src/inc/utils/HashlistUtils.class.php

@@ -208,7 +208,7 @@ public static function createWordlists($hashlistId, $user) {
     fclose($wordlistFile);
 
     //add file to files list
-    $file = new File(null, $wordlistName, Util::filesize($wordlistFilename), $hashlist->getIsSecret(), 0, $hashlist->getAccessGroupId());
+    $file = new File(null, $wordlistName, Util::filesize($wordlistFilename), $hashlist->getIsSecret(), 0, $hashlist->getAccessGroupId(), null);
     Factory::getFileFactory()->save($file);
     return [$wordCount, $wordlistName, $file];
   }
@@ -674,7 +674,7 @@ public static function export($hashlistId, $user) {
     fclose($file);
     usleep(1000000);
 
-    $file = new File(null, $tmpname, Util::filesize($tmpfile), $hashlist->getIsSecret(), 0, $hashlist->getAccessGroupId());
+    $file = new File(null, $tmpname, Util::filesize($tmpfile), $hashlist->getIsSecret(), 0, $hashlist->getAccessGroupId(), null);
     $file = Factory::getFileFactory()->save($file);
     return $file;
   }
@@ -759,6 +759,7 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
       Factory::getAgentFactory()->getDB()->rollback();
       throw new HTException("Required file does not exist!");
     }
+    // TODO: replace countLines with fileLineCount? Seems like a better option, not OS-dependent
     else if (Util::countLines($tmpfile) > SConfig::getInstance()->getVal(DConfig::MAX_HASHLIST_SIZE)) {
       Factory::getAgentFactory()->getDB()->rollback();
       throw new HTException("Hashlist has too many lines!");
@@ -1047,7 +1048,7 @@ public static function leftlist($hashlistId, $user) {
     fclose($file);
     usleep(1000000);
 
-    $file = new File(null, $tmpname, Util::filesize($tmpfile), $hashlist->getIsSecret(), 0, $hashlist->getAccessGroupId());
+    $file = new File(null, $tmpname, Util::filesize($tmpfile), $hashlist->getIsSecret(), 0, $hashlist->getAccessGroupId(), null);
     return Factory::getFileFactory()->save($file);
   }
 

src/inc/utils/SupertaskUtils.class.php

@@ -463,4 +463,38 @@ private static function prepareImportMasks(&$masks) {
       $masks[$i] = $mask;
     }
   }
+
+  /**
+   * @param $supertaskId
+   * @param $pretaskId
+   * @throws HTException
+   */
+  public static function removePretaskFromSupertask($supertaskId, $pretaskId) {
+    if ($supertaskId == null) {
+      throw new HTException("Invalid supertask ID!");
+    }
+    if ($pretaskId == null) {
+      throw new HTException("Invalid pretask ID!");
+    }
+    $qF1 = new QueryFilter(SupertaskPretask::SUPERTASK_ID, $supertaskId, "=");
+    $qF2 = new QueryFilter(SupertaskPretask::PRETASK_ID, $pretaskId, "=");
+    $supertaskPretask = Factory::getSupertaskPretaskFactory()->filter([Factory::FILTER => [$qF1, $qF2]], true);
+    Factory::getSupertaskPretaskFactory()->delete($supertaskPretask);
+  }
+
+  /**
+   * @param $supertaskId
+   * @param $pretaskId
+   * @throws HTException
+   */
+  public static function addPretaskToSupertask($supertaskId, $pretaskId) {
+    if ($supertaskId == null) {
+      throw new HTException("Invalid supertask ID!");
+    }
+    if ($pretaskId == null) {
+      throw new HTException("Invalid pretask ID!");
+    }
+    $supertaskPretask = new SupertaskPretask(null, $supertaskId, $pretaskId);
+    Factory::getSupertaskPretaskFactory()->save($supertaskPretask);
+  }
 }

src/inc/utils/TaskUtils.class.php

@@ -799,6 +799,7 @@ public static function createTask($hashlistId, $name, $attackCmd, $chunkTime, $s
   public static function splitByRules($task, $taskWrapper, $files, $splitFile, $split) {
     // calculate how much we need to split
     $numSplits = floor($split[1] / 1000 / $task->getChunkTime());
+    // TODO: replace countLines with fileLineCount? Could be a better option: not OS-dependent
     $numLines = Util::countLines(dirname(__FILE__) . "/../../files/" . $splitFile->getFilename());
     $linesPerFile = floor($numLines / $numSplits) + 1;
 

src/install/updates/update_v0.12.x_v0.x.x.php

@@ -82,3 +82,7 @@
   $EXECUTED["v0.12.x_agentBinariesUpdateTrack"] = true;
 }
 
+if (!isset($PRESENT["v0.12.x_fileLineCount"])) {
+  Factory::getFileFactory()->getDB()->query("ALTER TABLE `File` ADD `lineCount` INT NULL;");
+  $EXECUTED["v0.12.x_fileLineCount"] = true;
+}