Skip to content

Commit

Permalink
Merge branch 'dev'
Browse files Browse the repository at this point in the history
  • Loading branch information
@DariusIII
DariusIII committed Nov 14, 2018
2 parents f48b539 + 1b17744 commit 631fb5c
Show file tree
Hide file tree
Showing 704 changed files with 66,524 additions and 8,953 deletions.
8 changes: 1 addition & 7 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,4 @@ test.php
/resources/nzb/*
/resources/tmp/*
!/resources/tmp/unrar/
/public/assets/*
!/public/assets/js/
!/public/assets/css/
!/public/assets/images/
!/public/assets/Charisma/
!/public/assets/Gamma/
!/public/assets/Omicron/
!/public/assets/*
63 changes: 28 additions & 35 deletions Blacklight/Backfill.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

use App\Models\Group;
use App\Models\Settings;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;

class Backfill
Expand Down Expand Up @@ -55,6 +54,11 @@ class Backfill
*/
protected $_disableBackfillGroup;

/**
* @var \Blacklight\ColorCLI
*/
protected $colorCli;

/**
* Constructor.
*
Expand Down Expand Up @@ -83,38 +87,32 @@ public function __construct(array $options = [])
['NNTP' => $this->_nntp, 'Echo' => $this->_echoCLI]
);

$this->colorCli = new ColorCLI();

$this->_compressedHeaders = (int) Settings::settingValue('..compressedheaders') === 1;
$this->_safeBackFillDate = Settings::settingValue('..safebackfilldate') !== '' ? (string) Settings::settingValue('safebackfilldate') : '2008-08-14';
$this->_safePartRepair = (int) Settings::settingValue('..safepartrepair') === 1 ? 'update' : 'backfill';
$this->_disableBackfillGroup = (int) Settings::settingValue('..disablebackfillgroup') === 1;
}

/**
* Backfill all the groups up to user specified time/date.
*
* @param string $groupName
* @param string $groupName
* @param string|int $articles
* @param string $type
*
* @return void
* @throws \Exception
* @param string $type
* @throws \Throwable
*/
public function backfillAllGroups($groupName = '', $articles = '', $type = ''): void
{
$res = [];
if ($groupName !== '') {
$grp = Group::getByName($groupName);
if ($grp) {
$res = [$grp];
}
$grp[] = Group::getByName($groupName);
} else {
$res = Group::getActiveBackfill($type);
$grp = Group::getActiveBackfill($type);
}

$groupCount = \count($res);
$groupCount = \count($grp);
if ($groupCount > 0) {
$counter = 1;
$allTime = microtime(true);
$allTime = now();
$dMessage = (
'Backfilling: '.
$groupCount.
Expand All @@ -123,36 +121,36 @@ public function backfillAllGroups($groupName = '', $articles = '', $type = ''):
);

if ($this->_echoCLI) {
ColorCLI::header($dMessage);
$this->colorCli->header($dMessage);
}

if ($articles !== '' && ! is_numeric($articles)) {
$articles = 20000;
}

// Loop through groups.
foreach ($res as $groupArr) {
foreach ($grp as $groupArr) {
if ($groupName === '') {
$dMessage = 'Starting group '.$counter.' of '.$groupCount;

if ($this->_echoCLI) {
ColorCLI::header($dMessage);
$this->colorCli->header($dMessage);
}
}
$this->backfillGroup($groupArr, $groupCount - $counter, $articles);
$counter++;
}

$dMessage = 'Backfilling completed in '.number_format(microtime(true) - $allTime, 2).' seconds.';
$dMessage = 'Backfilling completed in '.now()->diffInSeconds($allTime).' seconds.';

if ($this->_echoCLI) {
ColorCLI::primary($dMessage);
$this->colorCli->primary($dMessage);
}
} else {
$dMessage = 'No groups specified. Ensure groups are added to database for updating.';

if ($this->_echoCLI) {
ColorCLI::warning($dMessage);
$this->colorCli->warning($dMessage);
}
}
}
Expand Down Expand Up @@ -184,7 +182,7 @@ public function backfillGroup($groupArr, $left, $articles = ''): void
'. Otherwise the group is dead, you must disable it.';

if ($this->_echoCLI) {
ColorCLI::error($dMessage);
$this->colorCli->error($dMessage);
}

return;
Expand All @@ -200,7 +198,7 @@ public function backfillGroup($groupArr, $left, $articles = ''): void
}

if ($this->_echoCLI) {
ColorCLI::primary('Processing '.$groupName);
$this->colorCli->primary('Processing '.$groupName);
}

// Check if this is days or post backfill.
Expand Down Expand Up @@ -233,14 +231,14 @@ public function backfillGroup($groupArr, $left, $articles = ''): void
}

if ($this->_echoCLI) {
ColorCLI::notice($dMessage);
$this->colorCli->notice($dMessage);
}

return;
}

if ($this->_echoCLI) {
ColorCLI::primary(
$this->colorCli->primary(
'Group '.
$groupName.
"'s oldest article is ".
Expand Down Expand Up @@ -268,7 +266,7 @@ public function backfillGroup($groupArr, $left, $articles = ''): void
$done = false;
while ($done === false) {
if ($this->_echoCLI) {
ColorCLI::header('Getting '.
$this->colorCli->header('Getting '.
number_format($last - $first + 1).
' articles from '.
$groupName.
Expand Down Expand Up @@ -315,7 +313,7 @@ public function backfillGroup($groupArr, $left, $articles = ''): void
}

if ($this->_echoCLI) {
ColorCLI::primary(
$this->colorCli->primary(
PHP_EOL.
'Group '.
$groupName.
Expand All @@ -327,18 +325,13 @@ public function backfillGroup($groupArr, $left, $articles = ''): void
}

/**
* Safe backfill using posts. Going back to a date specified by the user on the site settings.
* This does 1 group for x amount of parts until it reaches the date.
*
* @param string $articles
*
* @return void
* @throws \Exception
* @throws \Throwable
*/
public function safeBackfill($articles = ''): void
{
$groupname = Group::query()
->whereBetween('first_record_postdate', [$this->_safeBackFillDate, Carbon::now()])
->whereBetween('first_record_postdate', [$this->_safeBackFillDate, now()])
->where('backfill', '=', 1)
->select('name')
->orderBy('name')
Expand Down
62 changes: 31 additions & 31 deletions Blacklight/Binaries.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ class Binaries
/**
* @var \Blacklight\ColorCLI
*/
protected $_colorCLI;
protected $colorCli;

/**
* @var \Blacklight\CollectionsCleaning
Expand Down Expand Up @@ -241,8 +241,8 @@ public function __construct(array $options = [])
$this->_echoCLI = ($options['Echo'] && config('nntmux.echocli'));

$this->_pdo = DB::connection()->getPdo();
$this->_colorCLI = ($options['ColorCLI'] instanceof ColorCLI ? $options['ColorCLI'] : new ColorCLI());
$this->_nntp = ($options['NNTP'] instanceof NNTP ? $options['NNTP'] : new NNTP(['Echo' => $this->_colorCLI, 'ColorCLI' => $this->_colorCLI]));
$this->colorCli = ($options['ColorCLI'] instanceof ColorCLI ? $options['ColorCLI'] : new ColorCLI());
$this->_nntp = ($options['NNTP'] instanceof NNTP ? $options['NNTP'] : new NNTP(['Echo' => $this->colorCli, 'ColorCLI' => $this->colorCli]));
$this->_collectionsCleaning = ($options['CollectionsCleaning'] instanceof CollectionsCleaning ? $options['CollectionsCleaning'] : new CollectionsCleaning());

$this->messageBuffer = Settings::settingValue('..maxmssgs') !== '' ?
Expand Down Expand Up @@ -346,14 +346,14 @@ public function updateGroup($groupMySQL, $maxHeaders = 0): void
}

if ($this->_echoCLI) {
ColorCLI::primary('Processing '.$groupMySQL['name']);
$this->colorCli->primary('Processing '.$groupMySQL['name']);
}

// Attempt to repair any missing parts before grabbing new ones.
if ((int) $groupMySQL['last_record'] !== 0) {
if ($this->_partRepair) {
if ($this->_echoCLI) {
ColorCLI::primary('Part repair enabled. Checking for missing parts.');
$this->colorCli->primary('Part repair enabled. Checking for missing parts.');
}
$this->partRepair($groupMySQL);

Expand All @@ -363,7 +363,7 @@ public function updateGroup($groupMySQL, $maxHeaders = 0): void
$this->partRepair($groupMySQL, $tableNames);
}
} elseif ($this->_echoCLI) {
ColorCLI::primary('Part repair disabled by user.');
$this->colorCli->primary('Part repair disabled by user.');
}
}

Expand Down Expand Up @@ -431,7 +431,7 @@ public function updateGroup($groupMySQL, $maxHeaders = 0): void
// If total is bigger than 0 it means we have new parts in the newsgroup.
if ($total > 0) {
if ($this->_echoCLI) {
ColorCLI::primary(
$this->colorCli->primary(
(
(int) $groupMySQL['last_record'] === 0
? 'New group '.$groupNNTP['group'].' starting with '.
Expand Down Expand Up @@ -465,7 +465,7 @@ public function updateGroup($groupMySQL, $maxHeaders = 0): void
$first++;

if ($this->_echoCLI) {
ColorCLI::header(
$this->colorCli->header(
PHP_EOL.'Getting '.number_format($last - $first + 1).' articles ('.number_format($first).
' to '.number_format($last).') from '.$groupMySQL['name'].' - ('.
number_format($groupLast - $last).' articles in queue).'
Expand Down Expand Up @@ -534,13 +534,13 @@ public function updateGroup($groupMySQL, $maxHeaders = 0): void
}

if ($this->_echoCLI) {
ColorCLI::primary(
$this->colorCli->primary(
PHP_EOL.'Group '.$groupMySQL['name'].' processed in '.
number_format(microtime(true) - $startGroup, 2).' seconds.'
);
}
} elseif ($this->_echoCLI) {
ColorCLI::primary(
$this->colorCli->primary(
'No new articles for '.$groupMySQL['name'].' (first '.number_format($first).
', last '.number_format($last).', grouplast '.number_format($groupMySQL['last_record']).
', total '.number_format($total).")\n".'Server oldest: '.number_format($groupNNTP['first']).
Expand Down Expand Up @@ -771,7 +771,7 @@ public function scan($groupMySQL, $first, $last, $type = 'update', $missingParts
$this->addMissingParts($rangeNotReceived, $this->tableNames['prname'], $this->groupMySQL['id']);

if ($this->_echoCLI) {
ColorCLI::alternate(
$this->colorCli->alternate(
'Server did not return '.$notReceivedCount.
' articles from '.$this->groupMySQL['name'].'.'
);
Expand Down Expand Up @@ -1008,7 +1008,7 @@ protected function updateBlacklistUsage(): void
*/
protected function outputHeaderInitial(): void
{
ColorCLI::primary(
$this->colorCli->primary(
'Received '.\count($this->headersReceived).
' articles of '.number_format($this->last - $this->first + 1).' requested, '.
$this->headersBlackListed.' blacklisted, '.$this->notYEnc.' not yEnc.'
Expand All @@ -1022,16 +1022,16 @@ protected function outputHeaderDuration(): void
{
$currentMicroTime = microtime(true);
if ($this->_echoCLI) {
ColorCLI::alternateOver($this->timeHeaders.'s').
ColorCLI::primaryOver(' to download articles, ').
ColorCLI::alternateOver($this->timeCleaning.'s').
ColorCLI::primaryOver(' to process collections, ').
ColorCLI::alternateOver($this->timeInsert.'s').
ColorCLI::primaryOver(' to insert binaries/parts, ').
ColorCLI::alternateOver(number_format($currentMicroTime - $this->startPR, 2).'s').
ColorCLI::primaryOver(' for part repair, ').
ColorCLI::alternateOver(number_format($currentMicroTime - $this->startLoop, 2).'s').
ColorCLI::primary(' total.');
$this->colorCli->alternateOver($this->timeHeaders.'s').
$this->colorCli->primaryOver(' to download articles, ').
$this->colorCli->alternateOver($this->timeCleaning.'s').
$this->colorCli->primaryOver(' to process collections, ').
$this->colorCli->alternateOver($this->timeInsert.'s').
$this->colorCli->primaryOver(' to insert binaries/parts, ').
$this->colorCli->alternateOver(number_format($currentMicroTime - $this->startPR, 2).'s').
$this->colorCli->primaryOver(' for part repair, ').
$this->colorCli->alternateOver(number_format($currentMicroTime - $this->startLoop, 2).'s').
$this->colorCli->primary(' total.');
}
}

Expand Down Expand Up @@ -1086,15 +1086,15 @@ public function partRepair($groupArr, $tables = ''): void
DB::commit();
}
if ($e->getMessage() === 'SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction') {
ColorCLI::notice('Deadlock occurred');
$this->colorCli->notice('Deadlock occurred');
DB::rollBack();
}
}

$missingCount = \count($missingParts);
if ($missingCount > 0) {
if ($this->_echoCLI) {
ColorCLI::primary(
$this->colorCli->primary(
'Attempting to repair '.
number_format($missingCount).
' parts.'
Expand Down Expand Up @@ -1176,7 +1176,7 @@ public function partRepair($groupArr, $tables = ''): void
}

if ($this->_echoCLI) {
ColorCLI::primary(
$this->colorCli->primary(
PHP_EOL.
number_format($partsRepaired).
' parts repaired.'
Expand Down Expand Up @@ -1308,7 +1308,7 @@ public function daytopost($days, $data): string
}

if ($this->_echoCLI) {
ColorCLI::primary(
$this->colorCli->primary(
'Searching for an approximate article number for group '.$data['group'].' '.$days.' days back.'
);
}
Expand Down Expand Up @@ -1366,7 +1366,7 @@ public function daytopost($days, $data): string

$wantedArticle = (int) $wantedArticle;
if ($this->_echoCLI) {
ColorCLI::primary(
$this->colorCli->primary(
PHP_EOL.'Found article #'.$wantedArticle.' which has a date of '.date('r', $articleTime).
', vs wanted date of '.date('r', $goalTime).'. Difference from goal is '.Carbon::createFromTimestamp($goalTime)->diffInDays(Carbon::createFromTimestamp($articleTime)).'days.'
);
Expand Down Expand Up @@ -1623,7 +1623,7 @@ public function delete($collectionID): void
private function log($message, $method, $color): void
{
if ($this->_echoCLI) {
ColorCLI::$color($message.' ['.__CLASS__."::$method]");
$this->colorCli->$color($message.' ['.__CLASS__."::$method]");
}
}

Expand Down Expand Up @@ -1679,11 +1679,11 @@ protected function runQuery($query)
try {
return DB::insert($query);
} catch (QueryException $e) {
ColorCLI::debug('Query error occurred.');
$this->colorCli->debug('Query error occurred.');
} catch (\PDOException $e) {
ColorCLI::debug('Query error occurred.');
$this->colorCli->debug('Query error occurred.');
} catch (\Throwable $e) {
ColorCLI::debug('Query error occurred.');
$this->colorCli->debug('Query error occurred.');
}

return false;
Expand Down
Loading

0 comments on commit 631fb5c

Please sign in to comment.