diff --git a/Corals/core/Foundation/DataTables/BaseDataTable.php b/Corals/core/Foundation/DataTables/BaseDataTable.php index d091c50c..3c49cd55 100644 --- a/Corals/core/Foundation/DataTables/BaseDataTable.php +++ b/Corals/core/Foundation/DataTables/BaseDataTable.php @@ -27,9 +27,15 @@ public function __construct() $filters = array_merge($this->getFilters(), $this->getCustomRenderedFilters()); if ($this->usesQueryBuilderFilters) { - $this->addScope(new CoralsQueryBuilderDataTableScope($filters)); + $this->addScope(new CoralsQueryBuilderDataTableScope($filters, request('q'))); } else { - $this->addScope(new CoralsScope($filters)); + $this->addScope(new CoralsScope( + $this->getFilters(), + request()->filters, + $urlFilters ?? [], [ + 'class' => get_class($this), + 'parameters' => $this->resolveDataTableParameters() + ])); } if ($this->request->boolean('deleted')) { @@ -74,6 +80,22 @@ public function builder(): Builder return app(CoralsBuilder::class); } + /** + * @return array + */ + protected function resolveDataTableParameters() + { + $objectParameters = []; + + $objectReflection = new \ReflectionClass($this); + + foreach ($objectReflection->getConstructor()->getParameters() as $parameter) { + $objectParameters[$parameter->getName()] = $this->{$parameter->getName()}; + } + + return $objectParameters; + } + /** * Optional method if you want to use html builder. * diff --git a/Corals/core/Foundation/DataTables/CoralsQueryBuilderDataTableScope.php b/Corals/core/Foundation/DataTables/CoralsQueryBuilderDataTableScope.php index 47799c20..1ca153b8 100644 --- a/Corals/core/Foundation/DataTables/CoralsQueryBuilderDataTableScope.php +++ b/Corals/core/Foundation/DataTables/CoralsQueryBuilderDataTableScope.php @@ -9,21 +9,18 @@ class CoralsQueryBuilderDataTableScope implements DataTableScope { - - protected $filters; - /** * CoralsQueryBuilderDataTableScope constructor. * @param $filters + * @param $urlQuery */ - public function __construct($filters) + public function __construct(protected $filters, protected $urlQuery) { - $this->filters = $filters; } public function apply($query) { - if ($queryBuilderJson = request('q')) { + if ($queryBuilderJson = $this->urlQuery) { $queryBuilderParser = new QueryBuilderParser($this->filters); $query = $queryBuilderParser->parse(json_encode($queryBuilderJson), $query); } diff --git a/Corals/core/Foundation/DataTables/CoralsScope.php b/Corals/core/Foundation/DataTables/CoralsScope.php index d1c5d95d..fd81cd51 100644 --- a/Corals/core/Foundation/DataTables/CoralsScope.php +++ b/Corals/core/Foundation/DataTables/CoralsScope.php @@ -9,37 +9,55 @@ class CoralsScope implements DataTableScope { - public $filters; - - public function __construct($filters) + /** + * CoralsScope constructor. + * @param $filters + * @param $requestFilter + * @param array $urlFilters + * @param array $dataTable + */ + public function __construct( + protected $filters, + protected $requestFilter, + protected $urlFilters = [], + protected array $dataTable = [] + ) { - $this->filters = $filters; } public function apply($query) { + $ajaxRequestFilters = urldecode($this->requestFilter); + $filters = $this->filters; - if (empty($filters)) { - return $query; - } - $requestFilters = request()->get('filters'); + $filtersRequest = array_merge( + \Arr::dot($this->urlFilters), + getRequestFiltersArray($ajaxRequestFilters) + ); - if (!is_array($requestFilters)) { - $requestFilters = urldecode($requestFilters); - $requestFilters = get_request_filters_array($requestFilters); - } + if ($this->dataTable) { + $dt = app( + data_get($this->dataTable, 'class'), + data_get($this->dataTable, 'parameters'), + ); - $requestFilters = array_merge(request()->only(array_keys($filters)), $requestFilters); + if (method_exists($dt, 'applyDefaultFilters')) { + $dt->applyDefaultFilters($query, $filtersRequest, $this->urlFilters); + } + } - if (empty($requestFilters)) { + if (!$ajaxRequestFilters || empty($filters)) { return $query; } + request()->request->add($filtersRequest); + $baseTable = $query->getModel()->getTable(); - foreach ($requestFilters as $column => $value) { + + foreach ($filtersRequest as $column => $value) { $filter = Arr::get($filters, $column, Arr::get($filters, $column . "[]")); if (empty($filter) || !($value || $value === 0 || $value === 0.0 || $value === false)) { diff --git a/Corals/core/Foundation/FoundationServiceProvider.php b/Corals/core/Foundation/FoundationServiceProvider.php index 946ef638..8afe7325 100644 --- a/Corals/core/Foundation/FoundationServiceProvider.php +++ b/Corals/core/Foundation/FoundationServiceProvider.php @@ -44,6 +44,7 @@ use Illuminate\Contracts\Debug\ExceptionHandler; use Illuminate\Foundation\AliasLoader; use Illuminate\Notifications\Messages\MailMessage; +use Illuminate\Queue\Queue; use Illuminate\Queue\Worker; use Illuminate\Support\Arr; use Illuminate\Support\Facades\Lang; @@ -89,6 +90,8 @@ public function boot() $this->registerMissingHtmlMacro(); + + $this->registerCreateQueuePayloadCallback(); } /** @@ -210,10 +213,6 @@ public function register() }); - - - - Actions::do_action('post_coral_registration'); // Bind 'hashids' shared component to the IoC container @@ -251,6 +250,7 @@ protected function mobileResetPasswordConfiguration(): void $this->resetPasswordEmailCallback(); $this->extendPasswordBroker(); } + /** * */ @@ -322,4 +322,16 @@ protected function extendQueueWorker() ); }); } + + /** + * + */ + protected function registerCreateQueuePayloadCallback(): void + { + Queue::createPayloadUsing(function () { + return [ + 'user_id' => data_get(user(), 'id') + ]; + }); + } } diff --git a/Corals/core/Foundation/Helpers/helpers.php b/Corals/core/Foundation/Helpers/helpers.php index 9da96c82..5bf475cf 100644 --- a/Corals/core/Foundation/Helpers/helpers.php +++ b/Corals/core/Foundation/Helpers/helpers.php @@ -159,10 +159,10 @@ function format_time($time, $format = 'h:i A') if (!function_exists('log_exception')) { function log_exception( \Exception $exception = null, - $object = null, - $action = null, - $message = null, - $echo_message = false + $object = null, + $action = null, + $message = null, + $echo_message = false ) { logger(array_slice(debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 2), -1)); @@ -1120,3 +1120,44 @@ function getCurrentTimeForFileName() return now()->format('Y_dM_H_i'); } } + +if (!function_exists('getRequestFiltersArray')) { + /** + * @param $requestFilters + * @return array|string|string[] + */ + function getRequestFiltersArray($requestFilters) + { + $array = explode("&", $requestFilters); + + $array = str_replace('#amp#', '&', $array); + if (!(count($array) == 1 && $array[0] == "")) { + $index = 0; + + foreach ($array as $key => $value) { + $filter = explode("=", $value); + preg_match_all('/(.*)\[(.*?)\]/', $filter[0], $matches); + if (is_array($matches[0]) && (count($matches[0]) > 0)) { + if ($filter[1]) { + if (strpos($filter[1], ',') !== false) { + foreach (explode(',', $filter[1]) as $f) { + $array[$matches[1][0]][] = $f; + } + } else { + $nodeKey = empty(trim($matches[2][0], "'")) ? $index++ : trim($matches[2][0], "'"); + $array[$matches[1][0]][$nodeKey] = $filter[1]; + } + } + } else { + if ($filter[1]) { + $array[$filter[0]] = $filter[1]; + } + } + unset ($array[$key]); + } + return $array; + } else { + return []; + } + } +} diff --git a/Corals/core/Foundation/Jobs/GenerateExcelForDataTable.php b/Corals/core/Foundation/Jobs/GenerateExcelForDataTable.php index 720283eb..1bf4920d 100644 --- a/Corals/core/Foundation/Jobs/GenerateExcelForDataTable.php +++ b/Corals/core/Foundation/Jobs/GenerateExcelForDataTable.php @@ -2,16 +2,20 @@ namespace Corals\Foundation\Jobs; -use Corals\Foundation\Classes\ExcelWriter; use Corals\User\Models\User; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; +use Illuminate\Pagination\LengthAwarePaginator; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Arr; +use Illuminate\Support\Facades\Auth; +use League\Csv\Writer; +use League\Fractal\TransformerAbstract; use Yajra\DataTables\EloquentDataTable; use Yajra\DataTables\Transformers\DataArrayTransformer; +use Illuminate\Contracts\Database\Query\Builder as BuilderContract; class GenerateExcelForDataTable implements ShouldQueue { @@ -24,6 +28,25 @@ class GenerateExcelForDataTable implements ShouldQueue protected $tableID; protected $download; + /** + * @var int + */ + protected int $chunkSize = 200; + + /** + * @var int|mixed + */ + protected int $currentPage; + + /** + * @var string|mixed|null + */ + protected string|null $appendToFilePath; + + /** + * @var bool + */ + protected bool $headersProcessed = false; /** * GenerateExcelForDataTable constructor. @@ -33,8 +56,19 @@ class GenerateExcelForDataTable implements ShouldQueue * @param $tableID * @param User $user * @param false $download + * @param int $currentPage + * @param null $appendToFilePath */ - public function __construct($dataTable, $scopes, $columns, $tableID, User $user, $download = false) + public function __construct( + $dataTable, + $scopes, + $columns, + $tableID, + User $user, + $download = false, + $currentPage = 1, + $appendToFilePath = null + ) { $this->dataTable = $dataTable; $this->scopes = $scopes; @@ -42,6 +76,8 @@ public function __construct($dataTable, $scopes, $columns, $tableID, User $user, $this->user = $user; $this->tableID = str_replace('DataTable', '', $tableID); $this->download = $download; + $this->currentPage = $currentPage; + $this->appendToFilePath = $appendToFilePath; } /** @@ -52,6 +88,9 @@ public function __construct($dataTable, $scopes, $columns, $tableID, User $user, public function handle() { try { + + $this->login(); + logger('start exporting: ' . $this->dataTable); $dataTable = app()->make($this->dataTable); @@ -71,50 +110,37 @@ public function handle() $scope->apply($source); } - $rootPath = config('app.export_excel_base_path'); + $this->appendToFilePath = $this->generateFilePath(); - $exportName = join('_', [ - 'table_' . $this->tableID, - 'user_id_' . $this->user->id, - str_replace(['-', ':', ' '], '_', now()->toDateTimeString()) . '.xlsx' - ]); + $writer = Writer::createFromPath($this->appendToFilePath, 'a+') + ->setDelimiter(config('corals.csv_delimiter', ',')); - $filePath = storage_path($rootPath . $exportName); + $source = $this->write( + $writer, $source, $transformer, $modelTransformer + ); - if (!file_exists($rootPath = storage_path($rootPath))) { - mkdir($rootPath, 0755, true); - } - - if (file_exists($filePath)) { - unlink($filePath); + if ($this->download) { + logger($this->appendToFilePath . ' Completed'); + $this->logout(); + return response()->download($this->appendToFilePath); } - $writer = ExcelWriter::create($filePath); - - $source->chunk(100, function ($data) use ($transformer, $writer, $modelTransformer) { - foreach ($data as $row) { - $row = $modelTransformer->transform($row); - - $rowData = $transformer->transform($row, $this->columns, 'exportable'); - - $writer->addRow($rowData); - } - }); - - $writer->close(); - - if ($this->download) { - logger($exportName . ' Completed'); - return response()->download($filePath); + /** + * @var $source LengthAwarePaginator + */ + if ($source->hasMorePages()) { + $this->pushNextJob(); + } else { + event('notifications.user.send_excel_file', [ + 'file' => $this->appendToFilePath, + 'user' => $this->user, + 'table_id' => $this->tableID + ]); } - event('notifications.user.send_excel_file', [ - 'file' => $filePath, - 'user' => $this->user, - 'table_id' => $this->tableID - ]); + $this->logout(); - logger($exportName . ' Completed'); + logger($this->appendToFilePath . ' Completed'); } catch (\Exception $exception) { report($exception); } @@ -130,5 +156,153 @@ protected function getModelTransformer($dataTable, $source) return Arr::first($dataTable->dataTable($source)->getTransformer()); } + /** + * @param DataArrayTransformer $transformer + * @param Writer $writer + * @param TransformerAbstract $modelTransformer + * @return callable + */ + protected function exportItemsCallback( + DataArrayTransformer $transformer, + Writer $writer, + TransformerAbstract $modelTransformer + ): callable + { + return function ($data) use ($transformer, $writer, $modelTransformer) { + + foreach ($data as $row) { + $row = $modelTransformer->transform($row); + + $rowData = $transformer->transform($row, $this->columns, 'exportable'); + + if ($this->currentPage === 1 && !$this->headersProcessed) { + $writer->insertOne(array_keys($rowData)); + $this->headersProcessed = true; + } + + $writer->insertOne($rowData); + } + }; + } + + /** + * + */ + protected function pushNextJob(): void + { + self::dispatch( + $this->dataTable, + $this->scopes, + $this->columns, + $this->tableID, + $this->user, + $this->download, + $this->currentPage + 1, + $this->appendToFilePath + ); + } + + /** + * @param Writer $writer + * @param BuilderContract $source + * @param DataArrayTransformer $transformer + * @param TransformerAbstract $modelTransformer + * @return bool|LengthAwarePaginator + */ + protected function write( + Writer $writer, + BuilderContract $source, + DataArrayTransformer $transformer, + TransformerAbstract $modelTransformer + ): LengthAwarePaginator|bool + { + return $this->download + ? $source->chunk($this->chunkSize, $this->exportItemsCallback($transformer, $writer, $modelTransformer)) + : $this->paginate($writer, $source, $transformer, $modelTransformer); + } + + /** + * @param Writer $writer + * @param BuilderContract $source + * @param DataArrayTransformer $transformer + * @param TransformerAbstract $modelTransformer + * @return LengthAwarePaginator + */ + protected function paginate( + Writer $writer, + BuilderContract $source, + DataArrayTransformer $transformer, + TransformerAbstract $modelTransformer + ): LengthAwarePaginator + { + $source = $source->paginate(perPage: $this->chunkSize, page: $this->currentPage); + + /** + * @var $source LengthAwarePaginator + */ + $source->pipe($this->exportItemsCallback($transformer, $writer, $modelTransformer)); + + return $source; + } + + /** + * @return mixed|string|null + */ + protected function generateFilePath() + { + if ($this->appendToFilePath) { + return $this->appendToFilePath; + } + + $exportName = join('_', [ + 'table_' . $this->tableID, + 'user_id_' . $this->user->id, + str_replace(['-', ':', ' '], '_', now()->toDateTimeString()) . '.csv' + ]); + + $rootPath = config('app.export_excel_base_path'); + + $filePath = storage_path($rootPath . $exportName); + + if (file_exists($filePath)) { + unlink($filePath); + } + + if (!file_exists($rootPath = storage_path($rootPath))) { + mkdir($rootPath, 0755, true); + } + + return $filePath; + } + + /** + * + */ + protected function login(): void + { + if (!app()->runningInConsole()) { + return; + } + + $userId = data_get(optional($this->job)->payload(), 'user_id'); + + if ($userId) { + Auth::loginUsingId($userId); + } + } + + /** + * + */ + protected function logout(): void + { + if (!app()->runningInConsole()) { + return; + } + + if (Auth::user()) { + Auth::logout(); + } + } }