diff --git a/src/igz_controls/services/control-panel-logs-data.service.js b/src/igz_controls/services/control-panel-logs-data.service.js index 63e49c9a..ca4e21f7 100644 --- a/src/igz_controls/services/control-panel-logs-data.service.js +++ b/src/igz_controls/services/control-panel-logs-data.service.js @@ -4,8 +4,9 @@ angular.module('iguazio.dashboard-controls') .factory('ControlPanelLogsDataService', ControlPanelLogsDataService); - function ControlPanelLogsDataService($q, lodash, ElasticsearchService) { + function ControlPanelLogsDataService($q, lodash, ElasticsearchService, ElasticSearchDataService) { return { + collectLogs: collectLogs, entriesPaginated: search, logsPaginated: logsWidthReplicas }; @@ -52,8 +53,8 @@ if (queryParams.timeFrame) { searchFrom = 'now-' + queryParams.timeFrame; } else { - searchFrom = queryParams.customTimeFrame.from - searchTo = lodash.get(queryParams, 'customTimeFrame.to', 'now') + searchFrom = lodash.get(queryParams, 'customTimeFrame.from', '1970-01-01T00:00:00Z'); + searchTo = lodash.get(queryParams, 'customTimeFrame.to', 'now'); } } @@ -113,6 +114,10 @@ }); } + if (queryParams.trackTotalHits) { + config.track_total_hits = true; + } + return ElasticsearchService.search(config) .then(function (response) { // Saved log entry can be found in `_source` property @@ -140,6 +145,51 @@ }); } + /** + * Collects all the logs using chunks, and saved them as an array of strings. + * @returns {Promise.>} a promise resolving to an array of logs. + */ + function collectLogs(query) { + var keepAlive = '5m'; + var size = 10000; + var downloadLogsData = []; + var createPitConfig = { + index: 'filebeat*', + keepAlive: keepAlive + }; + + return ElasticSearchDataService.createPit(createPitConfig).then(function (pitId) { + return getNextPitLogs(pitId, null); + }); + + function getNextPitLogs(pitId, searchAfter) { + return ElasticSearchDataService.getNextPitLogs(size, query, keepAlive, pitId, searchAfter).then(function (response) { + var hits = response.hits.hits; + + if (hits.length > 0) { + var lastHit = lodash.last(hits); + + downloadLogsData = downloadLogsData.concat(prepareLogs(hits)); + + return getNextPitLogs(response.pit_id, lastHit.sort) + } else { + return downloadLogsData; + } + }).catch(function (error) { + throw error; + }); + } + + function prepareLogs(logs) { + return logs.map(function (logData) { + var log = lodash.get(logData, '_source', {}); + + return log['@timestamp'] + ' ' + log.name + ' (' + log.level + ') ' + + lodash.get(log, 'message', '') + ' ' + JSON.stringify(lodash.get(log, 'more', {})); + }); + } + } + /** * Mocks the real search function without the need for a running Elasticsearch service, for development * purposes. @@ -195,7 +245,7 @@ logs.replicas = replicas; } - console.info(queryParams); + // console.info(queryParams); return $q.when(logs); } diff --git a/src/nuclio/functions/version/version-execution-log/version-execution-log.component.js b/src/nuclio/functions/version/version-execution-log/version-execution-log.component.js index 51b832f4..33cace31 100644 --- a/src/nuclio/functions/version/version-execution-log/version-execution-log.component.js +++ b/src/nuclio/functions/version/version-execution-log/version-execution-log.component.js @@ -31,8 +31,9 @@ such restriction. function NclVersionExecutionLogController(lodash, $interval, i18next, $i18next, $rootScope, moment, ConfigService, ControlPanelLogsDataService, ExportService, LoginService,PaginationService) { var ctrl = this; - var lng = i18next.language; + var MAX_DOWNLOAD_LOGS = 2000000; + var refreshInterval = null; var initialTimeRange = { from: null, @@ -51,6 +52,7 @@ such restriction. } }; + ctrl.downloadButtonIsHidden = true; ctrl.isSplashShowed = { value: false }; @@ -67,6 +69,7 @@ such restriction. } }; ctrl.datePreset = initialDatePreset; + ctrl.logsAreDownloading = false; ctrl.timeRange = initialTimeRange; ctrl.searchStates = {}; ctrl.selectedReplicas = []; @@ -164,7 +167,7 @@ such restriction. ctrl.$onDestroy = onDestroy; ctrl.applyFilters = applyFilters; - // ctrl.downloadLogFiles = downloadLogFiles; + ctrl.downloadLogFiles = downloadLogFiles; ctrl.onCheckboxChange = onCheckboxChange; ctrl.onRefreshRateChange = onRefreshRateChange; ctrl.onTimeRangeChange = onTimeRangeChange; @@ -189,6 +192,12 @@ such restriction. ctrl.page.size = ctrl.perPageValues[0].id; + ControlPanelLogsDataService.logsPaginated(0, 100, { query: generateDefaultQuery(), trackTotalHits: true }) + .then(function (logs) { + if (logs.length > 0) { + ctrl.downloadButtonIsHidden = logs.total_logs_count >= MAX_DOWNLOAD_LOGS; + } + }); applyFilters(); } @@ -213,18 +222,18 @@ such restriction. searchWithParams(0, ctrl.page.size); } - // /** - // * Downloads log to the file - // */ - // function downloadLogFiles() { - // ExportService.exportLogs(prepareLogs(), ctrl.version.metadata.name); - // - // function prepareLogs() { - // return ctrl.logs.map(function (log) { - // return log['@timestamp'] + ' ' + log.name + ' (' + log.level + ') ' + log.message + ' ' + log.more; - // }); - // } - // } + /** + * Downloads log to the file + */ + function downloadLogFiles() { + stopAutoUpdate(); + + return ControlPanelLogsDataService.collectLogs(generateDefaultQuery()) + .then(function (response) { + ExportService.exportLogs(response, ctrl.version.metadata.name); + startAutoUpdate(); + }); + } /** * Triggered when search text was changed @@ -277,7 +286,7 @@ such restriction. function refreshLogs() { startAutoUpdate(); - ControlPanelLogsDataService.logsPaginated(ctrl.page.number, ctrl.page.size, queryParams(), true) + ControlPanelLogsDataService.logsPaginated(ctrl.page.number, ctrl.page.size, queryParams()) .then(function (logs) { if (logs.length > 0) { ctrl.logs = lodash.cloneDeep(logs); @@ -330,13 +339,12 @@ such restriction. return; } - ControlPanelLogsDataService.logsPaginated(ctrl.page.number, ctrl.page.size, queryParamsAutoUpdate(), true) + ControlPanelLogsDataService.logsPaginated(ctrl.page.number, ctrl.page.size, queryParamsAutoUpdate()) .then(function (logs) { if (logs.length > 0) { ctrl.logs = lodash.cloneDeep(logs); ctrl.page.total = logs['total_pages']; - // set lastItemTimeStamp and start autoupdate onChangePageCallback(); } }); @@ -412,12 +420,25 @@ such restriction. ctrl.filterQuery = lodash.join(queries, ' AND '); } + /** + * Generates query string without additional filters + */ + function generateDefaultQuery() { + var queries = ['system-id:"' + ConfigService.systemId + '"', '_exists_:nuclio']; + + if (!lodash.isEmpty(ctrl.version.metadata.name)) { + queries.push('name:' + ctrl.version.metadata.name); + } + return lodash.join(queries, ' AND '); + } + /** * Generates query params for ordinary request, for example, when page was changed * @returns {Object} */ function queryParams() { return { + trackTotalHits: true, query: ctrl.filterQuery, timeFrame: ctrl.datePreset, customTimeFrame: ctrl.timeRange diff --git a/src/nuclio/functions/version/version-execution-log/version-execution-log.tpl.html b/src/nuclio/functions/version/version-execution-log/version-execution-log.tpl.html index b3887cf5..e0608442 100644 --- a/src/nuclio/functions/version/version-execution-log/version-execution-log.tpl.html +++ b/src/nuclio/functions/version/version-execution-log/version-execution-log.tpl.html @@ -126,16 +126,17 @@ data-refresh="$ctrl.searchWithParams($ctrl.page.number, $ctrl.page.size)"> - - - - - - - - - - +
+
+
+
+