-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #101 from bcgov/fix/filecache
Implement support for automated cache pruning maintenance
- Loading branch information
Showing
8 changed files
with
199 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,5 +5,8 @@ features: | |
| persistentVolumeClaim: | ||
| enabled: false | ||
|
|
||
| cronJob: | ||
| enabled: false | ||
|
|
||
| fluentBit: | ||
| enabled: false | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| const config = require('config'); | ||
| const { readdirSync, realpathSync, rmSync, statSync } = require('fs-extra'); | ||
| const { tmpdir } = require('os'); | ||
| const { join } = require('path'); | ||
|
|
||
| const log = require('./src/components/log')(module.filename); | ||
|
|
||
| const RATIO = 0.8; // Best practice is to keep the cache no more than 80% full | ||
|
|
||
| const osTempDir = realpathSync(tmpdir()); | ||
| const cacheDir = (() => { | ||
| if (config.has('carbone.cacheDir')) { | ||
| return realpathSync(config.get('carbone.cacheDir')); | ||
| } else { | ||
| return osTempDir; | ||
| } | ||
| })(); | ||
|
|
||
| const cacheSize = (() => { | ||
| const parseRegex = /^(\d+(?:\.\d+)?) *([kmgtp]?b)$/i; | ||
| const unitMap = { | ||
| b: Math.pow(10, 0), | ||
| kb: Math.pow(10, 3), | ||
| mb: Math.pow(10, 6), | ||
| gb: Math.pow(10, 9), | ||
| tb: Math.pow(10, 12), | ||
| pb: Math.pow(10, 15) | ||
| }; | ||
|
|
||
| if (config.has('carbone.cacheSize')) { | ||
| const result = parseRegex.exec(config.get('carbone.cacheSize')); | ||
| if (result && Array.isArray(result)) { | ||
| return parseInt(result[1]) * unitMap[result[2].toLowerCase()]; | ||
| } | ||
| } else { | ||
| return null; | ||
| } | ||
| })(); | ||
| const cacheSizeLimit = Math.ceil(cacheSize * RATIO); | ||
|
|
||
| log.info(`Cache directory ${cacheDir} with max size of ${cacheSizeLimit}`); | ||
|
|
||
| // Short circuit exits | ||
| if (!cacheSize) { | ||
| log.info('Maximum cache size not defined - Exiting'); | ||
| process.exit(0); | ||
| } else if (cacheDir === osTempDir) { | ||
| log.info('Cache points to OS temp directory - Exiting'); | ||
| process.exit(0); | ||
| } | ||
|
|
||
| // Check cache size and prune oldest files away as needed | ||
| try { | ||
| const currCacheSize = dirSize(cacheDir); | ||
| const files = getSortedFiles(cacheDir); | ||
| const status = currCacheSize < cacheSizeLimit ? 'below' : 'above'; | ||
|
|
||
| log.info(`Current cache size ${currCacheSize} ${status} threshold of ${cacheSizeLimit}`, { | ||
| cacheLimit: cacheSizeLimit, | ||
| cacheSize: currCacheSize | ||
| }); | ||
|
|
||
| // Prune files if necessary | ||
| let rmCount = 0; | ||
| for (const file of files) { | ||
| if (dirSize(cacheDir) < cacheSizeLimit) break; | ||
| rmSync(`${cacheDir}/${file}`, { recursive: true, force: true }); | ||
| rmCount++; | ||
| } | ||
|
|
||
| log.info(`${rmCount} objects were pruned from the cache - Exiting`, { removeCount: rmCount }); | ||
| process.exit(0); | ||
| } catch(err) { | ||
| log.error(err.message); | ||
| process.exit(1); | ||
| } | ||
|
|
||
| /** | ||
| * @function dirSize | ||
| * Recursively calculates the size of directory `dir` | ||
| * @param {string} dir The directory to calculate | ||
| * @returns {number} The size of the directory in bytes | ||
| */ | ||
| function dirSize(dir) { | ||
| const files = readdirSync(dir, { withFileTypes: true }); | ||
| const paths = files.map(file => { | ||
| const path = join(dir, file.name); | ||
|
|
||
| if (file.isDirectory()) return dirSize(path); | ||
| if (file.isFile()) { | ||
| const { size } = statSync(path); | ||
| return size; | ||
| } | ||
| return 0; | ||
| }); | ||
|
|
||
| return paths.flat(Infinity).reduce((i, size) => i + size, 0); | ||
| } | ||
|
|
||
| /** | ||
| * @function getSortedFiles | ||
| * Acquires a list of files and directories ordered from oldest to newest modified | ||
| * @param {string} dir The directory to inspect | ||
| * @returns {Array<string>} The list of files and directories in directory `dir` | ||
| */ | ||
| function getSortedFiles(dir) { | ||
| const files = readdirSync(dir); | ||
| return files | ||
| .map(fileName => ({ | ||
| name: fileName, | ||
| time: statSync(`${dir}/${fileName}`).mtime.getTime(), | ||
| })) | ||
| .sort((a, b) => a.time - b.time) | ||
| .map(file => file.name); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| {{- if and .Values.cronJob.enabled .Values.persistentVolumeClaim.enabled }} | ||
| apiVersion: batch/v1 | ||
| kind: CronJob | ||
| metadata: | ||
| name: {{ template "cdogs.fullname" . }} | ||
| labels: {{ include "cdogs.labels" . | nindent 4 }} | ||
| spec: | ||
| concurrencyPolicy: Forbid | ||
| failedJobsHistoryLimit: 3 | ||
| successfulJobsHistoryLimit: 3 | ||
| startingDeadlineSeconds: 60 | ||
| jobTemplate: | ||
| metadata: | ||
| labels: {{ include "cdogs.labels" . | nindent 8 }} | ||
| spec: | ||
| backoffLimit: 6 | ||
| activeDeadlineSeconds: 300 | ||
| parallelism: 1 | ||
| completions: 1 | ||
| template: | ||
| metadata: | ||
| labels: {{ include "cdogs.labels" . | nindent 12 }} | ||
| spec: | ||
| {{- with .Values.imagePullSecrets }} | ||
| imagePullSecrets: {{ toYaml . | nindent 8 }} | ||
| {{- end }} | ||
| {{- if .Values.serviceAccount.create }} | ||
| serviceAccountName: {{ include "cdogs.serviceAccountName" . }} | ||
| {{- end }} | ||
| {{- with .Values.podSecurityContext }} | ||
| securityContext: {{ toYaml . | nindent 8 }} | ||
| {{- end }} | ||
| containers: | ||
| - name: job | ||
| {{- with .Values.securityContext }} | ||
| securityContext: {{ toYaml . | nindent 12 }} | ||
| {{- end }} | ||
| image: "{{ .Values.image.repository }}/{{ .Chart.Name }}:{{ .Values.image.tag | default .Chart.AppVersion }}" | ||
| imagePullPolicy: {{ .Values.image.pullPolicy }} | ||
| command: | ||
| - node | ||
| - ./cacheCleaner.js | ||
| resources: {{ toYaml .Values.resources | nindent 16 }} | ||
| env: | ||
| - name: NODE_ENV | ||
| value: production | ||
| envFrom: | ||
| - configMapRef: | ||
| name: {{ include "cdogs.configname" . }}-config | ||
| volumeMounts: | ||
| - name: file-cache-data | ||
| mountPath: /var/lib/file-cache/data | ||
| restartPolicy: Never | ||
| volumes: | ||
| - name: file-cache-data | ||
| persistentVolumeClaim: | ||
| claimName: {{ include "cdogs.configname" . }}-cache | ||
| schedule: {{ .Values.cronJob.schedule }} | ||
| suspend: {{ .Values.cronJob.suspend }} | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters