Skip to content

Commit

Permalink
Merge pull request #275 from FlowFuse/file-api
Browse files Browse the repository at this point in the history
First pass at file api
  • Loading branch information
hardillb authored Aug 28, 2024
2 parents a97648a + 05d767e commit 0c05f78
Show file tree
Hide file tree
Showing 9 changed files with 279 additions and 24 deletions.
2 changes: 2 additions & 0 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ if (NODE_MAJOR_VERSION > 14) {

const { Launcher } = require('./lib/launcher')
const { AdminInterface } = require('./lib/admin')
const { filesInterface } = require('./lib/files')

const cmdLineOptions = [
{ name: 'port', alias: 'p', type: Number },
Expand Down Expand Up @@ -111,6 +112,7 @@ async function main () {
await launcher.logAuditEvent('start-failed', { error })
}

filesInterface(adminInterface.app, launcher.settings)
// const wss = new ws.Server({ clientTracking: false, noServer: true })
//
// server.on('upgrade', (req, socket, head) => {
Expand Down
1 change: 1 addition & 0 deletions lib/admin.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ class AdminInterface {
this.launcher = launcher

const app = express()
this.app = app
this.server = http.createServer(app)

app.use(bodyParser.json({}))
Expand Down
134 changes: 134 additions & 0 deletions lib/files/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
const { mkdir, readdir, rm, rename, stat, writeFile } = require('fs/promises')
const { existsSync } = require('fs')
const { dirname, join, normalize } = require('path')
const multer = require('multer')

const filesInterface = (app, settings) => {
const BASE_PATH = join(settings.rootDir, settings.userDir, 'storage')
// not sure about this yet, need to check how file storage works
const storage = multer.memoryStorage()
const upload = multer({ storage })

const regex = /^\/flowforge\/files\/_\/(.*)$/

const listDirectory = async (path = '') => {
const fullPath = normalize(join(BASE_PATH, path))
if (fullPath.startsWith(BASE_PATH)) {
const files = await readdir(fullPath, { withFileTypes: true })
const response = {
meta: { },
files: [],
count: files.length
}
for (const i in files) {
const file = files[i]
const filePath = join(fullPath, file.name)
const s = await stat(filePath)
const rep = {
name: file.name,
lastModified: s.mtime
}
if (file.isFile()) {
rep.type = 'file'
rep.size = s.size
} else if (file.isDirectory()) {
rep.type = 'directory'
}
response.files.push(rep)
}
return response
} else {
return []
}
}

app.get(regex, async (request, reply) => {
try {
const files = await listDirectory(request.params[0])
reply.send(files)
} catch (err) {
if (err.code === 'ENOENT') {
reply.status(404).send()
} else if (err.code === 'ENOTDIR') {
reply.status(400).send()
} else {
reply.status(500).send()
}
}
})

app.put(regex, async (request, reply) => {
const fullPath = normalize(join(BASE_PATH, request.params[0]))
const newPath = normalize(join(BASE_PATH, request.body.path))
if (fullPath.startsWith(BASE_PATH) && newPath.startsWith(BASE_PATH)) {
if (existsSync(fullPath) && existsSync(dirname(newPath))) {
try {
await rename(fullPath, newPath)
reply.status(202).send()
} catch (err) {
reply.status(500).send()
}
} else {
reply.status(404).send()
}
}
reply.status(403).send()
})

app.post(regex, upload.single('file'), async (request, reply) => {
const startPath = normalize(join(BASE_PATH, request.params[0]))
if (startPath.startsWith(BASE_PATH)) {
if (request.get('content-type') === 'application/json') {
const newPath = request.body.path
const fullPath = normalize(join(startPath, newPath))
if (fullPath.startsWith(BASE_PATH)) {
try {
await mkdir(fullPath, { recursive: true })
reply.status(201).send()
} catch (err) {
reply.status(500).send()
}
} else {
reply.status(500).send()
}
} else if (request.get('content-type').startsWith('multipart/form-data')) {
const targetDir = dirname(startPath)
if (existsSync(targetDir)) {
await writeFile(startPath, request.file.buffer)
reply.status(201).send()
} else {
reply.status(404).send()
}
} else {
reply.status(406).send()
}
} else {
reply.status(403).send()
}
})

app.delete(regex, async (request, reply) => {
const fullPath = normalize(join(BASE_PATH, request.params[0]))
if (fullPath !== BASE_PATH && fullPath.startsWith(BASE_PATH)) {
if (existsSync(fullPath)) {
try {
await rm(fullPath, {
force: true,
recursive: true
})
reply.status(204).send()
} catch (err) {
reply.status(500).send()
}
} else {
reply.status(404).send()
}
} else {
reply.status(403).send()
}
})
}

module.exports = {
filesInterface
}
4 changes: 3 additions & 1 deletion lib/launcher.js
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,8 @@ class Launcher {
this.settings.broker = this.options.broker
this.settings.launcherVersion = this.options?.versions?.launcher || ''

this.settings.storageDir = path.normalize(path.join(this.settings.rootDir, this.settings.userDir, 'storage'))

// setup nodeDir to include the path to additional nodes and plugins
const nodesDir = []
if (Array.isArray(this.settings.nodesDir) && this.settings.nodesDir.length) {
Expand Down Expand Up @@ -367,7 +369,7 @@ class Launcher {
windowsHide: true,
env: appEnv,
stdio: ['ignore', 'pipe', 'pipe'],
cwd: path.join(this.settings.rootDir, this.settings.userDir, 'storage')
cwd: this.settings.storageDir
}

const processArguments = [
Expand Down
15 changes: 7 additions & 8 deletions lib/resources/sample.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,14 @@ let pptf
} catch (err) {
console.log(err)
}
})();

})()

let lastCPUTime = 0

async function sampleResources (url, time) {
const response = {}
try {
const res = await got.get(url, {
const res = await got.get(url, {
headers: {
pragma: 'no-cache',
'Cache-Control': 'max-age=0, must-revalidate, no-cache'
Expand All @@ -33,12 +32,12 @@ async function sampleResources (url, time) {
response.hu = parseInt(v.value)
})
} else if (metric.name === 'process_resident_memory_bytes') {
response.ps = parseInt(metric.metrics[0].value)/(1024*1024)
response.ps = parseInt(metric.metrics[0].value) / (1024 * 1024)
} else if (metric.name === 'process_cpu_seconds_total') {
cpuTime = parseFloat(metric.metrics[0].value)
if (lastCPUTime != 0) {
const cpuTime = parseFloat(metric.metrics[0].value)
if (lastCPUTime !== 0) {
const delta = cpuTime - lastCPUTime
response.cpu = (delta/time) * 100
response.cpu = (delta / time) * 100
}
lastCPUTime = cpuTime
} else if (metric.name === 'nodejs_eventloop_lag_mean_seconds') {
Expand All @@ -54,4 +53,4 @@ async function sampleResources (url, time) {
return response
}

module.exports = sampleResources
module.exports = sampleResources
18 changes: 7 additions & 11 deletions lib/resources/sampleBuffer.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
const os = require('node:os')
const crypto = require('crypto')

const instanceId = crypto.createHash('md5').update(os.hostname()).digest('hex').substring(0, 4)
class SampleBuffer {
constructor (size = 1000) {
this.size = size
Expand All @@ -17,7 +13,7 @@ class SampleBuffer {
sample.ts = Date.now()
}
this.buffer[this.head++] = sample
if (this.head == this.size) {
if (this.head === this.size) {
this.head = 0
this.wrapped = true
}
Expand All @@ -33,9 +29,9 @@ class SampleBuffer {
toArray () {
if (!this.wrapped) {
return this.buffer.slice(0, this.head)
} else {
} else {
const result = this.buffer.slice(this.head, this.size)
result.push(...this.buffer.slice(0,this.head))
result.push(...this.buffer.slice(0, this.head))
return result
}
}
Expand All @@ -56,7 +52,7 @@ class SampleBuffer {
}

avgLastX (x) {
const samples = this.lastX(x)
const samples = this.lastX(x)
const result = {}
let skipped = 0
samples.forEach(sample => {
Expand All @@ -68,18 +64,18 @@ class SampleBuffer {
} else {
result[key] = value
}
}
}
}
} else {
skipped++
}
})
for (const [key, value] of Object.entries(result)) {
result[key] = value/(samples.length-skipped)
result[key] = value / (samples.length - skipped)
}
result.count = samples.length
return result
}
}

module.exports = SampleBuffer
module.exports = SampleBuffer
21 changes: 19 additions & 2 deletions lib/runtimeSettings.js
Original file line number Diff line number Diff line change
Expand Up @@ -159,10 +159,26 @@ function getSettingsFile (settings) {

// all current drivers add settings.rootDir and settings.userDir
if (settings.rootDir) {
const uibRoot = path.join(settings.rootDir, settings.userDir, 'storage', 'uibuilder').split(path.sep).join(path.posix.sep)
const uibRoot = path.join(settings.storageDir, 'uibuilder').split(path.sep).join(path.posix.sep)
projectSettings.uibuilder = { uibRoot }
}

if (settings.settings?.httpStatic) {
// This is an array of httpStatic properties - however their path setting
// will currently be relative to cwd. For safety, map them to absolute paths
// and validate they are not traversing out of the storageDir
const httpStatic = []
settings.settings.httpStatic.forEach(staticSetting => {
staticSetting.path = path.normalize(path.join(settings.storageDir, staticSetting.path))
if (staticSetting.path.startsWith(settings.storageDir)) {
httpStatic.push(staticSetting)
}
})
if (httpStatic.length > 0) {
projectSettings.httpStatic = httpStatic
}
}

let contextStorage = ''
if (settings.fileStore?.url) {
// file nodes settings
Expand Down Expand Up @@ -358,7 +374,8 @@ module.exports = {
next()
},
httpAdminCookieOptions: ${JSON.stringify(httpAdminCookieOptions)},
${projectSettings.uibuilder ? 'uibuilder: ' + JSON.stringify(projectSettings.uibuilder) : ''}
${projectSettings.uibuilder ? 'uibuilder: ' + JSON.stringify(projectSettings.uibuilder) + ',' : ''}
${projectSettings.httpStatic ? 'httpStatic: ' + JSON.stringify(projectSettings.httpStatic) : ''}
}
`
return settingsTemplate
Expand Down
Loading

0 comments on commit 0c05f78

Please sign in to comment.