From e0179407e5b98b751c29df69db9f1fd1a26cbee5 Mon Sep 17 00:00:00 2001 From: Philip Heltweg Date: Tue, 6 Feb 2024 16:42:36 +0100 Subject: [PATCH 1/4] Moved changes to new branch Moved changes from old PR that is now out of sync with main (https://github.com/jvalue/jayvee/pull/478) to a fresh PR, based on latest main commit. Co-authored-by: OmarFourati --- libs/extensions/std/exec/src/extension.ts | 2 + .../src/local-file-extractor-executor.spec.ts | 131 ++++++++++++++++++ .../exec/src/local-file-extractor-executor.ts | 79 +++++++++++ .../invalid-file-not-found.jv | 18 +++ .../invalid-path-contains-traversal.jv | 18 +++ .../invalid-path-traversal.jv | 18 +++ .../local-file-test.csv | 9 ++ .../local-file-test.csv.license | 3 + .../valid-local-file.jv | 18 +++ .../blocktype-specific/property-assignment.ts | 28 ++++ .../builtin-blocktypes/LocalFileExtractor.jv | 22 +++ 11 files changed, 346 insertions(+) create mode 100644 libs/extensions/std/exec/src/local-file-extractor-executor.spec.ts create mode 100644 libs/extensions/std/exec/src/local-file-extractor-executor.ts create mode 100644 libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-file-not-found.jv create mode 100644 libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-contains-traversal.jv create mode 100644 libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal.jv create mode 100644 libs/extensions/std/exec/test/assets/local-file-extractor-executor/local-file-test.csv create mode 100644 libs/extensions/std/exec/test/assets/local-file-extractor-executor/local-file-test.csv.license create mode 100644 libs/extensions/std/exec/test/assets/local-file-extractor-executor/valid-local-file.jv create mode 100644 libs/language-server/src/stdlib/builtin-blocktypes/LocalFileExtractor.jv diff --git a/libs/extensions/std/exec/src/extension.ts b/libs/extensions/std/exec/src/extension.ts index e83192d4..86bf3526 100644 --- a/libs/extensions/std/exec/src/extension.ts +++ b/libs/extensions/std/exec/src/extension.ts @@ -13,6 +13,7 @@ import { ArchiveInterpreterExecutor } from './archive-interpreter-executor'; import { FilePickerExecutor } from './file-picker-executor'; import { GtfsRTInterpreterExecutor } from './gtfs-rt-interpreter-executor'; import { HttpExtractorExecutor } from './http-extractor-executor'; +import { LocalFileExtractorExecutor } from './local-file-extractor-executor'; import { TextFileInterpreterExecutor } from './text-file-interpreter-executor'; import { TextLineDeleterExecutor } from './text-line-deleter-executor'; import { TextRangeSelectorExecutor } from './text-range-selector-executor'; @@ -33,6 +34,7 @@ export class StdExecExtension implements JayveeExecExtension { ArchiveInterpreterExecutor, FilePickerExecutor, GtfsRTInterpreterExecutor, + LocalFileExtractorExecutor, ]; } } diff --git a/libs/extensions/std/exec/src/local-file-extractor-executor.spec.ts b/libs/extensions/std/exec/src/local-file-extractor-executor.spec.ts new file mode 100644 index 00000000..c3d4012f --- /dev/null +++ b/libs/extensions/std/exec/src/local-file-extractor-executor.spec.ts @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; +import * as nock from 'nock'; + +import { LocalFileExtractorExecutor } from './local-file-extractor-executor'; + +describe('Validation of LocalFileExtractorExecutor', () => { + let parse: (input: string) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../test/assets/local-file-extractor-executor/', + ); + + async function parseAndExecuteExecutor( + input: string, + ): Promise> { + const document = await parse(input); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new LocalFileExtractorExecutor().doExecute( + R.NONE, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + afterEach(() => { + nock.restore(); + }); + + beforeEach(() => { + if (!nock.isActive()) { + nock.activate(); + } + nock.cleanAll(); + }); + + it('should diagnose no error on valid local file path', async () => { + const text = readJvTestAsset('valid-local-file.jv'); + + const result = await parseAndExecuteExecutor(text); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right).toEqual( + expect.objectContaining({ + name: 'local-file-test.csv', + extension: 'csv', + ioType: IOType.FILE, + mimeType: R.MimeType.APPLICATION_OCTET_STREAM, + }), + ); + } + }); + + it('should diagnose error on file not found', async () => { + const text = readJvTestAsset('invalid-file-not-found.jv'); + + const result = await parseAndExecuteExecutor(text); + + expect(R.isErr(result)).toEqual(true); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + `File './does-not-exist.csv' not found.`, + ); + } + }); + + it('should diagnose error on path traversal restricted', async () => { + const text = readJvTestAsset('invalid-path-traversal.jv'); + + const result = await parseAndExecuteExecutor(text); + + expect(R.isErr(result)).toEqual(true); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + `File path cannot include "..". Path traversal is restricted.`, + ); + } + }); + + it('should diagnose error on path traversal restricted', async () => { + const text = readJvTestAsset('invalid-path-contains-traversal.jv'); + + const result = await parseAndExecuteExecutor(text); + + expect(R.isErr(result)).toEqual(true); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + `File path cannot include "..". Path traversal is restricted.`, + ); + } + }); +}); diff --git a/libs/extensions/std/exec/src/local-file-extractor-executor.ts b/libs/extensions/std/exec/src/local-file-extractor-executor.ts new file mode 100644 index 00000000..f27302fc --- /dev/null +++ b/libs/extensions/std/exec/src/local-file-extractor-executor.ts @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as fs from 'fs/promises'; +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { + AbstractBlockExecutor, + BinaryFile, + BlockExecutorClass, + ExecutionContext, + FileExtension, + MimeType, + None, + implementsStatic, + inferFileExtensionFromFileExtensionString, +} from '@jvalue/jayvee-execution'; +import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; + +@implementsStatic() +export class LocalFileExtractorExecutor extends AbstractBlockExecutor< + IOType.NONE, + IOType.FILE +> { + public static readonly type = 'LocalFileExtractor'; + + constructor() { + super(IOType.NONE, IOType.FILE); + } + + async doExecute( + input: None, + context: ExecutionContext, + ): Promise> { + const filePath = context.getPropertyValue( + 'filePath', + PrimitiveValuetypes.Text, + ); + + if (filePath.includes('..')) { + return R.err({ + message: 'File path cannot include "..". Path traversal is restricted.', + diagnostic: { node: context.getCurrentNode(), property: 'filePath' }, + }); + } + + try { + const rawData = await fs.readFile(filePath); + + // Infer FileName and FileExtension from filePath + const fileName = path.basename(filePath); + const extName = path.extname(fileName); + const fileExtension = + inferFileExtensionFromFileExtensionString(extName) ?? + FileExtension.NONE; + + // Infer Mimetype from FileExtension, if not inferrable, then default to application/octet-stream + const mimeType: MimeType | undefined = MimeType.APPLICATION_OCTET_STREAM; + + // Create file and return file + const file = new BinaryFile( + fileName, + fileExtension, + mimeType, + rawData.buffer as ArrayBuffer, + ); + + context.logger.logDebug(`Successfully extraced file ${filePath}`); + return R.ok(file); + } catch (error) { + return R.err({ + message: `File '${filePath}' not found.`, + diagnostic: { node: context.getCurrentNode(), property: 'filePath' }, + }); + } + } +} diff --git a/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-file-not-found.jv b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-file-not-found.jv new file mode 100644 index 00000000..6030d55a --- /dev/null +++ b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-file-not-found.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestFileExtractor { + } + + block TestBlock oftype LocalFileExtractor { + filePath: './does-not-exist.csv'; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-contains-traversal.jv b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-contains-traversal.jv new file mode 100644 index 00000000..e9447cda --- /dev/null +++ b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-contains-traversal.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestFileExtractor { + } + + block TestBlock oftype LocalFileExtractor { + filePath: './../jayvee/libs/extensions/std/exec/test/assets/local-file-extractor-executor/path-traversal-assets/path-traversal-file.csv'; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} \ No newline at end of file diff --git a/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal.jv b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal.jv new file mode 100644 index 00000000..7666bf5a --- /dev/null +++ b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestFileExtractor { + } + + block TestBlock oftype LocalFileExtractor { + filePath: '../jayvee/libs/extensions/std/exec/test/assets/local-file-extractor-executor/path-traversal-assets/path-traversal-file.csv'; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} \ No newline at end of file diff --git a/libs/extensions/std/exec/test/assets/local-file-extractor-executor/local-file-test.csv b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/local-file-test.csv new file mode 100644 index 00000000..e336857b --- /dev/null +++ b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/local-file-test.csv @@ -0,0 +1,9 @@ +HeaderExample1,HeaderExample2,HeaderExample3,HeaderExample4 +Example1,Example2,Example3,Example4 +Example1,Example2,Example3,Example4 +Example1,Example2,Example3,Example4 +Example1,Example2,Example3,Example4 +Example1,Example2,Example3,Example4 +Example1,Example2,Example3,Example4 +Example1,Example2,Example3,Example4 +Example1,Example2,Example3,Example4 \ No newline at end of file diff --git a/libs/extensions/std/exec/test/assets/local-file-extractor-executor/local-file-test.csv.license b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/local-file-test.csv.license new file mode 100644 index 00000000..42737858 --- /dev/null +++ b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/local-file-test.csv.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/libs/extensions/std/exec/test/assets/local-file-extractor-executor/valid-local-file.jv b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/valid-local-file.jv new file mode 100644 index 00000000..6ef89cfe --- /dev/null +++ b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/valid-local-file.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestFileExtractor { + } + + block TestBlock oftype LocalFileExtractor { + filePath: './libs/extensions/std/exec/test/assets/local-file-extractor-executor/local-file-test.csv'; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} \ No newline at end of file diff --git a/libs/language-server/src/lib/validation/checks/blocktype-specific/property-assignment.ts b/libs/language-server/src/lib/validation/checks/blocktype-specific/property-assignment.ts index d08bcd24..5e355d17 100644 --- a/libs/language-server/src/lib/validation/checks/blocktype-specific/property-assignment.ts +++ b/libs/language-server/src/lib/validation/checks/blocktype-specific/property-assignment.ts @@ -69,6 +69,13 @@ export function checkBlocktypeSpecificProperties( property, validationContext, ); + case 'LocalFileExtractor': + return checkLocalFileExtractorProperty( + propName, + propValue, + property, + validationContext, + ); case 'RowDeleter': return checkRowDeleterProperty( propName, @@ -250,6 +257,27 @@ function checkHttpExtractorProperty( } } +function checkLocalFileExtractorProperty( + propName: string, + propValue: InternalValueRepresentation, + property: PropertyAssignment, + validationContext: ValidationContext, +) { + if ( + propName === 'filePath' && + internalValueToString(propValue).includes('..') + ) { + validationContext.accept( + 'error', + 'File path cannot include "..". Path traversal is restricted.', + { + node: property, + property: 'value', + }, + ); + } +} + function checkRowDeleterProperty( propName: string, property: PropertyAssignment, diff --git a/libs/language-server/src/stdlib/builtin-blocktypes/LocalFileExtractor.jv b/libs/language-server/src/stdlib/builtin-blocktypes/LocalFileExtractor.jv new file mode 100644 index 00000000..b31641ae --- /dev/null +++ b/libs/language-server/src/stdlib/builtin-blocktypes/LocalFileExtractor.jv @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +/** +* Extracts a `File` from the local file system. +* +* @example Fetches a file from the given local path. +* block CarsFileExtractor oftype LocalFileExtractor { +* filePath: "cars.csv"; +* } +*/ +builtin blocktype LocalFileExtractor { + input default oftype None; + output default oftype File; + + /** + * The path to the file in the local file system to extract. + */ + property filePath oftype text; + +} \ No newline at end of file From ed92fe703d495cecf4ccff30d2186a1f55e5bc41 Mon Sep 17 00:00:00 2001 From: Philip Heltweg Date: Wed, 7 Feb 2024 11:24:07 +0100 Subject: [PATCH 2/4] feat: :sparkles: Correctly read file mime type from extensions --- .../std/exec/src/local-file-extractor-executor.spec.ts | 10 +++++----- .../std/exec/src/local-file-extractor-executor.ts | 5 ++++- ...traversal.jv => invalid-path-traversal-at-start.jv} | 2 +- ...-traversal.jv => invalid-path-traversal-in-path.jv} | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) rename libs/extensions/std/exec/test/assets/local-file-extractor-executor/{invalid-path-traversal.jv => invalid-path-traversal-at-start.jv} (71%) rename libs/extensions/std/exec/test/assets/local-file-extractor-executor/{invalid-path-contains-traversal.jv => invalid-path-traversal-in-path.jv} (71%) diff --git a/libs/extensions/std/exec/src/local-file-extractor-executor.spec.ts b/libs/extensions/std/exec/src/local-file-extractor-executor.spec.ts index c3d4012f..47a3d5e2 100644 --- a/libs/extensions/std/exec/src/local-file-extractor-executor.spec.ts +++ b/libs/extensions/std/exec/src/local-file-extractor-executor.spec.ts @@ -84,7 +84,7 @@ describe('Validation of LocalFileExtractorExecutor', () => { name: 'local-file-test.csv', extension: 'csv', ioType: IOType.FILE, - mimeType: R.MimeType.APPLICATION_OCTET_STREAM, + mimeType: R.MimeType.TEXT_CSV, }), ); } @@ -103,8 +103,8 @@ describe('Validation of LocalFileExtractorExecutor', () => { } }); - it('should diagnose error on path traversal restricted', async () => { - const text = readJvTestAsset('invalid-path-traversal.jv'); + it('should diagnose error on path traversal at the start of the path', async () => { + const text = readJvTestAsset('invalid-path-traversal-at-start.jv'); const result = await parseAndExecuteExecutor(text); @@ -116,8 +116,8 @@ describe('Validation of LocalFileExtractorExecutor', () => { } }); - it('should diagnose error on path traversal restricted', async () => { - const text = readJvTestAsset('invalid-path-contains-traversal.jv'); + it('should diagnose error on path traversal in the path', async () => { + const text = readJvTestAsset('invalid-path-traversal-in-path.jv'); const result = await parseAndExecuteExecutor(text); diff --git a/libs/extensions/std/exec/src/local-file-extractor-executor.ts b/libs/extensions/std/exec/src/local-file-extractor-executor.ts index f27302fc..04566390 100644 --- a/libs/extensions/std/exec/src/local-file-extractor-executor.ts +++ b/libs/extensions/std/exec/src/local-file-extractor-executor.ts @@ -16,6 +16,7 @@ import { None, implementsStatic, inferFileExtensionFromFileExtensionString, + inferMimeTypeFromFileExtensionString, } from '@jvalue/jayvee-execution'; import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; @@ -57,7 +58,9 @@ export class LocalFileExtractorExecutor extends AbstractBlockExecutor< FileExtension.NONE; // Infer Mimetype from FileExtension, if not inferrable, then default to application/octet-stream - const mimeType: MimeType | undefined = MimeType.APPLICATION_OCTET_STREAM; + const mimeType: MimeType | undefined = + inferMimeTypeFromFileExtensionString(fileExtension) ?? + MimeType.APPLICATION_OCTET_STREAM; // Create file and return file const file = new BinaryFile( diff --git a/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal.jv b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal-at-start.jv similarity index 71% rename from libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal.jv rename to libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal-at-start.jv index 7666bf5a..464751f1 100644 --- a/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal.jv +++ b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal-at-start.jv @@ -8,7 +8,7 @@ pipeline TestPipeline { } block TestBlock oftype LocalFileExtractor { - filePath: '../jayvee/libs/extensions/std/exec/test/assets/local-file-extractor-executor/path-traversal-assets/path-traversal-file.csv'; + filePath: '../non-existent-file.csv'; } block TestLoader oftype TestSheetLoader { diff --git a/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-contains-traversal.jv b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal-in-path.jv similarity index 71% rename from libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-contains-traversal.jv rename to libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal-in-path.jv index e9447cda..23a0aeed 100644 --- a/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-contains-traversal.jv +++ b/libs/extensions/std/exec/test/assets/local-file-extractor-executor/invalid-path-traversal-in-path.jv @@ -8,7 +8,7 @@ pipeline TestPipeline { } block TestBlock oftype LocalFileExtractor { - filePath: './../jayvee/libs/extensions/std/exec/test/assets/local-file-extractor-executor/path-traversal-assets/path-traversal-file.csv'; + filePath: './../non-existent-file.csv'; } block TestLoader oftype TestSheetLoader { From 81cd6cdb713c3ec3150c58f7ea751b06591707b3 Mon Sep 17 00:00:00 2001 From: Philip Heltweg Date: Wed, 7 Feb 2024 11:40:23 +0100 Subject: [PATCH 3/4] test: :white_check_mark: Added tests for property assignment of LocalFileExtractor --- .../property-assignment.spec.ts | 27 +++++++++++++++++++ .../invalid-invalid-filepath-param.jv | 17 ++++++++++++ .../valid-valid-filepath-param.jv | 17 ++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 libs/language-server/src/test/assets/property-assignment/blocktype-specific/local-file-extractor/invalid-invalid-filepath-param.jv create mode 100644 libs/language-server/src/test/assets/property-assignment/blocktype-specific/local-file-extractor/valid-valid-filepath-param.jv diff --git a/libs/language-server/src/lib/validation/checks/blocktype-specific/property-assignment.spec.ts b/libs/language-server/src/lib/validation/checks/blocktype-specific/property-assignment.spec.ts index 1b1d732b..7d379d0d 100644 --- a/libs/language-server/src/lib/validation/checks/blocktype-specific/property-assignment.spec.ts +++ b/libs/language-server/src/lib/validation/checks/blocktype-specific/property-assignment.spec.ts @@ -265,6 +265,33 @@ describe('Validation of blocktype specific properties', () => { }); }); + describe('LocalFileExtractor blocktype', () => { + it('should diagnose no error on valid filePath parameter value', async () => { + const text = readJvTestAsset( + 'property-assignment/blocktype-specific/local-file-extractor/valid-valid-filepath-param.jv', + ); + + await parseAndValidatePropertyAssignment(text); + + expect(validationAcceptorMock).toHaveBeenCalledTimes(0); + }); + + it('should diagnose error on invalid filePath parameter value', async () => { + const text = readJvTestAsset( + 'property-assignment/blocktype-specific/local-file-extractor/invalid-invalid-filepath-param.jv', + ); + + await parseAndValidatePropertyAssignment(text); + + expect(validationAcceptorMock).toHaveBeenCalledTimes(1); + expect(validationAcceptorMock).toHaveBeenCalledWith( + 'error', + 'File path cannot include "..". Path traversal is restricted.', + expect.any(Object), + ); + }); + }); + describe('RowDeleter blocktype', () => { it('should diagnose error on deleting partial row', async () => { const text = readJvTestAsset( diff --git a/libs/language-server/src/test/assets/property-assignment/blocktype-specific/local-file-extractor/invalid-invalid-filepath-param.jv b/libs/language-server/src/test/assets/property-assignment/blocktype-specific/local-file-extractor/invalid-invalid-filepath-param.jv new file mode 100644 index 00000000..0d7cd614 --- /dev/null +++ b/libs/language-server/src/test/assets/property-assignment/blocktype-specific/local-file-extractor/invalid-invalid-filepath-param.jv @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline Pipeline { + block Test oftype LocalFileExtractor { + filePath: './../invalid-path-traversal.csv'; + } + + block TestExtractor oftype TestFileExtractor { + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> Test -> TestLoader; +} diff --git a/libs/language-server/src/test/assets/property-assignment/blocktype-specific/local-file-extractor/valid-valid-filepath-param.jv b/libs/language-server/src/test/assets/property-assignment/blocktype-specific/local-file-extractor/valid-valid-filepath-param.jv new file mode 100644 index 00000000..affdbc18 --- /dev/null +++ b/libs/language-server/src/test/assets/property-assignment/blocktype-specific/local-file-extractor/valid-valid-filepath-param.jv @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline Pipeline { + block Test oftype LocalFileExtractor { + filePath: './valid-path-traversal.csv'; + } + + block TestExtractor oftype TestFileExtractor { + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> Test -> TestLoader; +} From 62a7c8e3df995a7f3a643a19dec1b23d76cc424c Mon Sep 17 00:00:00 2001 From: Philip Heltweg Date: Wed, 7 Feb 2024 11:41:02 +0100 Subject: [PATCH 4/4] docs: :memo: Updated documentation of LocalFileExtractor to call out forbidden path traversal --- .../src/stdlib/builtin-blocktypes/LocalFileExtractor.jv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/language-server/src/stdlib/builtin-blocktypes/LocalFileExtractor.jv b/libs/language-server/src/stdlib/builtin-blocktypes/LocalFileExtractor.jv index b31641ae..a4f7a3c5 100644 --- a/libs/language-server/src/stdlib/builtin-blocktypes/LocalFileExtractor.jv +++ b/libs/language-server/src/stdlib/builtin-blocktypes/LocalFileExtractor.jv @@ -5,7 +5,7 @@ /** * Extracts a `File` from the local file system. * -* @example Fetches a file from the given local path. +* @example Extracts a file from the given path on the local file system. * block CarsFileExtractor oftype LocalFileExtractor { * filePath: "cars.csv"; * } @@ -15,7 +15,7 @@ builtin blocktype LocalFileExtractor { output default oftype File; /** - * The path to the file in the local file system to extract. + * The path to the file in the local file system to extract. Path can not traverse up the directory tree. */ property filePath oftype text;