From 9f73090955542e47224ba25f2b106a762d011109 Mon Sep 17 00:00:00 2001 From: Alfonso Garcia-Caro Date: Mon, 22 Jun 2020 13:48:23 +0900 Subject: [PATCH 1/4] Add cache to fable-loader --- build.fsx | 6 +- src/Fable.Cli/ProjectCracker.fs | 2 +- src/fable-loader/.gitignore | 1 + src/fable-loader/.npmignore | 1 + src/fable-loader/index.js | 151 --------- src/fable-loader/package-lock.json | 63 +++- src/fable-loader/package.json | 4 +- src/fable-loader/src/index.ts | 307 ++++++++++++++++++ .../{net-client.js => src/tcpClient.ts} | 9 +- src/fable-loader/tsconfig.json | 58 ++++ src/fable-splitter/.npmignore | 2 +- 11 files changed, 428 insertions(+), 176 deletions(-) create mode 100644 src/fable-loader/.gitignore create mode 100644 src/fable-loader/.npmignore delete mode 100644 src/fable-loader/index.js create mode 100644 src/fable-loader/src/index.ts rename src/fable-loader/{net-client.js => src/tcpClient.ts} (70%) create mode 100644 src/fable-loader/tsconfig.json diff --git a/build.fsx b/build.fsx index 3576a653e7..2f866a8489 100644 --- a/build.fsx +++ b/build.fsx @@ -56,6 +56,9 @@ let buildTypescript projectDir = let buildFableSplitter() = buildTypescript "src/fable-splitter" +let buildFableLoader() = + buildTypescript "src/fable-loader" + let buildSplitterWithArgs projectDir args = if pathExists "src/fable-splitter/dist" |> not then buildFableSplitter() @@ -290,7 +293,7 @@ let packages = "fable-babel-plugins", doNothing "fable-compiler", buildCompiler "fable-compiler-js", buildCompilerJs - "fable-loader", doNothing + "fable-loader", buildFableLoader "fable-metadata", doNothing "fable-publish-utils", doNothing "fable-splitter", buildFableSplitter @@ -323,6 +326,7 @@ match argsLower with | ("fable-compiler"|"compiler")::_ -> buildCompiler() | ("fable-compiler-js"|"compiler-js")::_ -> buildCompilerJs() | ("fable-splitter"|"splitter")::_ -> buildFableSplitter() +| ("fable-loader"|"loader")::_ -> buildFableLoader() | ("fable-standalone"|"standalone")::_ -> buildStandalone() | "download-standalone"::_ -> downloadStandalone() | "publish"::restArgs -> publishPackages restArgs diff --git a/src/Fable.Cli/ProjectCracker.fs b/src/Fable.Cli/ProjectCracker.fs index 8b73d7e414..1f73b3f91e 100644 --- a/src/Fable.Cli/ProjectCracker.fs +++ b/src/Fable.Cli/ProjectCracker.fs @@ -357,7 +357,7 @@ let createFableDir rootDir = let fableDir = IO.Path.Combine(rootDir, Naming.fableHiddenDir) if isDirectoryEmpty fableDir then Directory.CreateDirectory(fableDir) |> ignore - File.WriteAllText(IO.Path.Combine(fableDir, ".gitignore"), "*.*") + File.WriteAllText(IO.Path.Combine(fableDir, ".gitignore"), "**/*") fableDir let copyDirIfDoesNotExist (source: string) (target: string) = diff --git a/src/fable-loader/.gitignore b/src/fable-loader/.gitignore new file mode 100644 index 0000000000..77738287f0 --- /dev/null +++ b/src/fable-loader/.gitignore @@ -0,0 +1 @@ +dist/ \ No newline at end of file diff --git a/src/fable-loader/.npmignore b/src/fable-loader/.npmignore new file mode 100644 index 0000000000..df6acf6d34 --- /dev/null +++ b/src/fable-loader/.npmignore @@ -0,0 +1 @@ +# Override .gitignore to include the `dist` folder \ No newline at end of file diff --git a/src/fable-loader/index.js b/src/fable-loader/index.js deleted file mode 100644 index 95a63302e1..0000000000 --- a/src/fable-loader/index.js +++ /dev/null @@ -1,151 +0,0 @@ -/// @ts-check - -var DEFAULT_COMPILER = "fable-compiler" -// var DEFAULT_COMPILER = require("../fable-compiler"); // testing - -var path = require("path"); -var babel = require("@babel/core"); -var babelPlugins = require("fable-babel-plugins"); - -function or(option, _default) { - return option !== void 0 ? option : _default; -} - -function ensureArray(obj) { - return Array.isArray(obj) ? obj : (obj != null ? [obj] : []); -} - -var customPlugins = [ - babelPlugins.getTransformMacroExpressions(babel.template) -]; - -var compilerCache = null; - -function log(opts, msg) { - if (!opts.silent) { - console.log(msg); - } -} - -function getTcpPort(opts) { - if (opts.port != null) { - return opts.port; - } else if (process.env.FABLE_SERVER_PORT != null) { - return parseInt(process.env.FABLE_SERVER_PORT, 10); - } else { - return null; - } -} - -function getCompiler(webpack, args, compiler) { - if (compilerCache == null) { - var fable = require(compiler); - compilerCache = fable.default(args); - if (!webpack.watchMode) { - webpack.hooks.done.tap("fable-loader", function() { - compilerCache.close(); - }); - } - } - return compilerCache; -} - -function transformBabelAst(babelAst, babelOptions, sourceMapOptions, callback) { - var fsCode = null; - if (sourceMapOptions != null) { - fsCode = sourceMapOptions.buffer.toString(); - babelOptions.sourceMaps = true; - var fileName = sourceMapOptions.path.replace(/\\/g, '/'); - babelOptions.filename = fileName; - babelOptions.sourceFileName = path.relative(process.cwd(), fileName); - } - babel.transformFromAst(babelAst, fsCode, babelOptions, callback); -} - -var Loader = function(buffer) { - var callback = this.async(); - - var opts = this.loaders[0].options || {}; - opts.cli = opts.cli || {}; - opts.cli.silent = opts.silent; - var babelOptions = opts.babel || {}; - babelOptions.plugins = customPlugins.concat(babelOptions.plugins || []); - - var define = ensureArray(or(opts.define, [])); - try { - if (this._compiler.options.mode === "development" && define.indexOf("DEBUG") === -1) { - define.push("DEBUG"); - } - } catch (er) {} - - var msg = { - path: this.resourcePath, - rootDir: process.cwd(), - define: define, - typedArrays: or(opts.typedArrays, true), - clampByteArrays: or(opts.clampByteArrays, false), - extra: opts.extra || {} - }; - - var port = getTcpPort(opts); - var command = port != null - ? require("./net-client").send("127.0.0.1", port, JSON.stringify(msg)).then(json => JSON.parse(json)) - : getCompiler(this._compiler, opts.cli, opts.compiler || DEFAULT_COMPILER).send(msg); - - command.then(data => { - if (data.error) { - callback(new Error(data.error)); - } - else { - try { - if (!msg.path.endsWith(".fsproj")) { - ensureArray(data.dependencies).forEach(p => { - // Fable normalizes path separator to '/' which causes issues in Windows - // Use `path.resolve` to restore the separator to the system default - this.addDependency(path.resolve(p)); - }); - } - if (typeof data.logs === "object") { - var isErrored = false; - Object.keys(data.logs).forEach(key => { - ensureArray(data.logs[key]).forEach(msg => { - switch (key) { - case "error": - isErrored = true; - this.emitError(new Error(msg)); - break; - case "warning": - this.emitWarning(new Error(msg)); - break; - default: - log(opts, msg) - } - }); - }); - this.cacheable(!isErrored); - } - var sourceMapOpts = this.sourceMap ? { - path: data.fileName, - buffer: buffer - } : null; - transformBabelAst(data, babelOptions, sourceMapOpts, function (err, babelParsed) { - if (err) { - callback(err); - } else { - log(opts, "fable: Compiled " + path.relative(process.cwd(), msg.path)); - callback(null, babelParsed.code, babelParsed.map); - } - }); - } - catch (err) { - callback(err) - } - } - }) - .catch(err => { - callback(new Error(err.message)) - }) -}; - -Loader.raw = true; -module.exports = Loader; \ No newline at end of file diff --git a/src/fable-loader/package-lock.json b/src/fable-loader/package-lock.json index 24a682f6cc..edef84a611 100644 --- a/src/fable-loader/package-lock.json +++ b/src/fable-loader/package-lock.json @@ -4,38 +4,69 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.10.3", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.3.tgz", + "integrity": "sha512-bU8JvtlYpJSBPuj1VUmKpFGaDZuLxASky3LhaKj3bmpSTY6VWooSM8msk+Z0CZoErFye2tlABF6yDkT3FOPAXw==", + "dev": true + }, "@babel/parser": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.1.6.tgz", - "integrity": "sha512-dWP6LJm9nKT6ALaa+bnL247GHHMWir3vSlZ2+IHgHgktZQx0L3Uvq2uAWcuzIe+fujRsYWBW2q622C5UvGK9iQ==", + "version": "7.10.3", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.10.3.tgz", + "integrity": "sha512-oJtNJCMFdIMwXGmx+KxuaD7i3b8uS7TTFYW/FNG2BT8m+fmGHoiPYoH0Pe3gya07WuFmM5FCDIr1x0irkD/hyA==", "dev": true }, "@babel/types": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.1.6.tgz", - "integrity": "sha512-DMiUzlY9DSjVsOylJssxLHSgj6tWM9PRFJOGW/RaOglVOK9nzTxoOMfTfRQXGUCUQ/HmlG2efwC+XqUEJ5ay4w==", + "version": "7.10.3", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.10.3.tgz", + "integrity": "sha512-nZxaJhBXBQ8HVoIcGsf9qWep3Oh3jCENK54V4mRF7qaJabVsAYdbTtmSD8WmAp1R6ytPiu5apMwSXyxB1WlaBA==", "dev": true, "requires": { - "esutils": "^2.0.2", - "lodash": "^4.17.10", + "@babel/helper-validator-identifier": "^7.10.3", + "lodash": "^4.17.13", "to-fast-properties": "^2.0.0" } }, "@types/babel__core": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.0.3.tgz", - "integrity": "sha512-F8E0lUeQ1uaprb5dEHLdOMElH5z+hk+L/DlQykXYOvhUyPQuH+Sj4Tm0sV3W0Za2sx1YkpdNyug7P2TNetWxKQ==", + "version": "7.1.9", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.9.tgz", + "integrity": "sha512-sY2RsIJ5rpER1u3/aQ8OFSI7qGIy8o1NEEbgb2UaJcvOtXOMpd39ko723NBpjQFg9SIX7TXtjejZVGeIMLhoOw==", "dev": true, "requires": { "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "@types/babel__generator": { + "version": "7.6.1", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.1.tgz", + "integrity": "sha512-bBKm+2VPJcMRVwNhxKu8W+5/zT7pwNEqeokFOmbvVSqGzFneNxYcEBro9Ac7/N9tlsaPYnZLK8J1LWKkMsLAew==", + "dev": true, + "requires": { "@babel/types": "^7.0.0" } }, - "esutils": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", - "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", - "dev": true + "@types/babel__template": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.0.2.tgz", + "integrity": "sha512-/K6zCpeW7Imzgab2bLkLEbz0+1JlFSrUMdw7KoIIu+IUdu51GWaBZpd3y1VXGVXzynvGa4DaIaxNZHiON3GXUg==", + "dev": true, + "requires": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "@types/babel__traverse": { + "version": "7.0.12", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.0.12.tgz", + "integrity": "sha512-t4CoEokHTfcyfb4hUaF9oOHu9RmmNWnm1CP0YmMqOOfClKascOmvlEM736vlqeScuGvBDsHkf8R2INd4DWreQA==", + "dev": true, + "requires": { + "@babel/types": "^7.3.0" + } }, "fable-babel-plugins": { "version": "2.3.0", diff --git a/src/fable-loader/package.json b/src/fable-loader/package.json index 3a2e72cefe..72cd0391e7 100644 --- a/src/fable-loader/package.json +++ b/src/fable-loader/package.json @@ -1,7 +1,7 @@ { "name": "fable-loader", "version": "2.1.9", - "main": "index.js", + "main": "dist/index.js", "description": "Webpack loader for Fable compiler", "keywords": [ "fable", @@ -25,7 +25,7 @@ "webpack": ">=4.23.0" }, "devDependencies": { - "@types/babel__core": "^7.0.3" + "@types/babel__core": "^7.1.6" }, "dependencies": { "fable-babel-plugins": "^2.3.0" diff --git a/src/fable-loader/src/index.ts b/src/fable-loader/src/index.ts new file mode 100644 index 0000000000..4f0e465810 --- /dev/null +++ b/src/fable-loader/src/index.ts @@ -0,0 +1,307 @@ +/// @ts-check + +const DEFAULT_COMPILER = "fable-compiler" +// const DEFAULT_COMPILER = require("../fable-compiler"); // testing + +import * as fs from "fs"; +import * as path from "path"; +import * as babel from "@babel/core"; +import * as babelPlugins from "fable-babel-plugins"; +import tcpClient from "./tcpClient"; + +let firstCompilationFinished = false; +const customPlugins: babel.PluginItem[] = [ + babelPlugins.getTransformMacroExpressions(babel.template) +]; + +interface Options { + cli?: any; + extra?: any; + babel?: babel.TransformOptions; + define?: string[]|string; + typedArrays?: boolean; + clampByteArrays?: boolean; + compiler?: string, + port?: number, + silent?: boolean, + cache?: boolean, + watch?: boolean, +} + +interface Compiler { + compile(req: CompilationRequest): Promise, +} + +interface CompilationRequest { + path: string, + rootDir: string, + define: string[], + typedArrays: boolean, + clampByteArrays: boolean, + extra: any +} + +interface CompilationResult { + fileName: string, + error?: string, + dependencies?: string[], + logs: { [key: string]: string[] } +} + +interface WebpackHelper { + addDependency(s: string): void + emitError(s: string): void, + emitWarning(s: string): void, + sourceMap?: boolean, + cacheable(b: boolean): void, + buffer: Buffer, + onCompiled(f: ()=>void): void +} + +type Option = T | undefined; +const None = undefined; + +interface FileCache { + getFile(path: string): Promise>, + setFile(path: string, content?: string): void +} + +function getTcpPort(opts: Options): Option { + if (opts.port != null) { + return opts.port; + } else if (process.env.FABLE_SERVER_PORT != null) { + return parseInt(process.env.FABLE_SERVER_PORT, 10); + } else { + return undefined; + } +} + +const getCompiler = (function() { + let compiler: Option = undefined; + return function(opts: Options, webpack: WebpackHelper): Compiler { + if (compiler == null) { + const port = getTcpPort(opts); + if (port != null) { + compiler = { + compile(req) { + return tcpClient("127.0.0.1", port, JSON.stringify(req)) + .then(json => JSON.parse(json)); + }, + } + } else { + const fableCompiler = require(opts.compiler ?? DEFAULT_COMPILER).default(opts.cli); + webpack.onCompiled(function() { + firstCompilationFinished = true; + if (!opts.watch) { + fableCompiler.close(); + } + }); + compiler = { + compile(req) { + return fableCompiler.send(req); + } + } + } + } + return compiler; + } +}()); + +const getFileCache = (function() { + let fileCache: Option = undefined; + return async function(options: Options): Promise { + if (options.watch && options.cache) { + if (fileCache == null) { + const hash = stringHash(JSON.stringify({ + client: "fable-loader", + compilerVersion: require("fable-compiler/package.json").version, + ...options + })).toString(16); + + const cacheDir = path.join(process.cwd(), ".fable", "cache", hash); + await createDirIfNotExists(cacheDir, () => + tryWriteFile(path.join(process.cwd(), ".fable", ".gitignore"), "**/*")); + + function toCachePath(filePath: string) { + filePath = path.resolve(filePath); // Normalize + const hash = stringHash(filePath).toString(16); + return path.join(cacheDir, hash + ".js"); + } + + fileCache = { + // Cache is only active during first watch compilation + getFile(path: string) { + return !path.endsWith(".fsproj") && !firstCompilationFinished + ? tryReadFile(toCachePath(path)) + : Promise.resolve(None); + }, + // Don't return the promise so we don't block compilation + setFile(path: string, content?: string) { + tryWriteFile(toCachePath(path), content); + } + }; + } + return fileCache + } else { + return { + getFile(_path) { return Promise.resolve(None) }, + setFile(_path, _content) { } + } + } + } +}()) + +function transformBabelAst(babelAst, babelOptions, sourceMapOptions): Promise { + let fsCode = undefined; + if (sourceMapOptions != null) { + fsCode = sourceMapOptions.buffer.toString(); + babelOptions.sourceMaps = true; + const fileName = sourceMapOptions.path.replace(/\\/g, '/'); + babelOptions.filename = fileName; + babelOptions.sourceFileName = path.relative(process.cwd(), fileName); + } + return new Promise(function(resolve, reject) { + babel.transformFromAst(babelAst, fsCode, babelOptions, function (err, result) { + err ? reject(err) : resolve(result ?? {}); + }); + }); +} + +async function compile(filePath: string, opts: Options, webpack: WebpackHelper) { + const fileCache = await getFileCache(opts); + const cachedFile = await fileCache.getFile(filePath); + if (cachedFile != null) { + log(opts, "fable: Cached " + path.relative(process.cwd(), filePath)); + return { code: cachedFile } + } + + const req: CompilationRequest = { + path: filePath, + rootDir: process.cwd(), + define: ensureArray(opts.define), + typedArrays: opts.typedArrays ?? false, + clampByteArrays: opts.clampByteArrays ?? false, + extra: opts.extra ?? {} + }; + const compiler = getCompiler(opts, webpack); + const data = await compiler.compile(req); + if (data.error) { + throw new Error(data.error); + } + + if (!req.path.endsWith(".fsproj")) { + ensureArray(data.dependencies).forEach(p => { + // Fable normalizes path separator to '/' which causes issues in Windows + // Use `path.resolve` to restore the separator to the system default + webpack.addDependency(path.resolve(p)); + }); + } + + let isErrored = false; + if (typeof data.logs === "object") { + Object.keys(data.logs).forEach(key => { + ensureArray(data.logs[key]).forEach(msg => { + switch (key) { + case "error": + isErrored = true; + webpack.emitError(msg); + break; + case "warning": + webpack.emitWarning(msg); + break; + default: + log(opts, msg) + } + }); + }); + } + webpack.cacheable(!isErrored); + + const babelParsed = await transformBabelAst(data, opts.babel, webpack.sourceMap ? { + path: data.fileName, + buffer: webpack.buffer + } : null); + + fileCache.setFile(req.path, babelParsed.code ?? undefined); + log(opts, "fable: Compiled " + path.relative(process.cwd(), req.path)); + return babelParsed; +} + +function Loader(buffer: Buffer) { + const callback = this.async(); + const webpackCompiler = this._compiler; + + const opts: Options = this.loaders[0].options ?? {}; + opts.cli = opts.cli ?? {}; + opts.cli.silent = opts.silent; + opts.babel = opts.babel ?? {}; + opts.babel.plugins = customPlugins.concat(opts.babel.plugins ?? []); + opts.extra = opts.extra ?? {} + opts.watch = webpackCompiler.watchMode; + opts.define = ensureArray(opts.define); + if (webpackCompiler?.options?.mode === "development" && opts.define.indexOf("DEBUG") === -1) { + opts.define.push("DEBUG"); + } + + const webpackHelper: WebpackHelper = { + addDependency: (s: string) => this.addDependency(s), + emitError: (s: string) => this.emitError(new Error(s)), + emitWarning: (s: string) => this.emitWarning(new Error(s)), + sourceMap: this.sourceMap, + cacheable: (b: boolean) => this.cacheable(b), + buffer: buffer, + onCompiled(f) { + webpackCompiler.hooks.done.tap("fable-loader", f); + } + } + + compile(this.resourcePath, opts, webpackHelper).then( + result => callback(null, result.code, result.map), + err => callback(err) + ); +}; + +Loader.raw = true; +export default Loader; + +// Helpers + +function ensureArray(x: Option) { + return x == null ? [] : Array.isArray(x) ? x : [x]; +} + +function log(opts: Options, msg: string) { + if (!opts.silent) { + console.log(msg); + } +} + +function createDirIfNotExists(dirPath: string, onCreation?: ()=>void): Promise { + return fs.promises.access(dirPath, fs.constants.F_OK).catch(_err => + fs.promises.mkdir(dirPath, { recursive: true }).then(() => { + if (onCreation != null) { + onCreation(); + } + })); +} + +function tryReadFile(filePath: string): Promise> { + return fs.promises.readFile(filePath).then(b => b.toString(), _err => None); +} + +function tryWriteFile(filePath: string, content?: string): void { + if (content != null) { + fs.promises.writeFile(filePath, content) + .catch(function (_err) {}); // Ignore errors + } +} + +function stringHash(str: string) { + let i = 0; + let h = 5381; + const len = str.length; + while (i < len) { + h = (h * 33) ^ str.charCodeAt(i++); + } + return h; + } \ No newline at end of file diff --git a/src/fable-loader/net-client.js b/src/fable-loader/src/tcpClient.ts similarity index 70% rename from src/fable-loader/net-client.js rename to src/fable-loader/src/tcpClient.ts index a62e977b3c..48a83e7f05 100644 --- a/src/fable-loader/net-client.js +++ b/src/fable-loader/src/tcpClient.ts @@ -1,9 +1,10 @@ -var net = require('net'); +import * as net from 'net'; -exports.send = function(host, port, msg) { +export default function send(host: string, port: number, msg: any): Promise { return new Promise((resolve, reject) => { - var buffer = ""; - var client = new net.Socket(), resolved = false; + let buffer = ""; + let resolved = false; + const client = new net.Socket(); client.connect(port, host, function() { client.write(msg); diff --git a/src/fable-loader/tsconfig.json b/src/fable-loader/tsconfig.json new file mode 100644 index 0000000000..05459f1848 --- /dev/null +++ b/src/fable-loader/tsconfig.json @@ -0,0 +1,58 @@ +{ + "include": [ + "src/**/*" + ], + "compilerOptions": { + /* Basic Options */ + "target": "es2015", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */ + "module": "commonjs", /* Specify module code generation: 'commonjs', 'amd', 'system', 'umd' or 'es2015'. */ + // "lib": [], /* Specify library files to be included in the compilation: */ + // "allowJs": true, /* Allow javascript files to be compiled. */ + // "checkJs": true, /* Report errors in .js files. */ + // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ + "declaration": true, /* Generates corresponding '.d.ts' file. */ + "sourceMap": false, /* Generates corresponding '.map' file. */ + "outDir": "./dist", /* Redirect output structure to the directory. */ + // "outFile": "./", /* Concatenate and emit output to single file. */ + // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ + // "removeComments": true, /* Do not emit comments to output. */ + // "noEmit": true, /* Do not emit outputs. */ + // "importHelpers": true, /* Import emit helpers from 'tslib'. */ + // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ + // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ + + /* Strict Type-Checking Options */ + "strict": true, /* Enable all strict type-checking options. */ + "noImplicitAny": false, /* Raise error on expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* Enable strict null checks. */ + "noImplicitThis": false, /* Raise error on 'this' expressions with an implied 'any' type. */ + // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ + + /* Additional Checks */ + "noUnusedLocals": true, /* Report errors on unused locals. */ + "noUnusedParameters": true, /* Report errors on unused parameters. */ + "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ + "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ + + /* Module Resolution Options */ + // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ + // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ + // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ + // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ + // "typeRoots": [], /* List of folders to include type definitions from. */ + // "types": [], /* Type declaration files to be included in compilation. */ + // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ + + /* Source Map Options */ + // "sourceRoot": "./", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ + // "mapRoot": "./", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ + // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ + + /* Experimental Options */ + // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ + // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ + "noEmitOnError": true, + "newLine": "LF" + } +} diff --git a/src/fable-splitter/.npmignore b/src/fable-splitter/.npmignore index 981aeb856a..df6acf6d34 100644 --- a/src/fable-splitter/.npmignore +++ b/src/fable-splitter/.npmignore @@ -1 +1 @@ -/out \ No newline at end of file +# Override .gitignore to include the `dist` folder \ No newline at end of file From 4b3acbf830c8ee686484153d3fcb9af948325174 Mon Sep 17 00:00:00 2001 From: Alfonso Garcia-Caro Date: Mon, 22 Jun 2020 23:33:46 +0900 Subject: [PATCH 2/4] Add noRestore option --- src/Fable.Cli/Agent.fs | 12 +++++++-- src/Fable.Cli/Parser.fs | 2 ++ src/Fable.Cli/ProjectCracker.fs | 43 ++++++++++++++++++++------------ src/Fable.Cli/Util.fs | 3 ++- src/fable-loader/src/index.ts | 36 +++++++++++--------------- src/quicktest/QuickTest.fsproj | 3 --- src/quicktest/splitter.config.js | 8 ++++-- 7 files changed, 61 insertions(+), 46 deletions(-) diff --git a/src/Fable.Cli/Agent.fs b/src/Fable.Cli/Agent.fs index 668adcbcd4..cc4f248997 100644 --- a/src/Fable.Cli/Agent.fs +++ b/src/Fable.Cli/Agent.fs @@ -133,7 +133,13 @@ let createProject (msg: Parser.Message) projFile (prevProject: ProjectExtra opti else proj | None -> let projectOptions, fableLibraryDir = - getFullProjectOpts msg.define msg.noReferences msg.rootDir projFile + getFullProjectOpts { + define = msg.define + noReferences = msg.noReferences + noRestore = msg.noRestore + rootDir = msg.rootDir + projFile = projFile + } Log.verbose(lazy let proj = getRelativePath projectOptions.ProjectFileName let opts = projectOptions.OtherOptions |> String.concat "\n " @@ -283,7 +289,9 @@ let startAgent () = MailboxProcessor.Start(fun agent -> Respond(res, msgHandler) |> agent.Post try let msg = Parser.parse msgHandler.Message - // lazy sprintf "Received message %A" msg |> Log.logVerbose + Log.verbose(lazy + if msg.path.EndsWith(".fsproj") then sprintf "Received %A" msg + else "") let newState, activeProject = updateState state msg let libDir = Path.getRelativePath msg.path activeProject.LibraryDir let com = Compiler(msg.path, activeProject.Project, Parser.toCompilerOptions msg, libDir) diff --git a/src/Fable.Cli/Parser.fs b/src/Fable.Cli/Parser.fs index 53d46c7b4a..f8518dc58e 100644 --- a/src/Fable.Cli/Parser.fs +++ b/src/Fable.Cli/Parser.fs @@ -10,6 +10,7 @@ type Message = rootDir: string define: string[] noReferences: bool + noRestore: bool typedArrays: bool clampByteArrays: bool classTypes: bool @@ -77,6 +78,7 @@ let parse (msg: string) = |> Array.append [|Naming.fableCompilerConstant|] |> Array.distinct noReferences = parseBoolean false "noReferences" json + noRestore = parseBoolean false "noRestore" json typedArrays = parseBoolean false "typedArrays" json clampByteArrays = parseBoolean false "clampByteArrays" json classTypes = parseBoolean false "classTypes" json diff --git a/src/Fable.Cli/ProjectCracker.fs b/src/Fable.Cli/ProjectCracker.fs index 1f73b3f91e..22c4b6fd02 100644 --- a/src/Fable.Cli/ProjectCracker.fs +++ b/src/Fable.Cli/ProjectCracker.fs @@ -11,6 +11,14 @@ open FSharp.Compiler.SourceCodeServices open Fable open Globbing.Operators +type Options = { + define: string[] + noReferences: bool + noRestore: bool + rootDir: string + projFile: string +} + let isSystemPackage (pkgName: string) = pkgName.StartsWith("System.") || pkgName.StartsWith("Microsoft.") @@ -234,12 +242,15 @@ let private isUsefulOption (opt : string) = /// and get F# compiler args from an .fsproj file. As we'll merge this /// later with other projects we'll only take the sources and the references, /// checking if some .dlls correspond to Fable libraries -let fullCrack noReferences (projFile: string): CrackedFsproj = +let fullCrack (opts: Options): CrackedFsproj = + let projFile = opts.projFile // Use case insensitive keys, as package names in .paket.resolved // may have a different case, see #1227 let dllRefs = Dictionary(StringComparer.OrdinalIgnoreCase) // Try restoring project - if projFile.EndsWith(".fsproj") then + if opts.noRestore then + Log.always "Skipping restore..." + else Process.runCmd Log.always (IO.Path.GetDirectoryName projFile) "dotnet" ["restore"; IO.Path.GetFileName projFile] @@ -265,7 +276,7 @@ let fullCrack noReferences (projFile: string): CrackedFsproj = else (Path.normalizeFullPath line)::src, otherOpts) let projRefs = - if noReferences then [] + if opts.noReferences then [] else projRefs |> List.choose (fun projRef -> // Remove dllRefs corresponding to project references @@ -307,7 +318,7 @@ let easyCrack (projFile: string): CrackedFsproj = PackageReferences = [] OtherCompilerOptions = [] } -let getCrackedProjectsFromMainFsproj noReferences (projFile: string) = +let getCrackedProjectsFromMainFsproj (opts: Options) = let rec crackProjects (acc: CrackedFsproj list) (projFile: string) = let crackedFsproj = match acc |> List.tryFind (fun x -> x.ProjectFile = projFile) with @@ -316,29 +327,29 @@ let getCrackedProjectsFromMainFsproj noReferences (projFile: string) = // Add always a reference to the front to preserve compilation order // Duplicated items will be removed later List.fold crackProjects (crackedFsproj::acc) crackedFsproj.ProjectReferences - let mainProj = fullCrack noReferences projFile + let mainProj = fullCrack opts let refProjs = List.fold crackProjects [] mainProj.ProjectReferences |> List.distinctBy (fun x -> x.ProjectFile) refProjs, mainProj -let getCrackedProjects define noReferences (projFile: string) = - match (Path.GetExtension projFile).ToLower() with +let getCrackedProjects (opts: Options) = + match (Path.GetExtension opts.projFile).ToLower() with | ".fsx" -> - getProjectOptionsFromScript define projFile + getProjectOptionsFromScript opts.define opts.projFile | ".fsproj" -> - getCrackedProjectsFromMainFsproj noReferences projFile + getCrackedProjectsFromMainFsproj opts | s -> failwithf "Unsupported project type: %s" s // It is common for editors with rich editing or 'intellisense' to also be watching the project // file for changes. In some cases that editor will lock the file which can cause fable to // get a read error. If that happens the lock is usually brief so we can reasonably wait // for it to be released. -let retryGetCrackedProjects define noReferences (projFile: string) = +let retryGetCrackedProjects opts = let retryUntil = (DateTime.Now + TimeSpan.FromSeconds 2.) let rec retry () = try - getCrackedProjects define noReferences projFile + getCrackedProjects opts with | :? IOException as ioex -> if retryUntil > DateTime.Now then @@ -398,13 +409,13 @@ let removeFilesInObjFolder sourceFiles = let reg = System.Text.RegularExpressions.Regex(@"[\\\/]obj[\\\/]") sourceFiles |> Array.filter (reg.IsMatch >> not) -let getFullProjectOpts (define: string[]) (noReferences: bool) (rootDir: string) (projFile: string) = - let projFile = Path.GetFullPath(projFile) +let getFullProjectOpts (opts: Options) = + let projFile = Path.GetFullPath(opts.projFile) if not(File.Exists(projFile)) then failwith ("File does not exist: " + projFile) - let projRefs, mainProj = retryGetCrackedProjects define noReferences projFile + let projRefs, mainProj = retryGetCrackedProjects opts let fableLibraryPath, pkgRefs = - copyFableLibraryAndPackageSources rootDir mainProj.PackageReferences + copyFableLibraryAndPackageSources opts.rootDir mainProj.PackageReferences let projOpts = let sourceFiles = let pkgSources = pkgRefs |> List.collect getSourcesFromFsproj @@ -433,7 +444,7 @@ let getFullProjectOpts (define: string[]) (noReferences: bool) (rootDir: string) mainProj.DllReferences |> List.mapToArray (fun r -> "-r:" + r) let otherOpts = mainProj.OtherCompilerOptions |> Array.ofList - [ getBasicCompilerArgs define + [ getBasicCompilerArgs opts.define otherOpts dllRefs ] |> Array.concat diff --git a/src/Fable.Cli/Util.fs b/src/Fable.Cli/Util.fs index 8ad669c240..0e50fb3ba2 100644 --- a/src/Fable.Cli/Util.fs +++ b/src/Fable.Cli/Util.fs @@ -125,7 +125,8 @@ module Log = let writerLock = obj() let always (msg: string) = - if GlobalParams.Singleton.Verbosity <> Fable.Verbosity.Silent then + if GlobalParams.Singleton.Verbosity <> Fable.Verbosity.Silent + && not(String.IsNullOrEmpty(msg)) then lock writerLock (fun () -> Console.Out.WriteLine(msg) Console.Out.Flush()) diff --git a/src/fable-loader/src/index.ts b/src/fable-loader/src/index.ts index 4f0e465810..8a163205d5 100644 --- a/src/fable-loader/src/index.ts +++ b/src/fable-loader/src/index.ts @@ -15,14 +15,13 @@ const customPlugins: babel.PluginItem[] = [ ]; interface Options { - cli?: any; + define?: string[]|string; extra?: any; + cli?: any; babel?: babel.TransformOptions; - define?: string[]|string; - typedArrays?: boolean; - clampByteArrays?: boolean; compiler?: string, port?: number, + verbose?: boolean, silent?: boolean, cache?: boolean, watch?: boolean, @@ -32,14 +31,10 @@ interface Compiler { compile(req: CompilationRequest): Promise, } -interface CompilationRequest { - path: string, - rootDir: string, - define: string[], - typedArrays: boolean, - clampByteArrays: boolean, - extra: any -} +type CompilationRequest = Options & { + path: string; + rootDir: string; +}; interface CompilationResult { fileName: string, @@ -175,14 +170,11 @@ async function compile(filePath: string, opts: Options, webpack: WebpackHelper) return { code: cachedFile } } - const req: CompilationRequest = { - path: filePath, - rootDir: process.cwd(), - define: ensureArray(opts.define), - typedArrays: opts.typedArrays ?? false, - clampByteArrays: opts.clampByteArrays ?? false, - extra: opts.extra ?? {} - }; + const req = Object.assign({}, + opts, + { path: filePath, rootDir: process.cwd() }, + { cli: undefined, babel: undefined } // Remove some unneeded options + ); const compiler = getCompiler(opts, webpack); const data = await compiler.compile(req); if (data.error) { @@ -233,10 +225,10 @@ function Loader(buffer: Buffer) { const opts: Options = this.loaders[0].options ?? {}; opts.cli = opts.cli ?? {}; - opts.cli.silent = opts.silent; + opts.cli.verbose = opts.cli.verbose ?? opts.verbose; + opts.cli.silent = opts.cli.silent ?? opts.silent; opts.babel = opts.babel ?? {}; opts.babel.plugins = customPlugins.concat(opts.babel.plugins ?? []); - opts.extra = opts.extra ?? {} opts.watch = webpackCompiler.watchMode; opts.define = ensureArray(opts.define); if (webpackCompiler?.options?.mode === "development" && opts.define.indexOf("DEBUG") === -1) { diff --git a/src/quicktest/QuickTest.fsproj b/src/quicktest/QuickTest.fsproj index 40a004dc99..51c47f03b4 100644 --- a/src/quicktest/QuickTest.fsproj +++ b/src/quicktest/QuickTest.fsproj @@ -5,9 +5,6 @@ Major preview - - - diff --git a/src/quicktest/splitter.config.js b/src/quicktest/splitter.config.js index be1f735677..7ba7fd4e36 100644 --- a/src/quicktest/splitter.config.js +++ b/src/quicktest/splitter.config.js @@ -1,12 +1,16 @@ const path = require("path"); module.exports = { - noReferences: true, - cli: { path: resolve("../Fable.Cli") }, + cli: { + // verbose: true, + path: resolve("../Fable.Cli") + }, entry: resolve("QuickTest.fsproj"), // outDir: resolve("temp"), // port: 61225, fable: { + noRestore: true, + noReferences: true, define: [] //["DEBUG"] }, babel: { From d4a0d83b1c555f040efaf3ed30a2204d53bdd383 Mon Sep 17 00:00:00 2001 From: Alfonso Garcia-Caro Date: Tue, 23 Jun 2020 12:51:17 +0900 Subject: [PATCH 3/4] Don't block agent when parsing project --- src/Fable.Cli/Agent.fs | 344 +++++++++++++++------------ src/Fable.Cli/ProjectCracker.fs | 7 +- src/Fable.Cli/Util.fs | 3 + src/Fable.Transforms/Replacements.fs | 2 +- src/Fable.Transforms/State.fs | 2 +- src/fable-loader/src/index.ts | 4 +- 6 files changed, 207 insertions(+), 155 deletions(-) diff --git a/src/Fable.Cli/Agent.fs b/src/Fable.Cli/Agent.fs index cc4f248997..2f10dcbdb9 100644 --- a/src/Fable.Cli/Agent.fs +++ b/src/Fable.Cli/Agent.fs @@ -1,4 +1,4 @@ -module Fable.Cli.Agent +module rec Fable.Cli.Agent open Fable open Fable.AST @@ -10,50 +10,77 @@ open FSharp.Compiler.SourceCodeServices open Newtonsoft.Json open ProjectCracker -type File(normalizedFullPath: string) = - let mutable sourceHash = None - member __.NormalizedFullPath = normalizedFullPath - member __.ReadSource() = - match sourceHash with - | Some h -> h, lazy File.readAllTextNonBlocking normalizedFullPath +let agentBody (agent: MailboxProcessor) = + let jsonSettings = + JsonSerializerSettings( + Converters=[|Json.ErasedUnionConverter()|], + ContractResolver=Serialization.CamelCasePropertyNamesContractResolver(), + NullValueHandling=NullValueHandling.Ignore) + // StringEscapeHandling=StringEscapeHandling.EscapeNonAscii) + + let rec loop (state: Map) = async { + match! agent.Receive() with + | Parsed(projFile, proj, checker) -> + match Map.tryFind projFile state with | None -> - let source = File.readAllTextNonBlocking normalizedFullPath - let h = hash source - sourceHash <- Some h - h, lazy source + Log.always("Project parsed but proj file not found in state: " + projFile) + return! loop state + | Some projWrapper -> + let stackedMessages, projWrapper = projWrapper.WithParsedProject(proj, checker) + for msg in List.rev stackedMessages do + Received msg |> agent.Post + return! addOrUpdateProject state projWrapper |> fst |> loop -/// Fable.Transforms.State.Project plus some properties only used here -type ProjectExtra(project: Project, checker: InteractiveChecker, - sourceFiles: File array, triggerFile: string, - fableLibraryDir: string) = - let timestamp = DateTime.Now - member __.TimeStamp = timestamp - member __.Checker = checker - member __.Project = project - member __.ProjectOptions = project.ProjectOptions - member __.ImplementationFiles = project.ImplementationFiles - member __.Errors = project.Errors - member __.ProjectFile = project.ProjectFile - member __.SourceFiles = sourceFiles - member __.TriggerFile = triggerFile - member __.LibraryDir = fableLibraryDir - member __.ContainsFile(file) = - sourceFiles |> Array.exists (fun file2 -> - file = file2.NormalizedFullPath) - static member Create checker sourceFiles triggerFile fableLibraryDir project = - ProjectExtra(project, checker, sourceFiles, triggerFile, fableLibraryDir) + | Respond(value, msgHandler) -> + msgHandler.Respond(fun writer -> + // CloseOutput=false is necessary to prevent closing the underlying stream + use jsonWriter = new JsonTextWriter(writer, CloseOutput=false) + let serializer = JsonSerializer.Create(jsonSettings) + serializer.Serialize(jsonWriter, value)) + return! loop state -let getSourceFiles (opts: FSharpProjectOptions) = - opts.OtherOptions |> Array.choose (fun path -> - if not(path.StartsWith("-")) then - // These should be already normalized, but just in case - // TODO: We should add a NormalizedFullPath type so we don't need normalize everywhere - Path.normalizeFullPath path |> Some - else None) + | Received msgHandler -> + let respond(res: obj) = + Respond(res, msgHandler) |> agent.Post + try + let msg = Parser.parse msgHandler.Message + Log.verbose(lazy + if msg.path.EndsWith(".fsproj") then sprintf "Received %A" msg + else "") + let state, (projWrapper: ProjectWrapper) = updateState state msg + match projWrapper.ProjectStatus with + | ParsedProject(proj,_,_) -> + let com = projWrapper.MakeCompiler(msg.path, proj) + addFSharpErrorLogs com proj msg.path + startCompilation respond com proj + return! loop state + | ParsingProject _ -> + if msg.path.EndsWith(".fsproj") then + // Quickly respond with a façade file so we don't block the client and cache can be activated + let sourceFiles = getSourceFiles projWrapper.ProjectOptions + Fable2Babel.Compiler.createFacade sourceFiles msg.path |> respond + return! loop state + else + return! projWrapper.StackMessage(msgHandler) + |> addOrUpdateProject state + |> fst |> loop + with ex -> + sendError respond ex + return! loop state + } + + loop Map.empty + +let private agent = new MailboxProcessor(agentBody) + +let startAgent() = agent.Start(); agent let getRelativePath path = Path.getRelativePath (IO.Directory.GetCurrentDirectory()) path +let getSourceFiles (opts: FSharpProjectOptions) = + opts.OtherOptions |> Array.filter (fun path -> path.StartsWith("-") |> not) + let hasFlag flagName (opts: IDictionary) = match opts.TryGetValue(flagName) with | true, value -> @@ -82,56 +109,94 @@ let checkFableCoreVersion (checkedProject: FSharpCheckProjectResults) = failwithf "Fable.Core v%i.%i detected, expecting v%i.%i" actualMajor actualMinor expectedMajor expectedMinor // else printfn "Fable.Core version matches" -let checkProject (msg: Parser.Message) - (opts: FSharpProjectOptions) - (fableLibraryDir: string) - (triggerFile: string) - (srcFiles: File[]) - (checker: InteractiveChecker) = - Log.always(sprintf "Parsing %s..." (getRelativePath opts.ProjectFileName)) - let checkedProject = - let fileDic = srcFiles |> Seq.map (fun f -> f.NormalizedFullPath, f) |> dict - let sourceReader f = fileDic.[f].ReadSource() - let filePaths = srcFiles |> Array.map (fun file -> file.NormalizedFullPath) - checker.ParseAndCheckProject(opts.ProjectFileName, filePaths, sourceReader) - // checkFableCoreVersion checkedProject - let optimized = GlobalParams.Singleton.Experimental.Contains("optimize-fcs") - let implFiles = - if not optimized - then checkedProject.AssemblyContents.ImplementationFiles - else checkedProject.GetOptimizedAssemblyContents().ImplementationFiles - if List.isEmpty implFiles then - Log.always "The list of files returned by F# compiler is empty" - let implFilesMap = - implFiles |> Seq.map (fun file -> Path.normalizePathAndEnsureFsExtension file.FileName, file) |> dict - tryGetOption "saveAst" msg.extra |> Option.iter (fun outDir -> - Printers.printAst outDir implFiles) - Project(opts, implFilesMap, checkedProject.Errors) - |> ProjectExtra.Create checker srcFiles triggerFile fableLibraryDir - -let createProject (msg: Parser.Message) projFile (prevProject: ProjectExtra option) = - match prevProject with - | Some proj -> - let mutable someDirtyFiles = false - // If now - proj.TimeStamp < 1s skip checking the lastwritetime for performance - if DateTime.Now - proj.TimeStamp < TimeSpan.FromSeconds(1.) then - proj - else - let sourceFiles = - proj.SourceFiles - |> Array.map (fun file -> - let path = file.NormalizedFullPath - // Assume files in .fable folder are stable - if path.Contains(".fable/") then file - else - let isDirty = IO.File.GetLastWriteTime(path) > proj.TimeStamp - someDirtyFiles <- someDirtyFiles || isDirty - if isDirty then File(path) // Clear the cached source hash - else file) - if someDirtyFiles then - checkProject msg proj.ProjectOptions proj.LibraryDir msg.path sourceFiles proj.Checker - else proj - | None -> +// TODO: Check the path is actually normalized? +type File(normalizedFullPath: string) = + let mutable sourceHash = None + member _.NormalizedFullPath = normalizedFullPath + member _.ReadSource() = + match sourceHash with + | Some h -> h, lazy File.readAllTextNonBlocking normalizedFullPath + | None -> + let source = File.readAllTextNonBlocking normalizedFullPath + let h = hash source + sourceHash <- Some h + h, lazy source + +type ProjectStatus = + | ParsedProject of Project * InteractiveChecker * DateTime + | ParsingProject of msgStack: IMessageHandler list + +type CompilerFactory(compilerOptions, fableLibraryDir) = + member _.Make(currentFile, project) = + let fableLibraryDir = Path.getRelativePath currentFile fableLibraryDir + Compiler(currentFile, project, compilerOptions, fableLibraryDir) + +type ProjectWrapper(sourceFiles: File array, + fsharpProjOptions: FSharpProjectOptions, + compilerFactory: CompilerFactory, + projectStatus: ProjectStatus) = + member _.ProjectStatus = projectStatus + member _.ProjectFile = fsharpProjOptions.ProjectFileName + member _.ProjectOptions = fsharpProjOptions + member _.SourceFiles = sourceFiles + + member _.MakeCompiler(currentFile, project) = + compilerFactory.Make(currentFile, project) + + member _.ContainsFile(file) = + sourceFiles |> Array.exists (fun file2 -> + file = file2.NormalizedFullPath) + + member _.ResetSourceFiles(sourceFiles) = + ProjectWrapper(sourceFiles, fsharpProjOptions, compilerFactory, ParsingProject []) + + member _.WithParsedProject(project, checker) = + let stackedMessages = + match projectStatus with + | ParsingProject stack -> stack + | _ -> [] + let proj = ParsedProject(project, checker, DateTime.Now) + stackedMessages, ProjectWrapper(sourceFiles, fsharpProjOptions, compilerFactory, proj) + + member this.StackMessage(msgHandler) = + match projectStatus with + | ParsingProject stack -> + let proj = ParsingProject(msgHandler::stack) + ProjectWrapper(sourceFiles, fsharpProjOptions, compilerFactory, proj) + | _ -> this + + member this.ParseProject(?checker, ?saveAstDir: string) = + Log.always(sprintf "Parsing %s..." (getRelativePath this.ProjectFile)) + Async.Start <| async { + let checker = + match checker with + | Some checker -> checker + | None -> InteractiveChecker.Create(this.ProjectOptions) + let checkedProject = + let fileDic = this.SourceFiles |> Seq.map (fun f -> f.NormalizedFullPath, f) |> dict + let sourceReader f = fileDic.[f].ReadSource() + let filePaths = this.SourceFiles |> Array.map (fun file -> file.NormalizedFullPath) + checker.ParseAndCheckProject(this.ProjectFile, filePaths, sourceReader) + // checkFableCoreVersion checkedProject + let optimized = GlobalParams.Singleton.Experimental.Contains("optimize-fcs") + let implFiles = + if not optimized + then checkedProject.AssemblyContents.ImplementationFiles + else checkedProject.GetOptimizedAssemblyContents().ImplementationFiles + match implFiles, saveAstDir with + | [], _ -> Log.always "The list of files returned by F# compiler is empty" + | _, Some saveAstDir -> Printers.printAst saveAstDir implFiles + | _ -> () + let implFilesMap = + implFiles + |> Seq.map (fun file -> Path.normalizePathAndEnsureFsExtension file.FileName, file) + |> dict + let proj = Project(this.ProjectOptions, implFilesMap, checkedProject.Errors) + Parsed(this.ProjectFile, proj, checker) |> agent.Post + } + this + + static member FromMessage(projFile: string, msg: Parser.Message) = let projectOptions, fableLibraryDir = getFullProjectOpts { define = msg.define @@ -141,19 +206,39 @@ let createProject (msg: Parser.Message) projFile (prevProject: ProjectExtra opti projFile = projFile } Log.verbose(lazy - let proj = getRelativePath projectOptions.ProjectFileName + let proj = getRelativePath projFile let opts = projectOptions.OtherOptions |> String.concat "\n " sprintf "F# PROJECT: %s\n %s" proj opts) let sourceFiles = getSourceFiles projectOptions |> Array.map File - InteractiveChecker.Create(projectOptions) - |> checkProject msg projectOptions fableLibraryDir projFile sourceFiles + let compilerFactory = CompilerFactory(Parser.toCompilerOptions msg, fableLibraryDir) + ProjectWrapper(sourceFiles, projectOptions, compilerFactory, ParsingProject []) -let jsonSettings = - JsonSerializerSettings( - Converters=[|Json.ErasedUnionConverter()|], - ContractResolver=Serialization.CamelCasePropertyNamesContractResolver(), - NullValueHandling=NullValueHandling.Ignore) - // StringEscapeHandling=StringEscapeHandling.EscapeNonAscii) +let createProject (msg: Parser.Message) projFile (prevProject: ProjectWrapper option) = + match prevProject with + | Some projWrapper -> + match projWrapper.ProjectStatus with + // Ignore requests when the project is still parsing, parsed less than a second ago or doesn't have dirty files + | ParsedProject(_, checker, timestamp) when DateTime.Now - timestamp >= TimeSpan.FromSeconds(1.) -> + let mutable someDirtyFiles = false + let sourceFiles = + projWrapper.SourceFiles + |> Array.map (fun file -> + let path = file.NormalizedFullPath + // Assume files in .fable folder are stable + if Naming.isInFableHiddenDir path then file + else + let isDirty = IO.File.GetLastWriteTime(path) > timestamp + someDirtyFiles <- someDirtyFiles || isDirty + if isDirty then File(path) // Clear the cached source hash + else file) + if someDirtyFiles then + projWrapper.ResetSourceFiles(sourceFiles) + .ParseProject(checker) + else projWrapper + | _ -> projWrapper + | None -> + ProjectWrapper.FromMessage(projFile, msg) + .ParseProject(?saveAstDir=tryGetOption "saveAst" msg.extra) let sendError (respond: obj->unit) (ex: Exception) = let rec innerStack (ex: Exception) = @@ -174,7 +259,7 @@ let findFsprojUpwards originalFile = | _ -> failwithf "Found more than one project file for %s, please disambiguate." originalFile IO.Path.GetDirectoryName(originalFile) |> innerLoop -let addOrUpdateProject state (project: ProjectExtra) = +let addOrUpdateProject state (project: ProjectWrapper) = let state = Map.add project.ProjectFile project state state, project @@ -189,10 +274,9 @@ let tryFindAndUpdateProject onNotFound state (msg: Parser.Message) sourceFile = // disambiguate files referenced by several projects, see #1116 match msg.extra.TryGetValue("projectFile") with | true, projFile -> - let projFile = Path.normalizeFullPath projFile checkIfProjectIsAlreadyInState state msg projFile | false, _ -> - state |> Map.tryPick (fun _ (project: ProjectExtra) -> + state |> Map.tryPick (fun _ (project: ProjectWrapper) -> if project.ContainsFile(sourceFile) then Some project else None) @@ -202,7 +286,7 @@ let tryFindAndUpdateProject onNotFound state (msg: Parser.Message) sourceFile = |> addOrUpdateProject state | None -> onNotFound() -let updateState (state: Map) (msg: Parser.Message) = +let updateState (state: Map) (msg: Parser.Message) = match IO.Path.GetExtension(msg.path).ToLower() with | ".fsproj" -> createProject msg msg.path None @@ -227,20 +311,20 @@ let updateState (state: Map) (msg: Parser.Message) = failwithf "Signature files cannot be compiled to JS: %s" msg.path | _ -> failwithf "Not an F# source file: %s" msg.path -let addFSharpErrorLogs (com: ICompiler) (proj: ProjectExtra) = +let addFSharpErrorLogs (com: ICompiler) (proj: Project) (triggerFile: string) = proj.Errors |> Seq.filter (fun er -> // Report warnings always in the corresponding file // but ignore those from packages in `.fable` folder if er.Severity = FSharpErrorSeverity.Warning then com.CurrentFile = er.FileName && not(Naming.isInFableHiddenDir er.FileName) // For errors, if the trigger is the .fsproj (first compilation), report them in the corresponding file - elif proj.TriggerFile.EndsWith(".fsproj") then + elif triggerFile.EndsWith(".fsproj") then com.CurrentFile = er.FileName - // If another file triggers the compilation report errors there so they don't go missing + // If another file triggers the compilation, report errors there so they don't go missing // But ignore errors from packages in `.fable` folder, as this is watch mode and users can't do anything // See https://github.com/fable-compiler/Fable/pull/1714#issuecomment-463137486 else - com.CurrentFile = proj.TriggerFile && not(Naming.isInFableHiddenDir er.FileName)) + com.CurrentFile = triggerFile && not(Naming.isInFableHiddenDir er.FileName)) |> Seq.map (fun er -> let severity = match er.Severity with @@ -256,51 +340,15 @@ let addFSharpErrorLogs (com: ICompiler) (proj: ProjectExtra) = com.AddLog(msg, severity, range, fileName, "FSHARP")) /// Don't await file compilation to let the agent receive more requests to implement files. -let startCompilation (respond: obj->unit) (com: Compiler) (project: ProjectExtra) = +let startCompilation (respond: obj->unit) (com: Compiler) (project: Project) = async { try - if com.CurrentFile.EndsWith(".fsproj") then - // If we compile the last file here, Webpack watcher will ignore changes in it - Fable2Babel.Compiler.createFacade (getSourceFiles project.ProjectOptions) com.CurrentFile - |> respond - else - let babel = - FSharp2Fable.Compiler.transformFile com project.ImplementationFiles - |> FableTransforms.transformFile com - |> Fable2Babel.Compiler.transformFile com - Babel.Program(babel.FileName, babel.Body, babel.Directives, com.GetFormattedLogs(), babel.Dependencies) - |> respond + let babel = + FSharp2Fable.Compiler.transformFile com project.ImplementationFiles + |> FableTransforms.transformFile com + |> Fable2Babel.Compiler.transformFile com + Babel.Program(babel.FileName, babel.Body, babel.Directives, com.GetFormattedLogs(), babel.Dependencies) + |> respond with ex -> sendError respond ex } |> Async.Start - -let startAgent () = MailboxProcessor.Start(fun agent -> - let rec loop (state: Map) = async { - match! agent.Receive() with - | Respond(value, msgHandler) -> - msgHandler.Respond(fun writer -> - // CloseOutput=false is necessary to prevent closing the underlying stream - use jsonWriter = new JsonTextWriter(writer, CloseOutput=false) - let serializer = JsonSerializer.Create(jsonSettings) - serializer.Serialize(jsonWriter, value)) - return! loop state - | Received msgHandler -> - let respond(res: obj) = - Respond(res, msgHandler) |> agent.Post - try - let msg = Parser.parse msgHandler.Message - Log.verbose(lazy - if msg.path.EndsWith(".fsproj") then sprintf "Received %A" msg - else "") - let newState, activeProject = updateState state msg - let libDir = Path.getRelativePath msg.path activeProject.LibraryDir - let com = Compiler(msg.path, activeProject.Project, Parser.toCompilerOptions msg, libDir) - addFSharpErrorLogs com activeProject - startCompilation respond com activeProject - return! loop newState - with ex -> - sendError respond ex - return! loop state - } - loop Map.empty - ) diff --git a/src/Fable.Cli/ProjectCracker.fs b/src/Fable.Cli/ProjectCracker.fs index 22c4b6fd02..2bd84895cc 100644 --- a/src/Fable.Cli/ProjectCracker.fs +++ b/src/Fable.Cli/ProjectCracker.fs @@ -410,9 +410,8 @@ let removeFilesInObjFolder sourceFiles = sourceFiles |> Array.filter (reg.IsMatch >> not) let getFullProjectOpts (opts: Options) = - let projFile = Path.GetFullPath(opts.projFile) - if not(File.Exists(projFile)) then - failwith ("File does not exist: " + projFile) + if not(File.Exists(opts.projFile)) then + failwith ("File does not exist: " + opts.projFile) let projRefs, mainProj = retryGetCrackedProjects opts let fableLibraryPath, pkgRefs = copyFableLibraryAndPackageSources opts.rootDir mainProj.PackageReferences @@ -448,5 +447,5 @@ let getFullProjectOpts (opts: Options) = otherOpts dllRefs ] |> Array.concat - makeProjectOptions projFile sourceFiles otherOptions + makeProjectOptions opts.projFile sourceFiles otherOptions projOpts, fableLibraryPath diff --git a/src/Fable.Cli/Util.fs b/src/Fable.Cli/Util.fs index 0e50fb3ba2..cbc7973078 100644 --- a/src/Fable.Cli/Util.fs +++ b/src/Fable.Cli/Util.fs @@ -55,6 +55,9 @@ type IMessageHandler = abstract Respond: write: (TextWriter->unit) -> unit type AgentMsg = + | Parsed of projectFile: string + * Fable.Transforms.State.Project + * FSharp.Compiler.SourceCodeServices.InteractiveChecker | Received of handler: IMessageHandler | Respond of response: obj * handler: IMessageHandler diff --git a/src/Fable.Transforms/Replacements.fs b/src/Fable.Transforms/Replacements.fs index 62ca46de01..d24f0a2ef6 100644 --- a/src/Fable.Transforms/Replacements.fs +++ b/src/Fable.Transforms/Replacements.fs @@ -719,7 +719,7 @@ let isCompatibleWithJsComparison = function // * `LanguagePrimitive.PhysicalHash` creates an identity hash no matter whether GetHashCode is implemented or not. let identityHash r (arg: Expr) = match arg.Type with - | Boolean | Char | String | Number _ | Enum _ | Option | Tuple | List + | Boolean | Char | String | Number _ | Enum _ | Option _ | Tuple _ | List _ | Builtin(BclInt64 | BclUInt64 | BclDecimal | BclBigInt) | Builtin(BclGuid | BclTimeSpan | BclDateTime | BclDateTimeOffset) | Builtin(FSharpSet _ | FSharpMap _ | FSharpChoice _ | FSharpResult _) -> diff --git a/src/Fable.Transforms/State.fs b/src/Fable.Transforms/State.fs index fded469a62..eaa2fba98b 100644 --- a/src/Fable.Transforms/State.fs +++ b/src/Fable.Transforms/State.fs @@ -24,7 +24,7 @@ open System.Collections.Concurrent type Project(projectOptions: FSharpProjectOptions, implFiles: IDictionary, errors: FSharpErrorInfo array) = - let projectFile = Path.normalizePath projectOptions.ProjectFileName + let projectFile = projectOptions.ProjectFileName let inlineExprs = ConcurrentDictionary() let rootModules = implFiles |> Seq.map (fun kv -> diff --git a/src/fable-loader/src/index.ts b/src/fable-loader/src/index.ts index 8a163205d5..4344a59b0c 100644 --- a/src/fable-loader/src/index.ts +++ b/src/fable-loader/src/index.ts @@ -166,7 +166,9 @@ async function compile(filePath: string, opts: Options, webpack: WebpackHelper) const fileCache = await getFileCache(opts); const cachedFile = await fileCache.getFile(filePath); if (cachedFile != null) { - log(opts, "fable: Cached " + path.relative(process.cwd(), filePath)); + if (opts.verbose) { + log(opts, "fable: Cached " + path.relative(process.cwd(), filePath)); + } return { code: cachedFile } } From 43943fcbdb46d20e6a4f70050724ca1faf2d756f Mon Sep 17 00:00:00 2001 From: Alfonso Garcia-Caro Date: Tue, 23 Jun 2020 23:51:09 +0900 Subject: [PATCH 4/4] Some fixes --- src/fable-compiler/src/index.ts | 2 +- src/fable-loader/src/index.ts | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/fable-compiler/src/index.ts b/src/fable-compiler/src/index.ts index a0cddd6d4d..50fc66d579 100644 --- a/src/fable-compiler/src/index.ts +++ b/src/fable-compiler/src/index.ts @@ -45,7 +45,7 @@ function processArgs(args?: {[x: string]: any}) { delete args.path; } for (const k of Object.keys(args)) { - if (args[k] !== false) { + if (args[k] != null && args[k] !== false) { cliArgs.push("--" + k.replace(/[A-Z]/g, (x) => "-" + x.toLowerCase())); if (args[k] !== true) { cliArgs.push(args[k]); diff --git a/src/fable-loader/src/index.ts b/src/fable-loader/src/index.ts index 4344a59b0c..03e5187346 100644 --- a/src/fable-loader/src/index.ts +++ b/src/fable-loader/src/index.ts @@ -84,7 +84,15 @@ const getCompiler = (function() { }, } } else { - const fableCompiler = require(opts.compiler ?? DEFAULT_COMPILER).default(opts.cli); + // We need to discard undefined/null values + // because of a bug in fable-compiler + const cliOptions = {}; + Object.keys(opts.cli).forEach(k => { + if (opts.cli[k] != null) { + cliOptions[k] = opts.cli[k]; + } + }); + const fableCompiler = require(opts.compiler ?? DEFAULT_COMPILER).default(cliOptions); webpack.onCompiled(function() { firstCompilationFinished = true; if (!opts.watch) { @@ -166,18 +174,16 @@ async function compile(filePath: string, opts: Options, webpack: WebpackHelper) const fileCache = await getFileCache(opts); const cachedFile = await fileCache.getFile(filePath); if (cachedFile != null) { - if (opts.verbose) { - log(opts, "fable: Cached " + path.relative(process.cwd(), filePath)); - } + log(opts, "fable: Cached " + path.relative(process.cwd(), filePath)); return { code: cachedFile } } + const compiler = getCompiler(opts, webpack); const req = Object.assign({}, opts, { path: filePath, rootDir: process.cwd() }, { cli: undefined, babel: undefined } // Remove some unneeded options ); - const compiler = getCompiler(opts, webpack); const data = await compiler.compile(req); if (data.error) { throw new Error(data.error);