[typer] Prepare for typer creation delaying (#11324)

* [debug] add trace utils * [debug] add compiler stage debug * [std] ensuring correct macro API usage is now mandatory * [xmldoc] update macro API usage * [dune] do not watch non source folders * [tests] add server tests utils * [tests] update macro API usage * [macro] update Context.onAfterInitMacros behavior * [std] update macro API usage for haxe.macro.CompilationServer * [std] mark more Context API as after init macros only * [macro] make Context.parse / Context.parseInlineString available in init macros too * [tests] fix macro-in-macro when test fails * [macro] make MacroStringTools.formatString work during init macros too * Move global_metadata to common context * [macro] make warning and current_module work (partially) with com_api * [macro] Compiler: delay type/meta patch instead of aborting * [tests] adjust for macro API changes * Revert "Move global_metadata to common context" This reverts commit 319e074.
HaxeFoundation · Oct 4, 2023 · 8fd670d · 8fd670d
1 parent 269fef4
commit 8fd670d
Show file tree

Hide file tree

Showing 46 changed files with 415 additions and 297 deletions.
diff --git a/dune b/dune
@@ -1 +1,2 @@
-(data_only_dirs extra lib std tests)
+(dirs :standard \ tests std extra)
+(data_only_dirs lib)
diff --git a/extra/ImportAll.hx b/extra/ImportAll.hx
@@ -69,42 +69,44 @@ class ImportAll {
 			if (!Context.defined("neko") && !Context.defined("cpp")) return;
 		case "tools", "build-tool", "jar-tool": return;
 		}
-		for( p in Context.getClassPath() ) {
-			if( p == "/" || p == "" )
-				continue;
-			// skip if we have a classpath to haxe
-			if( pack.length == 0 && sys.FileSystem.exists(p+"std") )
-				continue;
-			var p = p + pack.split(".").join("/");
-			if( StringTools.endsWith(p,"/") )
-				p = p.substr(0,-1);
-			if( !sys.FileSystem.exists(p) || !sys.FileSystem.isDirectory(p) )
-				continue;
-			for( file in sys.FileSystem.readDirectory(p) ) {
-				if( file == ".svn" || file == "_std" )
+		Context.onAfterInitMacros(() -> {
+			for( p in Context.getClassPath() ) {
+				if( p == "/" || p == "" )
 					continue;
-				var full = (pack == "") ? file : pack + "." + file;
-				if( StringTools.endsWith(file, ".hx") && file.substr(0, file.length - 3).indexOf(".") < 0 ) {
-					var cl = full.substr(0, full.length - 3);
-					switch( cl ) {
-					case "ImportAll", "neko.db.MacroManager": continue;
-					case "haxe.TimerQueue": if( Context.defined("neko") || Context.defined("php") || Context.defined("cpp") ) continue;
-					case "Sys": if(!isSysTarget()) continue;
-					case "haxe.web.Request": if( !(Context.defined("neko") || Context.defined("php") || Context.defined("js")) ) continue;
-					case "haxe.macro.ExampleJSGenerator","haxe.macro.Context", "haxe.macro.Compiler": if( !Context.defined("eval") ) continue;
-					case "haxe.remoting.SocketWrapper": if( !Context.defined("flash") ) continue;
-					case "haxe.remoting.SyncSocketConnection": if( !(Context.defined("neko") || Context.defined("php") || Context.defined("cpp")) ) continue;
-					case "neko.vm.Ui" | "sys.db.Sqlite" | "sys.db.Mysql" if ( Context.defined("interp") ): continue;
-					case "sys.db.Sqlite" | "sys.db.Mysql" | "cs.db.AdoNet" if ( Context.defined("cs") ): continue;
-					case "haxe.atomic.AtomicBool" if(!Context.defined("target.atomics")): continue;
-					case "haxe.atomic.AtomicInt" if(!Context.defined("target.atomics")): continue;
-					case "haxe.atomic.AtomicObject" if(!Context.defined("target.atomics") || Context.defined("js") || Context.defined("cpp")): continue;
-					}
-					Context.getModule(cl);
-				} else if( sys.FileSystem.isDirectory(p + "/" + file) )
-					run(full);
+				// skip if we have a classpath to haxe
+				if( pack.length == 0 && sys.FileSystem.exists(p+"std") )
+					continue;
+				var p = p + pack.split(".").join("/");
+				if( StringTools.endsWith(p,"/") )
+					p = p.substr(0,-1);
+				if( !sys.FileSystem.exists(p) || !sys.FileSystem.isDirectory(p) )
+					continue;
+				for( file in sys.FileSystem.readDirectory(p) ) {
+					if( file == ".svn" || file == "_std" )
+						continue;
+					var full = (pack == "") ? file : pack + "." + file;
+					if( StringTools.endsWith(file, ".hx") && file.substr(0, file.length - 3).indexOf(".") < 0 ) {
+						var cl = full.substr(0, full.length - 3);
+						switch( cl ) {
+						case "ImportAll", "neko.db.MacroManager": continue;
+						case "haxe.TimerQueue": if( Context.defined("neko") || Context.defined("php") || Context.defined("cpp") ) continue;
+						case "Sys": if(!isSysTarget()) continue;
+						case "haxe.web.Request": if( !(Context.defined("neko") || Context.defined("php") || Context.defined("js")) ) continue;
+						case "haxe.macro.ExampleJSGenerator","haxe.macro.Context", "haxe.macro.Compiler": if( !Context.defined("eval") ) continue;
+						case "haxe.remoting.SocketWrapper": if( !Context.defined("flash") ) continue;
+						case "haxe.remoting.SyncSocketConnection": if( !(Context.defined("neko") || Context.defined("php") || Context.defined("cpp")) ) continue;
+						case "neko.vm.Ui" | "sys.db.Sqlite" | "sys.db.Mysql" if ( Context.defined("interp") ): continue;
+						case "sys.db.Sqlite" | "sys.db.Mysql" | "cs.db.AdoNet" if ( Context.defined("cs") ): continue;
+						case "haxe.atomic.AtomicBool" if(!Context.defined("target.atomics")): continue;
+						case "haxe.atomic.AtomicInt" if(!Context.defined("target.atomics")): continue;
+						case "haxe.atomic.AtomicObject" if(!Context.defined("target.atomics") || Context.defined("js") || Context.defined("cpp")): continue;
+						}
+						Context.getModule(cl);
+					} else if( sys.FileSystem.isDirectory(p + "/" + file) )
+						run(full);
+				}
 			}
-		}
+		});
 	}
 
 }
diff --git a/src/compiler/compiler.ml b/src/compiler/compiler.ml
@@ -278,6 +278,7 @@ let do_type ctx tctx actx =
 	com.stage <- CInitMacrosStart;
 	List.iter (MacroContext.call_init_macro tctx) (List.rev actx.config_macros);
 	com.stage <- CInitMacrosDone;
+	ServerMessage.compiler_stage com;
 	check_defines ctx.com;
 	CommonCache.lock_signature com "after_init_macros";
 	com.callbacks#run com.callbacks#get_after_init_macros;
@@ -287,6 +288,7 @@ let do_type ctx tctx actx =
 		Finalization.finalize tctx;
 	) ();
 	com.stage <- CTypingDone;
+	ServerMessage.compiler_stage com;
 	(* If we are trying to find references, let's syntax-explore everything we know to check for the
 		identifier we are interested in. We then type only those modules that contain the identifier. *)
 	begin match com.display.dms_kind with
@@ -299,6 +301,7 @@ let finalize_typing ctx tctx =
 	let t = Timer.timer ["finalize"] in
 	let com = ctx.com in
 	com.stage <- CFilteringStart;
+	ServerMessage.compiler_stage com;
 	let main, types, modules = run_or_diagnose ctx Finalization.generate tctx in
 	com.main <- main;
 	com.types <- types;
@@ -343,13 +346,15 @@ let compile ctx actx callbacks =
 	List.iter (fun f -> f()) (List.rev (actx.pre_compilation));
 	t();
 	com.stage <- CInitialized;
+	ServerMessage.compiler_stage com;
 	if actx.classes = [([],"Std")] && not actx.force_typing then begin
 		if actx.cmds = [] && not actx.did_something then actx.raise_usage();
 	end else begin
 		(* Actual compilation starts here *)
 		let tctx = Setup.create_typer_context ctx actx.native_libs in
 		tctx.g.macros <- mctx;
 		com.stage <- CTyperCreated;
+		ServerMessage.compiler_stage com;
 		let display_file_dot_path = DisplayProcessing.maybe_load_display_file_before_typing tctx display_file_dot_path in
 		begin try
 			do_type ctx tctx actx
@@ -363,8 +368,10 @@ let compile ctx actx callbacks =
 		if ctx.has_error then raise Abort;
 		Generate.check_auxiliary_output com actx;
 		com.stage <- CGenerationStart;
+		ServerMessage.compiler_stage com;
 		if not actx.no_output then Generate.generate ctx tctx ext actx;
 		com.stage <- CGenerationDone;
+		ServerMessage.compiler_stage com;
 	end;
 	Sys.catch_break false;
 	com.callbacks#run com.callbacks#get_after_generation;

diff --git a/src/compiler/serverMessage.ml b/src/compiler/serverMessage.ml
@@ -4,6 +4,7 @@ open CompilationCache
 open Type
 
 type server_message_options = {
+	mutable print_compiler_stage : bool;
 	mutable print_added_directory : bool;
 	mutable print_found_directories : bool;
 	mutable print_changed_directories : bool;
@@ -30,6 +31,7 @@ type server_message_options = {
 }
 
 let config = {
+	print_compiler_stage = false;
 	print_added_directory = false;
 	print_found_directories = false;
 	print_changed_directories = false;
@@ -61,6 +63,9 @@ let sign_string com =
 	let	sign_id = (cs#get_context sign)#get_index in
 	Printf.sprintf "%2i,%3s: " sign_id (short_platform_name com.platform)
 
+let compiler_stage com =
+	if config.print_compiler_stage then print_endline (Printf.sprintf "compiler stage: %s" (s_compiler_stage com.stage))
+
 let added_directory com tabs dir =
 	if config.print_added_directory then print_endline (Printf.sprintf "%sadded directory %s" (sign_string com) dir)
 
@@ -156,6 +161,7 @@ let uncaught_error s =
 	if config.print_uncaught_error then print_endline ("Uncaught Error : " ^ s)
 
 let enable_all () =
+	config.print_compiler_stage <- true;
 	config.print_added_directory <- true;
 	config.print_found_directories <- true;
 	config.print_changed_directories <- true;
@@ -180,6 +186,7 @@ let enable_all () =
 	config.print_new_context <- true
 
 let set_by_name name value = match name with
+	| "compilerStage" -> config.print_compiler_stage <- value
 	| "addedDirectory" -> config.print_added_directory <- value
 	| "foundDirectories" -> config.print_found_directories <- value;
 	| "changedDirectories" -> config.print_changed_directories <- value;

diff --git a/src/context/common.ml b/src/context/common.ml
@@ -271,6 +271,24 @@ type compiler_stage =
 	| CGenerationStart  (* Generation is about to begin. *)
 	| CGenerationDone   (* Generation just finished. *)
 
+let s_compiler_stage = function
+	| CCreated          -> "CCreated"
+	| CInitialized      -> "CInitialized"
+	| CTyperCreated     -> "CTyperCreated"
+	| CInitMacrosStart  -> "CInitMacrosStart"
+	| CInitMacrosDone   -> "CInitMacrosDone"
+	| CTypingDone       -> "CTypingDone"
+	| CFilteringStart   -> "CFilteringStart"
+	| CAnalyzerStart    -> "CAnalyzerStart"
+	| CAnalyzerDone     -> "CAnalyzerDone"
+	| CSaveStart        -> "CSaveStart"
+	| CSaveDone         -> "CSaveDone"
+	| CDceStart         -> "CDceStart"
+	| CDceDone          -> "CDceDone"
+	| CFilteringDone    -> "CFilteringDone"
+	| CGenerationStart  -> "CGenerationStart"
+	| CGenerationDone   -> "CGenerationDone"
+
 type report_mode =
 	| RMNone
 	| RMDiagnostics of Path.UniqueKey.t list
@@ -1299,3 +1317,100 @@ let get_entry_point com =
 		let e = Option.get com.main in (* must be present at this point *)
 		(snd path, c, e)
 	) com.main_class
+
+let format_string com s p process_expr =
+	let e = ref None in
+	let pmin = ref p.pmin in
+	let min = ref (p.pmin + 1) in
+	let add_expr (enext,p) len =
+		min := !min + len;
+		let enext = process_expr enext p in
+		match !e with
+		| None -> e := Some enext
+		| Some prev ->
+			e := Some (EBinop (OpAdd,prev,enext),punion (pos prev) p)
+	in
+	let add enext len =
+		let p = { p with pmin = !min; pmax = !min + len } in
+		add_expr (enext,p) len
+	in
+	let add_sub start pos =
+		let len = pos - start in
+		if len > 0 || !e = None then add (EConst (String (String.sub s start len,SDoubleQuotes))) len
+	in
+	let len = String.length s in
+	let rec parse start pos =
+		if pos = len then add_sub start pos else
+		let c = String.unsafe_get s pos in
+		let pos = pos + 1 in
+		if c = '\'' then begin
+			incr pmin;
+			incr min;
+		end;
+		if c <> '$' || pos = len then parse start pos else
+		match String.unsafe_get s pos with
+		| '$' ->
+			(* double $ *)
+			add_sub start pos;
+			parse (pos + 1) (pos + 1)
+		| '{' ->
+			parse_group start pos '{' '}' "brace"
+		| 'a'..'z' | 'A'..'Z' | '_' ->
+			add_sub start (pos - 1);
+			incr min;
+			let rec loop i =
+				if i = len then i else
+				let c = String.unsafe_get s i in
+				match c with
+				| 'a'..'z' | 'A'..'Z' | '0'..'9' | '_' -> loop (i+1)
+				| _ -> i
+			in
+			let iend = loop (pos + 1) in
+			let len = iend - pos in
+			add (EConst (Ident (String.sub s pos len))) len;
+			parse (pos + len) (pos + len)
+		| _ ->
+			(* keep as-it *)
+			parse start pos
+	and parse_group start pos gopen gclose gname =
+		add_sub start (pos - 1);
+		let rec loop groups i =
+			if i = len then
+				match groups with
+				| [] -> die "" __LOC__
+				| g :: _ -> Error.raise_typing_error ("Unclosed " ^ gname) { p with pmin = !pmin + g + 1; pmax = !pmin + g + 2 }
+			else
+				let c = String.unsafe_get s i in
+				if c = gopen then
+					loop (i :: groups) (i + 1)
+				else if c = gclose then begin
+					let groups = List.tl groups in
+					if groups = [] then i else loop groups (i + 1)
+				end else
+					loop groups (i + 1)
+		in
+		let send = loop [pos] (pos + 1) in
+		let slen = send - pos - 1 in
+		let scode = String.sub s (pos + 1) slen in
+		min := !min + 2;
+		begin
+			let e =
+				let ep = { p with pmin = !pmin + pos + 2; pmax = !pmin + send + 1 } in
+				let error msg pos =
+					if Lexer.string_is_whitespace scode then Error.raise_typing_error "Expression cannot be empty" ep
+					else Error.raise_typing_error msg pos
+				in
+				match ParserEntry.parse_expr_string com.defines scode ep error true with
+					| ParseSuccess(data,_,_) -> data
+					| ParseError(_,(msg,p),_) -> error (Parser.error_msg msg) p
+			in
+			add_expr e slen
+		end;
+		min := !min + 1;
+		parse (send + 1) (send + 1)
+	in
+	parse 0 0;
+	match !e with
+	| None -> die "" __LOC__
+	| Some e -> e
+
diff --git a/src/core/globals.ml b/src/core/globals.ml
@@ -33,6 +33,41 @@ let version_pre = Some "alpha.1"
 
 let null_pos = { pfile = "?"; pmin = -1; pmax = -1 }
 
+let no_color = false
+let c_reset = if no_color then "" else "\x1b[0m"
+let c_dim = if no_color then "" else "\x1b[2m"
+
+let loc_short (loc:Printexc.location) =
+	Printf.sprintf "%s:%d" loc.filename loc.line_number
+
+let loc_to_string (loc:Printexc.location) =
+	Printf.sprintf "%s, line %d, characters %d-%d" loc.filename loc.line_number loc.start_char loc.end_char
+
+let trace s =
+	let stack = Printexc.get_callstack 2 in
+	match Printexc.backtrace_slots stack with
+	| Some [|_; item |] ->
+		(match Printexc.Slot.location item with
+		| Some loc -> print_endline (Printf.sprintf "%s%s:%s %s" c_dim (loc_short loc) c_reset s)
+		| _ -> ())
+	| _ ->
+		()
+
+let trace_call_stack ?(n:int = 5) () =
+	assert (n >= 0);
+	let stack = Printexc.get_callstack (n+2) in
+	let len = Printexc.raw_backtrace_length stack - 1 in
+
+	let slot = Printexc.convert_raw_backtrace_slot (Printexc.get_raw_backtrace_slot stack 1) in
+	let loc = Printexc.Slot.location slot in
+	Option.may (fun loc -> print_endline (Printf.sprintf "%s%s:%s" c_dim (loc_short loc) c_reset)) loc;
+
+	for i = 2 to len do
+		let slot = Printexc.convert_raw_backtrace_slot (Printexc.get_raw_backtrace_slot stack i) in
+		let loc = Printexc.Slot.location slot in
+		Option.may (fun loc -> print_endline (Printf.sprintf "  called from %s" (loc_to_string loc))) loc;
+	done
+
 let macro_platform = ref Neko
 
 let return_partial_type = ref false