updated to v5.9.0

README.md

@@ -4,28 +4,6 @@ A data dumper and typing generator for the KubeJS functions, constants and class
 
 Great thanks to @DAmNRelentless, @LatvianModder and @yesterday17 for invaluable suggestions during the development!
 
-For things not dumped, or maybe if you want to add some more comments to class, methods or fields, you can add static
-documents to override auto-generated typing!
-
-ProbeJS also has builtin documents to support some other mods' recipes, but it's not completed, if you want to add
-something to the document, maybe you can send a PR too!
-
-Current progress of mod documents:
-
-- Minecraft (Completed)
-- Create (Completed)
-- Integrated Dynamics (Completed)
-- ArsNouveau (Completed)
-- ArtisanWorktables (Completed)
-- Botania (Builtin in KubeJS Botania)
-- Cucumber (Completed)
-- DankStorage (Completed)
-- ExtendedCrafting (Completed)
-- Mekanism (Completed)
-- Immersive Engineering (Completed)
-- Thermal Expansion (Completed)
-- Others (Please PR)
-
 For the detailed information about documents, please refer to the wiki page.
 
 ## 1. Installation

common/src/main/java/com/probejs/ProbeCommands.java

@@ -3,6 +3,7 @@
 import com.mojang.brigadier.Command;
 import com.mojang.brigadier.CommandDispatcher;
 import com.probejs.docs.DocCompiler;
+import com.probejs.jdoc.jsgen.RemoteSchema;
 import com.probejs.specials.SnippetCompiler;
 import com.probejs.specials.SpecialCompiler;
 import com.probejs.docs.formatter.ClassResolver;
@@ -118,6 +119,9 @@ public static void triggerDump(ServerPlayer player) {
                 ClassResolver.init();
                 NameResolver.init();
                 DocCompiler.compile(sendMessage, event);
+                if (ProbeConfig.INSTANCE.pullSchema && ProbeConfig.INSTANCE.modChanged) {
+                    RemoteSchema.dumpSchemas(sendMessage);
+                }
             } catch (Exception e) {
                 ProbeJS.LOGGER.error("Uncaught exception has occurred!", e);
                 player.sendSystemMessage(Component.literal("Uncaught exception happened in wrapper, please report to the Github issue with complete latest.log."), false);
@@ -174,6 +178,7 @@ public static void register(CommandDispatcher<CommandSourceStack> dispatcher) {
                         .then(Commands.literal("clear_cache")
                                 .requires(source -> source.getServer().isSingleplayer())
                                 .executes(context -> {
+                                    ProbeConfig.INSTANCE.docsTimestamp = -1;
                                     for (File file : Objects.requireNonNull(ProbePaths.CACHE.toFile().listFiles())) {
                                         // delete everything, including folders, folders might not be empty
                                         try {

common/src/main/java/com/probejs/ProbeConfig.java

@@ -9,10 +9,11 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
 import java.util.Map;
 
 public class ProbeConfig {
-    public static final int MOD_COUNT = 100;
+    public static final int MOD_COUNT = 350;
 
     public static ProbeConfig INSTANCE = new ProbeConfig();
     private static final int CONFIG_VERSION = 1;
@@ -23,9 +24,11 @@ public class ProbeConfig {
     public boolean allowRegistryLiteralDumps = true;
     public boolean requireSingleAndPerm = true;
     public boolean enabled = true;
+    public boolean modChanged = false;
 
     public boolean disableRecipeJsonDump = true;
     public boolean dumpJSONIntermediates = false;
+    public boolean pullSchema = true;
 
     @SuppressWarnings("unchecked")
     private static <E> E fetchPropertyOrDefault(Object key, Map<?, ?> value, E defaultValue) {
@@ -51,6 +54,7 @@ private ProbeConfig() {
                 enabled = fetchPropertyOrDefault("enabled", obj, true);
                 disableRecipeJsonDump = fetchPropertyOrDefault("disableRecipeJsonDump", obj, true);
                 dumpJSONIntermediates = fetchPropertyOrDefault("dumpJSONIntermediates", obj, false);
+                pullSchema = fetchPropertyOrDefault("pullSchema", obj, true);
             } catch (IOException e) {
                 ProbeJS.LOGGER.warn("Cannot read config properties, falling back to defaults.");
             }

common/src/main/java/com/probejs/ProbeJS.java

@@ -3,11 +3,16 @@
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.probejs.jdoc.Serde;
+import com.probejs.recipe.component.ComponentConverter;
+import com.probejs.specials.assign.ClassAssignmentManager;
 import dev.architectury.event.events.common.CommandRegistrationEvent;
 import dev.architectury.event.events.common.PlayerEvent;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+
 public class ProbeJS {
     public static final String MOD_ID = "probejs";
     public static final Logger LOGGER = LogManager.getLogger("probejs");
@@ -19,10 +24,12 @@ public class ProbeJS {
             .setPrettyPrinting()
             .disableHtmlEscaping()
             .create();
+    public static final HttpClient HTTP_CLIENT = HttpClient.newHttpClient();
 
     public static void init() {
         CommandRegistrationEvent.EVENT.register((dispatcher, registry, selection) -> ProbeCommands.register(dispatcher));
         PlayerEvent.PLAYER_JOIN.register(ProbeJSEvents::playerJoined);
         Serde.init();
+        ClassAssignmentManager.init(ComponentConverter.PROBEJS_CONTEXT);
     }
 }

common/src/main/java/com/probejs/ProbeJSEvents.java

@@ -53,6 +53,7 @@ public static void playerJoined(ServerPlayer player) {
                 String hash = byte2Hex(digest.digest());
                 if (!hash.equals(ProbeConfig.getModHash())) {
                     ProbeConfig.writeModHash(hash);
+                    ProbeConfig.INSTANCE.modChanged = true;
                     player.sendSystemMessage(Component.literal("Mod list has changed, dumping new docs..."));
                     ProbeCommands.triggerDump(player);
                 }
@@ -68,8 +69,6 @@ public static void playerJoined(ServerPlayer player) {
                 ProbeConfig.INSTANCE.firstLoad = false;
                 if (Platform.getMods().size() > ProbeConfig.MOD_COUNT) {
                     player.sendSystemMessage(Component.literal("There are more than " + ProbeConfig.MOD_COUNT + " mods installed. Disabling some features to prevent lag..."));
-                    ProbeConfig.INSTANCE.allowRegistryLiteralDumps = false;
-                    player.sendSystemMessage(Component.literal("Registry literal dumps are disabled."));
                     ProbeConfig.INSTANCE.disableRecipeJsonDump = true;
                     player.sendSystemMessage(Component.literal("Recipe JSON dumps are disabled."));
                 }

common/src/main/java/com/probejs/ProbePaths.java

@@ -1,6 +1,7 @@
 package com.probejs;
 
 import dev.architectury.platform.Platform;
+import dev.latvian.mods.kubejs.KubeJS;
 import dev.latvian.mods.kubejs.KubeJSPaths;
 import dev.latvian.mods.kubejs.util.UtilsJS;
 
@@ -21,6 +22,8 @@ public class ProbePaths {
     public static Path RICH_ITEM = RICH.resolve("item");
     public static Path RICH_FLUID = RICH.resolve("fluid");
 
+    public static Path SCHEMA = KubeJSPaths.STARTUP_SCRIPTS.resolve("@recipes");
+
     public static Path KUBE_ASSETS = KubeJSPaths.DIRECTORY.resolve("assets");
 
     public static void init() {
@@ -48,6 +51,9 @@ public static void init() {
         if (Files.notExists(RICH)) {
             UtilsJS.tryIO(() -> Files.createDirectories(RICH));
         }
+        if (Files.notExists(ProbePaths.SCHEMA)) {
+            UtilsJS.tryIO(() -> Files.createDirectories(ProbePaths.SCHEMA));
+        }
     }
 
     static {

common/src/main/java/com/probejs/docs/DocCompiler.java

@@ -22,6 +22,8 @@
 import com.probejs.jdoc.jsgen.DocGenerationEventJS;
 import com.probejs.jdoc.property.PropertyComment;
 import com.probejs.specials.*;
+import com.probejs.specials.assign.ClassAssignmentManager;
+import com.probejs.specials.special.FormatterComponents;
 import com.probejs.util.PlatformSpecial;
 import dev.latvian.mods.kubejs.KubeJS;
 import dev.latvian.mods.kubejs.KubeJSPaths;
@@ -190,6 +192,8 @@ public static Set<Class<?>> fetchClasses(Set<Class<?>> typeMap, DummyBindingEven
         touchableClasses.addAll(CapturedClasses.getCapturedJavaClasses());
         touchableClasses.addAll(fetchRecipeClasses());
         touchableClasses.addAll(fetchComponentClasses());
+        touchableClasses.addAll(FormatterComponents.loadComponentsClasses());
+        touchableClasses.addAll(ClassAssignmentManager.ASSIGNMENTS.keys());
         Walker walker = new Walker(touchableClasses);
         return walker.walk();
     }
@@ -344,8 +348,7 @@ public static void compile(Consumer<String> sendMessage, DocGenerationEventJS ev
         cachedClasses.addAll(RegistryCompiler.getKJSRegistryClasses());
         if (ProbeConfig.INSTANCE.allowRegistryObjectDumps) cachedClasses.addAll(SpecialTypes.collectRegistryClasses());
         Set<Class<?>> typeMap = RecipeNamespace.getAll().values().stream().flatMap(namespace -> namespace.values().stream())
-                // TODO: add type -> recipe mapping once Lat got the new recipe system in
-                .map(type -> RecipeJS.class).collect(Collectors.toSet());
+                .map(type -> type.schema.recipeType).collect(Collectors.toSet());
 
         //Fetch all classes
         Set<Class<?>> globalClasses = DocCompiler.fetchClasses(typeMap, bindingEvent, cachedClasses);
@@ -362,16 +365,12 @@ public static void compile(Consumer<String> sendMessage, DocGenerationEventJS ev
         javaDocs.addAll(PlatformSpecial.INSTANCE.get().getPlatformDocuments(javaDocs));
 
         sendMessage.accept("Started downloading and merging docs...");
+
+        // Fuck IDEA
         Manager.downloadDocs();
         List<DocumentClass> fetchedDocs = Manager.loadFetchedClassDoc();
         List<DocumentClass> modDocs = Manager.loadModDocuments();
-        List<DocumentClass> userDocs = new ArrayList<>();
-        try {
-            userDocs = Manager.loadUserDocuments();
-        } catch (Exception e) {
-            ProbeJS.LOGGER.error("Error loading User-defined docs!");
-        }
-        Map<String, DocumentClass> mergedDocsMap = Manager.mergeDocuments(javaDocs, fetchedDocs, modDocs, userDocs);
+        Map<String, DocumentClass> mergedDocsMap = Manager.mergeDocuments(javaDocs, fetchedDocs, modDocs);
 
         event.getTransformers().forEach((key, transformer) -> {
             if (mergedDocsMap.containsKey(key)) {

common/src/main/java/com/probejs/docs/EventCompiler.java

@@ -2,18 +2,19 @@
 
 import com.mojang.datafixers.util.Pair;
 import com.probejs.ProbePaths;
-import com.probejs.jdoc.java.ClassInfo;
 import com.probejs.jdoc.Serde;
 import com.probejs.jdoc.document.DocumentClass;
+import com.probejs.jdoc.java.ClassInfo;
 import com.probejs.jdoc.property.AbstractProperty;
 import com.probejs.jdoc.property.PropertyComment;
-import com.probejs.jdoc.property.PropertyExtra;
 import com.probejs.jdoc.property.PropertyType;
+import com.probejs.recipe.component.ComponentConverter;
 import com.probejs.specials.RegistryCompiler;
 import com.probejs.specials.TagEventCompiler;
 import com.probejs.util.Util;
 import dev.latvian.mods.kubejs.event.EventGroup;
 import dev.latvian.mods.kubejs.event.EventHandler;
+import dev.latvian.mods.kubejs.typings.desc.TypeDescJS;
 
 import java.io.BufferedWriter;
 import java.io.IOException;
@@ -91,11 +92,12 @@ public static void compileEvents(Map<String, DocumentClass> globalClasses) throw
 
                 elements.addAll(comment.formatLines(4));
                 if (handler.extra != null) {
+                    TypeDescJS desc = handler.extra.describeType.apply(ComponentConverter.PROBEJS_CONTEXT);
+                    PropertyType<?> type = ComponentConverter.fromDescription(desc);
+
                     elements.add("%s(extra: %s, handler: (event: %s) => void):void,".formatted(
                             eventName,
-                            findProperty(globalClasses, document, PropertyExtra.class)
-                                    .map(extra -> Serde.getTypeFormatter(extra.getType()).formatFirst())
-                                    .orElse("string"), Util.formatMaybeParameterized(event)
+                            Serde.getTypeFormatter(type).formatFirst(), Util.formatMaybeParameterized(event)
                     ));
                 }
                 if (handler.extra == null || !handler.extra.required) {

common/src/main/java/com/probejs/docs/formatter/SpecialTypes.java

@@ -60,7 +60,7 @@ public static <T> void assignRegistries() {
             //We know that, all objects in Registry<T> must extend or implement T
             //So T must be the superclass or superinterface of all object in Registry<T>
             //And it must not be synthetic class unless some people are really crazy
-            ResourceKey<?> key = entry.key();
+            ResourceKey<? extends Registry<?>> key = entry.key();
             Registry<?> registry = entry.value();
             Class<?> putativeParent = null;
             //We assume it's class based first

common/src/main/java/com/probejs/jdoc/JsAnnotations.java

@@ -2,21 +2,16 @@
 
 import com.probejs.jdoc.java.ClassInfo;
 import com.probejs.jdoc.java.type.ITypeInfo;
-import com.probejs.jdoc.java.type.InfoTypeResolver;
 import com.probejs.jdoc.java.type.TypeInfoClass;
 import com.probejs.jdoc.java.type.TypeInfoParameterized;
 import com.probejs.jdoc.property.PropertyAssign;
 import com.probejs.jdoc.property.PropertyComment;
-import com.probejs.jdoc.property.PropertyType;
-import com.probejs.jdoc.wrapped.ClassType;
-import com.probejs.jdoc.wrapped.ListType;
-import com.probejs.jdoc.wrapped.LiteralType;
-import com.probejs.jdoc.wrapped.ObjectType;
+import com.probejs.recipe.component.ComponentConverter;
+import com.probejs.specials.assign.ClassAssignmentManager;
 import dev.latvian.mods.kubejs.typings.Generics;
 import dev.latvian.mods.kubejs.typings.Info;
 import dev.latvian.mods.kubejs.typings.Param;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -51,38 +46,10 @@ public static ITypeInfo fromGenerics(Generics generics) {
     }
 
     public static List<PropertyAssign> getClassAssignments(ClassInfo clazz) {
-        List<PropertyAssign> assigns = new ArrayList<>();
         Class<?> clazzRaw = clazz.getClazzRaw();
-        for (ClassType clazzType : clazzRaw.getAnnotationsByType(ClassType.class)) {
-            assigns.add(new PropertyAssign().type(Serde.deserializeFromJavaType(InfoTypeResolver.resolveType(clazzType.value()))));
-        }
-        for (LiteralType literalType : clazzRaw.getAnnotationsByType(LiteralType.class)) {
-            assigns.add(new PropertyAssign().type(new PropertyType.Native(literalType.value())));
-        }
-        for (ListType listType : clazzRaw.getAnnotationsByType(ListType.class)) {
-            assigns.add(new PropertyAssign().type(
-                    new PropertyType.JSArray(
-                            Arrays.stream(listType.value())
-                                    .map(cl -> Serde.deserializeFromJavaType(InfoTypeResolver.resolveType(cl.value())))
-                                    .collect(Collectors.toList())
-                    )
-            ));
-        }
-      for (ObjectType objectType : clazzRaw.getAnnotationsByType(ObjectType.class)) {
-            PropertyType.JSObject jsObject = new PropertyType.JSObject();
-            for (int i = 0; i < objectType.names().length; i++) {
-                var name = objectType.names()[i];
-                var optional = name.endsWith("?");
-                if (optional) {
-                    name = name.substring(0, name.length() - 1);
-                }
-                var type = objectType.types()[i];
-                jsObject.add(
-                        new PropertyType.JSObjectKey().optional(optional).withName(name),
-                        Serde.deserializeFromJavaType(InfoTypeResolver.resolveType(type.value()))
-                );
-            }
-        }
-        return assigns;
+        return ClassAssignmentManager.ASSIGNMENTS.get(clazzRaw).stream()
+                .map(ComponentConverter::fromDescription)
+                .map(type -> new PropertyAssign().type(type))
+                .collect(Collectors.toList());
     }
 }

common/src/main/java/com/probejs/jdoc/Manager.java

@@ -25,7 +25,6 @@
 
 public class Manager {
     private static boolean docsDownloaded = false;
-
     private static final String TIMESTAMP_API = "https://static.wolfgirl.moe/api/timestamp?path=probejs/docs-1.19.2-6.1.zip";
     private static final String DOWNLOAD_API = "https://static.wolfgirl.moe/object-service/checked/probejs/docs-1.19.2-6.1.zip?timestamp=%s";
 
@@ -42,9 +41,9 @@ public static void downloadDocs() throws IOException {
             ProbeJS.LOGGER.warn("The server might come back online later, this is not an error.");
             return;
         }
-        if (ProbeConfig.INSTANCE.docsTimestamp != remoteTimestamp) {
+        Path docsPath = ProbePaths.CACHE.resolve("docs");
+        if (ProbeConfig.INSTANCE.docsTimestamp != remoteTimestamp || !Files.exists(docsPath)) {
             ProbeJS.LOGGER.info("Found timestamp mismatch (local=%s, remote=%s), downloading docs from remote.".formatted(ProbeConfig.INSTANCE.docsTimestamp, remoteTimestamp));
-            Path docsPath = ProbePaths.CACHE.resolve("docs");
             if (Files.exists(docsPath)) {
                 FileUtils.deleteDirectory(docsPath.toFile());
             }