feat: adding the import solver

.github/workflows/ci.yml

@@ -154,7 +154,7 @@ jobs:
           -DCMAKE_C_COMPILER=${{ matrix.config.cc }} \
           -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} \
           -DARK_SANITIZERS=${{ matrix.config.sanitizers }} \
-          -DARK_BUILD_EXE=On -DARK_BUILD_MODULES=On -DARK_MOD_ALL=On
+          -DARK_BUILD_EXE=On -DARK_BUILD_MODULES=On -DARK_MOD_ALL=On -DARK_BUILD_PARSER_TESTS=On
 
     - name: Add SQLite deps
       if: startsWith(matrix.config.name, 'Windows')
@@ -179,17 +179,6 @@ jobs:
           -DARK_SANITIZERS=${{ matrix.config.sanitizers }}
         cmake --build build --config $BUILD_TYPE
 
-    - name: Configure & build CMake parser tests
-      shell: bash
-      run: |
-        cd tests/parser
-        cmake -Bbuild \
-          -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
-          -DCMAKE_C_COMPILER=${{ matrix.config.cc }} \
-          -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} \
-          -DARK_SANITIZERS=${{ matrix.config.sanitizers }}
-        cmake --build build --config $BUILD_TYPE
-
     - name: Organize files for upload
       shell: bash
       run: |
@@ -208,9 +197,13 @@ jobs:
     - name: Organize temp artifact
       shell: bash
       run: |
-        mkdir -p temp/parser/
+        mkdir -p temp/parser/build/
         cp -r tests/cpp temp/
         cp -r tests/parser temp/
+        cp build/parser temp/build/ || true
+        cp build/libArkReactor.* temp/build/ || true
+        cp build/$BUILD_TYPE/parser.exe temp/build/ || true
+        cp build/$BUILD_TYPE/ArkReactor.dll temp/build/  || true
 
     - name: Upload artifact
       uses: actions/[email protected]

.gitignore

@@ -4,7 +4,6 @@
 
 # Personal utilities
 warnings.log
-cformat.ps1
 
 # ArkScript
 include/Ark/Constants.hpp
@@ -28,6 +27,7 @@ afl/
 .cache/
 build/
 ninja/
+cmake-build-*/
 
 # Prerequisites
 *.d
@@ -47,10 +47,6 @@ ninja/
 *.dylib
 *.dll
 
-# Fortran module files
-*.mod
-*.smod
-
 # Compiled Static libraries
 *.lai
 *.la

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "lib/utf8_decoder"]
-	path = lib/utf8_decoder
-	url = https://github.com/PierrePharel/utf8_decoder.git
 [submodule "lib/std"]
 	path = lib/std
 	url = https://github.com/ArkScript-lang/std.git

CMakeLists.txt

@@ -46,8 +46,8 @@ if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANG OR APPLE)
         )
 
     if (CMAKE_COMPILER_IS_GNUCXX)
-        # The package utf8_decoder has an issues with constant overflow.
-        # Once thisis fixed remove this flag: 
+        # The package utf8 has an issue with constant overflow.
+        # Once this is fixed remove this flag:
         target_compile_options(ArkReactor PUBLIC -Wno-overflow)
     endif()
 
@@ -96,7 +96,6 @@ add_subdirectory("${ark_SOURCE_DIR}/lib/termcolor" EXCLUDE_FROM_ALL)
 
 target_include_directories(ArkReactor
     PUBLIC
-    "${ark_SOURCE_DIR}/lib/utf8_decoder/"
     "${ark_SOURCE_DIR}/lib/picosha2/"
     "${ark_SOURCE_DIR}/lib/fmt/include")
 
@@ -122,12 +121,7 @@ target_include_directories(ArkReactor
     PUBLIC
     ${ark_SOURCE_DIR}/include)
 
-# setting up project properties
-set_target_properties(
-    ArkReactor
-    PROPERTIES
-        CXX_STANDARD 17
-        CXX_STANDARD_REQUIRED ON)
+target_compile_features(ArkReactor PRIVATE cxx_std_17)
 
 # Installation rules
 
@@ -176,6 +170,12 @@ if (ARK_BUILD_MODULES)
     add_subdirectory(${ark_SOURCE_DIR}/lib/modules)
 endif()
 
+if (ARK_BUILD_PARSER_TESTS)
+    add_executable(parser ${ark_SOURCE_DIR}/tests/parser/main.cpp)
+    target_link_libraries(parser PUBLIC ArkReactor)
+    target_compile_features(parser PRIVATE cxx_std_17)
+endif()
+
 if (ARK_BUILD_EXE)
     # additional files needed for the exe (repl, command line and stuff)
     file(GLOB_RECURSE EXE_SOURCES

cmake/link_time_optimization.cmake

@@ -3,7 +3,7 @@ include(CheckIPOSupported)
 check_ipo_supported(RESULT ipo_supported)
 
 function(enable_lto target_name)
-    if (ipo_supported)
+    if (ipo_supported AND (${CMAKE_BUILD_TYPE} STREQUAL "Release"))
         if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND (CMAKE_CXX_COMPILER_VERSION MATCHES "^8\..+"))
             message(WARNING "LTO supported but not enabled to prevent https://github.com/ArkScript-lang/Ark/pull/385#issuecomment-1163597951")
         else()

include/Ark/Compiler/AST/BaseParser.hpp

@@ -74,6 +74,7 @@ namespace Ark::internal
         bool suffix(char c);
         bool number(std::string* s = nullptr);
         bool signedNumber(std::string* s = nullptr);
+        bool hexNumber(unsigned length, std::string* s = nullptr);
         bool name(std::string* s = nullptr);
         bool sequence(const std::string& s);
         bool packageName(std::string* s = nullptr);

include/Ark/Compiler/AST/Import.hpp

@@ -3,6 +3,7 @@
 
 #include <vector>
 #include <string>
+#include <numeric>
 
 #include <Ark/Platform.hpp>
 
@@ -15,25 +16,55 @@ namespace Ark::internal
          * @details Example: `(import foo.bar)` => `bar`
          *      `(import foo.bar.egg:*)` => `egg`
          *      `(import foo :a :b :c)` => `foo`
-         * 
+         *
          */
-        std::string package;
+        std::string prefix;
+
+        /**
+         * @brief Package with all the segments
+         * @details Example: `(import foo.bar)` => `{foo, bar}`
+         *      `(import foo.bar.egg:*)` => `{foo, bar, egg}`
+         *      `(import foo :a :b :c)` => `{foo}`
+         */
+        std::vector<std::string> package;
 
         /**
          * @brief Import with prefix (the package) or not
-         * 
+         *
          */
         bool with_prefix = true;
 
         /**
          * @brief List of symbols to import, can be empty if none provided
-         * 
+         *
          */
         std::vector<std::string> symbols;
 
+        inline std::string toPackageString() const
+        {
+            return std::accumulate(package.begin() + 1, package.end(), package.front(), [](const std::string& left, const std::string& right) {
+                return left + "." + right;
+            });
+        }
+
+        inline std::string packageToPath() const
+        {
+            std::size_t offset = 0;
+            if (package.front() == "std")
+                offset = 1;
+
+            return std::accumulate(
+                std::next(package.begin() + offset),
+                package.end(),
+                package[offset],
+                [](const std::string& a, const std::string& b) {
+                    return a + "/" + b;
+                });
+        }
+
         /**
          * @brief Check if we should import everything, given something like `(import foo.bar.egg:*)`
-         * 
+         *
          * @return true if all symbols of the file should be imported in the importer scope
          * @return false otherwise
          */
@@ -44,9 +75,9 @@ namespace Ark::internal
 
         /**
          * @brief Check if we should import everything with a prefix, given a `(import foo.bar.egg)`
-         * 
-         * @return true 
-         * @return false 
+         *
+         * @return true
+         * @return false
          */
         inline bool isBasic() const
         {

include/Ark/Compiler/AST/Module.hpp

@@ -0,0 +1,17 @@
+#ifndef ARK_MODULE_HPP
+#define ARK_MODULE_HPP
+
+#include <Ark/Compiler/AST/Node.hpp>
+
+namespace Ark::internal
+{
+    // TODO store something better than just the AST (AST+what we are importing as private/public/namespaced... vs all)
+    //      so that we can remember the order in which we encountered imports.
+    struct Module
+    {
+        Node ast;
+        bool has_been_processed = false;  // TODO document this
+    };
+}
+
+#endif  // ARK_MODULE_HPP

include/Ark/Compiler/AST/Parser.hpp

@@ -12,6 +12,8 @@
 #include <vector>
 #include <functional>
 
+#include <utf8.hpp>
+
 namespace Ark::internal
 {
     class ARK_API Parser : public BaseParser
@@ -28,6 +30,7 @@ namespace Ark::internal
     private:
         Node m_ast;
         std::vector<Import> m_imports;
+        unsigned m_allow_macro_behavior;  ///< Toggled on when inside a macro definition, off afterward
 
         void run();
 
@@ -38,7 +41,11 @@ namespace Ark::internal
         std::optional<Node> loop();
         std::optional<Node> import_();
         std::optional<Node> block();
+        std::optional<Node> functionArgs();
         std::optional<Node> function();
+        std::optional<Node> macroCondition();
+        std::optional<Node> macroBlock();
+        std::optional<Node> macroArgs();
         std::optional<Node> macro();
         std::optional<Node> functionCall();
         std::optional<Node> list();
@@ -89,6 +96,39 @@ namespace Ark::internal
                             res += '\b';
                         else if (accept(IsChar('0')))
                             res += '\0';
+                        else if (accept(IsChar('f')))
+                            res += '\f';
+                        else if (accept(IsChar('u')))
+                        {
+                            std::string seq;
+                            if (hexNumber(4, &seq))
+                            {
+                                char utf8_str[5];
+                                utf8::decode(seq.c_str(), utf8_str);
+                                if (*utf8_str == '\0')
+                                    error("Invalid escape sequence", "\\u" + seq);
+                                res += utf8_str;
+                            }
+                            else
+                                error("Invalid escape sequence", "\\u");
+                        }
+                        else if (accept(IsChar('U')))
+                        {
+                            std::string seq;
+                            if (hexNumber(8, &seq))
+                            {
+                                std::size_t begin = 0;
+                                for (; seq[begin] == '0'; ++begin)
+                                    ;
+                                char utf8_str[5];
+                                utf8::decode(seq.c_str() + begin, utf8_str);
+                                if (*utf8_str == '\0')
+                                    error("Invalid escape sequence", "\\U" + seq);
+                                res += utf8_str;
+                            }
+                            else
+                                error("Invalid escape sequence", "\\U");
+                        }
                         else
                         {
                             backtrack(getCount() - 1);
@@ -102,7 +142,6 @@ namespace Ark::internal
                         break;
                     else if (isEOF())
                         errorMissingSuffix('"', "string");
-                    // TODO accept(\Uxxxxx), accept(\uxxxxx)
                 }
 
                 return Node(NodeType::String, res);
@@ -121,7 +160,6 @@ namespace Ark::internal
 
             while (true)
             {
-                space();
                 if (leaf.list().size() == 1 && !accept(IsChar('.')))  // Symbol:abc
                     return std::nullopt;
 
@@ -144,6 +182,18 @@ namespace Ark::internal
             return Node(NodeType::Symbol, res);
         }
 
+        inline std::optional<Node> spread()
+        {
+            std::string res;
+            if (sequence("..."))
+            {
+                if (!name(&res))
+                    errorWithNextToken("Expected a name for the variadic");
+                return Node(NodeType::Spread, res);
+            }
+            return std::nullopt;
+        }
+
         inline std::optional<Node> nil()
         {
             if (!accept(IsChar('(')))

include/Ark/Compiler/AST/Predicates.hpp

@@ -48,6 +48,16 @@ namespace Ark::internal
         }
     } IsDigit;
 
+    inline struct IsHex : public CharPred
+    {
+        IsHex() :
+            CharPred("hex") {}
+        virtual bool operator()(const utf8_char_t::codepoint_t c) const override
+        {
+            return 0 <= c && c <= 255 && std::isxdigit(c) != 0;
+        }
+    } IsHex;
+
     inline struct IsUpper : public CharPred
     {
         IsUpper() :