feat: adding the import solver

.github/workflows/ci.yml

@@ -154,7 +154,7 @@ jobs:
           -DCMAKE_C_COMPILER=${{ matrix.config.cc }} \
           -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} \
           -DARK_SANITIZERS=${{ matrix.config.sanitizers }} \
-          -DARK_BUILD_EXE=On -DARK_BUILD_MODULES=On -DARK_MOD_ALL=On
+          -DARK_BUILD_EXE=On -DARK_BUILD_MODULES=On -DARK_MOD_ALL=On -DARK_BUILD_PARSER_TESTS=On
 
     - name: Add SQLite deps
       if: startsWith(matrix.config.name, 'Windows')
@@ -179,26 +179,17 @@ jobs:
           -DARK_SANITIZERS=${{ matrix.config.sanitizers }}
         cmake --build build --config $BUILD_TYPE
 
-    - name: Configure & build CMake parser tests
-      shell: bash
-      run: |
-        cd tests/parser
-        cmake -Bbuild \
-          -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
-          -DCMAKE_C_COMPILER=${{ matrix.config.cc }} \
-          -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} \
-          -DARK_SANITIZERS=${{ matrix.config.sanitizers }}
-        cmake --build build --config $BUILD_TYPE
-
     - name: Organize files for upload
       shell: bash
       run: |
         mkdir -p artifact/lib/std
         # Linux/MacOS
         cp build/arkscript artifact || true
+        cp build/parser artifact || true
         cp build/libArkReactor.* artifact || true
         # Windows
         cp build/$BUILD_TYPE/arkscript.exe artifact || true
+        cp build/$BUILD_TYPE/parser.exe artifact || true
         cp build/$BUILD_TYPE/ArkReactor.dll artifact  || true
         # Generic
         cp lib/*.arkm artifact/lib

.github/workflows/setup-tests/action.yaml

@@ -1,5 +1,6 @@
 ---
 name: "Setup tests"
+description: "Unpack necessary artifacts, updates compilers"
 
 runs:
   using: "composite"
@@ -31,9 +32,11 @@ runs:
         mv build/lib/*.arkm lib/
         chmod u+x build/arkscript tests/cpp/out/*
         cp -r artifact/parser/* tests/parser/
+        cp -r build/ tests/parser/
         chmod u+x tests/parser/build/parser
 
     - shell: bash
       if: startsWith(matrix.config.name, 'Windows')
       run: |
         cp build/*.dll tests/cpp/out/
+        cp build/*.dll tests/parser/build/

.gitignore

@@ -4,7 +4,6 @@
 
 # Personal utilities
 warnings.log
-cformat.ps1
 
 # ArkScript
 include/Ark/Constants.hpp
@@ -28,6 +27,7 @@ afl/
 .cache/
 build/
 ninja/
+cmake-build-*/
 
 # Prerequisites
 *.d
@@ -47,10 +47,6 @@ ninja/
 *.dylib
 *.dll
 
-# Fortran module files
-*.mod
-*.smod
-
 # Compiled Static libraries
 *.lai
 *.la

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "lib/utf8_decoder"]
-	path = lib/utf8_decoder
-	url = https://github.com/PierrePharel/utf8_decoder.git
 [submodule "lib/std"]
 	path = lib/std
 	url = https://github.com/ArkScript-lang/std.git

CMakeLists.txt

@@ -27,7 +27,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 # files needed for the library ArkReactor
 file(GLOB_RECURSE SOURCE_FILES
     ${ark_SOURCE_DIR}/src/arkreactor/*.cpp
-    ${ark_SOURCE_DIR}/lib/fmt/src/*.cc)
+    ${ark_SOURCE_DIR}/lib/fmt/src/format.cc)
 
 add_library(ArkReactor SHARED ${SOURCE_FILES})
 
@@ -46,8 +46,8 @@ if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANG OR APPLE)
         )
 
     if (CMAKE_COMPILER_IS_GNUCXX)
-        # The package utf8_decoder has an issues with constant overflow.
-        # Once thisis fixed remove this flag: 
+        # The package utf8 has an issue with constant overflow.
+        # Once this is fixed remove this flag:
         target_compile_options(ArkReactor PUBLIC -Wno-overflow)
     endif()
 
@@ -96,7 +96,6 @@ add_subdirectory("${ark_SOURCE_DIR}/lib/termcolor" EXCLUDE_FROM_ALL)
 
 target_include_directories(ArkReactor
     PUBLIC
-    "${ark_SOURCE_DIR}/lib/utf8_decoder/"
     "${ark_SOURCE_DIR}/lib/picosha2/"
     "${ark_SOURCE_DIR}/lib/fmt/include")
 
@@ -122,12 +121,7 @@ target_include_directories(ArkReactor
     PUBLIC
     ${ark_SOURCE_DIR}/include)
 
-# setting up project properties
-set_target_properties(
-    ArkReactor
-    PROPERTIES
-        CXX_STANDARD 17
-        CXX_STANDARD_REQUIRED ON)
+target_compile_features(ArkReactor PRIVATE cxx_std_17)
 
 # Installation rules
 
@@ -176,12 +170,18 @@ if (ARK_BUILD_MODULES)
     add_subdirectory(${ark_SOURCE_DIR}/lib/modules)
 endif()
 
+if (ARK_BUILD_PARSER_TESTS)
+    add_executable(parser ${ark_SOURCE_DIR}/tests/parser/main.cpp)
+    target_link_libraries(parser PUBLIC ArkReactor)
+    target_compile_features(parser PRIVATE cxx_std_17)
+endif()
+
 if (ARK_BUILD_EXE)
     # additional files needed for the exe (repl, command line and stuff)
     file(GLOB_RECURSE EXE_SOURCES
         ${ark_SOURCE_DIR}/src/arkscript/REPL/Utils.cpp
         ${ark_SOURCE_DIR}/src/arkscript/REPL/Repl.cpp
-        ${ark_SOURCE_DIR}/lib/fmt/src/*.cc
+        ${ark_SOURCE_DIR}/lib/fmt/src/format.cc
         ${ark_SOURCE_DIR}/src/arkscript/main.cpp)
 
     add_executable(arkscript ${EXE_SOURCES})

cmake/link_time_optimization.cmake

@@ -3,7 +3,7 @@ include(CheckIPOSupported)
 check_ipo_supported(RESULT ipo_supported)
 
 function(enable_lto target_name)
-    if (ipo_supported)
+    if (ipo_supported AND (${CMAKE_BUILD_TYPE} STREQUAL "Release"))
         if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND (CMAKE_CXX_COMPILER_VERSION MATCHES "^8\..+"))
             message(WARNING "LTO supported but not enabled to prevent https://github.com/ArkScript-lang/Ark/pull/385#issuecomment-1163597951")
         else()

examples/error.ark

@@ -2,7 +2,7 @@
 # very often, and this is a convention,
 # if an imported file starts with a capital letter,
 # it shall be a file in the standard library.
-(import "Exceptions.ark")
+(import std.Exceptions)
 
 # the function which should do a "safe number invertion"
 (let invert (fun (x) {

examples/macros.ark

@@ -1,12 +1,12 @@
-!{suffix-dup (sym x) {
-    !{if (> x 1)
-        (suffix-dup sym (- x 1))}
-    (symcat sym x)}}
+($ suffix-dup (sym x) {
+    ($if (> x 1)
+        (suffix-dup sym (- x 1)))
+    (symcat sym x)})
 
-!{partial (func ...defargs) {
-    !{bloc (suffix-dup a (- (argcount func) (len defargs)))}
+($ partial (func ...defargs) {
+    ($ bloc (suffix-dup a (- (argcount func) (len defargs))))
     (fun (bloc) (func ...defargs bloc))
-    !{undef bloc}}}
+    ($undef bloc)})
 
 (let test_func (fun (a b c) (* a b c)))
 (let test_func1 (partial test_func 1))
@@ -16,60 +16,60 @@
 (print "Expected arguments for test_func1: " (argcount test_func1) ", expected " 2)
 (print "Calling them: " (test_func 1 2 3) " " (test_func1 2 3))
 
-!{foo (a b) (+ a b)}
+($ foo (a b) (+ a b))
 (print "Using macro foo (a b) => (+ a b): " (foo 1 2))
 
-!{var 12}
+($ var 12)
 (print "Using macro constant var=12: " var)
 
-!{if (= var 12)
+($if (= var 12)
     (print "This was executed in a if macro, testing var == 12")
-    (print "You shouldn't see this")}
+    (print "You shouldn't see this"))
 
-!{if (and true true)
+($if (and true true)
     (print "This was executed in a if macro, testing (and true true)")
-    (print "You shouldn't see this (bis)")}
+    (print "You shouldn't see this (bis)"))
 
-!{defun (name args body) (let name (fun args body))}
+($ defun (name args body) (let name (fun args body)))
 (defun a_func (a b) (+ a b))
 (print "Generated a function with a macro, a_func (a b) => (+ a b)")
 (print "Calling (a_func 1 2): " (a_func 1 2))
 
-!{one (...args) (print "Macro 'one', returns the 2nd argument given in " args " => " (@ args 1))}
+($ one (...args) (print "Macro 'one', returns the 2nd argument given in " args " => " (@ args 1)))
 (one 1 2)
 (one 1 3 4)
 (one 1 5 6 7 8)
 
-!{last (...args) (print "Macro 'last', returns the last argument given in " args " => " (@ args -1))}
+($ last (...args) (print "Macro 'last', returns the last argument given in " args " => " (@ args -1)))
 (last 1 2)
 (last 1 3 4)
 (last 1 5 6 7 8)
 
 {
     (print "Testing macros in scopes and macro shadowing")
 
-    !{test (+ 1 2 3)}
+    ($ test (+ 1 2 3))
     (print "(global) Reading macro 'test', expected 6, " test)
 
     ((fun () {
-        !{test (- 1 2 3)}
+        ($ test (- 1 2 3))
         (print "(sub scope) Reading macro 'test', expected -4, " test)}))
 
     (print "(global) Reading macro 'test', expected 6, " test)
 
     {
-        !{test 555}
+        ($ test 555)
         (print "(subscope) Reading macro 'test', expected 555, " test)
-        !{undef test}
+        ($ undef test)
         (print "(subscope, undef test) Reading macro 'test', expected 6, " test)
-        !{undef a}}}
+        ($ undef a)}}
 
 (print "Demonstrating a threading macro")
 
-!{-> (arg fn1 ...fn) {
-    !{if (> (len fn) 0)
+($ -> (arg fn1 ...fn) {
+    ($if (> (len fn) 0)
         (-> (fn1 arg) ...fn)
-        (fn1 arg)}}}
+        (fn1 arg))})
 
 (let filename "hello.json")
 

include/Ark/Compiler/AST/BaseParser.hpp

@@ -74,6 +74,7 @@ namespace Ark::internal
         bool suffix(char c);
         bool number(std::string* s = nullptr);
         bool signedNumber(std::string* s = nullptr);
+        bool hexNumber(unsigned length, std::string* s = nullptr);
         bool name(std::string* s = nullptr);
         bool sequence(const std::string& s);
         bool packageName(std::string* s = nullptr);

include/Ark/Compiler/AST/Import.hpp

@@ -3,6 +3,7 @@
 
 #include <vector>
 #include <string>
+#include <numeric>
 
 #include <Ark/Platform.hpp>
 
@@ -15,25 +16,55 @@ namespace Ark::internal
          * @details Example: `(import foo.bar)` => `bar`
          *      `(import foo.bar.egg:*)` => `egg`
          *      `(import foo :a :b :c)` => `foo`
-         * 
+         *
          */
-        std::string package;
+        std::string prefix;
+
+        /**
+         * @brief Package with all the segments
+         * @details Example: `(import foo.bar)` => `{foo, bar}`
+         *      `(import foo.bar.egg:*)` => `{foo, bar, egg}`
+         *      `(import foo :a :b :c)` => `{foo}`
+         */
+        std::vector<std::string> package;
 
         /**
          * @brief Import with prefix (the package) or not
-         * 
+         *
          */
         bool with_prefix = true;
 
         /**
          * @brief List of symbols to import, can be empty if none provided
-         * 
+         *
          */
         std::vector<std::string> symbols;
 
+        inline std::string toPackageString() const
+        {
+            return std::accumulate(package.begin() + 1, package.end(), package.front(), [](const std::string& left, const std::string& right) {
+                return left + "." + right;
+            });
+        }
+
+        inline std::string packageToPath() const
+        {
+            std::size_t offset = 0;
+            if (package.front() == "std")
+                offset = 1;
+
+            return std::accumulate(
+                std::next(package.begin() + offset),
+                package.end(),
+                package[offset],
+                [](const std::string& a, const std::string& b) {
+                    return a + "/" + b;
+                });
+        }
+
         /**
          * @brief Check if we should import everything, given something like `(import foo.bar.egg:*)`
-         * 
+         *
          * @return true if all symbols of the file should be imported in the importer scope
          * @return false otherwise
          */
@@ -44,9 +75,9 @@ namespace Ark::internal
 
         /**
          * @brief Check if we should import everything with a prefix, given a `(import foo.bar.egg)`
-         * 
-         * @return true 
-         * @return false 
+         *
+         * @return true
+         * @return false
          */
         inline bool isBasic() const
         {

include/Ark/Compiler/AST/Module.hpp

@@ -0,0 +1,17 @@
+#ifndef ARK_MODULE_HPP
+#define ARK_MODULE_HPP
+
+#include <Ark/Compiler/AST/Node.hpp>
+
+namespace Ark::internal
+{
+    // TODO store something better than just the AST (AST+what we are importing as private/public/namespaced... vs all)
+    //      so that we can remember the order in which we encountered imports.
+    struct Module
+    {
+        Node ast;
+        bool has_been_processed = false;  // TODO document this
+    };
+}
+
+#endif  // ARK_MODULE_HPP