refactor!: prqlc-parser major reorg changes, remove prqlc-ast

.github/workflows/release.yaml

@@ -368,12 +368,12 @@ jobs:
       - uses: baptiste0928/cargo-install@v3
         with:
           crate: cargo-release
-      # Currently, we can only check prqlc-ast which is not dependent other local crates with --dry-run.
+      # Currently, we can only check prqlc-parser which is not dependent other local crates with --dry-run.
       # https://github.com/crate-ci/cargo-release/issues/691
       # --no-verify is required to prevent build.
       - run:
           cargo release publish --no-confirm ${{ github.event_name == 'release'
-          && '--execute' || '--no-verify --package prqlc-ast'}}
+          && '--execute' || '--no-verify --package prqlc-parser'}}
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
 

CHANGELOG.md

@@ -20,6 +20,16 @@
 
 **Internal changes**:
 
+- Major reorganization of `prqlc-parser` — `prqlc-ast` is merged into
+  `prqlc-parser`, and `prqlc-parser`'s files are rearranged, including its
+  exports. This is part of an effort to modularize the compiler by stage,
+  reducing the amount of context that's required to understand a single stage.
+  There will likely be some further changes (more detail in the PR description).
+  (@m-span, #4634)
+
+  - This is a breaking change for any libraries that use `prqlc-parser` which
+    should be fairly rare.
+
 - Renamed `prql-compiler-macros` to `prqlc-macros` for consistency with other
   crates (@max-sixty, #4565)
 

Cargo.toml

@@ -7,7 +7,6 @@ members = [
   "prqlc/bindings/prqlc-python",
   "prqlc/prqlc-macros",
   "prqlc/prql-compiler",
-  "prqlc/prqlc-ast",
   "prqlc/prqlc-parser",
   "prqlc/prqlc",
   "prqlc/prqlc/examples/compile-files", # An example

Taskfile.yaml

@@ -272,7 +272,7 @@ tasks:
           split(" ")[0]'
     cmds:
       - |
-        cargo test {{ range without (splitLines .PACKAGES) "prqlc-ast" "prqlc-parser" }} -p={{ . }} {{ end }}
+        cargo test {{ range without (splitLines .PACKAGES) "prqlc-parser" }} -p={{ . }} {{ end }}
 
   test-all:
     desc: Test everything, accepting snapshots.

lutra/lutra/src/compile.rs

@@ -4,9 +4,9 @@ use std::str::FromStr;
 use anyhow::Result;
 use prqlc::ir::decl::RootModule;
 use prqlc::ir::pl::{Ident, Literal};
+use prqlc::parser_error::Error;
 use prqlc::sql::Dialect;
-use prqlc::{semantic, ErrorMessages};
-use prqlc::{Error, Errors, Options, SourceTree, Target, WithErrorInfo};
+use prqlc::{semantic, ErrorMessages, Errors, Options, SourceTree, Target, WithErrorInfo};
 
 use crate::project::{DatabaseModule, ProjectCompiled, ProjectDiscovered, SqliteConnectionParams};
 

lutra/lutra/src/editing.rs

@@ -1,7 +1,7 @@
 use std::fs;
 
 use anyhow::Result;
-use prqlc::Error;
+use prqlc::parser_error::Error;
 
 use crate::ProjectCompiled;
 

lutra/lutra/src/execute.rs

@@ -1,5 +1,5 @@
 use anyhow::Result;
-use prqlc::{ir::pl::Ident, Error};
+use prqlc::{ir::pl::Ident, parser_error::Error};
 
 use super::ProjectCompiled;
 

lutra/lutra/src/pull_schema.rs

@@ -3,7 +3,7 @@ use arrow::datatypes::{DataType, SchemaRef};
 use connector_arrow::api::SchemaGet;
 use itertools::Itertools;
 use prqlc::ast::{PrimitiveSet, Stmt, Ty, TyKind, TyTupleField, VarDef};
-use prqlc::{Error, WithErrorInfo};
+use prqlc::{parser_error::Error, WithErrorInfo};
 
 use crate::ProjectCompiled;
 

prqlc/Taskfile.yaml

@@ -6,7 +6,7 @@ includes:
     dir: ./bindings/prqlc-python
 
 vars:
-  packages_core: -p prqlc-ast -p prqlc-parser -p prqlc
+  packages_core: -p prqlc-parser -p prqlc
   packages_addon: -p prqlc-macros -p compile-files
   packages_bindings: -p prql -p prql-java -p prqlc-js -p prqlc-c -p prqlc-python
 

prqlc/bindings/prqlc-python/src/lib.rs

@@ -148,7 +148,7 @@ impl CompileOptions {
 }
 
 fn convert_options(o: CompileOptions) -> Result<prqlc_lib::Options, prqlc_lib::ErrorMessages> {
-    use prqlc_lib::Error;
+    use prqlc_lib::parser_error::Error;
     let target = prqlc_lib::Target::from_str(&o.target).map_err(prqlc_lib::ErrorMessages::from)?;
 
     Ok(prqlc_lib::Options {

prqlc/prqlc-parser/Cargo.toml

@@ -12,12 +12,13 @@ version.workspace = true
 doctest = false
 
 [dependencies]
+enum-as-inner = {workspace = true}
 itertools = {workspace = true}
 log = {workspace = true}
-prqlc-ast = {path = "../prqlc-ast", version = "0.12.3"}
-semver = {workspace = true}
 serde = {workspace = true}
 serde_yaml = {workspace = true, optional = true}
+semver = {version = "1.0.23", features = ["serde"]}
+strum = {version = "0.26.2", features = ["std", "derive"]}
 
 # Chumsky's default features have issues when running in wasm (though we only
 # see it when compiling on macOS), so we only include features when running
@@ -31,3 +32,4 @@ chumsky = {version = "0.9.2", features = ["ahash", "std"], default-features = fa
 
 [dev-dependencies]
 insta = {workspace = true}
+serde_json = {workspace = true}

prqlc/prqlc-parser/src/err/error.rs → prqlc/prqlc-parser/src/error/mod.rs

@@ -1,12 +1,16 @@
-//! Error message produced by the compiler.
-
 use std::fmt::Debug;
 
 use chumsky::error::Cheap;
 use serde::Serialize;
 
-use crate::ast::span::Span;
-use crate::err::parse_error::PError;
+use crate::error::parse_error::PError;
+use crate::span::Span;
+
+/// Error message produced by the compiler.
+pub mod parse_error;
+
+#[cfg(test)]
+mod test;
 
 /// A prqlc error. Used internally, exposed as prqlc::ErrorMessage.
 #[derive(Debug, Clone)]
@@ -227,121 +231,3 @@ impl<T, E: WithErrorInfo> WithErrorInfo for Result<T, E> {
         self.map_err(|e| e.with_source(source))
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use insta::{assert_debug_snapshot, assert_snapshot};
-
-    use super::*;
-
-    // Helper function to create a simple Error object
-    fn create_simple_error() -> Error {
-        Error::new_simple("A simple error message")
-            .push_hint("take a hint")
-            .with_code("E001")
-    }
-
-    #[test]
-    fn display() {
-        assert_snapshot!(create_simple_error(),
-            @r###"Error { kind: Error, span: None, reason: Simple("A simple error message"), hints: ["take a hint"], code: Some("E001") }"###
-        );
-
-        let errors = Errors(vec![create_simple_error()]);
-        assert_snapshot!(errors,
-            @r###"Errors([Error { kind: Error, span: None, reason: Simple("A simple error message"), hints: ["take a hint"], code: Some("E001") }])"###
-        );
-        assert_debug_snapshot!(errors, @r###"
-        Errors(
-            [
-                Error {
-                    kind: Error,
-                    span: None,
-                    reason: Simple(
-                        "A simple error message",
-                    ),
-                    hints: [
-                        "take a hint",
-                    ],
-                    code: Some(
-                        "E001",
-                    ),
-                },
-            ],
-        )
-        "###)
-    }
-
-    #[test]
-    fn test_simple_error() {
-        let err = create_simple_error();
-        assert_debug_snapshot!(err, @r###"
-        Error {
-            kind: Error,
-            span: None,
-            reason: Simple(
-                "A simple error message",
-            ),
-            hints: [
-                "take a hint",
-            ],
-            code: Some(
-                "E001",
-            ),
-        }
-        "###);
-    }
-
-    #[test]
-    fn test_complex_error() {
-        assert_debug_snapshot!(
-        Error::new(Reason::Expected {
-            who: Some("Test".to_string()),
-            expected: "expected_value".to_string(),
-            found: "found_value".to_string(),
-        })
-        .with_code("E002"), @r###"
-        Error {
-            kind: Error,
-            span: None,
-            reason: Expected {
-                who: Some(
-                    "Test",
-                ),
-                expected: "expected_value",
-                found: "found_value",
-            },
-            hints: [],
-            code: Some(
-                "E002",
-            ),
-        }
-        "###);
-    }
-
-    #[test]
-    fn test_simple_error_with_result() {
-        let result: Result<(), Error> = Err(Error::new_simple("A simple error message"))
-            .with_hints(vec!["Take a hint"])
-            .push_hint("Take another hint")
-            .with_code("E001");
-        assert_debug_snapshot!(result, @r###"
-        Err(
-            Error {
-                kind: Error,
-                span: None,
-                reason: Simple(
-                    "A simple error message",
-                ),
-                hints: [
-                    "Take a hint",
-                    "Take another hint",
-                ],
-                code: Some(
-                    "E001",
-                ),
-            },
-        )
-        "###);
-    }
-}

prqlc/prqlc-parser/src/err/parse_error.rs → prqlc/prqlc-parser/src/error/parse_error.rs

@@ -3,10 +3,9 @@ use std::collections::HashSet;
 use std::fmt::Display;
 use std::hash::Hash;
 
-use prqlc_ast::TokenKind;
-
-use crate::ast::Span;
-use crate::err::error::{Error, ErrorSource, Reason, WithErrorInfo};
+use crate::error::{Error, ErrorSource, Reason, WithErrorInfo};
+use crate::lexer::lr::TokenKind;
+use crate::span::Span;
 
 #[derive(Clone, Debug)]
 pub struct ChumError<T: Hash + Eq> {