add operators into token kinds

vuvoth · Dec 28, 2024 · bb95297 · bb95297
1 parent 8e4853d
commit bb95297
Show file tree

Hide file tree

Showing 2 changed files with 105 additions and 77 deletions.
diff --git a/crates/lsp/src/handler/goto_definition.rs b/crates/lsp/src/handler/goto_definition.rs
@@ -228,7 +228,7 @@ template Y() {
         let file = FileDB::create(&source, Url::from_file_path(Path::new("/tmp")).unwrap());
 
         let syntax_node = SyntaxTreeBuilder::syntax_tree(&source);
-
+        
         if let Some(program_ast) = AstCircomProgram::cast(syntax_node) {
             for template in program_ast.template_list() {
                 println!("{template:?}");

diff --git a/crates/parser/src/token_kind.rs b/crates/parser/src/token_kind.rs
@@ -5,24 +5,29 @@ use serde::Serialize;
 #[allow(non_camel_case_types)]
 #[repr(u16)]
 pub enum TokenKind {
+    // Error
     #[error]
     Error = 0,
+    // Comments
     #[regex(r"//[^\n]*")]
     CommentLine,
     #[token("/*")]
     CommentBlockOpen,
     #[token("*/")]
     CommentBlockClose,
+    // Trivial
     #[regex("[ \t]+")]
     WhiteSpace,
     #[regex("[\n]")]
     EndLine,
+    // Circom
     #[token("pragma")]
     Pragma,
     #[token("circom")]
     Circom,
     #[regex("2.[0-9].[0-9]")]
     Version,
+    // Literals
     #[regex("[0-9]+")]
     Number,
     #[regex("[$_]*[a-zA-Z][a-zA-Z0-9_$]*")]
@@ -31,26 +36,7 @@ pub enum TokenKind {
     CircomString,
     #[token("template")]
     TemplateKw,
-    #[token("function")]
-    FunctionKw,
-    #[token("component")]
-    ComponentKw,
-    #[token("main")]
-    MainKw,
-    #[token("public")]
-    PublicKw,
-    #[token("signal")]
-    SignalKw,
-    #[token("var")]
-    VarKw,
-    #[token("include")]
-    IncludeKw,
-    #[token("input")]
-    InputKw,
-    #[token("output")]
-    OutputKw,
-    #[token("log")]
-    LogKw,
+    // Brackets
     #[token("(")]
     LParen,
     #[token(")")]
@@ -63,30 +49,94 @@ pub enum TokenKind {
     LBracket,
     #[token("]")]
     RBracket,
+    // Punctuation 
     #[token(";")]
     Semicolon,
     #[token(",")]
     Comma,
-    // TODO: review
-    #[token("++")]
-    UnitInc,
-    #[token("--")]
-    UnitDec,
+    #[token(".")]
+    Dot,
+    // Boolean operators
+    #[token("&&")]
+    BoolAnd,
+    #[token("||")]
+    BoolOr,
+    #[token("!")]
+    Not,
+    // Relational operators
+    #[token("==")]
+    Equal,
+    #[token("!=")]
+    NotEqual,
+    #[token("<")]
+    LessThan,
+    #[token(">")]
+    GreaterThan,
+    #[token("<=")]
+    LessThanAndEqual,
+    #[token(">=")]
+    GreaterThanAndEqual,
+    // Arithmetic operators
+    #[token("+")]
+    Add,
+    #[token("-")]
+    Sub,
+    #[token("*")]
+    Mul,
+    #[token("**")]
+    Power,
+    #[token("/")]
+    Div,
+    #[token("\\")]
+    IntDiv,
+    #[token("%")]
+    Mod,
+    // Combined arithmetic assignment
     #[token("+=")]
     AddAssign,
     #[token("-=")]
     SubAssign,
+    #[token("*=")]
+    MulAssign,
+    #[token("**=")]
+    PowerAssign,
     #[token("/=")]
     DivAssign,
     #[token(r"\=")]
     IntDivAssign,
     #[token("%=")]
     ModAssign,
-    #[token("*=")]
-    MulAssign,
-    #[token("**=")]
-    PowerAssign,
-    // end review
+    #[token("++")]
+    UnitInc,
+    #[token("--")]
+    UnitDec,
+    // Bitwise operators
+    #[token("&")]
+    BitAnd,
+    #[token("|")]
+    BitOr,
+    #[token("~")]
+    BitNot,
+    #[token("^")]
+    BitXor,
+    #[token(">>")]
+    ShiftR,
+    #[token("<<")]
+    ShiftL,
+    // Combined bitwise assignments 
+    #[token("&=")]
+    BitAndAssign,
+    #[token("|=")]
+    BitOrAssign,
+    #[token("~=")]
+    BitNotAssign,
+    #[token("^=")]
+    BitXorAssign,
+    #[token(">>=")]
+    ShiftRAssign,
+    #[token("<<=")]
+    ShiftLAssign,
+    // Assign
     #[token("=")]
     Assign,
     #[token("===")]
@@ -99,56 +149,33 @@ pub enum TokenKind {
     RAssignSignal,
     #[token("<==")]
     RAssignConstraintSignal,
-    #[token("+")]
-    Add,
-    #[token("-")]
-    Sub,
-    #[token("/")]
-    Div,
-    #[token("*")]
-    Mul,
-    #[token("!")]
-    Not,
-    #[token("~")]
-    BitNot,
-    #[token("**")]
-    Power,
-    #[token("\\")]
-    IntDiv,
-    #[token("%")]
-    Mod,
-    #[token("<<")]
-    ShiftL,
-    #[token(">>")]
-    ShiftR,
-    #[token("&")]
-    BitAnd,
-    #[token("|")]
-    BitOr,
-    #[token("^")]
-    BitXor,
-    #[token("==")]
-    Equal,
-    #[token("!=")]
-    NotEqual,
-    #[token("<")]
-    LessThan,
-    #[token(">")]
-    GreaterThan,
-    #[token("<=")]
-    LessThanAndEqual,
-    #[token(">=")]
-    GreaterThanAndEqual,
-    #[token("&&")]
-    BoolAnd,
-    #[token("||")]
-    BoolOr,
+    // Conditional expressions
     #[token("?")]
     MarkQuestion,
     #[token(":")]
     Colon,
-    #[token(".")]
-    Dot,
+    // Keywords
+    #[token("function")]
+    FunctionKw,
+    #[token("component")]
+    ComponentKw,
+    #[token("main")]
+    MainKw,
+    #[token("public")]
+    PublicKw,
+    #[token("signal")]
+    SignalKw,
+    #[token("var")]
+    VarKw,
+    #[token("include")]
+    IncludeKw,
+    #[token("input")]
+    InputKw,
+    #[token("output")]
+    OutputKw,
+    #[token("log")]
+    LogKw,
+    // Statement keywords
     #[token("if")]
     IfKw,
     #[token("else")]
@@ -161,6 +188,7 @@ pub enum TokenKind {
     ReturnKw,
     #[token("assert")]
     AssertKw,
+    // Complex token kind
     ForLoop,
     AssignStatement,
     CircomProgram,