start readerv2

cli/shared/src/main/scala/chester/cli/CLI.scala

@@ -6,7 +6,7 @@ import chester.error.Problem.Severity
 import chester.integrity.IntegrityCheck
 import chester.reader.{ChesterReader, FilePath, FilePathImpl}
 import chester.repl.REPLEngine
-import chester.tyck.{Reporter, TyckResult, Tycker}
+import chester.tyck.{TyckResult, Tycker}
 import chester.utils.env.Environment
 import chester.utils.io.*
 import chester.utils.term.{Terminal, TerminalInit}

reader/src/main/scala/chester/reader/Parser.scala

@@ -64,16 +64,16 @@ case class ReaderInternal(
   def maybeSpace1: P[Vector[Comment]] = P(delimiter1.?.map(_.toVector.flatten))
 
   def simpleId: P[String] = P(
-    (CharacterPred(identifierFirst).rep(1) ~ CharacterPred(
-      identifierMiddle
-    ).rep.? ~ CharacterPred(identifierEnd).?).!
+    (CharacterPred(isIdentifierFirst).rep(1) ~ CharacterPred(
+      isIdentifierMiddle
+    ).rep.? ~ CharacterPred(isIdentifierEnd).?).!
   )
 
   def id: P[String] = operatorId | simpleId
 
   def operatorId: P[String] = P(
-    (CharacterPred(operatorIdentifierFirst).rep(1) ~ CharacterPred(
-      operatorIdentifierRest
+    (CharacterPred(isOperatorIdentifierFirst).rep(1) ~ CharacterPred(
+      isOperatorIdentifierRest
     ).rep).!
   )
 

reader/src/main/scala/chester/readerv2/Token.scala

@@ -0,0 +1,88 @@
+package chester.readerv2
+
+import chester.error.Pos
+
+sealed trait Token {
+  def pos: Pos
+  def text: String
+}
+
+sealed trait TokenKind {
+  def text: String
+}
+
+sealed trait Literal extends TokenKind
+sealed trait Delimiter extends TokenKind
+sealed trait Operator extends TokenKind
+
+object TokenKind {
+  sealed trait Name {
+    def parts: Vector[NamePart]
+  }
+
+  sealed trait NamePart
+  case class IdentifierPart(value: Vector[Char]) extends NamePart
+  case class OperatorPart(value: Vector[Char]) extends NamePart
+
+  case class Identifier(parts: Vector[NamePart]) extends TokenKind with Name {
+    def text: String = parts.map {
+      case IdentifierPart(chars) => chars.mkString
+      case OperatorPart(chars) => chars.mkString
+    }.mkString
+  }
+
+  case class IntegerLiteral(value: BigInt, radix: Int) extends TokenKind with Literal {
+    def text: String = if (radix == 10) value.toString else s"0x${value.toString(16)}"
+  }
+
+  case class RationalLiteral(value: BigDecimal) extends TokenKind with Literal {
+    def text: String = value.toString
+  }
+
+  case class StringLiteral(segments: Vector[StringSegment]) extends TokenKind with Literal {
+    def text: String = s"\"${segments.map(_.text).mkString}\""
+  }
+
+  sealed trait StringSegment {
+    def text: String
+  }
+  case class StringChars(chars: Vector[Char]) extends StringSegment {
+    def text: String = chars.mkString
+  }
+  case class StringEscape(char: Char) extends StringSegment {
+    def text: String = s"\\$char"
+  }
+
+  case class SymbolLiteral(segments: Vector[StringSegment]) extends TokenKind with Literal {
+    def text: String = s"'${segments.map(_.text).mkString}"
+  }
+
+  // Delimiters
+  case object LParen extends TokenKind with Delimiter { def text = "(" }
+  case object RParen extends TokenKind with Delimiter { def text = ")" }
+  case object LBrace extends TokenKind with Delimiter { def text = "{" }
+  case object RBrace extends TokenKind with Delimiter { def text = "}" }
+  case object LBracket extends TokenKind with Delimiter { def text = "[" }
+  case object RBracket extends TokenKind with Delimiter { def text = "]" }
+
+  // Operators
+  case object Comma extends TokenKind with Operator { def text = "," }
+  case object Dot extends TokenKind with Operator { def text = "." }
+  case object Equal extends TokenKind with Operator { def text = "=" }
+  case object Arrow extends TokenKind with Operator { def text = "->" }
+
+  // Comments and Whitespace
+  case class Comment(content: Vector[Char]) extends TokenKind {
+    def text: String = s"//${content.mkString}"
+  }
+
+  case class Whitespace(chars: Vector[Char]) extends TokenKind {
+    def text: String = chars.mkString
+  }
+
+  case object EOF extends TokenKind { def text = "" }
+}
+
+case class TokenWithPos(kind: TokenKind, pos: Pos) extends Token {
+  def text: String = kind.text
+}