more work on readerv2

compatibility/jvm-native/src/main/scala/chester/reader/FilePathImplJVM.scala

@@ -13,7 +13,7 @@ object FilePathImplJVM extends FilePathImpl {
         Right(content)
       case Failure(exception) =>
         Left(
-          ParseError(s"Failed to read file: ${exception.getMessage}", Pos.Zero)
+          ParseError(s"Failed to read file: ${exception.getMessage}", Pos.zero)
         )
     }
   }

core/src/main/scala/chester/integrity/IntegrityCheck.scala

@@ -42,7 +42,7 @@ object IntegrityCheck {
       .fold(
         error =>
           fail(
-            s"Parsing failed for input: $input ${error.message} at index ${error.index}"
+            s"Parsing failed for input: $input ${error.message} at index ${error.pos}"
           ),
         value => assertEquals(value, expected, s"Failed for input: $input")
       )

core/src/test/scala/chester/reader/parseAndCheck.scala

@@ -16,7 +16,7 @@ def parseAndCheck(input: String, expected: Expr): Unit = {
     .fold(
       error =>
         fail(
-          s"Parsing failed for input: $input ${error.message} at index ${error.index}"
+          s"Parsing failed for input: $input ${error.message} at index ${error.pos}"
         ),
       { value =>
         assertEquals(read[Expr](write[Expr](value)), value)
@@ -46,7 +46,7 @@ def getParsed(input: String): Expr = {
     .fold(
       error =>
         fail(
-          s"Parsing failed for input: $input ${error.message} at index ${error.index}"
+          s"Parsing failed for input: $input ${error.message} at index ${error.pos}"
         ),
       value => value
     )

lsp/src/main/scala/chester/lsp/ChesterLanguageServer.scala

@@ -182,8 +182,8 @@ class ChesterLanguageServer extends LanguageServer with TextDocumentService with
     parseResult.fold(
       { parseError =>
         val range = new Range(
-          new Position(parseError.index.line, parseError.index.column.utf16),
-          new Position(parseError.index.line, parseError.index.column.utf16)
+          new Position(parseError.pos.line, parseError.pos.column.utf16),
+          new Position(parseError.pos.line, parseError.pos.column.utf16)
         )
         val diagnostic = new Diagnostic(
           range,

reader/src/main/scala/chester/reader/Parser.scala

@@ -65,7 +65,7 @@ case class ReaderInternal(
 
   def simpleId: P[String] = P(
     (CharacterPred(isIdentifierFirst).rep(1) ~ CharacterPred(
-      isIdentifierMiddle
+      isIdentifierPart
     ).rep.? ~ CharacterPred(isIdentifierEnd).?).!
   )
 

reader/src/main/scala/chester/reader/ReaderREPL.scala

@@ -79,7 +79,7 @@ object ReaderREPL {
         )(using p).exprEntrance
     ) match {
       case Parsed.Success(expr, _) => Right(expr)
-      case f: Parsed.Failure       => Left(ParseError(f.msg, Pos.Zero))
+      case f: Parsed.Failure       => Left(ParseError(f.msg, Pos.zero))
     }
   }
 }

reader/src/main/scala/chester/readerv2/Lexer.scala

@@ -0,0 +1,41 @@
+package chester.readerv2
+
+import chester.error.Pos
+import chester.reader.ParseError
+import Token._
+
+case class LexerState(
+  tokens: TokenStream,
+  current: Token,
+  errors: Vector[ParseError] = Vector.empty
+)
+
+class Lexer(tokens: TokenStream) {
+  def initialize: LexerState = {
+    tokens.headOption match {
+      case Some(Right(token)) => LexerState(tokens.tail, token)
+      case Some(Left(error)) => LexerState(tokens.tail, EOF(error.pos), Vector(error))
+      case None => LexerState(LazyList.empty, EOF(Pos.zero))
+    }
+  }
+
+  def advance(state: LexerState): LexerState = {
+    state.tokens.headOption match {
+      case Some(Right(token)) => 
+        state.copy(tokens = state.tokens.tail, current = token)
+      case Some(Left(error)) => 
+        state.copy(
+          tokens = state.tokens.tail, 
+          errors = state.errors :+ error
+        )
+      case None => state
+    }
+  }
+
+  def skipWhitespaceAndComments(state: LexerState): LexerState = {
+    state.current match {
+      case _: Whitespace | _: Comment => skipWhitespaceAndComments(advance(state))
+      case _ => state
+    }
+  }
+}

reader/src/main/scala/chester/readerv2/Token.scala

@@ -1,88 +1,76 @@
 package chester.readerv2
 
 import chester.error.Pos
+import chester.reader.ParseError
 
 sealed trait Token {
   def pos: Pos
   def text: String
 }
 
-sealed trait TokenKind {
-  def text: String
-}
-
-sealed trait Literal extends TokenKind
-sealed trait Delimiter extends TokenKind
-sealed trait Operator extends TokenKind
-
-object TokenKind {
-  sealed trait Name {
-    def parts: Vector[NamePart]
-  }
-
+object Token {
   sealed trait NamePart
   case class IdentifierPart(value: Vector[Char]) extends NamePart
   case class OperatorPart(value: Vector[Char]) extends NamePart
 
-  case class Identifier(parts: Vector[NamePart]) extends TokenKind with Name {
+  sealed trait StringSegment
+  case class StringChars(chars: Vector[Char]) extends StringSegment
+  case class StringEscape(char: Char) extends StringSegment
+  case class StringInterpolation(expr: Vector[Token]) extends StringSegment
+
+  case class Identifier(parts: Vector[NamePart], pos: Pos) extends Token {
     def text: String = parts.map {
       case IdentifierPart(chars) => chars.mkString
       case OperatorPart(chars) => chars.mkString
     }.mkString
   }
 
-  case class IntegerLiteral(value: BigInt, radix: Int) extends TokenKind with Literal {
+  case class IntegerLiteral(value: BigInt, radix: Int, pos: Pos) extends Token {
     def text: String = if (radix == 10) value.toString else s"0x${value.toString(16)}"
   }
 
-  case class RationalLiteral(value: BigDecimal) extends TokenKind with Literal {
+  case class RationalLiteral(value: BigDecimal, pos: Pos) extends Token {
     def text: String = value.toString
   }
 
-  case class StringLiteral(segments: Vector[StringSegment]) extends TokenKind with Literal {
-    def text: String = s"\"${segments.map(_.text).mkString}\""
+  case class StringLiteral(segments: Vector[StringSegment], pos: Pos) extends Token {
+    def text: String = {
+      val sb = new StringBuilder("\"")
+      segments.foreach {
+        case StringChars(chars) => sb.append(chars.mkString)
+        case StringEscape(c) => sb.append('\\').append(c)
+        case StringInterpolation(expr) => sb.append("${").append(expr.map(_.text).mkString).append("}")
+      }
+      sb.append("\"").toString
+    }
   }
 
-  sealed trait StringSegment {
-    def text: String
-  }
-  case class StringChars(chars: Vector[Char]) extends StringSegment {
-    def text: String = chars.mkString
-  }
-  case class StringEscape(char: Char) extends StringSegment {
-    def text: String = s"\\$char"
-  }
-
-  case class SymbolLiteral(segments: Vector[StringSegment]) extends TokenKind with Literal {
-    def text: String = s"'${segments.map(_.text).mkString}"
+  case class SymbolLiteral(name: String, pos: Pos) extends Token {
+    def text: String = s"'$name"
   }
 
   // Delimiters
-  case object LParen extends TokenKind with Delimiter { def text = "(" }
-  case object RParen extends TokenKind with Delimiter { def text = ")" }
-  case object LBrace extends TokenKind with Delimiter { def text = "{" }
-  case object RBrace extends TokenKind with Delimiter { def text = "}" }
-  case object LBracket extends TokenKind with Delimiter { def text = "[" }
-  case object RBracket extends TokenKind with Delimiter { def text = "]" }
-
-  // Operators
-  case object Comma extends TokenKind with Operator { def text = "," }
-  case object Dot extends TokenKind with Operator { def text = "." }
-  case object Equal extends TokenKind with Operator { def text = "=" }
-  case object Arrow extends TokenKind with Operator { def text = "->" }
+  case class LParen(pos: Pos) extends Token { def text = "(" }
+  case class RParen(pos: Pos) extends Token { def text = ")" }
+  case class LBrace(pos: Pos) extends Token { def text = "{" }
+  case class RBrace(pos: Pos) extends Token { def text = "}" }
+  case class LBracket(pos: Pos) extends Token { def text = "[" }
+  case class RBracket(pos: Pos) extends Token { def text = "]" }
+  case class Comma(pos: Pos) extends Token { def text = "," }
+  case class Dot(pos: Pos) extends Token { def text = "." }
+  case class Equal(pos: Pos) extends Token { def text = "=" }
+  case class Arrow(pos: Pos) extends Token { def text = "->" }
 
   // Comments and Whitespace
-  case class Comment(content: Vector[Char]) extends TokenKind {
+  case class Comment(content: Vector[Char], pos: Pos) extends Token {
     def text: String = s"//${content.mkString}"
   }
 
-  case class Whitespace(chars: Vector[Char]) extends TokenKind {
+  case class Whitespace(chars: Vector[Char], pos: Pos) extends Token {
     def text: String = chars.mkString
   }
 
-  case object EOF extends TokenKind { def text = "" }
+  case class EOF(pos: Pos) extends Token { def text = "" }
 }
 
-case class TokenWithPos(kind: TokenKind, pos: Pos) extends Token {
-  def text: String = kind.text
-}
+type TokenStream = LazyList[Either[ParseError, Token]]