Skip to content

Fix char round-trip above U+FFFF; cap \x escape at two digits #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hellerve merged 2 commits into from
Jun 28, 2026
Merged

Fix char round-trip above U+FFFF; cap \x escape at two digits #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d66ab6a
Fix char round-trip above U+FFFF; cap \x escape at two digits
Jun 28, 2026
ef1e728
Format carp-reader.carp with carp-fmt
Jun 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,15 @@ the project follows [Semantic Versioning](https://semver.org/).

## Unreleased

- Characters above U+FFFF now round-trip through `Form.str`. They are
rendered as `\U` followed by eight hex digits instead of `\u` with
five or six digits, which could not be re-parsed. Character literals
also accept `\U` escapes when reading.
- The `\x` escape inside string literals now consumes at most two hex
digits (one byte), matching C. Previously trailing hex digits were
silently folded into the low byte (e.g. `\x413` produced a single
byte instead of `A` followed by `3`).

## [0.3.6]

- String parser now rejects invalid escape sequences (e.g. `\z`,
Expand Down
211 changes: 120 additions & 91 deletions carp-reader.carp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,14 +95,25 @@ nested. Synthesized nodes (reader-macro expansions) carry
(hidden show-char)
(defn show-char [c]
(cond
(Char.= c \space) @"\\space"
(Char.= c \tab) @"\\tab"
(Char.= c \newline) @"\\newline"
(Char.= c \return) @"\\return"
(Char.= c \backspace) @"\\backspace"
(Char.= c \formfeed) @"\\formfeed"
(Char.= c \") @"\\\""
(Int.> (Char.to-int c) 127) (fmt "\\u%04x" (Char.to-int c))
(Char.= c \space)
@"\\space"
(Char.= c \tab)
@"\\tab"
(Char.= c \newline)
@"\\newline"
(Char.= c \return)
@"\\return"
(Char.= c \backspace)
@"\\backspace"
(Char.= c \formfeed)
@"\\formfeed"
(Char.= c \")
@"\\\""
; codepoints above the BMP need \U + 8 digits; \u only re-parses 4.
(Int.> (Char.to-int c) 65535)
(fmt "\\U%08x" (Char.to-int c))
(Int.> (Char.to-int c) 127)
(fmt "\\u%04x" (Char.to-int c))
(fmt "\\%c" c)))

(private double-round-trips?)
Expand Down Expand Up @@ -516,100 +527,103 @@ you need it.")
(Int.+ @(Cursor.pos &c) 1))]
(cond
(Char.= esc \n)
(do
(set! bytes (push-byte bytes \newline))
(set! c (Cursor.advance-by &c 2 false)))
(do
(set! bytes (push-byte bytes \newline))
(set! c (Cursor.advance-by &c 2 false)))
(Char.= esc \t)
(do
(set! bytes (push-byte bytes \tab))
(set! c (Cursor.advance-by &c 2 false)))
(do
(set! bytes (push-byte bytes \tab))
(set! c (Cursor.advance-by &c 2 false)))
(Char.= esc \r)
(do
(set! bytes (push-byte bytes \return))
(set! c (Cursor.advance-by &c 2 false)))
(do
(set! bytes (push-byte bytes \return))
(set! c (Cursor.advance-by &c 2 false)))
(Char.= esc \b)
(do
(set! bytes (push-byte bytes \backspace))
(set! c (Cursor.advance-by &c 2 false)))
(do
(set! bytes (push-byte bytes \backspace))
(set! c (Cursor.advance-by &c 2 false)))
(Char.= esc \f)
(do
(set! bytes (push-byte bytes \formfeed))
(set! c (Cursor.advance-by &c 2 false)))
(do
(set! bytes (push-byte bytes \formfeed))
(set! c (Cursor.advance-by &c 2 false)))
(Char.= esc \v)
(do
(set! bytes
(Array.push-back bytes
(Byte.from-int 11)))
(set! c (Cursor.advance-by &c 2 false)))
(do
(set! bytes
(Array.push-back bytes (Byte.from-int 11)))
(set! c (Cursor.advance-by &c 2 false)))
(Char.= esc \a)
(do
(set! bytes
(Array.push-back bytes (Byte.from-int 7)))
(set! c (Cursor.advance-by &c 2 false)))
(do
(set! bytes
(Array.push-back bytes (Byte.from-int 7)))
(set! c (Cursor.advance-by &c 2 false)))
(or (Char.= esc \\)
(or (Char.= esc \") (Char.= esc \')))
(do
(set! bytes (push-byte bytes esc))
(set! c (Cursor.advance-by &c 2 false)))
(do
(set! bytes (push-byte bytes esc))
(set! c (Cursor.advance-by &c 2 false)))
(Char.= esc \x)
(let-do [digits-start (Int.+ @(Cursor.pos &c) 2)
end digits-start]
(while-do (and (Int.< end len)
(hex-digit? (String.char-at src
end)))
(set! end (Int.inc end)))
(if (Int.> end digits-start)
; C semantics: \x maps to one byte, so read at
; most 2 hex digits (capping avoids silently
; folding extra digits down to the low byte).
(let-do [digits-start (Int.+ @(Cursor.pos &c) 2)
max-end (Int.min len
(Int.+ digits-start 2))
end digits-start]
(while-do (and (Int.< end max-end)
(hex-digit? (String.char-at src
end)))
(set! end (Int.inc end)))
(if (Int.> end digits-start)
(do
(set! bytes
(Array.push-back bytes
(Byte.from-int
(read-hex src
digits-start
(Int.- end
digits-start)))))
(set! c
(Cursor.advance-by &c
(Int.- end
@(Cursor.pos &c))
false)))
(set! err @"hex digits after \\x")))
(Char.= esc \u)
(if (Int.>= (Int.+ @(Cursor.pos &c) 6)
(Int.inc len))
(set! err @"4 hex digits in \\u escape")
(let-do [hex-start (Int.+ @(Cursor.pos &c) 2)
all-hex true]
(for [j hex-start (Int.+ hex-start 4)]
(unless (hex-digit? (String.char-at src j))
(set! all-hex false)))
(if all-hex
(do
(set! bytes
(Array.push-back bytes
(Byte.from-int
(Int.bit-and 255
(read-hex src
digits-start
(Int.- end
digits-start))))))
(set! c
(Cursor.advance-by &c
(Int.- end
@(Cursor.pos &c))
false)))
(set! err @"hex digits after \\x")))
(Char.= esc \u)
(if (Int.>= (Int.+ @(Cursor.pos &c) 6)
(Int.inc len))
(set! err @"4 hex digits in \\u escape")
(let-do [hex-start (Int.+ @(Cursor.pos &c) 2)
all-hex true]
(for [j hex-start (Int.+ hex-start 4)]
(unless (hex-digit? (String.char-at src j))
(set! all-hex false)))
(if all-hex
(do
(set! bytes
(push-utf8-bytes bytes
(read-hex src
hex-start
4)))
(set! c (Cursor.advance-by &c 6 false)))
(set! err @"hex digits in \\u escape"))))
(push-utf8-bytes bytes
(read-hex src
hex-start
4)))
(set! c (Cursor.advance-by &c 6 false)))
(set! err @"hex digits in \\u escape"))))
(Char.= esc \U)
(if (Int.>= (Int.+ @(Cursor.pos &c) 10)
(Int.inc len))
(set! err @"8 hex digits in \\U escape")
(let-do [hex-start (Int.+ @(Cursor.pos &c) 2)
all-hex true]
(for [j hex-start (Int.+ hex-start 8)]
(unless (hex-digit? (String.char-at src j))
(set! all-hex false)))
(if all-hex
(do
(set! bytes
(push-utf8-bytes bytes
(read-hex src
hex-start
8)))
(set! c (Cursor.advance-by &c 10 false)))
(set! err @"hex digits in \\U escape"))))
(if (Int.>= (Int.+ @(Cursor.pos &c) 10)
(Int.inc len))
(set! err @"8 hex digits in \\U escape")
(let-do [hex-start (Int.+ @(Cursor.pos &c) 2)
all-hex true]
(for [j hex-start (Int.+ hex-start 8)]
(unless (hex-digit? (String.char-at src j))
(set! all-hex false)))
(if all-hex
(do
(set! bytes
(push-utf8-bytes bytes
(read-hex src
hex-start
8)))
(set! c (Cursor.advance-by &c 10 false)))
(set! err @"hex digits in \\U escape"))))
(set! err @"valid escape sequence"))))
(do
(set! bytes (push-byte bytes ch))
Expand Down Expand Up @@ -708,13 +722,28 @@ you need it.")
(Parser.pure
(Form.Chr (Char.from-int (read-hex &hex 0 4)))))))))

(private unicode-char-long)
(hidden unicode-char-long)
(defn unicode-char-long []
(Parser.bind (Parser.byte \U)
(fn [_]
(Parser.bind
(Parser.slice-of
(Parser.count 8
(Parser.satisfy (fn [c] (hex-digit? c))
@"hex digit")))
(fn [hex]
(Parser.pure
(Form.Chr (Char.from-int (read-hex &hex 0 8)))))))))

(private char-)
(hidden char-)
(defn char- []
(Parser.bind (Parser.byte \\)
(fn [_]
(Parser.choice
[(Parser.try (unicode-char))
[(Parser.try (unicode-char-long))
(Parser.try (unicode-char))
(Parser.bind
(Parser.take-while1 (fn [c] (sym-byte? c)))
(fn [name]
Expand Down
28 changes: 28 additions & 0 deletions test/carp-reader.carp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,22 @@
"\\space"
&(Located.str &(first-form "\\space"))
"round-trip: named char")
(assert-equal t
1
(Array.length &(unsafe-parse "\\U0001f600"))
"char above BMP parses as a single form")
(assert-equal t
"\\U0001f600"
&(Located.str &(first-form "\\U0001f600"))
"round-trip: char above BMP via \\U")
(assert-equal t
"\\U00010000"
&(Located.str &(first-form "\\U00010000"))
"round-trip: first astral codepoint via \\U")
(assert-equal t
"\\uffff"
&(Located.str &(first-form "\\uffff"))
"round-trip: U+FFFF stays \\u at the BMP boundary")
(assert-equal t
"Foo.Bar.baz"
&(Located.str &(first-form "Foo.Bar.baz"))
Expand Down Expand Up @@ -193,6 +209,18 @@ b\"")) "string with literal newline stays as literal newline")
"\"A\""
&(Located.str &(first-form "\"\\u0041\""))
"unicode escape \\u0041 produces A")
(assert-equal t
"\"J\""
&(Located.str &(first-form "\"\\x4a\""))
"hex escape \\x4a produces J")
(assert-equal t
"\"A3\""
&(Located.str &(first-form "\"\\x413\""))
"hex escape caps at 2 digits: \\x413 is A then literal 3")
(assert-equal t
"\"😀\""
&(Located.str &(first-form "\"\\U0001F600\""))
"string \\U escape decodes to its UTF-8 bytes")

(assert-true t (parse-error? "\"\\z\"") "invalid escape \\z errors")
(assert-true t (parse-error? "\"\\q\"") "invalid escape \\q errors")
Expand Down
Loading