From d66ab6a0e737b294a19c7cc45e85de9d5f1a0323 Mon Sep 17 00:00:00 2001 From: "carpentry-heartbeat[bot]" Date: Sun, 28 Jun 2026 20:01:48 +0200 Subject: [PATCH 1/2] Fix char round-trip above U+FFFF; cap \x escape at two digits show-char emitted \u with five or six hex digits for codepoints above the BMP, which the four-digit \u reader could not re-parse. Emit \U + eight digits for those codepoints and teach the character reader to accept \U escapes so they round-trip. Also cap the \x string escape at two hex digits (one byte) to match C semantics; extra digits were previously folded silently into the low byte. Adds round-trip and edge-case tests for \U and \x. --- CHANGELOG.md | 9 +++++++++ carp-reader.carp | 35 ++++++++++++++++++++++++++++------- test/carp-reader.carp | 28 ++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31a83d3..def68a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ the project follows [Semantic Versioning](https://semver.org/). ## Unreleased +- Characters above U+FFFF now round-trip through `Form.str`. They are + rendered as `\U` followed by eight hex digits instead of `\u` with + five or six digits, which could not be re-parsed. Character literals + also accept `\U` escapes when reading. +- The `\x` escape inside string literals now consumes at most two hex + digits (one byte), matching C. Previously trailing hex digits were + silently folded into the low byte (e.g. `\x413` produced a single + byte instead of `A` followed by `3`). + ## [0.3.6] - String parser now rejects invalid escape sequences (e.g. `\z`, diff --git a/carp-reader.carp b/carp-reader.carp index 2c8eed2..71dd675 100644 --- a/carp-reader.carp +++ b/carp-reader.carp @@ -102,6 +102,8 @@ nested. Synthesized nodes (reader-macro expansions) carry (Char.= c \backspace) @"\\backspace" (Char.= c \formfeed) @"\\formfeed" (Char.= c \") @"\\\"" + ; codepoints above the BMP need \U + 8 digits; \u only re-parses 4. + (Int.> (Char.to-int c) 65535) (fmt "\\U%08x" (Char.to-int c)) (Int.> (Char.to-int c) 127) (fmt "\\u%04x" (Char.to-int c)) (fmt "\\%c" c))) @@ -552,9 +554,14 @@ you need it.") (set! bytes (push-byte bytes esc)) (set! c (Cursor.advance-by &c 2 false))) (Char.= esc \x) + ; C semantics: \x maps to one byte, so read at + ; most 2 hex digits (capping avoids silently + ; folding extra digits down to the low byte). (let-do [digits-start (Int.+ @(Cursor.pos &c) 2) + max-end (Int.min len + (Int.+ digits-start 2)) end digits-start] - (while-do (and (Int.< end len) + (while-do (and (Int.< end max-end) (hex-digit? (String.char-at src end))) (set! end (Int.inc end))) @@ -563,11 +570,10 @@ you need it.") (set! bytes (Array.push-back bytes (Byte.from-int - (Int.bit-and 255 - (read-hex src - digits-start - (Int.- end - digits-start)))))) + (read-hex src + digits-start + (Int.- end + digits-start))))) (set! c (Cursor.advance-by &c (Int.- end @@ -708,13 +714,28 @@ you need it.") (Parser.pure (Form.Chr (Char.from-int (read-hex &hex 0 4))))))))) + (private unicode-char-long) + (hidden unicode-char-long) + (defn unicode-char-long [] + (Parser.bind (Parser.byte \U) + (fn [_] + (Parser.bind + (Parser.slice-of + (Parser.count 8 + (Parser.satisfy (fn [c] (hex-digit? c)) + @"hex digit"))) + (fn [hex] + (Parser.pure + (Form.Chr (Char.from-int (read-hex &hex 0 8))))))))) + (private char-) (hidden char-) (defn char- [] (Parser.bind (Parser.byte \\) (fn [_] (Parser.choice - [(Parser.try (unicode-char)) + [(Parser.try (unicode-char-long)) + (Parser.try (unicode-char)) (Parser.bind (Parser.take-while1 (fn [c] (sym-byte? c))) (fn [name] diff --git a/test/carp-reader.carp b/test/carp-reader.carp index 0e909be..1a877b8 100644 --- a/test/carp-reader.carp +++ b/test/carp-reader.carp @@ -51,6 +51,22 @@ "\\space" &(Located.str &(first-form "\\space")) "round-trip: named char") + (assert-equal t + 1 + (Array.length &(unsafe-parse "\\U0001f600")) + "char above BMP parses as a single form") + (assert-equal t + "\\U0001f600" + &(Located.str &(first-form "\\U0001f600")) + "round-trip: char above BMP via \\U") + (assert-equal t + "\\U00010000" + &(Located.str &(first-form "\\U00010000")) + "round-trip: first astral codepoint via \\U") + (assert-equal t + "\\uffff" + &(Located.str &(first-form "\\uffff")) + "round-trip: U+FFFF stays \\u at the BMP boundary") (assert-equal t "Foo.Bar.baz" &(Located.str &(first-form "Foo.Bar.baz")) @@ -193,6 +209,18 @@ b\"")) "string with literal newline stays as literal newline") "\"A\"" &(Located.str &(first-form "\"\\u0041\"")) "unicode escape \\u0041 produces A") + (assert-equal t + "\"J\"" + &(Located.str &(first-form "\"\\x4a\"")) + "hex escape \\x4a produces J") + (assert-equal t + "\"A3\"" + &(Located.str &(first-form "\"\\x413\"")) + "hex escape caps at 2 digits: \\x413 is A then literal 3") + (assert-equal t + "\"😀\"" + &(Located.str &(first-form "\"\\U0001F600\"")) + "string \\U escape decodes to its UTF-8 bytes") (assert-true t (parse-error? "\"\\z\"") "invalid escape \\z errors") (assert-true t (parse-error? "\"\\q\"") "invalid escape \\q errors") From ef1e728931224f2ce231cce1b251d0fccc095b61 Mon Sep 17 00:00:00 2001 From: "carpentry-heartbeat[bot]" Date: Sun, 28 Jun 2026 23:01:28 +0200 Subject: [PATCH 2/2] Format carp-reader.carp with carp-fmt Run the formatter over the file so it passes the CI format check; no behaviour change. --- carp-reader.carp | 198 ++++++++++++++++++++++++----------------------- 1 file changed, 103 insertions(+), 95 deletions(-) diff --git a/carp-reader.carp b/carp-reader.carp index 71dd675..3856de3 100644 --- a/carp-reader.carp +++ b/carp-reader.carp @@ -95,16 +95,25 @@ nested. Synthesized nodes (reader-macro expansions) carry (hidden show-char) (defn show-char [c] (cond - (Char.= c \space) @"\\space" - (Char.= c \tab) @"\\tab" - (Char.= c \newline) @"\\newline" - (Char.= c \return) @"\\return" - (Char.= c \backspace) @"\\backspace" - (Char.= c \formfeed) @"\\formfeed" - (Char.= c \") @"\\\"" + (Char.= c \space) + @"\\space" + (Char.= c \tab) + @"\\tab" + (Char.= c \newline) + @"\\newline" + (Char.= c \return) + @"\\return" + (Char.= c \backspace) + @"\\backspace" + (Char.= c \formfeed) + @"\\formfeed" + (Char.= c \") + @"\\\"" ; codepoints above the BMP need \U + 8 digits; \u only re-parses 4. - (Int.> (Char.to-int c) 65535) (fmt "\\U%08x" (Char.to-int c)) - (Int.> (Char.to-int c) 127) (fmt "\\u%04x" (Char.to-int c)) + (Int.> (Char.to-int c) 65535) + (fmt "\\U%08x" (Char.to-int c)) + (Int.> (Char.to-int c) 127) + (fmt "\\u%04x" (Char.to-int c)) (fmt "\\%c" c))) (private double-round-trips?) @@ -518,104 +527,103 @@ you need it.") (Int.+ @(Cursor.pos &c) 1))] (cond (Char.= esc \n) - (do - (set! bytes (push-byte bytes \newline)) - (set! c (Cursor.advance-by &c 2 false))) + (do + (set! bytes (push-byte bytes \newline)) + (set! c (Cursor.advance-by &c 2 false))) (Char.= esc \t) - (do - (set! bytes (push-byte bytes \tab)) - (set! c (Cursor.advance-by &c 2 false))) + (do + (set! bytes (push-byte bytes \tab)) + (set! c (Cursor.advance-by &c 2 false))) (Char.= esc \r) - (do - (set! bytes (push-byte bytes \return)) - (set! c (Cursor.advance-by &c 2 false))) + (do + (set! bytes (push-byte bytes \return)) + (set! c (Cursor.advance-by &c 2 false))) (Char.= esc \b) - (do - (set! bytes (push-byte bytes \backspace)) - (set! c (Cursor.advance-by &c 2 false))) + (do + (set! bytes (push-byte bytes \backspace)) + (set! c (Cursor.advance-by &c 2 false))) (Char.= esc \f) - (do - (set! bytes (push-byte bytes \formfeed)) - (set! c (Cursor.advance-by &c 2 false))) + (do + (set! bytes (push-byte bytes \formfeed)) + (set! c (Cursor.advance-by &c 2 false))) (Char.= esc \v) - (do - (set! bytes - (Array.push-back bytes - (Byte.from-int 11))) - (set! c (Cursor.advance-by &c 2 false))) + (do + (set! bytes + (Array.push-back bytes (Byte.from-int 11))) + (set! c (Cursor.advance-by &c 2 false))) (Char.= esc \a) - (do - (set! bytes - (Array.push-back bytes (Byte.from-int 7))) - (set! c (Cursor.advance-by &c 2 false))) + (do + (set! bytes + (Array.push-back bytes (Byte.from-int 7))) + (set! c (Cursor.advance-by &c 2 false))) (or (Char.= esc \\) (or (Char.= esc \") (Char.= esc \'))) - (do - (set! bytes (push-byte bytes esc)) - (set! c (Cursor.advance-by &c 2 false))) + (do + (set! bytes (push-byte bytes esc)) + (set! c (Cursor.advance-by &c 2 false))) (Char.= esc \x) - ; C semantics: \x maps to one byte, so read at - ; most 2 hex digits (capping avoids silently - ; folding extra digits down to the low byte). - (let-do [digits-start (Int.+ @(Cursor.pos &c) 2) - max-end (Int.min len - (Int.+ digits-start 2)) - end digits-start] - (while-do (and (Int.< end max-end) - (hex-digit? (String.char-at src - end))) - (set! end (Int.inc end))) - (if (Int.> end digits-start) + ; C semantics: \x maps to one byte, so read at + ; most 2 hex digits (capping avoids silently + ; folding extra digits down to the low byte). + (let-do [digits-start (Int.+ @(Cursor.pos &c) 2) + max-end (Int.min len + (Int.+ digits-start 2)) + end digits-start] + (while-do (and (Int.< end max-end) + (hex-digit? (String.char-at src + end))) + (set! end (Int.inc end))) + (if (Int.> end digits-start) + (do + (set! bytes + (Array.push-back bytes + (Byte.from-int + (read-hex src + digits-start + (Int.- end + digits-start))))) + (set! c + (Cursor.advance-by &c + (Int.- end + @(Cursor.pos &c)) + false))) + (set! err @"hex digits after \\x"))) + (Char.= esc \u) + (if (Int.>= (Int.+ @(Cursor.pos &c) 6) + (Int.inc len)) + (set! err @"4 hex digits in \\u escape") + (let-do [hex-start (Int.+ @(Cursor.pos &c) 2) + all-hex true] + (for [j hex-start (Int.+ hex-start 4)] + (unless (hex-digit? (String.char-at src j)) + (set! all-hex false))) + (if all-hex (do (set! bytes - (Array.push-back bytes - (Byte.from-int - (read-hex src - digits-start - (Int.- end - digits-start))))) - (set! c - (Cursor.advance-by &c - (Int.- end - @(Cursor.pos &c)) - false))) - (set! err @"hex digits after \\x"))) - (Char.= esc \u) - (if (Int.>= (Int.+ @(Cursor.pos &c) 6) - (Int.inc len)) - (set! err @"4 hex digits in \\u escape") - (let-do [hex-start (Int.+ @(Cursor.pos &c) 2) - all-hex true] - (for [j hex-start (Int.+ hex-start 4)] - (unless (hex-digit? (String.char-at src j)) - (set! all-hex false))) - (if all-hex - (do - (set! bytes - (push-utf8-bytes bytes - (read-hex src - hex-start - 4))) - (set! c (Cursor.advance-by &c 6 false))) - (set! err @"hex digits in \\u escape")))) + (push-utf8-bytes bytes + (read-hex src + hex-start + 4))) + (set! c (Cursor.advance-by &c 6 false))) + (set! err @"hex digits in \\u escape")))) (Char.= esc \U) - (if (Int.>= (Int.+ @(Cursor.pos &c) 10) - (Int.inc len)) - (set! err @"8 hex digits in \\U escape") - (let-do [hex-start (Int.+ @(Cursor.pos &c) 2) - all-hex true] - (for [j hex-start (Int.+ hex-start 8)] - (unless (hex-digit? (String.char-at src j)) - (set! all-hex false))) - (if all-hex - (do - (set! bytes - (push-utf8-bytes bytes - (read-hex src - hex-start - 8))) - (set! c (Cursor.advance-by &c 10 false))) - (set! err @"hex digits in \\U escape")))) + (if (Int.>= (Int.+ @(Cursor.pos &c) 10) + (Int.inc len)) + (set! err @"8 hex digits in \\U escape") + (let-do [hex-start (Int.+ @(Cursor.pos &c) 2) + all-hex true] + (for [j hex-start (Int.+ hex-start 8)] + (unless (hex-digit? (String.char-at src j)) + (set! all-hex false))) + (if all-hex + (do + (set! bytes + (push-utf8-bytes bytes + (read-hex src + hex-start + 8))) + (set! c (Cursor.advance-by &c 10 false))) + (set! err @"hex digits in \\U escape")))) (set! err @"valid escape sequence")))) (do (set! bytes (push-byte bytes ch))