diff --git a/Cargo.toml b/Cargo.toml index 882b298..189df2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "litrs" -version = "0.4.0" +version = "0.4.1" authors = ["Lukas Kalbertodt "] edition = "2018" rust-version = "1.54" @@ -25,9 +25,11 @@ exclude = [".github"] [features] -default = ["proc-macro2"] +default = ["proc-macro2", "printing"] check_suffix = ["unicode-xid"] +printing = ["dep:quote"] [dependencies] -proc-macro2 = { version = "1", optional = true } +proc-macro2 = { version = "1", optional = true } unicode-xid = { version = "0.2.4", optional = true } +quote = { version = "1", optional = true, no-default-features = true } diff --git a/src/bool/mod.rs b/src/bool/mod.rs index d7b54a1..b43ea26 100644 --- a/src/bool/mod.rs +++ b/src/bool/mod.rs @@ -1,7 +1,9 @@ use std::fmt; -use crate::{ParseError, err::{perr, ParseErrorKind::*}}; - +use crate::{ + err::{perr, ParseErrorKind::*}, + ParseError, +}; /// A bool literal: `true` or `false`. Also see [the reference][ref]. /// @@ -50,6 +52,5 @@ impl fmt::Display for BoolLit { } } - #[cfg(test)] mod tests; diff --git a/src/bool/tests.rs b/src/bool/tests.rs index 4b82924..f5b6fab 100644 --- a/src/bool/tests.rs +++ b/src/bool/tests.rs @@ -1,18 +1,17 @@ -use crate::{ - Literal, BoolLit, - test_util::assert_parse_ok_eq, -}; +use crate::{test_util::assert_parse_ok_eq, BoolLit, Literal}; macro_rules! assert_bool_parse { ($input:literal, $expected:expr) => { assert_parse_ok_eq( - $input, Literal::parse($input), Literal::Bool($expected), "Literal::parse"); + $input, + Literal::parse($input), + Literal::Bool($expected), + "Literal::parse", + ); assert_parse_ok_eq($input, BoolLit::parse($input), $expected, "BoolLit::parse"); }; } - - #[test] fn parse_ok() { assert_bool_parse!("false", BoolLit::False); diff --git a/src/byte/mod.rs b/src/byte/mod.rs index ffdff5d..05eae71 100644 --- a/src/byte/mod.rs +++ b/src/byte/mod.rs @@ -1,13 +1,12 @@ use core::fmt; use crate::{ - Buffer, ParseError, err::{perr, ParseErrorKind::*}, escape::unescape, parse::check_suffix, + Buffer, ParseError, }; - /// A (single) byte literal, e.g. `b'k'` or `b'!'`. /// /// See [the reference][ref] for more information. @@ -33,7 +32,11 @@ impl ByteLit { } let (value, start_suffix) = parse_impl(&input)?; - Ok(Self { raw: input, value, start_suffix }) + Ok(Self { + raw: input, + value, + start_suffix, + }) } /// Returns the byte value that this literal represents. @@ -55,7 +58,6 @@ impl ByteLit { pub fn into_raw_input(self) -> B { self.raw } - } impl ByteLit<&str> { @@ -80,7 +82,9 @@ impl fmt::Display for ByteLit { #[inline(never)] pub(crate) fn parse_impl(input: &str) -> Result<(u8, usize), ParseError> { let input_bytes = input.as_bytes(); - let first = input_bytes.get(2).ok_or(perr(None, UnterminatedByteLiteral))?; + let first = input_bytes + .get(2) + .ok_or(perr(None, UnterminatedByteLiteral))?; let (c, len) = match first { b'\'' if input_bytes.get(3) == Some(&b'\'') => return Err(perr(2, UnescapedSingleQuote)), b'\'' => return Err(perr(None, EmptyByteLiteral)), diff --git a/src/byte/tests.rs b/src/byte/tests.rs index 3cf16b5..11c2e0e 100644 --- a/src/byte/tests.rs +++ b/src/byte/tests.rs @@ -1,9 +1,14 @@ -use crate::{ByteLit, Literal, test_util::{assert_parse_ok_eq, assert_roundtrip}}; +use crate::{ + test_util::{assert_parse_ok_eq, assert_roundtrip}, + ByteLit, Literal, +}; // ===== Utility functions ======================================================================= macro_rules! check { - ($lit:literal) => { check!($lit, stringify!($lit), "") }; + ($lit:literal) => { + check!($lit, stringify!($lit), "") + }; ($lit:literal, $input:expr, $suffix:literal) => { let input = $input; let expected = ByteLit { @@ -12,8 +17,18 @@ macro_rules! check { value: $lit, }; - assert_parse_ok_eq(input, ByteLit::parse(input), expected.clone(), "ByteLit::parse"); - assert_parse_ok_eq(input, Literal::parse(input), Literal::Byte(expected), "Literal::parse"); + assert_parse_ok_eq( + input, + ByteLit::parse(input), + expected.clone(), + "ByteLit::parse", + ); + assert_parse_ok_eq( + input, + Literal::parse(input), + Literal::Byte(expected), + "Literal::parse", + ); let lit = ByteLit::parse(input).unwrap(); assert_eq!(lit.value(), $lit); assert_eq!(lit.suffix(), $suffix); @@ -21,7 +36,6 @@ macro_rules! check { }; } - // ===== Actual tests ============================================================================ #[test] diff --git a/src/bytestr/mod.rs b/src/bytestr/mod.rs index a0e0972..b05a735 100644 --- a/src/bytestr/mod.rs +++ b/src/bytestr/mod.rs @@ -1,12 +1,11 @@ use std::{fmt, ops::Range}; use crate::{ - Buffer, ParseError, err::{perr, ParseErrorKind::*}, escape::{scan_raw_string, unescape_string}, + Buffer, ParseError, }; - /// A byte string or raw byte string literal, e.g. `b"hello"` or `br#"abc"def"#`. /// /// See [the reference][ref] for more information. @@ -41,13 +40,20 @@ impl ByteStringLit { } let (value, num_hashes, start_suffix) = parse_impl(&input)?; - Ok(Self { raw: input, value, num_hashes, start_suffix }) + Ok(Self { + raw: input, + value, + num_hashes, + start_suffix, + }) } /// Returns the string value this literal represents (where all escapes have /// been turned into their respective values). pub fn value(&self) -> &[u8] { - self.value.as_deref().unwrap_or(&self.raw.as_bytes()[self.inner_range()]) + self.value + .as_deref() + .unwrap_or(&self.raw.as_bytes()[self.inner_range()]) } /// Like `value` but returns a potentially owned version of the value. @@ -57,7 +63,9 @@ impl ByteStringLit { pub fn into_value(self) -> B::ByteCow { let inner_range = self.inner_range(); let Self { raw, value, .. } = self; - value.map(B::ByteCow::from).unwrap_or_else(|| raw.cut(inner_range).into_byte_cow()) + value + .map(B::ByteCow::from) + .unwrap_or_else(|| raw.cut(inner_range).into_byte_cow()) } /// The optional suffix. Returns `""` if the suffix is empty/does not exist. @@ -109,7 +117,6 @@ impl fmt::Display for ByteStringLit { } } - /// Precondition: input has to start with either `b"` or `br`. #[inline(never)] fn parse_impl(input: &str) -> Result<(Option>, Option, usize), ParseError> { diff --git a/src/bytestr/tests.rs b/src/bytestr/tests.rs index 2afef5a..3eea7db 100644 --- a/src/bytestr/tests.rs +++ b/src/bytestr/tests.rs @@ -1,4 +1,7 @@ -use crate::{Literal, ByteStringLit, test_util::{assert_parse_ok_eq, assert_roundtrip}}; +use crate::{ + test_util::{assert_parse_ok_eq, assert_roundtrip}, + ByteStringLit, Literal, +}; // ===== Utility functions ======================================================================= @@ -10,15 +13,27 @@ macro_rules! check { let input = $input; let expected = ByteStringLit { raw: input, - value: if $has_escapes { Some($lit.to_vec()) } else { None }, + value: if $has_escapes { + Some($lit.to_vec()) + } else { + None + }, num_hashes: $num_hashes, start_suffix: input.len() - $suffix.len(), }; assert_parse_ok_eq( - input, ByteStringLit::parse(input), expected.clone(), "ByteStringLit::parse"); + input, + ByteStringLit::parse(input), + expected.clone(), + "ByteStringLit::parse", + ); assert_parse_ok_eq( - input, Literal::parse(input), Literal::ByteString(expected.clone()), "Literal::parse"); + input, + Literal::parse(input), + Literal::ByteString(expected.clone()), + "Literal::parse", + ); let lit = ByteStringLit::parse(input).unwrap(); assert_eq!(lit.value(), $lit); assert_eq!(lit.suffix(), $suffix); @@ -27,7 +42,6 @@ macro_rules! check { }; } - // ===== Actual tests ============================================================================ #[test] @@ -52,11 +66,22 @@ fn special_whitespace() { start_suffix: input.len(), }; assert_parse_ok_eq( - &input, ByteStringLit::parse(&*input), expected.clone(), "ByteStringLit::parse"); + &input, + ByteStringLit::parse(&*input), + expected.clone(), + "ByteStringLit::parse", + ); assert_parse_ok_eq( - &input, Literal::parse(&*input), Literal::ByteString(expected), "Literal::parse"); + &input, + Literal::parse(&*input), + Literal::ByteString(expected), + "Literal::parse", + ); assert_eq!(ByteStringLit::parse(&*input).unwrap().value(), s.as_bytes()); - assert_eq!(ByteStringLit::parse(&*input).unwrap().into_value(), s.as_bytes()); + assert_eq!( + ByteStringLit::parse(&*input).unwrap().into_value(), + s.as_bytes() + ); } } @@ -87,22 +112,38 @@ fn simple_escapes() { #[test] fn string_continue() { - check!(b"foo\ - bar", true, None); - check!(b"foo\ -bar", true, None); - - check!(b"foo\ - - banana", true, None); + check!( + b"foo\ + bar", + true, + None + ); + check!( + b"foo\ +bar", + true, + None + ); + + check!( + b"foo\ + + banana", + true, + None + ); // Weird whitespace characters let lit = ByteStringLit::parse("b\"foo\\\n\r\t\n \n\tbar\"").expect("failed to parse"); assert_eq!(lit.value(), b"foobar"); // Raw strings do not handle "string continues" - check!(br"foo\ - bar", false, Some(0)); + check!( + br"foo\ + bar", + false, + Some(0) + ); } #[test] @@ -137,7 +178,11 @@ fn raw_byte_string() { check!(br#"a"#, false, Some(1)); check!(br##"peter"##, false, Some(2)); check!(br###"Greetings # Jason!"###, false, Some(3)); - check!(br########"we ## need #### more ####### hashtags"########, false, Some(8)); + check!( + br########"we ## need #### more ####### hashtags"########, + false, + Some(8) + ); check!(br#"foo " bar"#, false, Some(1)); check!(br##"foo " bar"##, false, Some(2)); @@ -159,7 +204,13 @@ fn suffixes() { check!(b"hello", r###"b"hello"suffix"###, false, None, "suffix"); check!(b"fox", r#"b"fox"peter"#, false, None, "peter"); check!(b"a\x0cb\\", r#"b"a\x0cb\\"_jürgen"#, true, None, "_jürgen"); - check!(br"a\x0cb\\", r###"br#"a\x0cb\\"#_jürgen"###, false, Some(1), "_jürgen"); + check!( + br"a\x0cb\\", + r###"br#"a\x0cb\\"#_jürgen"###, + false, + Some(1), + "_jürgen" + ); } #[test] @@ -176,7 +227,12 @@ fn parse_err() { assert_err!(ByteStringLit, "b\"fo\rx\"", IsolatedCr, 4); assert_err!(ByteStringLit, r##"br####""##, UnterminatedRawString, None); - assert_err!(ByteStringLit, r#####"br##"foo"#bar"#####, UnterminatedRawString, None); + assert_err!( + ByteStringLit, + r#####"br##"foo"#bar"#####, + UnterminatedRawString, + None + ); assert_err!(ByteStringLit, r##"br####"##, InvalidLiteral, None); assert_err!(ByteStringLit, r##"br####x"##, InvalidLiteral, None); } @@ -204,21 +260,106 @@ fn invalid_escapes() { #[test] fn unicode_escape_not_allowed() { - assert_err!(ByteStringLit, r#"b"\u{0}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{00}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{b}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{B}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{7e}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{E4}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{e4}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{fc}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{Fc}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{fC}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{FC}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{b10}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{B10}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{0b10}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{2764}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{1f602}""#, UnicodeEscapeInByteLiteral, 2..4); - assert_err!(ByteStringLit, r#"b"\u{1F602}""#, UnicodeEscapeInByteLiteral, 2..4); + assert_err!( + ByteStringLit, + r#"b"\u{0}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{00}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{b}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{B}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{7e}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{E4}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{e4}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{fc}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{Fc}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{fC}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{FC}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{b10}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{B10}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{0b10}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{2764}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{1f602}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); + assert_err!( + ByteStringLit, + r#"b"\u{1F602}""#, + UnicodeEscapeInByteLiteral, + 2..4 + ); } diff --git a/src/char/mod.rs b/src/char/mod.rs index 54f6f11..a3cd005 100644 --- a/src/char/mod.rs +++ b/src/char/mod.rs @@ -1,13 +1,12 @@ use std::fmt; use crate::{ - Buffer, ParseError, err::{perr, ParseErrorKind::*}, escape::unescape, - parse::{first_byte_or_empty, check_suffix}, + parse::{check_suffix, first_byte_or_empty}, + Buffer, ParseError, }; - /// A character literal, e.g. `'g'` or `'🦊'`. /// /// See [the reference][ref] for more information. @@ -28,8 +27,12 @@ impl CharLit { match first_byte_or_empty(&input)? { b'\'' => { let (value, start_suffix) = parse_impl(&input)?; - Ok(Self { raw: input, value, start_suffix }) - }, + Ok(Self { + raw: input, + value, + start_suffix, + }) + } _ => Err(perr(0, DoesNotStartWithQuote)), } } @@ -53,7 +56,6 @@ impl CharLit { pub fn into_raw_input(self) -> B { self.raw } - } impl CharLit<&str> { @@ -77,12 +79,14 @@ impl fmt::Display for CharLit { /// Precondition: first character in input must be `'`. #[inline(never)] pub(crate) fn parse_impl(input: &str) -> Result<(char, usize), ParseError> { - let first = input.chars().nth(1).ok_or(perr(None, UnterminatedCharLiteral))?; + let first = input + .chars() + .nth(1) + .ok_or(perr(None, UnterminatedCharLiteral))?; let (c, len) = match first { '\'' if input.chars().nth(2) == Some('\'') => return Err(perr(1, UnescapedSingleQuote)), '\'' => return Err(perr(None, EmptyCharLiteral)), - '\n' | '\t' | '\r' - => return Err(perr(1, UnescapedSpecialWhitespace)), + '\n' | '\t' | '\r' => return Err(perr(1, UnescapedSpecialWhitespace)), '\\' => unescape::(&input[1..], 1)?, other => (other, other.len_utf8()), diff --git a/src/char/tests.rs b/src/char/tests.rs index 19219db..5b8277f 100644 --- a/src/char/tests.rs +++ b/src/char/tests.rs @@ -1,10 +1,15 @@ -use crate::{Literal, test_util::{assert_parse_ok_eq, assert_roundtrip}}; use super::CharLit; +use crate::{ + test_util::{assert_parse_ok_eq, assert_roundtrip}, + Literal, +}; // ===== Utility functions ======================================================================= macro_rules! check { - ($lit:literal) => { check!($lit, stringify!($lit), "") }; + ($lit:literal) => { + check!($lit, stringify!($lit), "") + }; ($lit:literal, $input:expr, $suffix:literal) => { let input = $input; let expected = CharLit { @@ -13,8 +18,18 @@ macro_rules! check { value: $lit, }; - assert_parse_ok_eq(input, CharLit::parse(input), expected.clone(), "CharLit::parse"); - assert_parse_ok_eq(input, Literal::parse(input), Literal::Char(expected), "Literal::parse"); + assert_parse_ok_eq( + input, + CharLit::parse(input), + expected.clone(), + "CharLit::parse", + ); + assert_parse_ok_eq( + input, + Literal::parse(input), + Literal::Char(expected), + "Literal::parse", + ); let lit = CharLit::parse(input).unwrap(); assert_eq!(lit.value(), $lit); assert_eq!(lit.suffix(), $suffix); @@ -22,7 +37,6 @@ macro_rules! check { }; } - // ===== Actual tests ============================================================================ #[test] @@ -196,7 +210,12 @@ fn invalid_unicode_escapes() { assert_err!(CharLit, r"'\u{1234567}'", TooManyDigitInUnicodeEscape, 10); assert_err!(CharLit, r"'\u{1234567}'", TooManyDigitInUnicodeEscape, 10); - assert_err!(CharLit, r"'\u{1_23_4_56_7}'", TooManyDigitInUnicodeEscape, 14); + assert_err!( + CharLit, + r"'\u{1_23_4_56_7}'", + TooManyDigitInUnicodeEscape, + 14 + ); assert_err!(CharLit, r"'\u{abcdef123}'", TooManyDigitInUnicodeEscape, 10); assert_err!(CharLit, r"'\u{110000}'", InvalidUnicodeEscapeChar, 1..10); diff --git a/src/err.rs b/src/err.rs index 86d51dc..db45e1f 100644 --- a/src/err.rs +++ b/src/err.rs @@ -1,6 +1,5 @@ use std::{fmt, ops::Range}; - /// An error signaling that a different kind of token was expected. Returned by /// the various `TryFrom` impls. #[derive(Debug, Clone, Copy)] @@ -15,7 +14,7 @@ impl InvalidToken { /// `"msg"` is the output of `self.to_string()`. **Panics if called outside /// of a proc-macro context!** pub fn to_compile_error(&self) -> proc_macro::TokenStream { - use proc_macro::{Delimiter, Ident, Group, Punct, Spacing, TokenTree}; + use proc_macro::{Delimiter, Group, Ident, Punct, Spacing, TokenTree}; let span = match self.span { Span::One(s) => s, @@ -32,8 +31,13 @@ impl InvalidToken { )), ]; - - tokens.into_iter().map(|mut t| { t.set_span(span); t }).collect() + tokens + .into_iter() + .map(|mut t| { + t.set_span(span); + t + }) + .collect() } /// Like [`to_compile_error`][Self::to_compile_error], but returns a token @@ -41,7 +45,7 @@ impl InvalidToken { /// context. #[cfg(feature = "proc-macro2")] pub fn to_compile_error2(&self) -> proc_macro2::TokenStream { - use proc_macro2::{Delimiter, Ident, Group, Punct, Spacing, TokenTree}; + use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, TokenTree}; let span = match self.span { Span::One(s) => proc_macro2::Span::from(s), @@ -57,8 +61,13 @@ impl InvalidToken { )), ]; - - tokens.into_iter().map(|mut t| { t.set_span(span); t }).collect() + tokens + .into_iter() + .map(|mut t| { + t.set_span(span); + t + }) + .collect() } } @@ -82,7 +91,12 @@ impl fmt::Display for InvalidToken { } } - write!(f, "expected {}, but found {}", kind_desc(self.expected), kind_desc(self.actual)) + write!( + f, + "expected {}, but found {}", + kind_desc(self.expected), + kind_desc(self.actual) + ) } } @@ -198,7 +212,6 @@ impl SpanLike for usize { } } - /// Kinds of errors. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] diff --git a/src/escape.rs b/src/escape.rs index 5eb8382..8126963 100644 --- a/src/escape.rs +++ b/src/escape.rs @@ -1,9 +1,14 @@ -use crate::{ParseError, err::{perr, ParseErrorKind::*}, parse::{hex_digit_value, check_suffix}}; - +use crate::{ + err::{perr, ParseErrorKind::*}, + parse::{check_suffix, hex_digit_value}, + ParseError, +}; /// Must start with `\` pub(crate) fn unescape(input: &str, offset: usize) -> Result<(E, usize), ParseError> { - let first = input.as_bytes().get(1) + let first = input + .as_bytes() + .get(1) .ok_or(perr(offset, UnterminatedEscape))?; let out = match first { // Quote escapes @@ -17,13 +22,14 @@ pub(crate) fn unescape(input: &str, offset: usize) -> Result<(E, usi b'\\' => (E::from_byte(b'\\'), 2), b'0' => (E::from_byte(b'\0'), 2), b'x' => { - let hex_string = input.get(2..4) + let hex_string = input + .get(2..4) .ok_or(perr(offset..offset + input.len(), UnterminatedEscape))? .as_bytes(); - let first = hex_digit_value(hex_string[0]) - .ok_or(perr(offset..offset + 4, InvalidXEscape))?; - let second = hex_digit_value(hex_string[1]) - .ok_or(perr(offset..offset + 4, InvalidXEscape))?; + let first = + hex_digit_value(hex_string[0]).ok_or(perr(offset..offset + 4, InvalidXEscape))?; + let second = + hex_digit_value(hex_string[1]).ok_or(perr(offset..offset + 4, InvalidXEscape))?; let value = second + 16 * first; if E::SUPPORTS_UNICODE && value > 0x7F { @@ -31,7 +37,7 @@ pub(crate) fn unescape(input: &str, offset: usize) -> Result<(E, usi } (E::from_byte(value), 4) - }, + } // Unicode escape b'u' => { @@ -43,8 +49,10 @@ pub(crate) fn unescape(input: &str, offset: usize) -> Result<(E, usi return Err(perr(offset..offset + 2, UnicodeEscapeWithoutBrace)); } - let closing_pos = input.bytes().position(|b| b == b'}') - .ok_or(perr(offset..offset + input.len(), UnterminatedUnicodeEscape))?; + let closing_pos = input.bytes().position(|b| b == b'}').ok_or(perr( + offset..offset + input.len(), + UnterminatedUnicodeEscape, + ))?; let inner = &input[3..closing_pos]; if inner.as_bytes().first() == Some(&b'_') { @@ -54,12 +62,12 @@ pub(crate) fn unescape(input: &str, offset: usize) -> Result<(E, usi let mut v: u32 = 0; let mut digit_count = 0; for (i, b) in inner.bytes().enumerate() { - if b == b'_'{ + if b == b'_' { continue; } - let digit = hex_digit_value(b) - .ok_or(perr(offset + 3 + i, NonHexDigitInUnicodeEscape))?; + let digit = + hex_digit_value(b).ok_or(perr(offset + 3 + i, NonHexDigitInUnicodeEscape))?; if digit_count == 6 { return Err(perr(offset + 3 + i, TooManyDigitInUnicodeEscape)); @@ -129,7 +137,8 @@ pub(crate) fn unescape_string( value.push_str(&input[end_last_escape..i]); // Find the first non-whitespace character. - let end_escape = input[i + 2..].bytes() + let end_escape = input[i + 2..] + .bytes() .position(|b| !is_string_continue_skipable_whitespace(b)) .ok_or(perr(None, UnterminatedString))?; @@ -150,15 +159,16 @@ pub(crate) fn unescape_string( i += 2; end_last_escape = i; } else { - return Err(perr(i, IsolatedCr)) + return Err(perr(i, IsolatedCr)); } } b'"' => { closing_quote_pos = Some(i); break; - }, - b if !E::SUPPORTS_UNICODE && !b.is_ascii() - => return Err(perr(i, NonAsciiInByteLiteral)), + } + b if !E::SUPPORTS_UNICODE && !b.is_ascii() => { + return Err(perr(i, NonAsciiInByteLiteral)) + } _ => i += 1, } } @@ -193,7 +203,9 @@ pub(crate) fn scan_raw_string( offset: usize, ) -> Result<(Option, u32, usize), ParseError> { // Raw string literal - let num_hashes = input[offset..].bytes().position(|b| b != b'#') + let num_hashes = input[offset..] + .bytes() + .position(|b| b != b'#') .ok_or(perr(None, InvalidLiteral))?; if input.as_bytes().get(offset + num_hashes) != Some(&b'"') { @@ -227,7 +239,7 @@ pub(crate) fn scan_raw_string( } else if E::SUPPORTS_UNICODE { // If no \n follows the \r and we are scanning a raw string // (not raw byte string), we error. - return Err(perr(i, IsolatedCr)) + return Err(perr(i, IsolatedCr)); } } diff --git a/src/float/mod.rs b/src/float/mod.rs index 7e5fa1e..77763f1 100644 --- a/src/float/mod.rs +++ b/src/float/mod.rs @@ -1,13 +1,11 @@ use std::{fmt, str::FromStr}; use crate::{ - Buffer, ParseError, err::{perr, ParseErrorKind::*}, - parse::{end_dec_digits, first_byte_or_empty, check_suffix}, + parse::{check_suffix, end_dec_digits, first_byte_or_empty}, + Buffer, ParseError, }; - - /// A floating point literal, e.g. `3.14`, `8.`, `135e12`, `27f32` or `1.956e2f64`. /// /// This kind of literal has several forms, but generally consists of a main @@ -70,8 +68,13 @@ impl FloatLit { .. } = parse_impl(&s)?; - Ok(Self { raw: s, end_integer_part, end_fractional_part, end_number_part }) - }, + Ok(Self { + raw: s, + end_integer_part, + end_fractional_part, + end_number_part, + }) + } _ => Err(perr(0, DoesNotStartWithDigit)), } } @@ -148,7 +151,6 @@ pub(crate) fn parse_impl(input: &str) -> Result, ParseError> { let end_integer_part = end_dec_digits(input.as_bytes()); let rest = &input[end_integer_part..]; - // Fractional part. let end_fractional_part = if rest.as_bytes().get(0) == Some(&b'.') { // The fractional part must not start with `_`. @@ -178,7 +180,10 @@ pub(crate) fn parse_impl(input: &str) -> Result, ParseError> { // Find end of exponent and make sure there is at least one digit. let end_exponent = end_dec_digits(rest[exp_number_start..].as_bytes()) + exp_number_start; - if !rest[exp_number_start..end_exponent].bytes().any(|b| matches!(b, b'0'..=b'9')) { + if !rest[exp_number_start..end_exponent] + .bytes() + .any(|b| matches!(b, b'0'..=b'9')) + { return Err(perr( end_fractional_part..end_fractional_part + end_exponent, NoExponentDigits, @@ -208,7 +213,6 @@ pub(crate) fn parse_impl(input: &str) -> Result, ParseError> { }) } - /// All possible float type suffixes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] @@ -251,6 +255,5 @@ impl fmt::Display for FloatType { } } - #[cfg(test)] mod tests; diff --git a/src/float/tests.rs b/src/float/tests.rs index f22443b..8e1915b 100644 --- a/src/float/tests.rs +++ b/src/float/tests.rs @@ -1,9 +1,8 @@ +use super::{FloatLit, FloatType}; use crate::{ - Literal, ParseError, test_util::{assert_parse_ok_eq, assert_roundtrip}, + Literal, ParseError, }; -use super::{FloatLit, FloatType}; - // ===== Utility functions ======================================================================= @@ -36,7 +35,6 @@ macro_rules! check { (@stringify_suffix $suffix:ident) => { stringify!($suffix) }; } - // ===== Actual tests =========================================================================== #[test] @@ -175,7 +173,10 @@ fn non_standard_suffixes() { let lit = match Literal::parse(input) { Ok(Literal::Float(f)) => f, - other => panic!("Expected float literal, but got {:?} for '{}'", other, input), + other => panic!( + "Expected float literal, but got {:?} for '{}'", + other, input + ), }; assert_eq!(lit.integer_part(), integer_part); assert_eq!(lit.fractional_part(), fractional_part); @@ -237,8 +238,8 @@ fn parse_err() { assert_err!(FloatLit, "3.7-2", UnexpectedChar, 3..5); assert_err!(FloatLit, "3.7e+", NoExponentDigits, 3..5); assert_err!(FloatLit, "3.7e-", NoExponentDigits, 3..5); - assert_err!(FloatLit, "3.7e-+3", NoExponentDigits, 3..5); // suboptimal error - assert_err!(FloatLit, "3.7e+-3", NoExponentDigits, 3..5); // suboptimal error + assert_err!(FloatLit, "3.7e-+3", NoExponentDigits, 3..5); // suboptimal error + assert_err!(FloatLit, "3.7e+-3", NoExponentDigits, 3..5); // suboptimal error assert_err_single!(FloatLit::parse("0x44.5"), InvalidSuffix, 1..6); assert_err_single!(FloatLit::parse("3"), UnexpectedIntegerLit, None); diff --git a/src/impls.rs b/src/impls.rs index 61a314d..eec70cd 100644 --- a/src/impls.rs +++ b/src/impls.rs @@ -1,7 +1,9 @@ use std::convert::TryFrom; -use crate::{Literal, err::{InvalidToken, TokenKind}}; - +use crate::{ + err::{InvalidToken, TokenKind}, + Literal, +}; /// Helper macro to call a `callback` macro four times for all combinations of /// `proc_macro`/`proc_macro2` and `&`/owned. @@ -25,7 +27,6 @@ macro_rules! helper_no_refs { }; } - // ============================================================================================== // ===== `From<*Lit> for Literal` // ============================================================================================== @@ -48,13 +49,10 @@ impl_specific_lit_to_lit!(crate::StringLit, String); impl_specific_lit_to_lit!(crate::ByteLit, Byte); impl_specific_lit_to_lit!(crate::ByteStringLit, ByteString); - - // ============================================================================================== // ===== `From for Literal` // ============================================================================================== - macro_rules! impl_tt_to_lit { ([$($prefix:tt)*] => ) => { impl From<$($prefix)* Literal> for Literal { @@ -69,8 +67,7 @@ macro_rules! impl_tt_to_lit { } } -helper!(impl_tt_to_lit, ); - +helper!(impl_tt_to_lit,); // ============================================================================================== // ===== `TryFrom for Literal` @@ -106,8 +103,7 @@ macro_rules! impl_tt_to_lit { } } -helper!(impl_tt_to_lit, ); - +helper!(impl_tt_to_lit,); // ============================================================================================== // ===== `TryFrom`, `TryFrom` for non-bool `*Lit` @@ -167,13 +163,32 @@ macro_rules! impl_for_specific_lit { }; } -helper!(impl_for_specific_lit, crate::IntegerLit, Integer, IntegerLit); -helper!(impl_for_specific_lit, crate::FloatLit, Float, FloatLit); +helper!( + impl_for_specific_lit, + crate::IntegerLit, + Integer, + IntegerLit +); +helper!( + impl_for_specific_lit, + crate::FloatLit, + Float, + FloatLit +); helper!(impl_for_specific_lit, crate::CharLit, Char, CharLit); -helper!(impl_for_specific_lit, crate::StringLit, String, StringLit); +helper!( + impl_for_specific_lit, + crate::StringLit, + String, + StringLit +); helper!(impl_for_specific_lit, crate::ByteLit, Byte, ByteLit); -helper!(impl_for_specific_lit, crate::ByteStringLit, ByteString, ByteStringLit); - +helper!( + impl_for_specific_lit, + crate::ByteStringLit, + ByteString, + ByteStringLit +); // ============================================================================================== // ===== `From<*Lit> for pm::Literal` @@ -204,8 +219,12 @@ helper_no_refs!(impl_specific_lit_to_pm_lit, FloatLit, Float, FloatLit); helper_no_refs!(impl_specific_lit_to_pm_lit, CharLit, Char, CharLit); helper_no_refs!(impl_specific_lit_to_pm_lit, StringLit, String, StringLit); helper_no_refs!(impl_specific_lit_to_pm_lit, ByteLit, Byte, ByteLit); -helper_no_refs!(impl_specific_lit_to_pm_lit, ByteStringLit, ByteString, ByteStringLit); - +helper_no_refs!( + impl_specific_lit_to_pm_lit, + ByteStringLit, + ByteString, + ByteStringLit +); // ============================================================================================== // ===== `TryFrom for BoolLit` @@ -239,7 +258,7 @@ macro_rules! impl_from_tt_for_bool { }; } -helper!(impl_from_tt_for_bool, ); +helper!(impl_from_tt_for_bool,); // ============================================================================================== // ===== `From for pm::Ident` @@ -255,8 +274,7 @@ macro_rules! impl_bool_lit_to_pm_lit { }; } -helper_no_refs!(impl_bool_lit_to_pm_lit, ); - +helper_no_refs!(impl_bool_lit_to_pm_lit,); mod tests { //! # Tests diff --git a/src/integer/mod.rs b/src/integer/mod.rs index 2a8d3a2..549ca96 100644 --- a/src/integer/mod.rs +++ b/src/integer/mod.rs @@ -1,12 +1,11 @@ use std::{fmt, str::FromStr}; use crate::{ - Buffer, ParseError, err::{perr, ParseErrorKind::*}, - parse::{first_byte_or_empty, hex_digit_value, check_suffix}, + parse::{check_suffix, first_byte_or_empty, hex_digit_value}, + Buffer, ParseError, }; - /// An integer literal, e.g. `27`, `0x7F`, `0b101010u8` or `5_000_000i64`. /// /// An integer literal consists of an optional base prefix (`0b`, `0o`, `0x`), @@ -47,10 +46,15 @@ impl IntegerLit { end_main_part, base, .. - } = parse_impl(&input, digit)?; + } = parse_impl(&input, digit)?; - Ok(Self { raw: input, start_main_part, end_main_part, base }) - }, + Ok(Self { + raw: input, + start_main_part, + end_main_part, + base, + }) + } _ => Err(perr(0, DoesNotStartWithDigit)), } } @@ -199,7 +203,6 @@ pub(crate) fn parse_impl(input: &str, first: u8) -> Result, Par }; let without_prefix = &input[end_prefix..]; - // Scan input to find the first character that's not a valid digit. let is_valid_digit = match base { IntegerBase::Binary => |b| matches!(b, b'0' | b'1' | b'_'), @@ -207,7 +210,8 @@ pub(crate) fn parse_impl(input: &str, first: u8) -> Result, Par IntegerBase::Decimal => |b| matches!(b, b'0'..=b'9' | b'_'), IntegerBase::Hexadecimal => |b| matches!(b, b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F' | b'_'), }; - let end_main = without_prefix.bytes() + let end_main = without_prefix + .bytes() .position(|b| !is_valid_digit(b)) .unwrap_or(without_prefix.len()); let (main_part, suffix) = without_prefix.split_at(end_main); @@ -239,7 +243,6 @@ pub(crate) fn parse_impl(input: &str, first: u8) -> Result, Par }) } - /// The bases in which an integer can be specified. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum IntegerBase { @@ -344,6 +347,5 @@ impl fmt::Display for IntegerType { } } - #[cfg(test)] mod tests; diff --git a/src/integer/tests.rs b/src/integer/tests.rs index e6dad3f..0f0aa38 100644 --- a/src/integer/tests.rs +++ b/src/integer/tests.rs @@ -1,9 +1,10 @@ -use std::fmt::{Debug, Display}; use crate::{ - FromIntegerLiteral, Literal, IntegerLit, IntegerType as Ty, IntegerBase, IntegerBase::*, test_util::{assert_parse_ok_eq, assert_roundtrip}, + FromIntegerLiteral, IntegerBase, + IntegerBase::*, + IntegerLit, IntegerType as Ty, Literal, }; - +use std::fmt::{Debug, Display}; // ===== Utility functions ======================================================================= @@ -22,11 +23,22 @@ fn check( base, }; assert_parse_ok_eq( - input, IntegerLit::parse(input), expected_integer.clone(), "IntegerLit::parse"); + input, + IntegerLit::parse(input), + expected_integer.clone(), + "IntegerLit::parse", + ); assert_parse_ok_eq( - input, Literal::parse(input), Literal::Integer(expected_integer), "Literal::parse"); + input, + Literal::parse(input), + Literal::Integer(expected_integer), + "Literal::parse", + ); assert_roundtrip(expected_integer.to_owned(), input); - assert_eq!(Ty::from_suffix(IntegerLit::parse(input).unwrap().suffix()), type_suffix); + assert_eq!( + Ty::from_suffix(IntegerLit::parse(input).unwrap().suffix()), + type_suffix + ); let actual_value = IntegerLit::parse(input) .unwrap() @@ -35,14 +47,11 @@ fn check( if actual_value != value { panic!( "Parsing int literal `{}` should give value `{}`, but actually resulted in `{}`", - input, - value, - actual_value, + input, value, actual_value, ); } } - // ===== Actual tests =========================================================================== #[test] @@ -96,7 +105,13 @@ fn parse_binary() { check("0b01", 0b01, Binary, "01", None); check("0b101010", 0b101010, Binary, "101010", None); check("0b10_10_10", 0b10_10_10, Binary, "10_10_10", None); - check("0b01101110____", 0b01101110____, Binary, "01101110____", None); + check( + "0b01101110____", + 0b01101110____, + Binary, + "01101110____", + None, + ); check("0b10010u8", 0b10010u8, Binary, "10010", Some(Ty::U8)); check("0b10010i8", 0b10010u8, Binary, "10010", Some(Ty::I8)); @@ -175,12 +190,42 @@ fn starting_underscore() { #[test] fn parse_overflowing_just_fine() { check("256u8", 256u16, Decimal, "256", Some(Ty::U8)); - check("123_456_789u8", 123_456_789u32, Decimal, "123_456_789", Some(Ty::U8)); - check("123_456_789u16", 123_456_789u32, Decimal, "123_456_789", Some(Ty::U16)); + check( + "123_456_789u8", + 123_456_789u32, + Decimal, + "123_456_789", + Some(Ty::U8), + ); + check( + "123_456_789u16", + 123_456_789u32, + Decimal, + "123_456_789", + Some(Ty::U16), + ); - check("123_123_456_789u8", 123_123_456_789u64, Decimal, "123_123_456_789", Some(Ty::U8)); - check("123_123_456_789u16", 123_123_456_789u64, Decimal, "123_123_456_789", Some(Ty::U16)); - check("123_123_456_789u32", 123_123_456_789u64, Decimal, "123_123_456_789", Some(Ty::U32)); + check( + "123_123_456_789u8", + 123_123_456_789u64, + Decimal, + "123_123_456_789", + Some(Ty::U8), + ); + check( + "123_123_456_789u16", + 123_123_456_789u64, + Decimal, + "123_123_456_789", + Some(Ty::U16), + ); + check( + "123_123_456_789u32", + 123_123_456_789u64, + Decimal, + "123_123_456_789", + Some(Ty::U32), + ); } #[test] @@ -196,8 +241,13 @@ fn suffixes() { ("123u32", Ty::U32), ("123u64", Ty::U64), ("123u128", Ty::U128), - ].iter().for_each(|&(s, ty)| { - assert_eq!(Ty::from_suffix(IntegerLit::parse(s).unwrap().suffix()), Some(ty)); + ] + .iter() + .for_each(|&(s, ty)| { + assert_eq!( + Ty::from_suffix(IntegerLit::parse(s).unwrap().suffix()), + Some(ty) + ); }); } @@ -226,8 +276,14 @@ fn overflow_u128() { #[test] fn overflow_u8() { let inputs = [ - "256", "0x100", "0o400", "0b100000000", - "257", "0x101", "0o401", "0b100000001", + "256", + "0x100", + "0o400", + "0b100000000", + "257", + "0x101", + "0o401", + "0b100000001", "300", "1548", "2548985", diff --git a/src/lib.rs b/src/lib.rs index 64ed781..405892d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -154,6 +154,9 @@ mod test_util; #[cfg(test)] mod tests; +#[cfg(feature = "printing")] +mod printing; + mod bool; mod byte; mod bytestr; @@ -166,8 +169,11 @@ mod integer; mod parse; mod string; - -use std::{borrow::{Borrow, Cow}, fmt, ops::{Deref, Range}}; +use std::{ + borrow::{Borrow, Cow}, + fmt, + ops::{Deref, Range}, +}; pub use self::{ bool::BoolLit, @@ -176,11 +182,10 @@ pub use self::{ char::CharLit, err::{InvalidToken, ParseError}, float::{FloatLit, FloatType}, - integer::{FromIntegerLiteral, IntegerLit, IntegerBase, IntegerType}, + integer::{FromIntegerLiteral, IntegerBase, IntegerLit, IntegerType}, string::StringLit, }; - // ============================================================================================== // ===== `Literal` and type defs // ============================================================================================== @@ -291,7 +296,6 @@ impl fmt::Display for Literal { } } - // ============================================================================================== // ===== Buffer // ============================================================================================== diff --git a/src/parse.rs b/src/parse.rs index efc6b87..781d6c6 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -1,18 +1,12 @@ use crate::{ - BoolLit, - Buffer, - ByteLit, - ByteStringLit, - CharLit, - ParseError, - FloatLit, - IntegerLit, - Literal, + err::{ + perr, + ParseErrorKind::{self, *}, + }, + BoolLit, Buffer, ByteLit, ByteStringLit, CharLit, FloatLit, IntegerLit, Literal, ParseError, StringLit, - err::{perr, ParseErrorKind::{*, self}}, }; - pub fn parse(input: B) -> Result, ParseError> { let (first, rest) = input.as_bytes().split_first().ok_or(perr(None, Empty))?; let second = input.as_bytes().get(1).copied(); @@ -32,25 +26,24 @@ pub fn parse(input: B) -> Result, ParseError> { // The first non-decimal char in a float literal must // be '.', 'e' or 'E'. match input.as_bytes().get(1 + end_dec_digits(rest)) { - Some(b'.') | Some(b'e') | Some(b'E') - => FloatLit::parse(input).map(Literal::Float), + Some(b'.') | Some(b'e') | Some(b'E') => FloatLit::parse(input).map(Literal::Float), _ => IntegerLit::parse(input).map(Literal::Integer), } - }, + } b'\'' => CharLit::parse(input).map(Literal::Char), b'"' | b'r' => StringLit::parse(input).map(Literal::String), b'b' if second == Some(b'\'') => ByteLit::parse(input).map(Literal::Byte), - b'b' if second == Some(b'r') || second == Some(b'"') - => ByteStringLit::parse(input).map(Literal::ByteString), + b'b' if second == Some(b'r') || second == Some(b'"') => { + ByteStringLit::parse(input).map(Literal::ByteString) + } _ => Err(perr(None, InvalidLiteral)), } } - pub(crate) fn first_byte_or_empty(s: &str) -> Result { s.as_bytes().get(0).copied().ok_or(perr(None, Empty)) } @@ -58,7 +51,8 @@ pub(crate) fn first_byte_or_empty(s: &str) -> Result { /// Returns the index of the first non-underscore, non-decimal digit in `input`, /// or the `input.len()` if all characters are decimal digits. pub(crate) fn end_dec_digits(input: &[u8]) -> usize { - input.iter() + input + .iter() .position(|b| !matches!(b, b'_' | b'0'..=b'9')) .unwrap_or(input.len()) } @@ -98,8 +92,7 @@ pub(crate) fn check_suffix(s: &str) -> Result<(), ParseErrorKind> { fn is_valid_suffix(first: char, rest: &str) -> bool { use unicode_xid::UnicodeXID; - (first == '_' || first.is_xid_start()) - && rest.chars().all(|c| c.is_xid_continue()) + (first == '_' || first.is_xid_start()) && rest.chars().all(|c| c.is_xid_continue()) } // When avoiding the dependency on `unicode_xid`, we just do a best effort diff --git a/src/printing/mod.rs b/src/printing/mod.rs new file mode 100644 index 0000000..1ada3b4 --- /dev/null +++ b/src/printing/mod.rs @@ -0,0 +1,44 @@ +use quote::{ToTokens, TokenStreamExt}; + +use crate::Buffer; + +macro_rules! to_tokens_simple { + ($id:ident) => { + impl ToTokens for crate::$id { + fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { + tokens.append(>::into(self.clone())) + } + + fn into_token_stream(self) -> proc_macro2::TokenStream + where + Self: Sized, + { + proc_macro2::TokenStream::from(proc_macro2::TokenTree::from(>::into(self))) + } + } + }; +} +to_tokens_simple!(ByteLit); +to_tokens_simple!(ByteStringLit); +to_tokens_simple!(CharLit); +to_tokens_simple!(FloatLit); +to_tokens_simple!(IntegerLit); +to_tokens_simple!(StringLit); + +impl ToTokens for crate::BoolLit { + fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { + use crate::BoolLit::*; + tokens.append(proc_macro2::Ident::new( + match self { + True => "true", + False => "false", + }, + proc_macro2::Span::call_site(), + )) + } +} + +#[cfg(test)] +mod tests; diff --git a/src/printing/tests.rs b/src/printing/tests.rs new file mode 100644 index 0000000..b5dc0b3 --- /dev/null +++ b/src/printing/tests.rs @@ -0,0 +1,82 @@ +use proc_macro2::{Literal, TokenTree}; +use quote::ToTokens; + +use crate::{ByteLit, ByteStringLit, CharLit, FloatLit, IntegerLit, StringLit}; + +#[test] +fn it_preserves_bool_true() { + let other = crate::BoolLit::True + .to_token_stream() + .into_iter() + .next() + .and_then(|v| { + if let TokenTree::Ident(v) = v { + Some(v) + } else { + None + } + }) + .unwrap(); + assert_eq!(other, "true") +} +#[test] +fn it_preserves_bool_false() { + let other = crate::BoolLit::False + .to_token_stream() + .into_iter() + .next() + .and_then(|v| { + if let TokenTree::Ident(v) = v { + Some(v) + } else { + None + } + }) + .unwrap(); + assert_eq!(other, "false") +} + +// NOTE: sometimes the round-trip to quote simplifies literals (for example float literals) +// this is something that has to be taken into consideration when writing tests +macro_rules! preservation { + ($name:ident : $ty:ident : $($content:tt)+) => { + #[test] + fn $name() { + use std::convert::TryFrom; + + let lit = Literal::$($content)+; + + let lhs = $ty::try_from(lit).unwrap(); + + let rhs = $ty::try_from( + lhs.to_token_stream() + .into_iter() + .next() + .and_then(|v| { + if let TokenTree::Literal(v) = v { + Some(v) + } else { + None + } + }) + .unwrap(), + ) + .unwrap(); + + assert_eq!(lhs, rhs); + } + } +} + +preservation!(it_preserves_u8_suffixed : IntegerLit : u8_suffixed(10)); +preservation!(it_preserves_u8_unsuffixed : IntegerLit : u8_unsuffixed(10)); + +preservation!(it_preserves_f64_suffixed : FloatLit : f64_suffixed(1.1)); +preservation!(it_preserves_f64_unsuffixed : FloatLit : f64_unsuffixed(1.1)); + +preservation!(it_preserves_strings : StringLit : string("hello world")); +preservation!(it_preserves_raw_strings : StringLit : string(r#"hello world"#)); + +preservation!(it_preserves_char : CharLit : character('1')); + +preservation!(it_preserves_bytestring : ByteStringLit : byte_string(b"hello world")); diff --git a/src/string/mod.rs b/src/string/mod.rs index d2034a6..5aca050 100644 --- a/src/string/mod.rs +++ b/src/string/mod.rs @@ -1,13 +1,12 @@ use std::{fmt, ops::Range}; use crate::{ - Buffer, ParseError, err::{perr, ParseErrorKind::*}, escape::{scan_raw_string, unescape_string}, parse::first_byte_or_empty, + Buffer, ParseError, }; - /// A string or raw string literal, e.g. `"foo"`, `"Grüße"` or `r#"a🦊c"d🦀f"#`. /// /// See [the reference][ref] for more information. @@ -37,7 +36,12 @@ impl StringLit { match first_byte_or_empty(&input)? { b'r' | b'"' => { let (value, num_hashes, start_suffix) = parse_impl(&input)?; - Ok(Self { raw: input, value, num_hashes, start_suffix }) + Ok(Self { + raw: input, + value, + num_hashes, + start_suffix, + }) } _ => Err(perr(0, InvalidStringLiteralStart)), } @@ -46,7 +50,9 @@ impl StringLit { /// Returns the string value this literal represents (where all escapes have /// been turned into their respective values). pub fn value(&self) -> &str { - self.value.as_deref().unwrap_or(&self.raw[self.inner_range()]) + self.value + .as_deref() + .unwrap_or(&self.raw[self.inner_range()]) } /// Like `value` but returns a potentially owned version of the value. @@ -56,7 +62,9 @@ impl StringLit { pub fn into_value(self) -> B::Cow { let inner_range = self.inner_range(); let Self { raw, value, .. } = self; - value.map(B::Cow::from).unwrap_or_else(|| raw.cut(inner_range).into_cow()) + value + .map(B::Cow::from) + .unwrap_or_else(|| raw.cut(inner_range).into_cow()) } /// The optional suffix. Returns `""` if the suffix is empty/does not exist. @@ -115,11 +123,9 @@ pub(crate) fn parse_impl(input: &str) -> Result<(Option, Option, us scan_raw_string::(&input, 1) .map(|(v, hashes, start_suffix)| (v, Some(hashes), start_suffix)) } else { - unescape_string::(&input, 1) - .map(|(v, start_suffix)| (v, None, start_suffix)) + unescape_string::(&input, 1).map(|(v, start_suffix)| (v, None, start_suffix)) } } - #[cfg(test)] mod tests; diff --git a/src/string/tests.rs b/src/string/tests.rs index 1c0cb63..bed7f27 100644 --- a/src/string/tests.rs +++ b/src/string/tests.rs @@ -1,4 +1,7 @@ -use crate::{Literal, StringLit, test_util::{assert_parse_ok_eq, assert_roundtrip}}; +use crate::{ + test_util::{assert_parse_ok_eq, assert_roundtrip}, + Literal, StringLit, +}; // ===== Utility functions ======================================================================= @@ -10,14 +13,27 @@ macro_rules! check { let input = $input; let expected = StringLit { raw: input, - value: if $has_escapes { Some($lit.to_string()) } else { None }, + value: if $has_escapes { + Some($lit.to_string()) + } else { + None + }, num_hashes: $num_hashes, start_suffix: input.len() - $suffix.len(), }; - assert_parse_ok_eq(input, StringLit::parse(input), expected.clone(), "StringLit::parse"); assert_parse_ok_eq( - input, Literal::parse(input), Literal::String(expected.clone()), "Literal::parse"); + input, + StringLit::parse(input), + expected.clone(), + "StringLit::parse", + ); + assert_parse_ok_eq( + input, + Literal::parse(input), + Literal::String(expected.clone()), + "Literal::parse", + ); let lit = StringLit::parse(input).unwrap(); assert_eq!(lit.value(), $lit); assert_eq!(lit.suffix(), $suffix); @@ -26,7 +42,6 @@ macro_rules! check { }; } - // ===== Actual tests ============================================================================ #[test] @@ -56,9 +71,17 @@ fn special_whitespace() { start_suffix: input.len(), }; assert_parse_ok_eq( - &input, StringLit::parse(&*input), expected.clone(), "StringLit::parse"); + &input, + StringLit::parse(&*input), + expected.clone(), + "StringLit::parse", + ); assert_parse_ok_eq( - &input, Literal::parse(&*input), Literal::String(expected), "Literal::parse"); + &input, + Literal::parse(&*input), + Literal::String(expected), + "Literal::parse", + ); assert_eq!(StringLit::parse(&*input).unwrap().value(), s); assert_eq!(StringLit::parse(&*input).unwrap().into_value(), s); } @@ -117,14 +140,26 @@ fn unicode_escapes() { #[test] fn string_continue() { - check!("నక్క\ - bar", true, None); - check!("foo\ -🦊", true, None); - - check!("foo\ - - banana", true, None); + check!( + "నక్క\ + bar", + true, + None + ); + check!( + "foo\ +🦊", + true, + None + ); + + check!( + "foo\ + + banana", + true, + None + ); // Weird whitespace characters let lit = StringLit::parse("\"foo\\\n\r\t\n \n\tbar\"").expect("failed to parse"); @@ -135,8 +170,12 @@ fn string_continue() { assert_eq!(lit.value(), "foo\u{a0}bar"); // Raw strings do not handle "string continues" - check!(r"foo\ - bar", false, Some(0)); + check!( + r"foo\ + bar", + false, + Some(0) + ); } #[test] @@ -168,7 +207,11 @@ fn raw_string() { check!(r"Sei gegrüßt, Bärthelt!", false, Some(0)); check!(r"أنا لا أتحدث العربية", false, Some(0)); check!(r"お前はもう死んでいる", false, Some(0)); - check!(r"Пушки - интересные музыкальные инструменты", false, Some(0)); + check!( + r"Пушки - интересные музыкальные инструменты", + false, + Some(0) + ); check!(r"lit 👌 😂 af", false, Some(0)); check!(r#""#, false, Some(1)); @@ -195,10 +238,22 @@ fn raw_string() { #[test] fn suffixes() { check!("hello", r###""hello"suffix"###, false, None, "suffix"); - check!(r"お前はもう死んでいる", r###"r"お前はもう死んでいる"_banana"###, false, Some(0), "_banana"); + check!( + r"お前はもう死んでいる", + r###"r"お前はもう死んでいる"_banana"###, + false, + Some(0), + "_banana" + ); check!("fox", r#""fox"peter"#, false, None, "peter"); check!("🦊", r#""🦊"peter"#, false, None, "peter"); - check!("నక్క\\\\u{0b10}", r###""నక్క\\\\u{0b10}"jü_rgen"###, true, None, "jü_rgen"); + check!( + "నక్క\\\\u{0b10}", + r###""నక్క\\\\u{0b10}"jü_rgen"###, + true, + None, + "jü_rgen" + ); } #[test] @@ -217,7 +272,12 @@ fn parse_err() { assert_err!(StringLit, "r\"fo\rx\"", IsolatedCr, 4); assert_err!(StringLit, r##"r####""##, UnterminatedRawString, None); - assert_err!(StringLit, r#####"r##"foo"#bar"#####, UnterminatedRawString, None); + assert_err!( + StringLit, + r#####"r##"foo"#bar"#####, + UnterminatedRawString, + None + ); assert_err!(StringLit, r##"r####"##, InvalidLiteral, None); assert_err!(StringLit, r##"r####x"##, InvalidLiteral, None); } @@ -258,7 +318,12 @@ fn invalid_unicode_escapes() { assert_err!(StringLit, r#""\u{""#, UnterminatedUnicodeEscape, 1..4); assert_err!(StringLit, r#""\u{12""#, UnterminatedUnicodeEscape, 1..6); assert_err!(StringLit, r#""🦊\u{a0b""#, UnterminatedUnicodeEscape, 5..11); - assert_err!(StringLit, r#""\u{a0_b ""#, UnterminatedUnicodeEscape, 1..10); + assert_err!( + StringLit, + r#""\u{a0_b ""#, + UnterminatedUnicodeEscape, + 1..10 + ); assert_err!(StringLit, r#""\u{_}నక్క""#, InvalidStartOfUnicodeEscape, 4); assert_err!(StringLit, r#""\u{_5f}""#, InvalidStartOfUnicodeEscape, 4); @@ -266,13 +331,43 @@ fn invalid_unicode_escapes() { assert_err!(StringLit, r#""fox\u{x}""#, NonHexDigitInUnicodeEscape, 7); assert_err!(StringLit, r#""\u{0x}🦊""#, NonHexDigitInUnicodeEscape, 5); assert_err!(StringLit, r#""నక్క\u{3bx}""#, NonHexDigitInUnicodeEscape, 18); - assert_err!(StringLit, r#""\u{3b_x}лиса""#, NonHexDigitInUnicodeEscape, 7); + assert_err!( + StringLit, + r#""\u{3b_x}лиса""#, + NonHexDigitInUnicodeEscape, + 7 + ); assert_err!(StringLit, r#""\u{4x_}""#, NonHexDigitInUnicodeEscape, 5); - assert_err!(StringLit, r#""\u{1234567}""#, TooManyDigitInUnicodeEscape, 10); - assert_err!(StringLit, r#""నక్క\u{1234567}🦊""#, TooManyDigitInUnicodeEscape, 22); - assert_err!(StringLit, r#""నక్క\u{1_23_4_56_7}""#, TooManyDigitInUnicodeEscape, 26); - assert_err!(StringLit, r#""\u{abcdef123}лиса""#, TooManyDigitInUnicodeEscape, 10); - - assert_err!(StringLit, r#""\u{110000}fox""#, InvalidUnicodeEscapeChar, 1..10); + assert_err!( + StringLit, + r#""\u{1234567}""#, + TooManyDigitInUnicodeEscape, + 10 + ); + assert_err!( + StringLit, + r#""నక్క\u{1234567}🦊""#, + TooManyDigitInUnicodeEscape, + 22 + ); + assert_err!( + StringLit, + r#""నక్క\u{1_23_4_56_7}""#, + TooManyDigitInUnicodeEscape, + 26 + ); + assert_err!( + StringLit, + r#""\u{abcdef123}лиса""#, + TooManyDigitInUnicodeEscape, + 10 + ); + + assert_err!( + StringLit, + r#""\u{110000}fox""#, + InvalidUnicodeEscapeChar, + 1..10 + ); } diff --git a/src/test_util.rs b/src/test_util.rs index fd284e9..cce4c9f 100644 --- a/src/test_util.rs +++ b/src/test_util.rs @@ -1,7 +1,6 @@ use crate::*; use std::fmt::{Debug, Display}; - #[track_caller] pub(crate) fn assert_parse_ok_eq( input: &str, @@ -14,26 +13,20 @@ pub(crate) fn assert_parse_ok_eq( if actual.to_string() != input { panic!( "formatting does not yield original input `{}`: {:?}", - input, - actual, + input, actual, ); } } Ok(actual) => { panic!( "unexpected parsing result (with `{}`) for `{}`:\nactual: {:?}\nexpected: {:?}", - parse_method, - input, - actual, - expected, + parse_method, input, actual, expected, ); } Err(e) => { panic!( "expected `{}` to be parsed (with `{}`) successfully, but it failed: {:?}", - input, - parse_method, - e, + input, parse_method, e, ); } } @@ -52,7 +45,8 @@ where proc_macro2::Literal: From, >::Error: std::fmt::Display, { - let pm_lit = input.parse::() + let pm_lit = input + .parse::() .expect("failed to parse input as proc_macro2::Literal"); let t_name = std::any::type_name::(); @@ -70,16 +64,17 @@ where match T::try_from(pm_lit) { Err(e) => { - panic!("Trying to convert proc_macro2::Literal to {} results in error: {}", t_name, e); + panic!( + "Trying to convert proc_macro2::Literal to {} results in error: {}", + t_name, e + ); } Ok(res) => { if res != ours { panic!( "Converting proc_macro2::Literal to {} has unexpected result:\ \nactual: {:?}\nexpected: {:?}", - t_name, - res, - ours, + t_name, res, ours, ); } } diff --git a/src/tests.rs b/src/tests.rs index 613b429..c784ee2 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,6 +1,5 @@ use crate::Literal; - #[test] fn empty() { assert_err!(Literal, "", Empty, None); @@ -91,14 +90,13 @@ fn never_panic_len_4() { #[cfg(feature = "proc-macro2")] #[test] fn proc_macro() { - use std::convert::TryFrom; - use proc_macro2::{ - self as pm2, TokenTree, Group, TokenStream, Delimiter, Spacing, Punct, Span, Ident, - }; use crate::{ - BoolLit, ByteLit, ByteStringLit, CharLit, FloatLit, IntegerLit, StringLit, err::TokenKind + err::TokenKind, BoolLit, ByteLit, ByteStringLit, CharLit, FloatLit, IntegerLit, StringLit, }; - + use proc_macro2::{ + self as pm2, Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree, + }; + use std::convert::TryFrom; macro_rules! assert_invalid_token { ($input:expr, expected: $expected:path, actual: $actual:path $(,)?) => { @@ -106,17 +104,18 @@ fn proc_macro() { if err.expected != $expected { panic!( "err.expected was expected to be {:?}, but is {:?}", - $expected, - err.expected, + $expected, err.expected, ); } if err.actual != $actual { - panic!("err.actual was expected to be {:?}, but is {:?}", $actual, err.actual); + panic!( + "err.actual was expected to be {:?}, but is {:?}", + $actual, err.actual + ); } }; } - let pm_u16_lit = pm2::Literal::u16_suffixed(2700); let pm_i16_lit = pm2::Literal::i16_unsuffixed(3912); let pm_f32_lit = pm2::Literal::f32_unsuffixed(3.14); @@ -141,7 +140,6 @@ fn proc_macro() { assert_eq!(Literal::from(&pm_bytestr_lit), bytestr_lit); assert_eq!(Literal::from(&pm_char_lit), char_lit); - let group = TokenTree::from(Group::new(Delimiter::Brace, TokenStream::new())); let punct = TokenTree::from(Punct::new(':', Spacing::Alone)); let ident = TokenTree::from(Ident::new("peter", Span::call_site())); @@ -166,17 +164,34 @@ fn proc_macro() { actual: TokenKind::Ident, ); - - assert_eq!(Literal::from(IntegerLit::try_from(pm_u16_lit.clone()).unwrap()), u16_lit); - assert_eq!(Literal::from(IntegerLit::try_from(pm_i16_lit.clone()).unwrap()), i16_lit); - assert_eq!(Literal::from(FloatLit::try_from(pm_f32_lit.clone()).unwrap()), f32_lit); - assert_eq!(Literal::from(FloatLit::try_from(pm_f64_lit.clone()).unwrap()), f64_lit); - assert_eq!(Literal::from(StringLit::try_from(pm_string_lit.clone()).unwrap()), string_lit); + assert_eq!( + Literal::from(IntegerLit::try_from(pm_u16_lit.clone()).unwrap()), + u16_lit + ); + assert_eq!( + Literal::from(IntegerLit::try_from(pm_i16_lit.clone()).unwrap()), + i16_lit + ); + assert_eq!( + Literal::from(FloatLit::try_from(pm_f32_lit.clone()).unwrap()), + f32_lit + ); + assert_eq!( + Literal::from(FloatLit::try_from(pm_f64_lit.clone()).unwrap()), + f64_lit + ); + assert_eq!( + Literal::from(StringLit::try_from(pm_string_lit.clone()).unwrap()), + string_lit + ); assert_eq!( Literal::from(ByteStringLit::try_from(pm_bytestr_lit.clone()).unwrap()), bytestr_lit, ); - assert_eq!(Literal::from(CharLit::try_from(pm_char_lit.clone()).unwrap()), char_lit); + assert_eq!( + Literal::from(CharLit::try_from(pm_char_lit.clone()).unwrap()), + char_lit + ); assert_invalid_token!( StringLit::try_from(pm_u16_lit.clone()), @@ -209,7 +224,6 @@ fn proc_macro() { actual: TokenKind::CharLit, ); - assert_eq!( Literal::from(IntegerLit::try_from(TokenTree::from(pm_u16_lit.clone())).unwrap()), u16_lit, @@ -290,15 +304,20 @@ fn proc_macro() { #[cfg(feature = "proc-macro2")] #[test] fn bool_try_from_tt() { - use std::convert::TryFrom; - use proc_macro2::{Ident, Span, TokenTree}; use crate::BoolLit; - + use proc_macro2::{Ident, Span, TokenTree}; + use std::convert::TryFrom; let ident = |s: &str| Ident::new(s, Span::call_site()); - assert_eq!(BoolLit::try_from(TokenTree::Ident(ident("true"))).unwrap(), BoolLit::True); - assert_eq!(BoolLit::try_from(TokenTree::Ident(ident("false"))).unwrap(), BoolLit::False); + assert_eq!( + BoolLit::try_from(TokenTree::Ident(ident("true"))).unwrap(), + BoolLit::True + ); + assert_eq!( + BoolLit::try_from(TokenTree::Ident(ident("false"))).unwrap(), + BoolLit::False + ); assert!(BoolLit::try_from(TokenTree::Ident(ident("falsex"))).is_err()); assert!(BoolLit::try_from(TokenTree::Ident(ident("_false"))).is_err()); @@ -306,7 +325,6 @@ fn bool_try_from_tt() { assert!(BoolLit::try_from(TokenTree::Ident(ident("True"))).is_err()); assert!(BoolLit::try_from(TokenTree::Ident(ident("ltrue"))).is_err()); - assert_eq!( Literal::try_from(TokenTree::Ident(ident("true"))).unwrap(), Literal::Bool(BoolLit::True), @@ -326,7 +344,7 @@ fn bool_try_from_tt() { #[cfg(feature = "proc-macro2")] #[test] fn invalid_token_display() { - use crate::{InvalidToken, err::TokenKind}; + use crate::{err::TokenKind, InvalidToken}; let span = crate::err::Span::Two(proc_macro2::Span::call_site()); assert_eq!( @@ -334,7 +352,8 @@ fn invalid_token_display() { actual: TokenKind::StringLit, expected: TokenKind::FloatLit, span, - }.to_string(), + } + .to_string(), r#"expected a float literal (e.g. `3.14`), but found a string literal (e.g. "Ferris")"#, ); @@ -343,7 +362,8 @@ fn invalid_token_display() { actual: TokenKind::Punct, expected: TokenKind::Literal, span, - }.to_string(), + } + .to_string(), r#"expected a literal, but found a punctuation character"#, ); }