Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(parser)!: Allow literals to be parsers of partial inputs #171

Merged
merged 2 commits into from
Feb 16, 2023
Merged

fix(parser)!: Allow literals to be parsers of partial inputs #171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7878863
fix(parser)!: Allow literals to be parsers of partial inputs
epage Feb 16, 2023
0f49bc8
docs(example): Show that `one_of` isnt needed anymore
epage Feb 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions benches/number.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,20 @@ fn number(c: &mut Criterion) {
fn float_bytes(c: &mut Criterion) {
println!(
"float_bytes result: {:?}",
float::<_, f64, Error<_>, false>(&b"-1.234E-12"[..])
float::<_, f64, Error<_>>(&b"-1.234E-12"[..])
);
c.bench_function("float bytes", |b| {
b.iter(|| float::<_, f64, Error<_>, false>(&b"-1.234E-12"[..]));
b.iter(|| float::<_, f64, Error<_>>(&b"-1.234E-12"[..]));
});
}

fn float_str(c: &mut Criterion) {
println!(
"float_str result: {:?}",
float::<_, f64, Error<_>, false>("-1.234E-12")
float::<_, f64, Error<_>>("-1.234E-12")
);
c.bench_function("float str", |b| {
b.iter(|| float::<_, f64, Error<_>, false>("-1.234E-12"));
b.iter(|| float::<_, f64, Error<_>>("-1.234E-12"));
});
}

Expand Down
36 changes: 11 additions & 25 deletions examples/json/parser_partial.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use std::str;
use winnow::prelude::*;
use winnow::{
branch::alt,
bytes::{any, none_of, one_of, tag, take, take_while0},
bytes::{any, none_of, tag, take, take_while0},
character::float,
combinator::{cut_err, rest},
error::{ContextError, ParseError},
Expand All @@ -13,15 +13,7 @@ use winnow::{
stream::Partial,
};

#[derive(Debug, PartialEq, Clone)]
pub enum JsonValue {
Null,
Boolean(bool),
Str(String),
Num(f64),
Array(Vec<JsonValue>),
Object(HashMap<String, JsonValue>),
}
use crate::json::JsonValue;

pub type Stream<'i> = Partial<&'i str>;

Expand Down Expand Up @@ -89,7 +81,7 @@ fn string<'i, E: ParseError<Stream<'i>> + ContextError<Stream<'i>, &'static str>
input: Stream<'i>,
) -> IResult<Stream<'i>, String, E> {
preceded(
one_of('\"'),
'\"',
// `cut_err` transforms an `ErrMode::Backtrack(e)` to `ErrMode::Cut(e)`, signaling to
// combinators like `alt` that they should not try other parsers. We were in the
// right branch (since we found the `"` character) but encountered an error when
Expand All @@ -99,7 +91,7 @@ fn string<'i, E: ParseError<Stream<'i>> + ContextError<Stream<'i>, &'static str>
string.push(c);
string
}),
one_of('\"'),
'\"',
)),
)
// `context` lets you add a static string to errors to provide more information in the
Expand All @@ -125,7 +117,7 @@ fn character<'i, E: ParseError<Stream<'i>>>(input: Stream<'i>) -> IResult<Stream
_ => return None,
})
}),
preceded(one_of('u'), unicode_escape),
preceded('u', unicode_escape),
))(input)
} else {
Ok((input, c))
Expand All @@ -141,7 +133,7 @@ fn unicode_escape<'i, E: ParseError<Stream<'i>>>(
.verify(|cp| !(0xD800..0xE000).contains(cp))
.map(|cp| cp as u32),
// See https://en.wikipedia.org/wiki/UTF-16#Code_points_from_U+010000_to_U+10FFFF for details
separated_pair(u16_hex, tag("\\u"), u16_hex)
separated_pair(u16_hex, "\\u", u16_hex)
.verify(|(high, low)| (0xD800..0xDC00).contains(high) && (0xDC00..0xE000).contains(low))
.map(|(high, low)| {
let high_ten = (high as u32) - 0xD800;
Expand Down Expand Up @@ -170,11 +162,8 @@ fn array<'i, E: ParseError<Stream<'i>> + ContextError<Stream<'i>, &'static str>>
input: Stream<'i>,
) -> IResult<Stream<'i>, Vec<JsonValue>, E> {
preceded(
(one_of('['), ws),
cut_err(terminated(
separated0(json_value, (ws, one_of(','), ws)),
(ws, one_of(']')),
)),
('[', ws),
cut_err(terminated(separated0(json_value, (ws, ',', ws)), (ws, ']'))),
)
.context("array")
.parse_next(input)
Expand All @@ -184,11 +173,8 @@ fn object<'i, E: ParseError<Stream<'i>> + ContextError<Stream<'i>, &'static str>
input: Stream<'i>,
) -> IResult<Stream<'i>, HashMap<String, JsonValue>, E> {
preceded(
(one_of('{'), ws),
cut_err(terminated(
separated0(key_value, (ws, one_of(','), ws)),
(ws, one_of('}')),
)),
('{', ws),
cut_err(terminated(separated0(key_value, (ws, ',', ws)), (ws, '}'))),
)
.context("object")
.parse_next(input)
Expand All @@ -197,7 +183,7 @@ fn object<'i, E: ParseError<Stream<'i>> + ContextError<Stream<'i>, &'static str>
fn key_value<'i, E: ParseError<Stream<'i>> + ContextError<Stream<'i>, &'static str>>(
input: Stream<'i>,
) -> IResult<Stream<'i>, (String, JsonValue), E> {
separated_pair(string, cut_err((ws, one_of(':'), ws)), json_value)(input)
separated_pair(string, cut_err((ws, ':', ws)), json_value)(input)
}

/// Parser combinators are constructed from the bottom up:
Expand Down
18 changes: 9 additions & 9 deletions examples/json_iterator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ impl<'a, 'b: 'a> JsonValue<'a, 'b> {

pub fn number(&self) -> Option<f64> {
println!("number()");
match float::<_, _, (), false>(self.data()) {
match float::<_, _, ()>(self.data()) {
Ok((i, o)) => {
self.offset(i);
println!("-> {}", o);
Expand All @@ -75,7 +75,7 @@ impl<'a, 'b: 'a> JsonValue<'a, 'b> {
pub fn array(&self) -> Option<impl Iterator<Item = JsonValue<'a, 'b>>> {
println!("array()");

match tag::<_, _, (), false>("[")(self.data()) {
match tag::<_, _, ()>("[")(self.data()) {
Err(_) => None,
Ok((i, _)) => {
println!("[");
Expand All @@ -99,7 +99,7 @@ impl<'a, 'b: 'a> JsonValue<'a, 'b> {
}
}

if let Ok((i, _)) = tag::<_, _, (), false>("]")(v.data()) {
if let Ok((i, _)) = tag::<_, _, ()>("]")(v.data()) {
println!("]");
v.offset(i);
done = true;
Expand All @@ -109,7 +109,7 @@ impl<'a, 'b: 'a> JsonValue<'a, 'b> {
if first {
first = false;
} else {
match tag::<_, _, (), false>(",")(v.data()) {
match tag::<_, _, ()>(",")(v.data()) {
Ok((i, _)) => {
println!(",");
v.offset(i);
Expand All @@ -131,7 +131,7 @@ impl<'a, 'b: 'a> JsonValue<'a, 'b> {

pub fn object(&self) -> Option<impl Iterator<Item = (&'a str, JsonValue<'a, 'b>)>> {
println!("object()");
match tag::<_, _, (), false>("{")(self.data()) {
match tag::<_, _, ()>("{")(self.data()) {
Err(_) => None,
Ok((i, _)) => {
self.offset(i);
Expand All @@ -157,7 +157,7 @@ impl<'a, 'b: 'a> JsonValue<'a, 'b> {
}
}

if let Ok((i, _)) = tag::<_, _, (), false>("}")(v.data()) {
if let Ok((i, _)) = tag::<_, _, ()>("}")(v.data()) {
println!("}}");
v.offset(i);
done = true;
Expand All @@ -167,7 +167,7 @@ impl<'a, 'b: 'a> JsonValue<'a, 'b> {
if first {
first = false;
} else {
match tag::<_, _, (), false>(",")(v.data()) {
match tag::<_, _, ()>(",")(v.data()) {
Ok((i, _)) => {
println!(",");
v.offset(i);
Expand All @@ -183,7 +183,7 @@ impl<'a, 'b: 'a> JsonValue<'a, 'b> {
Ok((i, key)) => {
v.offset(i);

match tag::<_, _, (), false>(":")(v.data()) {
match tag::<_, _, ()>(":")(v.data()) {
Err(_) => None,
Ok((i, _)) => {
v.offset(i);
Expand Down Expand Up @@ -258,7 +258,7 @@ fn value(i: &str) -> IResult<&str, ()> {
hash,
array,
string.map(|_| ()),
float::<_, f64, _, false>.map(|_| ()),
float::<_, f64, _>.map(|_| ()),
boolean.map(|_| ()),
)),
)(i)
Expand Down
14 changes: 6 additions & 8 deletions src/bits/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,11 +158,11 @@ where
/// assert_eq!(parser((stream(&[0b00010010]), 0), 12), Err(winnow::error::ErrMode::Backtrack(Error{input: (stream(&[0b00010010]), 0), kind: ErrorKind::Eof })));
/// ```
#[inline(always)]
pub fn take<I, O, C, E: ParseError<(I, usize)>, const PARTIAL: bool>(
pub fn take<I, O, C, E: ParseError<(I, usize)>>(
count: C,
) -> impl FnMut((I, usize)) -> IResult<(I, usize), O, E>
where
I: Stream<Token = u8> + AsBytes + StreamIsPartial<PARTIAL>,
I: Stream<Token = u8> + AsBytes + StreamIsPartial,
C: ToUsize,
O: From<u8> + AddAssign + Shl<usize, Output = O> + Shr<usize, Output = O>,
{
Expand Down Expand Up @@ -230,12 +230,12 @@ where
/// );
/// ```
#[inline(always)]
pub fn tag<I, O, C, E: ParseError<(I, usize)>, const PARTIAL: bool>(
pub fn tag<I, O, C, E: ParseError<(I, usize)>>(
pattern: O,
count: C,
) -> impl FnMut((I, usize)) -> IResult<(I, usize), O, E>
where
I: Stream<Token = u8> + AsBytes + StreamIsPartial<PARTIAL>,
I: Stream<Token = u8> + AsBytes + StreamIsPartial,
C: ToUsize,
O: From<u8> + AddAssign + Shl<usize, Output = O> + Shr<usize, Output = O> + PartialEq,
{
Expand Down Expand Up @@ -272,11 +272,9 @@ where
/// assert_eq!(parse((stream(&[0b10000000]), 0)), Ok(((stream(&[0b10000000]), 1), true)));
/// assert_eq!(parse((stream(&[0b10000000]), 1)), Ok(((stream(&[0b10000000]), 2), false)));
/// ```
pub fn bool<I, E: ParseError<(I, usize)>, const PARTIAL: bool>(
input: (I, usize),
) -> IResult<(I, usize), bool, E>
pub fn bool<I, E: ParseError<(I, usize)>>(input: (I, usize)) -> IResult<(I, usize), bool, E>
where
I: Stream<Token = u8> + AsBytes + StreamIsPartial<PARTIAL>,
I: Stream<Token = u8> + AsBytes + StreamIsPartial,
{
#![allow(deprecated)]
trace("bool", |input: (I, usize)| {
Expand Down
Loading