First version of tokenizer construct

src/construct/mod.rs

@@ -194,5 +194,6 @@ pub mod partial_whitespace;
 pub mod raw_flow;
 pub mod raw_text;
 pub mod string;
+pub mod task;
 pub mod text;
 pub mod thematic_break;

src/construct/task.rs

@@ -0,0 +1,131 @@
+use crate::construct::partial_space_or_tab::space_or_tab;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
+use crate::tokenizer::Tokenizer;
+
+pub fn start(tokenizer: &mut Tokenizer) -> State {
+    if matches!(tokenizer.current, Some(b':'))
+    {
+        log::debug!("---start--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+        tokenizer.tokenize_state.marker = tokenizer.current.unwrap();
+        tokenizer.enter(Name::Task);
+        tokenizer.enter(Name::TaskFence);
+        tokenizer.enter(Name::TaskSequence);
+        State::Retry(StateName::TaskOpenSequence)
+    } else {
+        State::Nok
+    }
+}
+
+pub fn open_sequence(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---open_sequence--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    if tokenizer.current == Some(tokenizer.tokenize_state.marker) {
+        tokenizer.tokenize_state.size += 1;
+        tokenizer.consume();
+        State::Next(StateName::TaskOpenSequence)
+    } else if tokenizer.tokenize_state.size == 2 {
+        tokenizer.tokenize_state.size = 0;
+        tokenizer.exit(Name::TaskSequence);
+        tokenizer.exit(Name::TaskFence);
+
+        if matches!(tokenizer.current, Some(b'\t' | b' ')) {
+            tokenizer.attempt(State::Next(StateName::TaskContentStart), State::Nok);
+            State::Retry(space_or_tab(tokenizer))
+        } else {
+            State::Retry(StateName::TaskContentStart)
+        }
+    } else {
+        tokenizer.tokenize_state.marker = 0;
+        tokenizer.tokenize_state.size = 0;
+        State::Nok
+    }
+}
+
+pub fn content_start(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---content_start--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    match tokenizer.current {
+        None | Some(b':') => {
+          State::Retry(StateName::TaskContentEnd)
+        },
+        Some(_) => {
+            tokenizer.enter(Name::TaskContent);
+            State::Retry(StateName::TaskContentInside)
+        }
+    }
+}
+
+pub fn content_inside(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---content_inside--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    match tokenizer.current {
+        None => {
+            tokenizer.tokenize_state.marker = 0;
+            State::Nok
+        }
+        Some(b':') => {
+            tokenizer.exit(Name::TaskContent);
+            State::Retry(StateName::TaskContentEnd)
+        }
+        Some(_) => {
+            tokenizer.consume();
+            State::Next(StateName::TaskContentInside)
+        }
+    }
+}
+
+pub fn content_end(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---content_end--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    match tokenizer.current {
+        None => {
+            tokenizer.tokenize_state.marker = 0;
+            State::Nok
+        }
+        Some(b':') => {
+          State::Retry(StateName::TaskCloseStart)
+        }
+        Some(_) => {
+          unreachable!("expected eof/eol")
+        }
+    }
+}
+
+pub fn close_start(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---close_start--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    if tokenizer.current == Some(tokenizer.tokenize_state.marker) {
+        tokenizer.enter(Name::TaskFence);
+        tokenizer.enter(Name::TaskSequence);
+        State::Retry(StateName::TaskCloseSequence)
+    } else {
+        State::Nok
+    }
+}
+
+pub fn close_sequence(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---close_sequence--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    if tokenizer.current == Some(tokenizer.tokenize_state.marker) {
+        tokenizer.tokenize_state.size += 1;
+        tokenizer.consume();
+        State::Next(StateName::TaskCloseSequence)
+    } else if tokenizer.tokenize_state.size == 1 {
+      log::debug!("else if");
+      tokenizer.tokenize_state.size = 0;
+      tokenizer.exit(Name::TaskSequence);
+
+      State::Retry(StateName::TaskCloseAfter)
+    } else {
+        log::debug!("else");
+        tokenizer.tokenize_state.size = 0;
+        State::Nok
+    }
+}
+
+pub fn close_after(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---close_after--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    match tokenizer.current {
+        None => State::Nok,
+        _ => {
+            tokenizer.exit(Name::TaskFence);
+            tokenizer.exit(Name::Task);
+            State::Ok
+        }
+    }
+}