Merge branch 'custom-task-todo-support'

* custom-task-todo-support:
  Added logging by default
  Look for the new syntax inside text
  Implement abstract syntax tree compilation of the new syntax
  Add events and state for tokenizer
  First version of tokenizer construct

Cargo.toml

@@ -21,7 +21,7 @@ swc_core = { version = "22", features = [
 ] }
 
 [features]
-default = []
+default = ["log"]
 json = ["serde"]
 log = ["dep:log"]
 serde = ["dep:serde"]

src/construct/mod.rs

@@ -194,5 +194,6 @@ pub mod partial_whitespace;
 pub mod raw_flow;
 pub mod raw_text;
 pub mod string;
+pub mod task;
 pub mod text;
 pub mod thematic_break;

src/construct/task.rs

@@ -0,0 +1,131 @@
+use crate::construct::partial_space_or_tab::space_or_tab;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
+use crate::tokenizer::Tokenizer;
+
+pub fn start(tokenizer: &mut Tokenizer) -> State {
+    if matches!(tokenizer.current, Some(b':'))
+    {
+        log::debug!("---start--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+        tokenizer.tokenize_state.marker = tokenizer.current.unwrap();
+        tokenizer.enter(Name::Task);
+        tokenizer.enter(Name::TaskFence);
+        tokenizer.enter(Name::TaskSequence);
+        State::Retry(StateName::TaskOpenSequence)
+    } else {
+        State::Nok
+    }
+}
+
+pub fn open_sequence(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---open_sequence--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    if tokenizer.current == Some(tokenizer.tokenize_state.marker) {
+        tokenizer.tokenize_state.size += 1;
+        tokenizer.consume();
+        State::Next(StateName::TaskOpenSequence)
+    } else if tokenizer.tokenize_state.size == 2 {
+        tokenizer.tokenize_state.size = 0;
+        tokenizer.exit(Name::TaskSequence);
+        tokenizer.exit(Name::TaskFence);
+
+        if matches!(tokenizer.current, Some(b'\t' | b' ')) {
+            tokenizer.attempt(State::Next(StateName::TaskContentStart), State::Nok);
+            State::Retry(space_or_tab(tokenizer))
+        } else {
+            State::Retry(StateName::TaskContentStart)
+        }
+    } else {
+        tokenizer.tokenize_state.marker = 0;
+        tokenizer.tokenize_state.size = 0;
+        State::Nok
+    }
+}
+
+pub fn content_start(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---content_start--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    match tokenizer.current {
+        None | Some(b':') => {
+          State::Retry(StateName::TaskContentEnd)
+        },
+        Some(_) => {
+            tokenizer.enter(Name::TaskContent);
+            State::Retry(StateName::TaskContentInside)
+        }
+    }
+}
+
+pub fn content_inside(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---content_inside--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    match tokenizer.current {
+        None => {
+            tokenizer.tokenize_state.marker = 0;
+            State::Nok
+        }
+        Some(b':') => {
+            tokenizer.exit(Name::TaskContent);
+            State::Retry(StateName::TaskContentEnd)
+        }
+        Some(_) => {
+            tokenizer.consume();
+            State::Next(StateName::TaskContentInside)
+        }
+    }
+}
+
+pub fn content_end(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---content_end--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    match tokenizer.current {
+        None => {
+            tokenizer.tokenize_state.marker = 0;
+            State::Nok
+        }
+        Some(b':') => {
+          State::Retry(StateName::TaskCloseStart)
+        }
+        Some(_) => {
+          unreachable!("expected eof/eol")
+        }
+    }
+}
+
+pub fn close_start(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---close_start--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    if tokenizer.current == Some(tokenizer.tokenize_state.marker) {
+        tokenizer.enter(Name::TaskFence);
+        tokenizer.enter(Name::TaskSequence);
+        State::Retry(StateName::TaskCloseSequence)
+    } else {
+        State::Nok
+    }
+}
+
+pub fn close_sequence(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---close_sequence--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    if tokenizer.current == Some(tokenizer.tokenize_state.marker) {
+        tokenizer.tokenize_state.size += 1;
+        tokenizer.consume();
+        State::Next(StateName::TaskCloseSequence)
+    } else if tokenizer.tokenize_state.size == 1 {
+      log::debug!("else if");
+      tokenizer.tokenize_state.size = 0;
+      tokenizer.exit(Name::TaskSequence);
+
+      State::Retry(StateName::TaskCloseAfter)
+    } else {
+        log::debug!("else");
+        tokenizer.tokenize_state.size = 0;
+        State::Nok
+    }
+}
+
+pub fn close_after(tokenizer: &mut Tokenizer) -> State {
+    log::debug!("---close_after--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
+    match tokenizer.current {
+        None => State::Nok,
+        _ => {
+            tokenizer.exit(Name::TaskFence);
+            tokenizer.exit(Name::Task);
+            State::Ok
+        }
+    }
+}

src/construct/text.rs

@@ -32,7 +32,7 @@ use crate::subtokenize::Subresult;
 use crate::tokenizer::Tokenizer;
 
 /// Characters that can start something in text.
-const MARKERS: [u8; 16] = [
+const MARKERS: [u8; 17] = [
     b'!',  // `label_start_image`
     b'$',  // `raw_text` (math (text))
     b'&',  // `character_reference`
@@ -49,6 +49,7 @@ const MARKERS: [u8; 16] = [
     b'w',  // `gfm_autolink_literal` (`www.` kind)
     b'{',  // `mdx_expression_text`
     b'~',  // `attention` (gfm strikethrough)
+    b':',  // `task` (custom task)
 ];
 
 /// Start of text.
@@ -163,6 +164,13 @@ pub fn before(tokenizer: &mut Tokenizer) -> State {
             );
             State::Retry(StateName::MdxExpressionTextStart)
         }
+        Some(b':') => {
+            tokenizer.attempt(
+                State::Next(StateName::TextBefore),
+                State::Next(StateName::TextBeforeData),
+            );
+            State::Retry(StateName::TaskStart)
+        }
         _ => State::Retry(StateName::TextBeforeData),
     }
 }

src/event.rs

@@ -6,6 +6,14 @@ use crate::util::constant::TAB_SIZE;
 /// Semantic label of a span.
 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
 pub enum Name {
+    // Whole task
+    Task,
+    // Content between starting :: and ending ::
+    TaskContent,
+    // Just the starting :: and ending ::
+    TaskFence,
+    // Either starting :: or ending ::
+    TaskSequence,
     /// Attention sequence.
     ///
     /// > 👉 **Note**: this is used while parsing but compiled away.

src/mdast.rs

@@ -193,6 +193,7 @@ pub enum Node {
     Break(Break),
     /// Code (phrasing).
     InlineCode(InlineCode),
+    Task(Task),
     /// Math (phrasing).
     InlineMath(InlineMath),
     /// Delete.
@@ -269,6 +270,7 @@ impl fmt::Debug for Node {
             Node::Yaml(x) => x.fmt(f),
             Node::Break(x) => x.fmt(f),
             Node::InlineCode(x) => x.fmt(f),
+            Node::Task(x) => x.fmt(f),
             Node::InlineMath(x) => x.fmt(f),
             Node::Delete(x) => x.fmt(f),
             Node::Emphasis(x) => x.fmt(f),
@@ -330,6 +332,7 @@ impl ToString for Node {
             Node::Toml(x) => x.value.clone(),
             Node::Yaml(x) => x.value.clone(),
             Node::InlineCode(x) => x.value.clone(),
+            Node::Task(x) => x.value.clone(),
             Node::InlineMath(x) => x.value.clone(),
             Node::MdxTextExpression(x) => x.value.clone(),
             Node::Html(x) => x.value.clone(),
@@ -414,6 +417,7 @@ impl Node {
             Node::Yaml(x) => x.position.as_ref(),
             Node::Break(x) => x.position.as_ref(),
             Node::InlineCode(x) => x.position.as_ref(),
+            Node::Task(x) => x.position.as_ref(),
             Node::InlineMath(x) => x.position.as_ref(),
             Node::Delete(x) => x.position.as_ref(),
             Node::Emphasis(x) => x.position.as_ref(),
@@ -453,6 +457,7 @@ impl Node {
             Node::Yaml(x) => x.position.as_mut(),
             Node::Break(x) => x.position.as_mut(),
             Node::InlineCode(x) => x.position.as_mut(),
+            Node::Task(x) => x.position.as_mut(),
             Node::InlineMath(x) => x.position.as_mut(),
             Node::Delete(x) => x.position.as_mut(),
             Node::Emphasis(x) => x.position.as_mut(),
@@ -492,6 +497,7 @@ impl Node {
             Node::Yaml(x) => x.position = position,
             Node::Break(x) => x.position = position,
             Node::InlineCode(x) => x.position = position,
+            Node::Task(x) => x.position = position,
             Node::InlineMath(x) => x.position = position,
             Node::Delete(x) => x.position = position,
             Node::Emphasis(x) => x.position = position,
@@ -892,6 +898,23 @@ pub struct InlineCode {
     pub position: Option<Position>,
 }
 
+/// Code (phrasing).
+///
+/// ```markdown
+/// > | `a`
+///     ^^^
+/// ```
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+pub struct Task {
+    // Text.
+    /// Content model.
+    pub value: String,
+    /// Positional info.
+    #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
+    pub position: Option<Position>,
+}
+
 /// Math (phrasing).
 ///
 /// ```markdown

src/state.rs

@@ -466,12 +466,34 @@ pub enum Name {
     TitleEscape,
     TitleInside,
     TitleNok,
+
+    TaskStart,
+    TaskOpenSequence,
+    TaskCloseSequence,
+    // TaskOpenAfter,
+    TaskContentStart,
+    TaskContentInside,
+    TaskContentEnd,
+    TaskCloseStart,
+    TaskCloseAfter,
+    // TaskAfter,
 }
 
 #[allow(clippy::too_many_lines)]
 /// Call the corresponding state for a state name.
 pub fn call(tokenizer: &mut Tokenizer, name: Name) -> State {
     let func = match name {
+        Name::TaskStart => construct::task::start,
+        Name::TaskOpenSequence => construct::task::open_sequence,
+        // Name::TaskOpenAfter => construct::task::open_after,
+        // Name::TaskAfter => construct::task::after,
+        Name::TaskContentStart => construct::task::content_start,
+        Name::TaskContentInside => construct::task::content_inside,
+        Name::TaskContentEnd => construct::task::content_end,
+        Name::TaskCloseStart => construct::task::close_start,
+        Name::TaskCloseSequence => construct::task::close_sequence,
+        Name::TaskCloseAfter => construct::task::close_after,
+
         Name::AttentionStart => construct::attention::start,
         Name::AttentionInside => construct::attention::inside,
 

src/to_html.rs

@@ -333,6 +333,7 @@ fn enter(context: &mut CompileContext) {
         Name::CodeIndented => on_enter_code_indented(context),
         Name::CodeFenced | Name::MathFlow => on_enter_raw_flow(context),
         Name::CodeText | Name::MathText => on_enter_raw_text(context),
+        Name::Task => on_enter_task(context),
         Name::Definition => on_enter_definition(context),
         Name::DefinitionDestinationString => on_enter_definition_destination_string(context),
         Name::Emphasis => on_enter_emphasis(context),
@@ -371,7 +372,7 @@ fn exit(context: &mut CompileContext) {
             on_exit_drop(context);
         }
         Name::MdxEsm | Name::MdxFlowExpression | Name::MdxJsxFlowTag => on_exit_drop_slurp(context),
-        Name::CharacterEscapeValue | Name::CodeTextData | Name::Data | Name::MathTextData => {
+        Name::CharacterEscapeValue | Name::CodeTextData | Name::TaskContent | Name::Data | Name::MathTextData => {
             on_exit_data(context);
         }
         Name::AutolinkEmail => on_exit_autolink_email(context),
@@ -391,6 +392,7 @@ fn exit(context: &mut CompileContext) {
         Name::CodeFencedFenceInfo => on_exit_raw_flow_fence_info(context),
         Name::CodeFlowChunk | Name::MathFlowChunk => on_exit_raw_flow_chunk(context),
         Name::CodeText | Name::MathText => on_exit_raw_text(context),
+        Name::Task => on_exit_task(context),
         Name::Definition => on_exit_definition(context),
         Name::DefinitionDestinationString => on_exit_definition_destination_string(context),
         Name::DefinitionLabelString => on_exit_definition_label_string(context),
@@ -455,6 +457,13 @@ fn on_enter_block_quote(context: &mut CompileContext) {
     context.push("<blockquote>");
 }
 
+/// Handle [`Enter`][Kind::Enter]:[`Task`][Name::Task].
+fn on_enter_task(context: &mut CompileContext) {
+    context.tight_stack.push(false);
+    // context.line_ending_if_needed();
+    context.push("<task-todo>");
+}
+
 /// Handle [`Enter`][Kind::Enter]:[`CodeIndented`][Name::CodeIndented].
 fn on_enter_code_indented(context: &mut CompileContext) {
     context.raw_flow_seen_data = Some(false);
@@ -757,6 +766,14 @@ fn on_exit_block_quote(context: &mut CompileContext) {
     context.push("</blockquote>");
 }
 
+/// Handle [`Exit`][Kind::Exit]:[`Task`][Name::Task].
+fn on_exit_task(context: &mut CompileContext) {
+    context.tight_stack.pop();
+    // context.line_ending_if_needed();
+    // context.slurp_one_line_ending = false;
+    context.push("</task-todo>");
+}
+
 /// Handle [`Exit`][Kind::Exit]:[`CharacterReferenceMarker`][Name::CharacterReferenceMarker].
 fn on_exit_character_reference_marker(context: &mut CompileContext) {
     context.character_reference_marker = Some(b'&');

src/to_mdast.rs

@@ -7,7 +7,7 @@ use crate::mdast::{
     ImageReference, InlineCode, InlineMath, Link, LinkReference, List, ListItem, Math,
     MdxFlowExpression, MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement,
     MdxJsxTextElement, MdxTextExpression, MdxjsEsm, Node, Paragraph, ReferenceKind, Root, Strong,
-    Table, TableCell, TableRow, Text, ThematicBreak, Toml, Yaml,
+    Table, TableCell, TableRow, Text, Task, ThematicBreak, Toml, Yaml,
 };
 use crate::message;
 use crate::unist::{Point, Position};
@@ -267,6 +267,7 @@ fn enter(context: &mut CompileContext) -> Result<(), message::Message> {
         | Name::CharacterReference
         | Name::CodeFlowChunk
         | Name::CodeTextData
+        | Name::TaskContent
         | Name::Data
         | Name::FrontmatterChunk
         | Name::HtmlFlowData
@@ -291,6 +292,7 @@ fn enter(context: &mut CompileContext) -> Result<(), message::Message> {
         Name::CodeFenced => on_enter_code_fenced(context),
         Name::CodeIndented => on_enter_code_indented(context),
         Name::CodeText => on_enter_code_text(context),
+        Name::Task => on_enter_task(context),
         Name::Definition => on_enter_definition(context),
         Name::Emphasis => on_enter_emphasis(context),
         Name::Frontmatter => on_enter_frontmatter(context),
@@ -359,6 +361,7 @@ fn exit(context: &mut CompileContext) -> Result<(), message::Message> {
         Name::CharacterEscapeValue
         | Name::CodeFlowChunk
         | Name::CodeTextData
+        | Name::TaskContent
         | Name::Data
         | Name::FrontmatterChunk
         | Name::HtmlFlowData
@@ -387,6 +390,7 @@ fn exit(context: &mut CompileContext) -> Result<(), message::Message> {
         Name::CodeFenced | Name::MathFlow => on_exit_raw_flow(context)?,
         Name::CodeIndented => on_exit_code_indented(context)?,
         Name::CodeText | Name::MathText => on_exit_raw_text(context)?,
+        Name::Task => on_exit_task(context)?,
         Name::DefinitionDestinationString => on_exit_definition_destination_string(context),
         Name::DefinitionLabelString | Name::GfmFootnoteDefinitionLabelString => {
             on_exit_definition_id(context);
@@ -501,6 +505,15 @@ fn on_enter_code_text(context: &mut CompileContext) {
     context.buffer();
 }
 
+/// Handle [`Enter`][Kind::Enter]:[`Task`][Name::Task].
+fn on_enter_task(context: &mut CompileContext) {
+    context.tail_push(Node::Task(Task {
+        value: String::new(),
+        position: None,
+    }));
+    context.buffer();
+}
+
 /// Handle [`Enter`][Kind::Enter]:[`MathText`][Name::MathText].
 fn on_enter_math_text(context: &mut CompileContext) {
     context.tail_push(Node::InlineMath(InlineMath {
@@ -1103,6 +1116,29 @@ fn on_exit_raw_text(context: &mut CompileContext) -> Result<(), message::Message
     Ok(())
 }
 
+/// Handle [`Exit`][Kind::Exit]:{[`CodeText`][Name::CodeText],[`MathText`][Name::MathText]}.
+fn on_exit_task(context: &mut CompileContext) -> Result<(), message::Message> {
+    let mut value = context.resume().to_string();
+
+    let value_bytes = value.as_bytes();
+    if value.len() > 2
+        && value_bytes[0] == b' '
+        && value_bytes[value.len() - 1] == b' '
+        && !value_bytes.iter().all(|b| *b == b' ')
+    {
+        value.remove(0);
+        value.pop();
+    }
+
+    match context.tail_mut() {
+        Node::Task(node) => node.value = value,
+        _ => unreachable!("expected task on stack for value"),
+    }
+
+    on_exit(context)?;
+    Ok(())
+}
+
 /// Handle [`Exit`][Kind::Exit]:[`Data`][Name::Data] (and many text things).
 fn on_exit_data(context: &mut CompileContext) -> Result<(), message::Message> {
     let value = Slice::from_position(