lusk/markdown-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'custom-task-todo-support'

Browse Source 
* custom-task-todo-support:
  Added logging by default
  Look for the new syntax inside text
  Implement abstract syntax tree compilation of the new syntax
  Add events and state for tokenizer
  First version of tokenizer construct
This commit is contained in:
Lukas Stancik
2026-04-04 11:57:19 +00:00
parent 5d3afabd24 9c64096e0a
commit e3334705cf
9 changed files with 250 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.toml
+1 -1
View File
@@ -21,7 +21,7 @@ swc_core = { version = "22", features = [
] }
[features]
default = []
default = ["log"]
json = ["serde"]
log = ["dep:log"]
serde = ["dep:serde"]
src/construct/mod.rs
+1
View File
@@ -194,5 +194,6 @@ pub mod partial_whitespace;
pub mod raw_flow;
pub mod raw_text;
pub mod string;
pub mod task;
pub mod text;
pub mod thematic_break;
src/construct/task.rs
+131
View File
@@ -0,0 +1,131 @@
use crate::construct::partial_space_or_tab::space_or_tab;
use crate::event::Name;
use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
pub fn start(tokenizer: &mut Tokenizer) -> State {
if matches!(tokenizer.current, Some(b':'))
{
log::debug!("---start--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
tokenizer.tokenize_state.marker = tokenizer.current.unwrap();
tokenizer.enter(Name::Task);
tokenizer.enter(Name::TaskFence);
tokenizer.enter(Name::TaskSequence);
State::Retry(StateName::TaskOpenSequence)
} else {
State::Nok
}
}
pub fn open_sequence(tokenizer: &mut Tokenizer) -> State {
log::debug!("---open_sequence--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
if tokenizer.current == Some(tokenizer.tokenize_state.marker) {
tokenizer.tokenize_state.size += 1;
tokenizer.consume();
State::Next(StateName::TaskOpenSequence)
} else if tokenizer.tokenize_state.size == 2 {
tokenizer.tokenize_state.size = 0;
tokenizer.exit(Name::TaskSequence);
tokenizer.exit(Name::TaskFence);
if matches!(tokenizer.current, Some(b'\t' | b' ')) {
tokenizer.attempt(State::Next(StateName::TaskContentStart), State::Nok);
State::Retry(space_or_tab(tokenizer))
} else {
State::Retry(StateName::TaskContentStart)
}
} else {
tokenizer.tokenize_state.marker = 0;
tokenizer.tokenize_state.size = 0;
State::Nok
}
}
pub fn content_start(tokenizer: &mut Tokenizer) -> State {
log::debug!("---content_start--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
match tokenizer.current {
None | Some(b':') => {
State::Retry(StateName::TaskContentEnd)
},
Some(_) => {
tokenizer.enter(Name::TaskContent);
State::Retry(StateName::TaskContentInside)
}
}
}
pub fn content_inside(tokenizer: &mut Tokenizer) -> State {
log::debug!("---content_inside--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
match tokenizer.current {
None => {
tokenizer.tokenize_state.marker = 0;
State::Nok
}
Some(b':') => {
tokenizer.exit(Name::TaskContent);
State::Retry(StateName::TaskContentEnd)
}
Some(_) => {
tokenizer.consume();
State::Next(StateName::TaskContentInside)
}
}
}
pub fn content_end(tokenizer: &mut Tokenizer) -> State {
log::debug!("---content_end--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
match tokenizer.current {
None => {
tokenizer.tokenize_state.marker = 0;
State::Nok
}
Some(b':') => {
State::Retry(StateName::TaskCloseStart)
}
Some(_) => {
unreachable!("expected eof/eol")
}
}
}
pub fn close_start(tokenizer: &mut Tokenizer) -> State {
log::debug!("---close_start--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
if tokenizer.current == Some(tokenizer.tokenize_state.marker) {
tokenizer.enter(Name::TaskFence);
tokenizer.enter(Name::TaskSequence);
State::Retry(StateName::TaskCloseSequence)
} else {
State::Nok
}
}
pub fn close_sequence(tokenizer: &mut Tokenizer) -> State {
log::debug!("---close_sequence--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
if tokenizer.current == Some(tokenizer.tokenize_state.marker) {
tokenizer.tokenize_state.size += 1;
tokenizer.consume();
State::Next(StateName::TaskCloseSequence)
} else if tokenizer.tokenize_state.size == 1 {
log::debug!("else if");
tokenizer.tokenize_state.size = 0;
tokenizer.exit(Name::TaskSequence);
State::Retry(StateName::TaskCloseAfter)
} else {
log::debug!("else");
tokenizer.tokenize_state.size = 0;
State::Nok
}
}
pub fn close_after(tokenizer: &mut Tokenizer) -> State {
log::debug!("---close_after--- {:?} {:?}", tokenizer.tokenize_state.size, tokenizer.current.unwrap() as char);
match tokenizer.current {
None => State::Nok,
_ => {
tokenizer.exit(Name::TaskFence);
tokenizer.exit(Name::Task);
State::Ok
}
}
}
src/construct/text.rs
+9 -1
View File
@@ -32,7 +32,7 @@ use crate::subtokenize::Subresult;
use crate::tokenizer::Tokenizer;
/// Characters that can start something in text.
const MARKERS: [u8; 16] = [
const MARKERS: [u8; 17] = [
b'!', // `label_start_image`
b'$', // `raw_text` (math (text))
b'&', // `character_reference`
@@ -49,6 +49,7 @@ const MARKERS: [u8; 16] = [
b'w', // `gfm_autolink_literal` (`www.` kind)
b'{', // `mdx_expression_text`
b'~', // `attention` (gfm strikethrough)
b':', // `task` (custom task)
];
/// Start of text.
@@ -163,6 +164,13 @@ pub fn before(tokenizer: &mut Tokenizer) -> State {
);
State::Retry(StateName::MdxExpressionTextStart)
}
Some(b':') => {
tokenizer.attempt(
State::Next(StateName::TextBefore),
State::Next(StateName::TextBeforeData),
);
State::Retry(StateName::TaskStart)
}
_ => State::Retry(StateName::TextBeforeData),
}
}
src/event.rs
+8
View File
@@ -6,6 +6,14 @@ use crate::util::constant::TAB_SIZE;
/// Semantic label of a span.
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub enum Name {
// Whole task
Task,
// Content between starting :: and ending ::
TaskContent,
// Just the starting :: and ending ::
TaskFence,
// Either starting :: or ending ::
TaskSequence,
/// Attention sequence.
///
/// > 👉 **Note**: this is used while parsing but compiled away.
src/mdast.rs
+23
View File
@@ -193,6 +193,7 @@ pub enum Node {
Break(Break),
/// Code (phrasing).
InlineCode(InlineCode),
Task(Task),
/// Math (phrasing).
InlineMath(InlineMath),
/// Delete.
@@ -269,6 +270,7 @@ impl fmt::Debug for Node {
Node::Yaml(x) => x.fmt(f),
Node::Break(x) => x.fmt(f),
Node::InlineCode(x) => x.fmt(f),
Node::Task(x) => x.fmt(f),
Node::InlineMath(x) => x.fmt(f),
Node::Delete(x) => x.fmt(f),
Node::Emphasis(x) => x.fmt(f),
@@ -330,6 +332,7 @@ impl ToString for Node {
Node::Toml(x) => x.value.clone(),
Node::Yaml(x) => x.value.clone(),
Node::InlineCode(x) => x.value.clone(),
Node::Task(x) => x.value.clone(),
Node::InlineMath(x) => x.value.clone(),
Node::MdxTextExpression(x) => x.value.clone(),
Node::Html(x) => x.value.clone(),
@@ -414,6 +417,7 @@ impl Node {
Node::Yaml(x) => x.position.as_ref(),
Node::Break(x) => x.position.as_ref(),
Node::InlineCode(x) => x.position.as_ref(),
Node::Task(x) => x.position.as_ref(),
Node::InlineMath(x) => x.position.as_ref(),
Node::Delete(x) => x.position.as_ref(),
Node::Emphasis(x) => x.position.as_ref(),
@@ -453,6 +457,7 @@ impl Node {
Node::Yaml(x) => x.position.as_mut(),
Node::Break(x) => x.position.as_mut(),
Node::InlineCode(x) => x.position.as_mut(),
Node::Task(x) => x.position.as_mut(),
Node::InlineMath(x) => x.position.as_mut(),
Node::Delete(x) => x.position.as_mut(),
Node::Emphasis(x) => x.position.as_mut(),
@@ -492,6 +497,7 @@ impl Node {
Node::Yaml(x) => x.position = position,
Node::Break(x) => x.position = position,
Node::InlineCode(x) => x.position = position,
Node::Task(x) => x.position = position,
Node::InlineMath(x) => x.position = position,
Node::Delete(x) => x.position = position,
Node::Emphasis(x) => x.position = position,
@@ -892,6 +898,23 @@ pub struct InlineCode {
pub position: Option<Position>,
}
/// Code (phrasing).
///
/// ```markdown
/// > | `a`
/// ^^^
/// ```
#[derive(Clone, Debug, Eq, PartialEq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct Task {
// Text.
/// Content model.
pub value: String,
/// Positional info.
#[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
pub position: Option<Position>,
}
/// Math (phrasing).
///
/// ```markdown
src/state.rs
+22
View File
@@ -466,12 +466,34 @@ pub enum Name {
TitleEscape,
TitleInside,
TitleNok,
TaskStart,
TaskOpenSequence,
TaskCloseSequence,
// TaskOpenAfter,
TaskContentStart,
TaskContentInside,
TaskContentEnd,
TaskCloseStart,
TaskCloseAfter,
// TaskAfter,
}
#[allow(clippy::too_many_lines)]
/// Call the corresponding state for a state name.
pub fn call(tokenizer: &mut Tokenizer, name: Name) -> State {
let func = match name {
Name::TaskStart => construct::task::start,
Name::TaskOpenSequence => construct::task::open_sequence,
// Name::TaskOpenAfter => construct::task::open_after,
// Name::TaskAfter => construct::task::after,
Name::TaskContentStart => construct::task::content_start,
Name::TaskContentInside => construct::task::content_inside,
Name::TaskContentEnd => construct::task::content_end,
Name::TaskCloseStart => construct::task::close_start,
Name::TaskCloseSequence => construct::task::close_sequence,
Name::TaskCloseAfter => construct::task::close_after,
Name::AttentionStart => construct::attention::start,
Name::AttentionInside => construct::attention::inside,
src/to_html.rs
+18 -1
View File
@@ -333,6 +333,7 @@ fn enter(context: &mut CompileContext) {
Name::CodeIndented => on_enter_code_indented(context),
Name::CodeFenced | Name::MathFlow => on_enter_raw_flow(context),
Name::CodeText | Name::MathText => on_enter_raw_text(context),
Name::Task => on_enter_task(context),
Name::Definition => on_enter_definition(context),
Name::DefinitionDestinationString => on_enter_definition_destination_string(context),
Name::Emphasis => on_enter_emphasis(context),
@@ -371,7 +372,7 @@ fn exit(context: &mut CompileContext) {
on_exit_drop(context);
}
Name::MdxEsm | Name::MdxFlowExpression | Name::MdxJsxFlowTag => on_exit_drop_slurp(context),
Name::CharacterEscapeValue | Name::CodeTextData | Name::Data | Name::MathTextData => {
Name::CharacterEscapeValue | Name::CodeTextData | Name::TaskContent | Name::Data | Name::MathTextData => {
on_exit_data(context);
}
Name::AutolinkEmail => on_exit_autolink_email(context),
@@ -391,6 +392,7 @@ fn exit(context: &mut CompileContext) {
Name::CodeFencedFenceInfo => on_exit_raw_flow_fence_info(context),
Name::CodeFlowChunk | Name::MathFlowChunk => on_exit_raw_flow_chunk(context),
Name::CodeText | Name::MathText => on_exit_raw_text(context),
Name::Task => on_exit_task(context),
Name::Definition => on_exit_definition(context),
Name::DefinitionDestinationString => on_exit_definition_destination_string(context),
Name::DefinitionLabelString => on_exit_definition_label_string(context),
@@ -455,6 +457,13 @@ fn on_enter_block_quote(context: &mut CompileContext) {
context.push("<blockquote>");
}
/// Handle [`Enter`][Kind::Enter]:[`Task`][Name::Task].
fn on_enter_task(context: &mut CompileContext) {
context.tight_stack.push(false);
// context.line_ending_if_needed();
context.push("<task-todo>");
}
/// Handle [`Enter`][Kind::Enter]:[`CodeIndented`][Name::CodeIndented].
fn on_enter_code_indented(context: &mut CompileContext) {
context.raw_flow_seen_data = Some(false);
@@ -757,6 +766,14 @@ fn on_exit_block_quote(context: &mut CompileContext) {
context.push("</blockquote>");
}
/// Handle [`Exit`][Kind::Exit]:[`Task`][Name::Task].
fn on_exit_task(context: &mut CompileContext) {
context.tight_stack.pop();
// context.line_ending_if_needed();
// context.slurp_one_line_ending = false;
context.push("</task-todo>");
}
/// Handle [`Exit`][Kind::Exit]:[`CharacterReferenceMarker`][Name::CharacterReferenceMarker].
fn on_exit_character_reference_marker(context: &mut CompileContext) {
context.character_reference_marker = Some(b'&');
src/to_mdast.rs
+37 -1
View File
@@ -7,7 +7,7 @@ use crate::mdast::{
ImageReference, InlineCode, InlineMath, Link, LinkReference, List, ListItem, Math,
MdxFlowExpression, MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement,
MdxJsxTextElement, MdxTextExpression, MdxjsEsm, Node, Paragraph, ReferenceKind, Root, Strong,
Table, TableCell, TableRow, Text, ThematicBreak, Toml, Yaml,
Table, TableCell, TableRow, Text, Task, ThematicBreak, Toml, Yaml,
};
use crate::message;
use crate::unist::{Point, Position};
@@ -267,6 +267,7 @@ fn enter(context: &mut CompileContext) -> Result<(), message::Message> {
| Name::CharacterReference
| Name::CodeFlowChunk
| Name::CodeTextData
| Name::TaskContent
| Name::Data
| Name::FrontmatterChunk
| Name::HtmlFlowData
@@ -291,6 +292,7 @@ fn enter(context: &mut CompileContext) -> Result<(), message::Message> {
Name::CodeFenced => on_enter_code_fenced(context),
Name::CodeIndented => on_enter_code_indented(context),
Name::CodeText => on_enter_code_text(context),
Name::Task => on_enter_task(context),
Name::Definition => on_enter_definition(context),
Name::Emphasis => on_enter_emphasis(context),
Name::Frontmatter => on_enter_frontmatter(context),
@@ -359,6 +361,7 @@ fn exit(context: &mut CompileContext) -> Result<(), message::Message> {
Name::CharacterEscapeValue
| Name::CodeFlowChunk
| Name::CodeTextData
| Name::TaskContent
| Name::Data
| Name::FrontmatterChunk
| Name::HtmlFlowData
@@ -387,6 +390,7 @@ fn exit(context: &mut CompileContext) -> Result<(), message::Message> {
Name::CodeFenced | Name::MathFlow => on_exit_raw_flow(context)?,
Name::CodeIndented => on_exit_code_indented(context)?,
Name::CodeText | Name::MathText => on_exit_raw_text(context)?,
Name::Task => on_exit_task(context)?,
Name::DefinitionDestinationString => on_exit_definition_destination_string(context),
Name::DefinitionLabelString | Name::GfmFootnoteDefinitionLabelString => {
on_exit_definition_id(context);
@@ -501,6 +505,15 @@ fn on_enter_code_text(context: &mut CompileContext) {
context.buffer();
}
/// Handle [`Enter`][Kind::Enter]:[`Task`][Name::Task].
fn on_enter_task(context: &mut CompileContext) {
context.tail_push(Node::Task(Task {
value: String::new(),
position: None,
}));
context.buffer();
}
/// Handle [`Enter`][Kind::Enter]:[`MathText`][Name::MathText].
fn on_enter_math_text(context: &mut CompileContext) {
context.tail_push(Node::InlineMath(InlineMath {
@@ -1103,6 +1116,29 @@ fn on_exit_raw_text(context: &mut CompileContext) -> Result<(), message::Message
Ok(())
}
/// Handle [`Exit`][Kind::Exit]:{[`CodeText`][Name::CodeText],[`MathText`][Name::MathText]}.
fn on_exit_task(context: &mut CompileContext) -> Result<(), message::Message> {
let mut value = context.resume().to_string();
let value_bytes = value.as_bytes();
if value.len() > 2
&& value_bytes[0] == b' '
&& value_bytes[value.len() - 1] == b' '
&& !value_bytes.iter().all(|b| *b == b' ')
{
value.remove(0);
value.pop();
}
match context.tail_mut() {
Node::Task(node) => node.value = value,
_ => unreachable!("expected task on stack for value"),
}
on_exit(context)?;
Ok(())
}
/// Handle [`Exit`][Kind::Exit]:[`Data`][Name::Data] (and many text things).
fn on_exit_data(context: &mut CompileContext) -> Result<(), message::Message> {
let value = Slice::from_position(