From e3f33fe66a54a852d223ee74d420d5576c310747 Mon Sep 17 00:00:00 2001 From: Lukas Stancik Date: Sat, 4 Apr 2026 07:46:04 +0000 Subject: [PATCH] Add events and state for tokenizer --- src/event.rs | 8 ++++++++ src/state.rs | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/event.rs b/src/event.rs index 87254f7..38a699f 100644 --- a/src/event.rs +++ b/src/event.rs @@ -6,6 +6,14 @@ use crate::util::constant::TAB_SIZE; /// Semantic label of a span. #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub enum Name { + // Whole task + Task, + // Content between starting :: and ending :: + TaskContent, + // Just the starting :: and ending :: + TaskFence, + // Either starting :: or ending :: + TaskSequence, /// Attention sequence. /// /// > 👉 **Note**: this is used while parsing but compiled away. diff --git a/src/state.rs b/src/state.rs index 5e765c7..8296fe4 100644 --- a/src/state.rs +++ b/src/state.rs @@ -466,12 +466,34 @@ pub enum Name { TitleEscape, TitleInside, TitleNok, + + TaskStart, + TaskOpenSequence, + TaskCloseSequence, + // TaskOpenAfter, + TaskContentStart, + TaskContentInside, + TaskContentEnd, + TaskCloseStart, + TaskCloseAfter, + // TaskAfter, } #[allow(clippy::too_many_lines)] /// Call the corresponding state for a state name. pub fn call(tokenizer: &mut Tokenizer, name: Name) -> State { let func = match name { + Name::TaskStart => construct::task::start, + Name::TaskOpenSequence => construct::task::open_sequence, + // Name::TaskOpenAfter => construct::task::open_after, + // Name::TaskAfter => construct::task::after, + Name::TaskContentStart => construct::task::content_start, + Name::TaskContentInside => construct::task::content_inside, + Name::TaskContentEnd => construct::task::content_end, + Name::TaskCloseStart => construct::task::close_start, + Name::TaskCloseSequence => construct::task::close_sequence, + Name::TaskCloseAfter => construct::task::close_after, + Name::AttentionStart => construct::attention::start, Name::AttentionInside => construct::attention::inside,