2 files changed, 32 insertions, 0 deletions
diff --git a/tokens/Cargo.toml b/tokens/Cargo.toml
new file mode 100644
index 0000000..9fc30b2
--- /dev/null
+++ b/tokens/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "tokens"
+version = "0.1.2"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+grammar = { path = "../grammar" }
+graph = { path = "../graph" }
+graph_macro = { path = "../graph_macro" }
+nfa = { path = "../nfa" }
diff --git a/tokens/src/lib.rs b/tokens/src/lib.rs
new file mode 100644
index 0000000..ba4f7ac
--- /dev/null
+++ b/tokens/src/lib.rs
@@ -0,0 +1,20 @@
+//! This package defines the trait for tokenizers that are expected
+//! from the parser.
+//!
+//! This is not the essential part of the package, and users might
+//! want to tweak this part.
+
+pub fn add(left: usize, right: usize) -> usize {
+    left + right
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        let result = add(2, 2);
+        assert_eq!(result, 4);
+    }
+}