summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJSDurand <mmemmew@gmail.com>2023-08-04 16:36:50 +0800
committerJSDurand <mmemmew@gmail.com>2023-08-04 16:36:50 +0800
commitbfb85bf0cd7a75b910d7125c69d5e5eae2bd51c1 (patch)
tree4c6b3acd208909a4539da2c9acdebb8f3a7f95a2
parent7033187abaf42772097377c0a1ffc2cd4cefdada (diff)
add a crate 'tokens'
This crate is responsible for the users to conveniently define their own tokenizers. Right now the user has to tokenize the input string manually, which is not quite what is expected from a parser, from the view point of users.
-rw-r--r--Cargo.toml2
-rw-r--r--src/test.c2
-rw-r--r--tokens/Cargo.toml12
-rw-r--r--tokens/src/lib.rs20
4 files changed, 34 insertions, 2 deletions
diff --git a/Cargo.toml b/Cargo.toml
index 071575e..8e2ffb5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,7 +13,7 @@ rust-version = "1.65"
[workspace]
members = [ "graph", "receme", "nfa", "chain", "viz", "grammar",
- "forest", "semiring", "graph_macro" ]
+ "forest", "semiring", "graph_macro", "tokens" ]
# testing the new resolver, even though this has no dependencies ;p
resolver = "2"
diff --git a/src/test.c b/src/test.c
index f9ab43b..c1043c8 100644
--- a/src/test.c
+++ b/src/test.c
@@ -25,7 +25,7 @@ main(int argc, char **argv)
"\n"
"item =/ header [ note ] *1( price )\n"
"\n"
-"header = star \"SP\" title %xA *( \"SP\" / %xA )\n"
+"header = *1star \"SP\" title %xA *( \"SP\" / %xA )\n"
"\n"
"title = 1*\"TEXT\"\n"
"\n"
diff --git a/tokens/Cargo.toml b/tokens/Cargo.toml
new file mode 100644
index 0000000..9fc30b2
--- /dev/null
+++ b/tokens/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "tokens"
+version = "0.1.2"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+grammar = { path = "../grammar" }
+graph = { path = "../graph" }
+graph_macro = { path = "../graph_macro" }
+nfa = { path = "../nfa" }
diff --git a/tokens/src/lib.rs b/tokens/src/lib.rs
new file mode 100644
index 0000000..ba4f7ac
--- /dev/null
+++ b/tokens/src/lib.rs
@@ -0,0 +1,20 @@
+//! This package defines the trait for tokenizers that are expected
+//! from the parser.
+//!
+//! This is not the essential part of the package, and users might
+//! want to tweak this part.
+
+pub fn add(left: usize, right: usize) -> usize {
+ left + right
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn it_works() {
+ let result = add(2, 2);
+ assert_eq!(result, 4);
+ }
+}