Ch 1.3
This commit is contained in:
commit
9ffab45bd6
6 changed files with 416 additions and 0 deletions
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
/target
|
220
Cargo.lock
generated
Normal file
220
Cargo.lock
generated
Normal file
|
@ -0,0 +1,220 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-executor",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366"
|
||||
|
||||
[[package]]
|
||||
name = "futures-timer"
|
||||
version = "3.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-macro",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
||||
|
||||
[[package]]
|
||||
name = "moose"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"rstest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
|
||||
|
||||
[[package]]
|
||||
name = "pin-utils"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.51"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rstest"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b07f2d176c472198ec1e6551dc7da28f1c089652f66a7b722676c2238ebc0edf"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"futures-timer",
|
||||
"rstest_macros",
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rstest_macros"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7229b505ae0706e64f37ffc54a9c163e11022a6636d58fe1f3f52018257ff9f7"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustc_version",
|
||||
"syn",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
|
||||
dependencies = [
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
|
9
Cargo.toml
Normal file
9
Cargo.toml
Normal file
|
@ -0,0 +1,9 @@
|
|||
[package]
|
||||
name = "moose"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
|
||||
[dev-dependencies]
|
||||
rstest = "0.16.0"
|
156
src/lexer/mod.rs
Normal file
156
src/lexer/mod.rs
Normal file
|
@ -0,0 +1,156 @@
|
|||
mod tokens;
|
||||
|
||||
use std::{iter::Peekable, str::Chars};
|
||||
|
||||
use tokens::Token;
|
||||
|
||||
pub fn tokenize(input: &str) -> Vec<Token> {
|
||||
let mut input = input.chars().into_iter().peekable();
|
||||
|
||||
let mut toks = Vec::new();
|
||||
while let Some(tok) = next_token(&mut input) {
|
||||
toks.push(tok)
|
||||
}
|
||||
|
||||
toks
|
||||
}
|
||||
|
||||
fn next_token(input: &mut Peekable<Chars>) -> Option<Token> {
|
||||
let tok = match input.next()? {
|
||||
'=' => Token::Assign,
|
||||
'+' => Token::Plus,
|
||||
|
||||
',' => Token::Comma,
|
||||
';' => Token::Semicolon,
|
||||
|
||||
'(' => Token::LeftParenthesis,
|
||||
')' => Token::RightParenthesis,
|
||||
'{' => Token::LeftBrace,
|
||||
'}' => Token::RightBrace,
|
||||
|
||||
// Parse multicharacter tokens
|
||||
tok if tok.is_ascii_alphabetic() => read_ident(input, tok),
|
||||
tok if tok.is_ascii_digit() => read_int(input, tok),
|
||||
|
||||
// Skip whitespace
|
||||
tok if tok.is_ascii_whitespace() => next_token(input)?,
|
||||
|
||||
_ => Token::Illegal,
|
||||
};
|
||||
|
||||
Some(tok)
|
||||
}
|
||||
|
||||
fn read_ident(input: &mut Peekable<Chars>, first: char) -> Token {
|
||||
// Read the entire ident
|
||||
let mut toks = vec![first];
|
||||
while let Some(tok) = input.peek() {
|
||||
if !tok.is_ascii_alphabetic() {
|
||||
break;
|
||||
}
|
||||
|
||||
let tok = input.next().unwrap();
|
||||
toks.push(tok);
|
||||
}
|
||||
|
||||
// Check if our ident is a keyword
|
||||
let ident = toks.iter().cloned().collect::<String>();
|
||||
match ident.as_str() {
|
||||
"fn" => Token::Function,
|
||||
"let" => Token::Let,
|
||||
|
||||
ident => Token::Ident(ident.to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_int(input: &mut Peekable<Chars>, first: char) -> Token {
|
||||
let mut toks = vec![first];
|
||||
while let Some(tok) = input.peek() {
|
||||
if !tok.is_ascii_digit() {
|
||||
break;
|
||||
}
|
||||
|
||||
let tok = input.next().unwrap();
|
||||
toks.push(tok);
|
||||
}
|
||||
|
||||
let int = toks
|
||||
.iter()
|
||||
.cloned()
|
||||
.collect::<String>()
|
||||
.parse::<i64>()
|
||||
.unwrap();
|
||||
Token::Int(int)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rstest::rstest;
|
||||
|
||||
#[rstest]
|
||||
#[case(
|
||||
"=+(){},;",
|
||||
vec![
|
||||
Token::Assign,
|
||||
Token::Plus,
|
||||
Token::LeftParenthesis,
|
||||
Token::RightParenthesis,
|
||||
Token::LeftBrace,
|
||||
Token::RightBrace,
|
||||
Token::Comma,
|
||||
Token::Semicolon,
|
||||
])]
|
||||
#[case(
|
||||
"
|
||||
let five = 5;
|
||||
let ten = 10;
|
||||
|
||||
let add = fn(x, y) {
|
||||
x + y;
|
||||
}
|
||||
|
||||
let result = add(five, ten);
|
||||
",
|
||||
vec![
|
||||
Token::Let,
|
||||
Token::Ident("five".into()),
|
||||
Token::Assign,
|
||||
Token::Int(5),
|
||||
Token::Semicolon,
|
||||
Token::Let,
|
||||
Token::Ident("ten".into()),
|
||||
Token::Assign,
|
||||
Token::Int(10),
|
||||
Token::Semicolon,
|
||||
Token::Let,
|
||||
Token::Ident("add".into()),
|
||||
Token::Assign,
|
||||
Token::Function,
|
||||
Token::LeftParenthesis,
|
||||
Token::Ident("x".into()),
|
||||
Token::Comma,
|
||||
Token::Ident("y".into()),
|
||||
Token::RightParenthesis,
|
||||
Token::LeftBrace,
|
||||
Token::Ident("x".into()),
|
||||
Token::Plus,
|
||||
Token::Ident("y".into()),
|
||||
Token::Semicolon,
|
||||
Token::RightBrace,
|
||||
Token::Let,
|
||||
Token::Ident("result".into()),
|
||||
Token::Assign,
|
||||
Token::Ident("add".into()),
|
||||
Token::LeftParenthesis,
|
||||
Token::Ident("five".into()),
|
||||
Token::Comma,
|
||||
Token::Ident("ten".into()),
|
||||
Token::RightParenthesis,
|
||||
Token::Semicolon,
|
||||
])]
|
||||
fn test_next_token(#[case] input: &str, #[case] expected: Vec<Token>) {
|
||||
let res = tokenize(input);
|
||||
assert_eq!(res, expected);
|
||||
}
|
||||
}
|
25
src/lexer/tokens.rs
Normal file
25
src/lexer/tokens.rs
Normal file
|
@ -0,0 +1,25 @@
|
|||
#[derive(Debug, PartialEq, PartialOrd)]
|
||||
pub enum Token {
|
||||
Illegal,
|
||||
|
||||
// Ident + Literals
|
||||
Ident(String),
|
||||
Int(i64),
|
||||
|
||||
// Operators
|
||||
Assign,
|
||||
Plus,
|
||||
|
||||
// Delimiters
|
||||
Comma,
|
||||
Semicolon,
|
||||
|
||||
LeftParenthesis,
|
||||
RightParenthesis,
|
||||
LeftBrace,
|
||||
RightBrace,
|
||||
|
||||
// Keywords
|
||||
Function,
|
||||
Let,
|
||||
}
|
5
src/main.rs
Normal file
5
src/main.rs
Normal file
|
@ -0,0 +1,5 @@
|
|||
mod lexer;
|
||||
|
||||
fn main() {
|
||||
println!("{:?}", lexer::tokenize("asdf"));
|
||||
}
|
Loading…
Reference in a new issue