aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDimitri Sokolyuk <demon@dim13.org>2017-05-20 16:26:54 +0200
committerDimitri Sokolyuk <demon@dim13.org>2017-05-20 16:26:54 +0200
commit562c190b394b0ad76d4d591be74e2f37513c964a (patch)
treed66f5e308e1130c2dbb0b57351c84526725d8839
01
-rw-r--r--lexer/lexer.go135
-rw-r--r--lexer/lexer_test.go126
-rw-r--r--main.go19
-rw-r--r--repl/repl.go30
-rw-r--r--token/token.go66
5 files changed, 376 insertions, 0 deletions
diff --git a/lexer/lexer.go b/lexer/lexer.go
new file mode 100644
index 0000000..cffd925
--- /dev/null
+++ b/lexer/lexer.go
@@ -0,0 +1,135 @@
+package lexer
+
+import "monkey/token"
+
+type Lexer struct {
+ input string
+ position int // current position in input (points to current char)
+ readPosition int // current reading position in input (after current char)
+ ch byte // current char under examination
+}
+
+func New(input string) *Lexer {
+ l := &Lexer{input: input}
+ l.readChar()
+ return l
+}
+
+func (l *Lexer) NextToken() token.Token {
+ var tok token.Token
+
+ l.skipWhitespace()
+
+ switch l.ch {
+ case '=':
+ if l.peekChar() == '=' {
+ ch := l.ch
+ l.readChar()
+ tok = token.Token{Type: token.EQ, Literal: string(ch) + string(l.ch)}
+ } else {
+ tok = newToken(token.ASSIGN, l.ch)
+ }
+ case '+':
+ tok = newToken(token.PLUS, l.ch)
+ case '-':
+ tok = newToken(token.MINUS, l.ch)
+ case '!':
+ if l.peekChar() == '=' {
+ ch := l.ch
+ l.readChar()
+ tok = token.Token{Type: token.NOT_EQ, Literal: string(ch) + string(l.ch)}
+ } else {
+ tok = newToken(token.BANG, l.ch)
+ }
+ case '/':
+ tok = newToken(token.SLASH, l.ch)
+ case '*':
+ tok = newToken(token.ASTERISK, l.ch)
+ case '<':
+ tok = newToken(token.LT, l.ch)
+ case '>':
+ tok = newToken(token.GT, l.ch)
+ case ';':
+ tok = newToken(token.SEMICOLON, l.ch)
+ case ',':
+ tok = newToken(token.COMMA, l.ch)
+ case '{':
+ tok = newToken(token.LBRACE, l.ch)
+ case '}':
+ tok = newToken(token.RBRACE, l.ch)
+ case '(':
+ tok = newToken(token.LPAREN, l.ch)
+ case ')':
+ tok = newToken(token.RPAREN, l.ch)
+ case 0:
+ tok.Literal = ""
+ tok.Type = token.EOF
+ default:
+ if isLetter(l.ch) {
+ tok.Literal = l.readIdentifier()
+ tok.Type = token.LookupIdent(tok.Literal)
+ return tok
+ } else if isDigit(l.ch) {
+ tok.Type = token.INT
+ tok.Literal = l.readNumber()
+ return tok
+ } else {
+ tok = newToken(token.ILLEGAL, l.ch)
+ }
+ }
+
+ l.readChar()
+ return tok
+}
+
+func (l *Lexer) skipWhitespace() {
+ for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' {
+ l.readChar()
+ }
+}
+
+func (l *Lexer) readChar() {
+ if l.readPosition >= len(l.input) {
+ l.ch = 0
+ } else {
+ l.ch = l.input[l.readPosition]
+ }
+ l.position = l.readPosition
+ l.readPosition += 1
+}
+
+func (l *Lexer) peekChar() byte {
+ if l.readPosition >= len(l.input) {
+ return 0
+ } else {
+ return l.input[l.readPosition]
+ }
+}
+
+func (l *Lexer) readIdentifier() string {
+ position := l.position
+ for isLetter(l.ch) {
+ l.readChar()
+ }
+ return l.input[position:l.position]
+}
+
+func (l *Lexer) readNumber() string {
+ position := l.position
+ for isDigit(l.ch) {
+ l.readChar()
+ }
+ return l.input[position:l.position]
+}
+
+func isLetter(ch byte) bool {
+ return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_'
+}
+
+func isDigit(ch byte) bool {
+ return '0' <= ch && ch <= '9'
+}
+
+func newToken(tokenType token.TokenType, ch byte) token.Token {
+ return token.Token{Type: tokenType, Literal: string(ch)}
+}
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
new file mode 100644
index 0000000..0a7f248
--- /dev/null
+++ b/lexer/lexer_test.go
@@ -0,0 +1,126 @@
+package lexer
+
+import (
+ "testing"
+
+ "monkey/token"
+)
+
+func TestNextToken(t *testing.T) {
+ input := `let five = 5;
+let ten = 10;
+
+let add = fn(x, y) {
+ x + y;
+};
+
+let result = add(five, ten);
+!-/*5;
+5 < 10 > 5;
+
+if (5 < 10) {
+ return true;
+} else {
+ return false;
+}
+
+10 == 10;
+10 != 9;
+`
+
+ tests := []struct {
+ expectedType token.TokenType
+ expectedLiteral string
+ }{
+ {token.LET, "let"},
+ {token.IDENT, "five"},
+ {token.ASSIGN, "="},
+ {token.INT, "5"},
+ {token.SEMICOLON, ";"},
+ {token.LET, "let"},
+ {token.IDENT, "ten"},
+ {token.ASSIGN, "="},
+ {token.INT, "10"},
+ {token.SEMICOLON, ";"},
+ {token.LET, "let"},
+ {token.IDENT, "add"},
+ {token.ASSIGN, "="},
+ {token.FUNCTION, "fn"},
+ {token.LPAREN, "("},
+ {token.IDENT, "x"},
+ {token.COMMA, ","},
+ {token.IDENT, "y"},
+ {token.RPAREN, ")"},
+ {token.LBRACE, "{"},
+ {token.IDENT, "x"},
+ {token.PLUS, "+"},
+ {token.IDENT, "y"},
+ {token.SEMICOLON, ";"},
+ {token.RBRACE, "}"},
+ {token.SEMICOLON, ";"},
+ {token.LET, "let"},
+ {token.IDENT, "result"},
+ {token.ASSIGN, "="},
+ {token.IDENT, "add"},
+ {token.LPAREN, "("},
+ {token.IDENT, "five"},
+ {token.COMMA, ","},
+ {token.IDENT, "ten"},
+ {token.RPAREN, ")"},
+ {token.SEMICOLON, ";"},
+ {token.BANG, "!"},
+ {token.MINUS, "-"},
+ {token.SLASH, "/"},
+ {token.ASTERISK, "*"},
+ {token.INT, "5"},
+ {token.SEMICOLON, ";"},
+ {token.INT, "5"},
+ {token.LT, "<"},
+ {token.INT, "10"},
+ {token.GT, ">"},
+ {token.INT, "5"},
+ {token.SEMICOLON, ";"},
+ {token.IF, "if"},
+ {token.LPAREN, "("},
+ {token.INT, "5"},
+ {token.LT, "<"},
+ {token.INT, "10"},
+ {token.RPAREN, ")"},
+ {token.LBRACE, "{"},
+ {token.RETURN, "return"},
+ {token.TRUE, "true"},
+ {token.SEMICOLON, ";"},
+ {token.RBRACE, "}"},
+ {token.ELSE, "else"},
+ {token.LBRACE, "{"},
+ {token.RETURN, "return"},
+ {token.FALSE, "false"},
+ {token.SEMICOLON, ";"},
+ {token.RBRACE, "}"},
+ {token.INT, "10"},
+ {token.EQ, "=="},
+ {token.INT, "10"},
+ {token.SEMICOLON, ";"},
+ {token.INT, "10"},
+ {token.NOT_EQ, "!="},
+ {token.INT, "9"},
+ {token.SEMICOLON, ";"},
+ {token.EOF, ""},
+ }
+
+ l := New(input)
+
+ for i, tt := range tests {
+ tok := l.NextToken()
+
+ if tok.Type != tt.expectedType {
+ t.Fatalf("tests[%d] - tokentype wrong. expected=%q, got=%q",
+ i, tt.expectedType, tok.Type)
+ }
+
+ if tok.Literal != tt.expectedLiteral {
+ t.Fatalf("tests[%d] - literal wrong. expected=%q, got=%q",
+ i, tt.expectedLiteral, tok.Literal)
+ }
+ }
+}
diff --git a/main.go b/main.go
new file mode 100644
index 0000000..1d27138
--- /dev/null
+++ b/main.go
@@ -0,0 +1,19 @@
+package main
+
+import (
+ "fmt"
+ "monkey/repl"
+ "os"
+ "os/user"
+)
+
+func main() {
+ user, err := user.Current()
+ if err != nil {
+ panic(err)
+ }
+ fmt.Printf("Hello %s! This is the Monkey programming language!\n",
+ user.Username)
+ fmt.Printf("Feel free to type in commands\n")
+ repl.Start(os.Stdin, os.Stdout)
+}
diff --git a/repl/repl.go b/repl/repl.go
new file mode 100644
index 0000000..337be46
--- /dev/null
+++ b/repl/repl.go
@@ -0,0 +1,30 @@
+package repl
+
+import (
+ "bufio"
+ "fmt"
+ "io"
+ "monkey/lexer"
+ "monkey/token"
+)
+
+const PROMPT = ">> "
+
+func Start(in io.Reader, out io.Writer) {
+ scanner := bufio.NewScanner(in)
+
+ for {
+ fmt.Printf(PROMPT)
+ scanned := scanner.Scan()
+ if !scanned {
+ return
+ }
+
+ line := scanner.Text()
+ l := lexer.New(line)
+
+ for tok := l.NextToken(); tok.Type != token.EOF; tok = l.NextToken() {
+ fmt.Printf("%+v\n", tok)
+ }
+ }
+}
diff --git a/token/token.go b/token/token.go
new file mode 100644
index 0000000..12158fa
--- /dev/null
+++ b/token/token.go
@@ -0,0 +1,66 @@
+package token
+
+type TokenType string
+
+const (
+ ILLEGAL = "ILLEGAL"
+ EOF = "EOF"
+
+ // Identifiers + literals
+ IDENT = "IDENT" // add, foobar, x, y, ...
+ INT = "INT" // 1343456
+
+ // Operators
+ ASSIGN = "="
+ PLUS = "+"
+ MINUS = "-"
+ BANG = "!"
+ ASTERISK = "*"
+ SLASH = "/"
+
+ LT = "<"
+ GT = ">"
+
+ EQ = "=="
+ NOT_EQ = "!="
+
+ // Delimiters
+ COMMA = ","
+ SEMICOLON = ";"
+
+ LPAREN = "("
+ RPAREN = ")"
+ LBRACE = "{"
+ RBRACE = "}"
+
+ // Keywords
+ FUNCTION = "FUNCTION"
+ LET = "LET"
+ TRUE = "TRUE"
+ FALSE = "FALSE"
+ IF = "IF"
+ ELSE = "ELSE"
+ RETURN = "RETURN"
+)
+
+type Token struct {
+ Type TokenType
+ Literal string
+}
+
+var keywords = map[string]TokenType{
+ "fn": FUNCTION,
+ "let": LET,
+ "true": TRUE,
+ "false": FALSE,
+ "if": IF,
+ "else": ELSE,
+ "return": RETURN,
+}
+
+func LookupIdent(ident string) TokenType {
+ if tok, ok := keywords[ident]; ok {
+ return tok
+ }
+ return IDENT
+}