From 523f40968b9c1a23da1f4a1c2f125197d7611fef Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Sat, 20 May 2017 16:28:10 +0200 Subject: 04 --- parser/parser.go | 78 +++++++++++++-- parser/parser_test.go | 266 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 335 insertions(+), 9 deletions(-) (limited to 'parser') diff --git a/parser/parser.go b/parser/parser.go index 0c4a9b7..e02595c 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -17,6 +17,7 @@ const ( PRODUCT // * PREFIX // -X or !X CALL // myFunction(X) + INDEX // array[index] ) var precedences = map[token.TokenType]int{ @@ -29,6 +30,7 @@ var precedences = map[token.TokenType]int{ token.SLASH: PRODUCT, token.ASTERISK: PRODUCT, token.LPAREN: CALL, + token.LBRACKET: INDEX, } type ( @@ -56,6 +58,7 @@ func New(l *lexer.Lexer) *Parser { p.prefixParseFns = make(map[token.TokenType]prefixParseFn) p.registerPrefix(token.IDENT, p.parseIdentifier) p.registerPrefix(token.INT, p.parseIntegerLiteral) + p.registerPrefix(token.STRING, p.parseStringLiteral) p.registerPrefix(token.BANG, p.parsePrefixExpression) p.registerPrefix(token.MINUS, p.parsePrefixExpression) p.registerPrefix(token.TRUE, p.parseBoolean) @@ -63,6 +66,8 @@ func New(l *lexer.Lexer) *Parser { p.registerPrefix(token.LPAREN, p.parseGroupedExpression) p.registerPrefix(token.IF, p.parseIfExpression) p.registerPrefix(token.FUNCTION, p.parseFunctionLiteral) + p.registerPrefix(token.LBRACKET, p.parseArrayLiteral) + p.registerPrefix(token.LBRACE, p.parseHashLiteral) p.infixParseFns = make(map[token.TokenType]infixParseFn) p.registerInfix(token.PLUS, p.parseInfixExpression) @@ -75,6 +80,7 @@ func New(l *lexer.Lexer) *Parser { p.registerInfix(token.GT, p.parseInfixExpression) p.registerInfix(token.LPAREN, p.parseCallExpression) + p.registerInfix(token.LBRACKET, p.parseIndexExpression) // Read two tokens, so curToken and peekToken are both set p.nextToken() @@ -254,6 +260,10 @@ func (p *Parser) parseIntegerLiteral() ast.Expression { return lit } +func (p *Parser) parseStringLiteral() ast.Expression { + return &ast.StringLiteral{Token: p.curToken, Value: p.curToken.Literal} +} + func (p *Parser) parsePrefixExpression() ast.Expression { expression := &ast.PrefixExpression{ Token: p.curToken, @@ -394,32 +404,82 @@ func (p *Parser) parseFunctionParameters() []*ast.Identifier { func (p *Parser) parseCallExpression(function ast.Expression) ast.Expression { exp := &ast.CallExpression{Token: p.curToken, Function: function} - exp.Arguments = p.parseCallArguments() + exp.Arguments = p.parseExpressionList(token.RPAREN) return exp } -func (p *Parser) parseCallArguments() []ast.Expression { - args := []ast.Expression{} +func (p *Parser) parseExpressionList(end token.TokenType) []ast.Expression { + list := []ast.Expression{} - if p.peekTokenIs(token.RPAREN) { + if p.peekTokenIs(end) { p.nextToken() - return args + return list } p.nextToken() - args = append(args, p.parseExpression(LOWEST)) + list = append(list, p.parseExpression(LOWEST)) for p.peekTokenIs(token.COMMA) { p.nextToken() p.nextToken() - args = append(args, p.parseExpression(LOWEST)) + list = append(list, p.parseExpression(LOWEST)) } - if !p.expectPeek(token.RPAREN) { + if !p.expectPeek(end) { + return nil + } + + return list +} + +func (p *Parser) parseArrayLiteral() ast.Expression { + array := &ast.ArrayLiteral{Token: p.curToken} + + array.Elements = p.parseExpressionList(token.RBRACKET) + + return array +} + +func (p *Parser) parseIndexExpression(left ast.Expression) ast.Expression { + exp := &ast.IndexExpression{Token: p.curToken, Left: left} + + p.nextToken() + exp.Index = p.parseExpression(LOWEST) + + if !p.expectPeek(token.RBRACKET) { + return nil + } + + return exp +} + +func (p *Parser) parseHashLiteral() ast.Expression { + hash := &ast.HashLiteral{Token: p.curToken} + hash.Pairs = make(map[ast.Expression]ast.Expression) + + for !p.peekTokenIs(token.RBRACE) { + p.nextToken() + key := p.parseExpression(LOWEST) + + if !p.expectPeek(token.COLON) { + return nil + } + + p.nextToken() + value := p.parseExpression(LOWEST) + + hash.Pairs[key] = value + + if !p.peekTokenIs(token.RBRACE) && !p.expectPeek(token.COMMA) { + return nil + } + } + + if !p.expectPeek(token.RBRACE) { return nil } - return args + return hash } func (p *Parser) registerPrefix(tokenType token.TokenType, fn prefixParseFn) { diff --git a/parser/parser_test.go b/parser/parser_test.go index fa080aa..e187c9f 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -345,6 +345,14 @@ func TestOperatorPrecedenceParsing(t *testing.T) { "add(a + b + c * d / f + g)", "add((((a + b) + ((c * d) / f)) + g))", }, + { + "a * [1, 2, 3, 4][b * c] * d", + "((a * ([1, 2, 3, 4][(b * c)])) * d)", + }, + { + "add(a * b[2], b[1], 2 * [1, 2][1])", + "add((a * (b[2])), (b[1]), (2 * ([1, 2][1])))", + }, } for _, tt := range tests { @@ -674,6 +682,264 @@ func TestCallExpressionParameterParsing(t *testing.T) { } } +func TestStringLiteralExpression(t *testing.T) { + input := `"hello world";` + + l := lexer.New(input) + p := New(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + stmt := program.Statements[0].(*ast.ExpressionStatement) + literal, ok := stmt.Expression.(*ast.StringLiteral) + if !ok { + t.Fatalf("exp not *ast.StringLiteral. got=%T", stmt.Expression) + } + + if literal.Value != "hello world" { + t.Errorf("literal.Value not %q. got=%q", "hello world", literal.Value) + } +} + +func TestParsingEmptyArrayLiterals(t *testing.T) { + input := "[]" + + l := lexer.New(input) + p := New(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + stmt, ok := program.Statements[0].(*ast.ExpressionStatement) + array, ok := stmt.Expression.(*ast.ArrayLiteral) + if !ok { + t.Fatalf("exp not ast.ArrayLiteral. got=%T", stmt.Expression) + } + + if len(array.Elements) != 0 { + t.Errorf("len(array.Elements) not 0. got=%d", len(array.Elements)) + } +} + +func TestParsingArrayLiterals(t *testing.T) { + input := "[1, 2 * 2, 3 + 3]" + + l := lexer.New(input) + p := New(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + stmt, ok := program.Statements[0].(*ast.ExpressionStatement) + array, ok := stmt.Expression.(*ast.ArrayLiteral) + if !ok { + t.Fatalf("exp not ast.ArrayLiteral. got=%T", stmt.Expression) + } + + if len(array.Elements) != 3 { + t.Fatalf("len(array.Elements) not 3. got=%d", len(array.Elements)) + } + + testIntegerLiteral(t, array.Elements[0], 1) + testInfixExpression(t, array.Elements[1], 2, "*", 2) + testInfixExpression(t, array.Elements[2], 3, "+", 3) +} + +func TestParsingIndexExpressions(t *testing.T) { + input := "myArray[1 + 1]" + + l := lexer.New(input) + p := New(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + stmt, ok := program.Statements[0].(*ast.ExpressionStatement) + indexExp, ok := stmt.Expression.(*ast.IndexExpression) + if !ok { + t.Fatalf("exp not *ast.IndexExpression. got=%T", stmt.Expression) + } + + if !testIdentifier(t, indexExp.Left, "myArray") { + return + } + + if !testInfixExpression(t, indexExp.Index, 1, "+", 1) { + return + } +} + +func TestParsingEmptyHashLiteral(t *testing.T) { + input := "{}" + + l := lexer.New(input) + p := New(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + stmt := program.Statements[0].(*ast.ExpressionStatement) + hash, ok := stmt.Expression.(*ast.HashLiteral) + if !ok { + t.Fatalf("exp is not ast.HashLiteral. got=%T", stmt.Expression) + } + + if len(hash.Pairs) != 0 { + t.Errorf("hash.Pairs has wrong length. got=%d", len(hash.Pairs)) + } +} + +func TestParsingHashLiteralsStringKeys(t *testing.T) { + input := `{"one": 1, "two": 2, "three": 3}` + + l := lexer.New(input) + p := New(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + stmt := program.Statements[0].(*ast.ExpressionStatement) + hash, ok := stmt.Expression.(*ast.HashLiteral) + if !ok { + t.Fatalf("exp is not ast.HashLiteral. got=%T", stmt.Expression) + } + + expected := map[string]int64{ + "one": 1, + "two": 2, + "three": 3, + } + + if len(hash.Pairs) != len(expected) { + t.Errorf("hash.Pairs has wrong length. got=%d", len(hash.Pairs)) + } + + for key, value := range hash.Pairs { + literal, ok := key.(*ast.StringLiteral) + if !ok { + t.Errorf("key is not ast.StringLiteral. got=%T", key) + continue + } + + expectedValue := expected[literal.String()] + testIntegerLiteral(t, value, expectedValue) + } +} + +func TestParsingHashLiteralsBooleanKeys(t *testing.T) { + input := `{true: 1, false: 2}` + + l := lexer.New(input) + p := New(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + stmt := program.Statements[0].(*ast.ExpressionStatement) + hash, ok := stmt.Expression.(*ast.HashLiteral) + if !ok { + t.Fatalf("exp is not ast.HashLiteral. got=%T", stmt.Expression) + } + + expected := map[string]int64{ + "true": 1, + "false": 2, + } + + if len(hash.Pairs) != len(expected) { + t.Errorf("hash.Pairs has wrong length. got=%d", len(hash.Pairs)) + } + + for key, value := range hash.Pairs { + boolean, ok := key.(*ast.Boolean) + if !ok { + t.Errorf("key is not ast.BooleanLiteral. got=%T", key) + continue + } + + expectedValue := expected[boolean.String()] + testIntegerLiteral(t, value, expectedValue) + } +} + +func TestParsingHashLiteralsIntegerKeys(t *testing.T) { + input := `{1: 1, 2: 2, 3: 3}` + + l := lexer.New(input) + p := New(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + stmt := program.Statements[0].(*ast.ExpressionStatement) + hash, ok := stmt.Expression.(*ast.HashLiteral) + if !ok { + t.Fatalf("exp is not ast.HashLiteral. got=%T", stmt.Expression) + } + + expected := map[string]int64{ + "1": 1, + "2": 2, + "3": 3, + } + + if len(hash.Pairs) != len(expected) { + t.Errorf("hash.Pairs has wrong length. got=%d", len(hash.Pairs)) + } + + for key, value := range hash.Pairs { + integer, ok := key.(*ast.IntegerLiteral) + if !ok { + t.Errorf("key is not ast.IntegerLiteral. got=%T", key) + continue + } + + expectedValue := expected[integer.String()] + + testIntegerLiteral(t, value, expectedValue) + } +} + +func TestParsingHashLiteralsWithExpressions(t *testing.T) { + input := `{"one": 0 + 1, "two": 10 - 8, "three": 15 / 5}` + + l := lexer.New(input) + p := New(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + stmt := program.Statements[0].(*ast.ExpressionStatement) + hash, ok := stmt.Expression.(*ast.HashLiteral) + if !ok { + t.Fatalf("exp is not ast.HashLiteral. got=%T", stmt.Expression) + } + + if len(hash.Pairs) != 3 { + t.Errorf("hash.Pairs has wrong length. got=%d", len(hash.Pairs)) + } + + tests := map[string]func(ast.Expression){ + "one": func(e ast.Expression) { + testInfixExpression(t, e, 0, "+", 1) + }, + "two": func(e ast.Expression) { + testInfixExpression(t, e, 10, "-", 8) + }, + "three": func(e ast.Expression) { + testInfixExpression(t, e, 15, "/", 5) + }, + } + + for key, value := range hash.Pairs { + literal, ok := key.(*ast.StringLiteral) + if !ok { + t.Errorf("key is not ast.StringLiteral. got=%T", key) + continue + } + + testFunc, ok := tests[literal.String()] + if !ok { + t.Errorf("No test function for key %q found", literal.String()) + continue + } + + testFunc(value) + } +} + func testLetStatement(t *testing.T, s ast.Statement, name string) bool { if s.TokenLiteral() != "let" { t.Errorf("s.TokenLiteral not 'let'. got=%q", s.TokenLiteral()) -- cgit v1.2.3