From ce64ad22a11308c8fafa1bc8b40e3e21210cc3ee Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Sun, 22 Mar 2015 15:09:01 +0100 Subject: Initial import --- lexer.go | 184 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 lexer.go (limited to 'lexer.go') diff --git a/lexer.go b/lexer.go new file mode 100644 index 0000000..bdbca7b --- /dev/null +++ b/lexer.go @@ -0,0 +1,184 @@ +package main + +import ( + "log" + "unicode/utf8" +) + +type item struct { + typ int + val string +} + +type yyLex struct { + input string + start int + pos int + width int + items chan item +} + +func (y *yyLex) Error(s string) { + log.Println(s) +} + +/* +type yySymType struct{} +func (y *yyLex) Lex(lval *yySymType) (ret int) { + item := <-y.items + return item.typ +} +*/ + +func (y *yyLex) Lex() (int, string) { + item := <-y.items + return item.typ, item.val +} + +func lex(input string) *yyLex { + l := &yyLex{ + input: input, + items: make(chan item), + } + go l.run() + return l +} + +func (y *yyLex) run() { + defer close(y.items) + for { + c := y.next() + + if t, ok := Tockens[c]; ok { + switch t { + case COMMENT, EOF: + return + case BLANK: + y.ignore() + case QUOTE: + y.lexQuoted() + case DIGIT, NEG: + y.lexNumber() + case CHAR: + y.lexString() + default: + y.emit(t) + } + } else { + y.ignore() + } + } +} + +func (y *yyLex) lexQuoted() { + y.ignore() + defer func() { + y.emit(QUOTED) + y.next() + y.ignore() + }() + for { + switch Tockens[y.next()] { + case EOF: + return + case QUOTE: + if !y.acceptTocken(QUOTE) { + y.backup() + return + } + y.next() + } + } +} + +func (y *yyLex) lexString() { + defer y.emit(STRING) + for { + if Tockens[y.next()] != CHAR { + y.backup() + return + } + } +} + +func (y *yyLex) lexNumber() { + t := INTEGER + y.acceptTocken(NEG) + y.acceptDigits() + if y.acceptTocken(DOT) { + t = FLOAT + y.acceptDigits() + } + if y.acceptRune('e', 'E') { + t = FLOAT + y.acceptTocken(NEG) + y.acceptDigits() + } + if y.acceptRune('j', 'J') { + t = COMPLEX + y.acceptTocken(NEG) + y.acceptDigits() + if y.acceptTocken(DOT) { + y.acceptDigits() + } + if y.acceptRune('e', 'E') { + y.acceptTocken(NEG) + y.acceptDigits() + } + } + y.emit(t) +} + +func (y *yyLex) acceptDigits() { + for y.acceptTocken(DIGIT) { + } +} + +func (y *yyLex) emit(t int) { + y.items <- item{ + typ: t, + val: y.input[y.start:y.pos], + } + y.start = y.pos +} + +func (y *yyLex) next() (r rune) { + if y.pos >= len(y.input) { + y.width = 0 + return EOF + } + r, y.width = utf8.DecodeRuneInString(y.input[y.pos:]) + y.pos += y.width + return +} + +func (y *yyLex) ignore() { + y.start = y.pos +} + +func (y *yyLex) backup() { + y.pos -= y.width +} + +func (y *yyLex) peek() rune { + defer y.backup() + return y.next() +} + +func (y *yyLex) acceptRune(valid ...rune) bool { + for _, r := range valid { + if y.next() == r { + return true + } + y.backup() + } + return false +} + +func (y *yyLex) acceptTocken(valid int) bool { + if t, ok := Tockens[y.next()]; ok && t == valid { + return true + } + y.backup() + return false +} -- cgit v1.2.3