lexer.rkt

#lang racket

(require parser-tools/lex
         (prefix-in : parser-tools/lex-sre))

(define-tokens value-tokens
  (CHAR VAR STRING))

(define-empty-tokens op-tokens
  (EOF OR
       LPAREN
       RPAREN
       STAR
       NOT
       SEMI
       DASH
       TIL
       UP
       EPSILON
       ARROW
       START
       GRM
       ANY
       PLUS
       OPTION
       AND
       LBRACK
       RBRACK
       COMMA
       AT))

(define next-token
  (lexer-src-pos
   [(eof) (token-EOF)]
   [(:+ whitespace #\newline) (return-without-pos (next-token input-port))]
   ["." (token-ANY)]
   ["," (token-COMMA)]
   ["[" (token-LBRACK)]
   ["]" (token-RBRACK)]
   ["/" (token-OR)]
   ["+" (token-PLUS)]
   ["?" (token-OPTION)]
   ["*" (token-STAR)]
   ["&" (token-AND)]
   ["<--" (token-ARROW)]
   ["!" (token-NOT)]
   [";" (token-SEMI)]
   ["-" (token-DASH)]
   ["~" (token-TIL)]
   ["^" (token-UP)]
   ["@" (token-AT)]
   ["epsilon" (token-EPSILON)]
   ["start:" (token-START)]
   ["grammar:" (token-GRM)]
   [#\( (token-LPAREN)]
   [#\) (token-RPAREN)]
   [(:seq #\" (complement (:seq any-string #\" any-string)) #\")
    (token-STRING (let* ([s lexeme]
                         [n (string-length s)])
                    (substring s 1 (- n 1))))]
   [(:seq alphabetic (:* (:+ alphabetic numeric)))
    (token-VAR lexeme)]
   [(:seq #\' any-char #\') (token-CHAR (let* ([s lexeme]
                                               [n (string-length s)])
                                          (substring s 1 (- n 1))))]
   [(:seq #\' #\\ #\n #\')  (token-CHAR "\n")]
   [(:seq #\' #\\ #\t #\')  (token-CHAR "\t")]
   [(:seq #\' #\\ #\r #\')  (token-CHAR "\r")]
   [(:seq #\' #\\ #\b #\')  (token-CHAR "\b")]
   [(:seq #\' #\\ #\" #\')  (token-CHAR "\"")]
   [(:seq #\' #\\ #\' #\')  (token-CHAR "'")]

   [(:seq #\' #\\ (:+ numeric) #\')  (token-CHAR  (let* ([s lexeme]
                                                         [n (string-length s)])
                                                         (string (integer->char
                                                                  (string->number (substring s 2 (- n 1)))))))]
   [(:seq #\' #\\ any-char #\')  (token-CHAR (let* ([s lexeme]
                                                    [n (string-length s)])
                                                    (substring s 2 (- n 1))))] 
   [(:seq (:seq #\% #\%) (:* (:~ #\012 #\015)))
    (return-without-pos (next-token input-port))]
))

(provide value-tokens op-tokens next-token)