shithub: martian9

ref: 24cf0e8cd6799c80652dd2f7ecf41e0209dfe159
dir: /utils.ml/

View raw version 
exception Syntax_error of string
exception Runtime_error of string

let token_re = Str.regexp "~@\\|[][{}()'`~^@]\\|\"\\(\\\\.\\|[^\"]\\)*\"?\\|;.*\\$\\|[^][  \n{}('\"`,;)]*"

(* copied verbatim - must needs grok *)
let gsub re f str =
  String.concat "" (List.map (function Str.Delim x -> f x | Str.Text x -> x) (Str.full_split re str))

let tokenize str =
  List.map
    (function Str.Delim x -> String.trim x (* move trim to regex for speed? *) | Str.Text _ -> "tokenize botch")
    (List.filter (function Str.Delim _ -> true | Str.Text _ -> false) (Str.full_split token_re str))