shithub: martian9

ref: 4f535f4a39ed10d5aef7a2f568c02fa3b40775ff
dir: /utils.ml/

exception Syntax_error of string
exception Runtime_error of string

let token_re = Str.regexp "~@\\|[][{}()'`~^@]\\|\"\\(\\\\.\\|[^\"]\\)*\"?\\|;.*\\$\\|[^][  \n{}('\"`,;)]*"

(* copied verbatim - must needs grok *)
let gsub re f str =
  String.concat ""
    (List.map
       (function
         | Str.Delim x -> f x
         | Str.Text x -> x )
       (Str.full_split re str) )

let tokenize str =
  List.map
    (function
      | Str.Delim x -> String.trim x (* move trim to regex for speed? *)
      | Str.Text _ -> "tokenize botch" )
    (List.filter
       (function
         | Str.Delim _ -> true
         | Str.Text _ -> false )
       (Str.full_split token_re str) )