# I got this code on OpenAI and optimised the memory later.
# but I did not found the source of this regexp.
# Very useful however, but I don't know the license for this...
module TokenParser
  # limits: do not support quoted token if not starting with "
  def self.parse_tokens(string)
    # Use a regular expression to match tokens in the input string
    tokens = string.scan(/"((?:[^"\\]|\\.)*)"|([^\s]+)/)
    tokens.flatten!
    tokens.compact!

    # Unescape any tokens that contain escape characters
    tokens.each { |token| token.gsub!(/\\(.)/, '\1') }

    tokens
  end
end