18 lines
586 B
Ruby
18 lines
586 B
Ruby
# I got this code on OpenAI and optimised the memory later.
|
|
# but I did not found the source of this regexp.
|
|
# Very useful however, but I don't know the license for this...
|
|
module TokenParser
|
|
# limits: do not support quoted token if not starting with "
|
|
def self.parse_tokens(string)
|
|
# Use a regular expression to match tokens in the input string
|
|
tokens = string.scan(/"((?:[^"\\]|\\.)*)"|([^\s]+)/)
|
|
tokens.flatten!
|
|
tokens.compact!
|
|
|
|
# Unescape any tokens that contain escape characters
|
|
tokens.each { |token| token.gsub!(/\\(.)/, '\1') }
|
|
|
|
tokens
|
|
end
|
|
end
|