# Copyright 2005-2007 Kevin Reid, under the terms of the MIT X license
# found at http://www.opensource.org/licenses/mit-license.html ................

  ? def <qu> := <import:org.quasiliteral.*>; null

  ? def makeTermLexer := <qu:term.makeTermLexer>; null

  ? def lex(twine) :any { return EIO.takeAll(makeTermLexer(false)(twine.asStream())) }; null

  ? lex("foo")
  # value: [["Tag", [["Ident", "foo"]]]]
  
  ? lex("(,)[{:]}+*?")
  # value: [["("], [","], [")"], ["["], ["{"], [":"], ["]"], ["}"], ["+"], ["*"], ["?"]]
  
  ? lex("foo(bar)")
  # value: [["Tag", [["Ident", "foo"]]], ["("], ["Tag", [["Ident", "bar"]]], [")"]]

  ? lex(".foo. .bar.baz")
  # value: [["Tag", [["Ident", ".foo."]]], ["Tag", [["Ident", ".bar.baz"]]]]
  
  ? lex("${9} @{10} ")
  # value: [["Quasi$", 9], ["Quasi@", 10]]
  
  ? lex("0 1 2 107 -1 3 000")
  # value: [["Integer", 0], ["Integer", 1], ["Integer", 2], ["Integer", 107], ["Integer", -1], ["Integer", 3], ["Integer", 0]]
  
  ? lex("+3")
  # value: [["+"], ["Integer", 3]]

  ? lex("\"foo\"")
  # value: [["String", "foo"]]

  ? lex("\"foo\\t\"")
  # value: [["String", "foo\t"]]

  ? lex("\"foo\\\"\"")
  # value: [["String", "foo\""]]
XXX EoJ term lexer returns terms
XXX token types: float, char, more strings, tag, punct, chars
XXX consequence of loose ::, or malformed tags in general
XXX consequence of non-token characters