\ 4tH library - TOKENIZE - Copyright 2004,2009 J.L. Bezemer
\ You can redistribute this file and/or modify it under
\ the terms of the GNU General Public License

\ Load definitions when needed
[UNDEFINED] ?NOT [IF]
[NEEDS lib/yesno.4th]
[THEN]

[UNDEFINED] IS-TYPE [IF]
DEFER IS-TYPE                          ( c -- f)

: (TOKENIZE)                           ( a1 n2 xt -- a2 n2)
  IS ?NOT BEGIN DUP IF OVER C@ IS-TYPE ?NOT ELSE DUP THEN
  WHILE CHOP REPEAT
;

: (-TOKENIZE)                          ( a1 n2 xt -- a2 n2 )
  IS ?NOT BEGIN DUP IF 2DUP 1- CHARS + C@ IS-TYPE ?NOT ELSE DUP THEN
  WHILE 1- REPEAT
;

: SCAN> ['] (NO)  (TOKENIZE) ;         ( a1 n1 -- a2 n2 )
: SCAN< ['] (NO)  (-TOKENIZE) ;        ( a1 n1 -- a2 n2 )
: SKIP> ['] (YES) (TOKENIZE) ;         ( a1 n1 -- a2 n2 )
: SKIP< ['] (YES) (-TOKENIZE) ;        ( a1 n1 -- a2 n2 )
: SPLIT> 2DUP SCAN> ROT >R ROT >R DUP 2R> ROT - ;
: SPLIT< TUCK SCAN< TUCK 2DUP 2>R CHARS + -ROT - 2R> ;
                                       ( a1 n1 -- a2 n2 a3 n3)

[DEFINED] 4TH# [IF]
hide (TOKENIZE)
hide (-TOKENIZE)
[THEN]
[THEN]
