page    55,132
title   TK - Token Parsing filter
;
;       TK --- A Simple Token Parsing Filter for DOS 2.0
;
;       (c) Copyright 1984 by   Jim Mott
;                               3710 Slopeview Drive
;                               Sunnyvale, CA 95148
;                               (408) 274-2620
;       All rights reserved. Permission granted to use this software for
;       personal, noncommercial purposes only.
;
;
;       This program is designed to be a filter for DOS 2.0.
;       It will tokenize its input and allow subsetting and/or
;       single token per line output.
;
;       The format of the command is:
;
;       TK {/RJx | /LJx} {/0} {{/v} | {/v/v}}
;         where /RJx means right justify all tokens to x positions
;               /LJx means left justify all tokens to x positions
;                    In the two entries x must be in [1..15]
;               /0   means output one token per line
;               /v   means select token v for output. You may select any
;                    number, up to 255, of tokens to output. Repeats are
;                    allowed and you may change the order of the input tokens
;                    on the output line.
;
;       For example, to extract the list of users from a VM directory file and
;       write a sorted list of them without passwords to the printer the
;       following command line would be used.
;
;       FIND "USER " < DIRECT.VM | TK/LJ8/2/4/5/6/7/8/9 | SORT > PRN
;
;
;       For example, to find a list of all sub-directories of the current
;       directory sorted by sub-directory name we would use the following
;       command line:
;
;       DIR | FIND "<DIR>" | TK/LJ8/1/3/4 | SORT | MORE
;
;
;       For example, to generate a sorted list of all words used in a document
;       with one word per line we could use the following command line:
;
;       TK/RJ8/0 < FOOBAR.DOC | SORT | MORE
;
;
;
;
;
;
;
;
;
;
;
;
stak   segment para    stack   'STACK'
       db  8 dup('Jim Mott  (408) 274-2620')
stak   ends
;
;
dsect   segment para    'DATA'
buffer  db      255 dup('?')            ; where to put the data
        db      ' '                     ; be sure to end scan correctly
;
glen    dw      0                       ; length of gbuff
gbptr   dw      gbuff                   ; point to start of buffer
gbuff   db      255 dup('G')            ; buffer used by bufget
;
flag1   db      ?                       ; options enabled
f1rj    equ     01h                     ; right justify tokens
f1lj    equ     02h                     ; left justify tokens
f1one   equ     04h                     ; output one token per line
f1sub   equ     08h                     ; substring function requested
f1work  equ     10h                     ; fill trailing spaces
f1oerr  equ     20h                     ; error in options string
f1eof   equ     40h                     ; end of file on standard input device
f1qeof  equ     80h                     ; queue the end of file
;
spaces  db      ?                       ; number of trailing spaces required
;
toksiz  db      ?                       ; token size if (f1rj or f1lj)
;
three   db      3                       ; length of each entry
tokcnt  db      ?                       ; count of tokens in table
toktbl  db      3*255 dup('0')          ; table of token pointers and lengths
;
outcnt  db      ?                       ; number of subsetting entries in
                                        ; outers
outers  db      255 dup('1')            ; list of token numbers to output
;
tokptr  dw      ?                       ; pointer to free token space
tokens  db      900 dup('2')            ; string space of tokens
;
msgver  db      'TK: Incorrect DOS version. Must be at least 2.00.'
        db      0dh,0ah,'$'
optmsg  db      'TK: Incorrect parameters given.'
        db      0dh,0ah
optmgl  equ     $ - optmsg
noroom  db      'TK: No room for user on device.'
        db      0dh,0ah
lnoroom equ     $ - noroom
chrspa  db      ' '                     ; a space to output
chrclf  db      0dh,0ah                 ; <cr><lf> sequence
dsect   ends
;
;
csect   segment para    'CODE'
    assume cs:csect,ds:dsect,ss:stak
;
main    proc    far
;
        push    ds                      ; set up a return address
        sub     ax,ax                   ; we want to return to DS:0000
        push    ax
        mov     ax,dsect                ; point to start of data area
        mov     ds,ax                   ; make assume and reality agree
        mov     ah,30h                  ; get DOS version number
        int     21h                     ; call OS to get it
        cmp     al,2                    ; is it at least 2.00?
        jnl     main00                  ; yesy - good enough
        lea     dx,msgver               ; no - point to the "Bad DOS version"
        mov     ah,9                    ; message and use DOS 1.?? function
        int     21h                     ; call to print it.
        ret                             ; and do a long return
;
main00: call    options                 ; parse the options (at ES:80) and set
                                        ; flags
        mov     ax,ds                   ; make ES and DS the same now
        mov     es,ax                   ; so the string moves work nicely.
        test    flag1,f1oerr            ; was there and error in the options
        jz      main01                  ; no - then go with this baby
        lea     dx,optmsg               ; yes - point to options error message
        mov     cx,optmgl               ; get length of message
        mov     bx,2                    ; error output device handle
        mov     ah,40h                  ; set DOS function number for
        int     21h                     ; "write to file or device" & call DOS
        jmp     short   main03          ; and return as done
;
main01: mov     tokcnt,0                ; no tokens in the table
        lea     ax,tokens               ; point to start of token work area
        mov     tokptr,ax               ; save pointer to next free byte
        call    bufget                  ; read in a buffer
        test    flag1,f1eof             ; is there any data in the read buffer
        jnz     main03                  ; no - we are done with this pgm then
        dec     cx                      ; yes - ignore the trailing <cr>
        jle     main01                  ; if length is =<0 just get next line
        lea     bx,buffer               ; point to the first byte of the data
;
main02: call    nextok                  ; get the next token
        or      cx,cx                   ; are we done with this line yet
        jnz     main02                  ; no - get yet another token
        call    write                   ; write the lines
        jmp     short   main01          ; and loop for the next line
;
main03: call    crlf                    ; write a final <cr><lf> sequence
        mov     al,0                    ; put 0 in al - return code to post
        mov     ah,4ch                  ; terminate a process code
        int     21h                     ; end this program
;
main    endp
;
;
;       OPTIONS - This subroutine will parse the options passed to the
;                 program and set the required bits in flag1. No registers
;                 are preserved since we are called only once, before the
;                 program has really started.
;
options proc    near
;
        mov     outcnt,0                ; initialize outers count
        mov     si,81h                  ; point to the first parms character
;
opt01:  mov     al,byte ptr es:0[si]    ; get a byte from the parm string
        inc     si                      ; point to the next byte
        cmp     al,0dh                  ; is it the end of the string?
        jne     opt02                   ; no - goody, more data to process
        ret                             ; yes, return to the caller then
;
opt02:  cmp     al,' '                  ; allow spaces anywhere before slashes
        je      opt01                   ; ignore them though
        cmp     al,'/'                  ; we have to start with a slash now
        je      opt04                   ; if it is a slash then process it
;
opterr: or      flag1,f1oerr            ; otherwise set the options error flag
        ret                             ; and return
;
opt04:  mov     al,byte ptr es:0[si]    ; get the next character after slash
        inc     si                      ; point to next character in parms
        cmp     al,'a'                  ; is it lower case or funny?
        jl      opt4a                   ; no - process it normally then
        sub     al,'a'-'A'              ; yes - map lower case to upper
;
opt4a:  cmp     al,'L'                  ; might it be left justify or numeric
        jl      optnum                  ; perhaps numeric - check it out
        jne     opt05                   ; it is not LJ for sure
        or      flag1,f1lj              ; assume it is LJ for the moment
        test    flag1,f1rj              ; make sure this isn't a duplicate
        jnz     opterr                  ; if RJ already then big problems
        jmp     short   opt06           ; and rejoin common justify code
;
opt05:  cmp     al,'R'                  ; might it be right justify (RJ)?
        jne     opterr                  ; no - then it is an error
        or      flag1,f1rj              ; yes - assume for the moment it is
        test    flag1,f1lj              ; make sure we aren't trying to left
        jnz     opterr                  ; justify too - if we are we are in
                                        ; deep s..t
opt06:  mov     al,byte ptr es:0[si]    ; get the next character
        inc     si                      ; point to the next character in parms
        cmp     al,'J'                  ; is it the J we expect?
        je      opt6a                   ; yes - process it normally then
        cmp     al,'j'                  ; is it a lower case J
        jne     opterr                  ; no - that's too bad.
;
opt6a:  mov     al,byte ptr es:0[si]    ; get the first byte of the number
        inc     si                      ; point to next character in parms
        call    decbin                  ; is it a number?
        jc      opterr                  ; no - then we have an error
        or      al,al                   ; is the field size 0?
        je      opterr                  ; yes - it is in error then
        cmp     al,15                   ; is field size more than 15?
        jg      opterr                  ; yes - it is in error then
        mov     toksiz,al               ; save the justified field size
        jmp     short opt01             ; and process further options
;
optnum: call    decbin                  ; is it a number after slash?
        jc      opterr                  ; no - then it is an error
        or      al,al                   ; zero is special
        jne     opt08                   ; not zero - save it in array then
        or      flag1,f1one             ; zero means one token per line
        jmp     short   opt01           ; process some other token then
;
opt08:  sub     cx,cx                   ; get a zeroed double register
        mov     cl,outcnt               ; get offset into outers for this guy
        lea     bx,outers               ; point just before list of outers
        add     bx,cx                   ; bx points to origin 1 save spot
        mov     byte ptr [bx],al        ; save the token position to write
        inc     outcnt                  ; add one to outcnt
        or      flag1,f1sub             ; make sure substitute flag is on
        jmp     opt01                   ; and play it again Sam.
;
options endp
;
;
;       nextok - This subroutine will find the next token in the string
;                pointed to by bx, with length contained in cx, and move
;                it to the end of the token space. An entry in toktbl will
;                be created for this token. When the subroutine returns cx
;                will be zero if the source data string is empty. bx will
;                point to the first character past the last token.
;
nextok  proc    near
;
        mov     di,tokptr               ; get pointer to where to put token
;
next01: mov     al,byte ptr 0[bx]       ; loop past junk
        cmp     al,' '                  ; is it a leading space?
        jne     next03                  ; no - then we have a token
        inc     bx                      ; yes - point to the next character
        loop    next01                  ; and try that one
        ret                             ; return if we are done with output
;
next03: mov     si,bx                   ; save pointer to start of token
        mov     ah,1                    ; initial guess for token length is 1
;
next04: inc     bx                      ; point to the next character in input
        mov     al,byte ptr [bx]        ; get the character
        cmp     al,' '                  ; is it the end of the token?
        je      next05                  ; yes - we have some good numbers
        inc     ah                      ; no - increment count of contiguous
                                        ; characters.
        loop    next04                  ; continue till out of chars or a
                                        ; field separator
        dec     ah                      ; shouldn't get here but correct for
                                        ; it anyway
next05: push    cx                      ; save number of chars left in source
                                        ; string
        test    flag1,f1rj + f1lj       ; do we have a maximum token length?
        jz      next09                  ; no - just a normal token write then
        cmp     ah,toksiz               ; yes - is this token just right?
        je      next09                  ; it sure is. we will keep it as is
        jl      next06                  ; if token size < max token size - pad
        mov     ah,toksiz               ; otherwise take max token size as own
        jmp     short   next09          ; and continue normally
;
next06: mov     al,toksiz               ; get the toekn size we must pad to
        sub     al,ah                   ; al contains number of spaces needed
        test    flag1,f1lj              ; left justify? (pad right with space)
        jz      next07                  ; no - must pad to the left with space
        mov     spaces,al               ; yes - save how many spaces to fill
        or      flag1,f1work            ; mark as work to do later on
        jmp     short   next09          ; and join mainline code
;
next07: mov     cl,al                   ; cx contains number of leading spaces
;
next08: mov     byte ptr [di],' '       ; put a leading space in this token
        inc     di                      ; point to the next slot
        loop    next08                  ; and fill in all needed spaces
;
next09: mov     cl,ah                   ; cx now contains total number of
                                        ; chars in token
        cld                             ; make the direction ever upward
        rep     movsb                   ; move the token to its spot
        test    flag1,f1work            ; is it left justified (need spaces)
        jz      next11                  ; no - we are done with hard part then
        mov     cl,spaces               ; get count of spaces needed
;
next10: mov     byte ptr [di],' '       ; move in a trailing space
        inc     di                      ; point to the next slot and
        loop    next10                  ; cont. till all trailing spaces done
        and     flag1,255-f1work        ; reset the work to do bit
;
next11: mov     dx,tokptr               ; get pointer to start of this token
        mov     tokptr,di               ; save pointer to next free token byte
        test    flag1,f1lj + f1rj       ; do we have fixed length tokens?
        jz      next12                  ; no - take them as we get them
        mov     ah,toksiz               ; yes - set this tokens length
;
next12: mov     cl,ah                   ; save length of token
        mov     al,3                    ; number of bytes per entry
        mul     tokcnt                  ; ax is now an offset in toktbl
        lea     si,toktbl               ; point to start of token table
        add     si,ax                   ; si points to an entry in toktbl
        mov     byte ptr [si],cl        ; move in length of entry
        mov     word ptr 1[si],dx       ; save pointer to start of token
        inc     tokcnt                  ; count one more token
        pop     cx                      ; cx contains number of source chars
        or      cx,cx                   ; left. Are we done yet?
        jz      next13                  ; yes - return
        dec     cx                      ; no - correct for undercounting by 1
;
next13: ret                             ; and return
;
nextok  endp
;
;
;       write - This routine will write the tokens to the standard output
;               device. It is controlled by the settings of flags in flag1.
;
write   proc    near
;
        sub     cx,cx                   ; get an empty loop counter
        mov     cl,tokcnt               ; cl contains total number tokens read
        or      cx,cx                   ; do we have anything to write out?
        jnz     write1                  ; yes - then go for it
        ret                             ; no - we are done before we begin
;
write1: test    flag1,f1sub             ; are we changing their order?
        jnz     write3                  ; yes - then use different write logic
        sub     dl,dl                   ; no - just output them all in order
;
write2: call    tout                    ; write the sucker
        inc     dl                      ; point to the next token
        loop    write2                  ; and go through them all
        jmp     short   write6          ; return. A job well done
;
write3: mov     cl,outcnt               ; get the number tokens to write
        lea     bx,outers               ; point to the first one to output
;
write4: mov     dl,byte ptr [bx]        ; get a token to write
        cmp     dl,tokcnt               ; is it <= max token?
        jg      write5                  ; no - don't write it then
        dec     dl                      ; yes - adjust for origin one and
        call    tout                    ; write this token then
;
write5: inc     bx                      ; point to the next token count to
        loop    write4                  ; write and loop through whole list
;
write6: test    flag1,f1one             ; are we outputting one token/line?
        jnz     write7                  ; yes - the last <cr><lf> was written
        call    crlf                    ; no - write a trailing <cr><lf>
;
write7: ret                             ; Done. Go home now.
;
write   endp
;
;
;       tout -  This routine will find and write the token from the input
;               line that is in position dl on that line.
;
tout    proc    near
;
        push    bx                      ; save the registers
        push    cx
        push    dx
        mov     al,3                    ; number of bytes per toktbl entry
        mul     dl                      ; get offset into toktbl for token
        lea     bx,toktbl               ; point to the start of the table
        add     bx,ax                   ; point to the correct 3 byte entry
        sub     cx,cx                   ; zero the counter
        mov     cl,byte ptr [bx]        ; get number of chars in this token
        mov     dx,word ptr 1[bx]       ; and point to first byte of token
        call    oswrite                 ; write to standard output device
        test    flag1,f1one             ; only one token per line?
        jz      tout02                  ; no - write a space then
        call    crlf                    ; yes - write a <cr><lf> sequence
        jmp     short   tout03          ; and return
;
tout02: mov     cx,1                    ; length of space is one
        lea     dx,chrspa               ; point to a space
        call    oswrite                 ; write to standard output device
;
tout03: pop     dx                      ; restore the registers
        pop     cx
        pop     bx
        ret                             ; and return
;
tout    endp
;
;
;       crlf -  Everybody knows what this routine does.
;
crlf    proc    near
;
        push    ax                      ; save the registers
        push    bx
        push    cx
        push    dx
        mov     cx,2                    ; length of <cr><lf> string
        lea     dx,chrclf               ; point to the data to write
        call    oswrite                 ; write to the standard output device
        pop     dx                      ; restore the registers
        pop     cx
        pop     bx
        pop     ax
        ret                             ; and return
;
crlf    endp
;
;
;       oswrite - This routine will write characters pointed to by ds:dx
;                 of length contained in cx, to the standard output device
;                 I any errors are detected a message will be written to the
;                 standard error device and flag f1eof will be set.
;
oswrite proc    near
;
        mov     bx,1                    ; file handle of standard output
        mov     ah,40h                  ; write to file or device DOS function
        int     21h                     ; call DOS
        jc      oswr01                  ; if error 5 or 6 then end
        cmp     cx,ax                   ; as many chars as we wanted written?
        je      oswr99                  ; return if all went well
;
oswr01: lea     dx,noroom               ; point to the "no space" message
        mov     cx,lnoroom              ; get the length of the message
        mov     bx,2                    ; get handle for standard error device
        mov     ah,40h                  ; write to file or device DOS function
        int     21h                     ; let him know we erred
        or      flag1,f1eof             ; pretend eof on input device so
;
oswr99: ret                             ; program stops and return
;
oswrite endp
;
;
;       decbin -  On entry this routine has the first character to convert
;                 to binary in al. si points to additional characters. On exit
;                 si points to the first non-numeric character found.
;                 al contains the binary value and carry isn't set. If carry
;                 is set on return then an invalid number was found.
;
decbin  proc    near
;
        push    bx                      ; save a register
        call    decb04                  ; check for numeric in al
        jnc     decb02                  ; if al was numeric it is now 0 .. 9
;
decb01: stc                             ; make sure carry flag set
        pop     bx                      ; restore the register
        ret                             ; and return indicating an error
;
decb02: mov     bl,al                   ; get total so far
        mov     al,byte ptr es:0[si]    ; get a byte from the input stream
        inc     si
        call    decb04                  ; check it for numeric
        jnc     decb03                  ; if numeric then juggle some
        mov     al,bl                   ; otherwise get the value to return
        clc                             ; clear carry flag to say it worked
        dec     si                      ; make sure next char is non-numeric
        pop     bx                      ; restore the register
        ret                             ; and return
;
decb03: mov     bh,al                   ; save the number for a minute
        mov     al,10                   ; get the base
        mul     bl                      ; shift left one position (base al)
        mov     bl,bh                   ; make bx a good number
        sub     bh,bh                   ; bx now contains16 bit value of digit
        add     ax,bx                   ; add in the latest digit
        or      ah,ah                   ; make sure no overflow
        jne     decb01                  ; if there was this is an error
        jmp     short   decb02          ; continue on our way
;
decb04: sub     al,'0'                  ; is it less than a number?
        jl      decb05                  ; yes - return with carry set
        cmp     al,9                    ; is it more than a number?
        jg      decb05                  ; yes - return with carry set
        clc                             ; no - make sure carry is off
        ret                             ; then return the number
;
decb05: stc                             ; set carry on
        ret                             ; and return
;
decbin  endp
;
;
;       bufget -  This routine will read one 'line' from the standard input
;                 device to buffer. On exit cx contains the count of chars
;                 read. f1eof is set if an end of file condition is
;                 encountered.
;
bufget  proc    near
;
        push    ax                      ; save the registers
        push    bx
        push    dx
        push    di
        push    si
        test    flag1,f1qeof            ; should we reflect an immediate eof?
        jz      bufg00                  ; no - standard logic here then
        or      flag1,f1eof             ; yes - set the end of file bit
        and     flag1,255 - f1qeof      ; and say it is no longer pending
        jmp     short   bufret          ; return now
;
bufg00: sub     cx,cx                   ; count of characters gotten
        lea     di,buffer               ; point destination to buffer
;
bufg01: call    cget                    ; get one character
        test    flag1,f1eof             ; did we get an eof on that try?
        jz      bufg02                  ; yes - let's hope it is an error
        or      cl,cl                   ; is there anything in the buffer?
        jz      bufret                  ; no - just return with cx=0 and f1eof
        mov     ah,0dh                  ; yes - slap a <cr> on the end
        call    cput                    ; put it at end of buffer
        and     flag1,255 - f1eof       ; clear the end of file bit
        or      flag1,f1qeof            ; say next time turn on eof for sure
        jmp     short   bufret          ; and return this last buffer
;
bufg02: cmp     ah,0dh                  ; is the record terminator character?
        je      bufg03                  ; yes - don't turn that into a space
        cmp     ah,20h                  ; no - if not <cr>
        jge     bufg03                  ; if >= 20h then use as is
        mov     ah,' '                  ; otherwise make it a space
;
bufg03: call    cput                    ; write char to output buffer
        cmp     ah,0dh                  ; just write the record terminator?
        je      bufret                  ; yes - then return
        cmp     cl,255                  ; written 255 characters yet?
        jne     bufg01                  ; no - get the next character
        mov     byte ptr [di],0dh       ; make it a terminator
;
bufret: pop     si                      ; restore the registers
        pop     di
        pop     dx
        pop     bx
        pop     ax
        ret                             ; and return
;
bufget  endp
;
;
;
cget    proc    near
;
        mov     dx,glen                 ; is there any data in gbuff?
        or      dx,dx                   ; if count is zero there isn't
        jnz     cget01                  ; there is data so read it
        push    cx                      ; save the registers we might need
        push    di
        mov     ah,3fh                  ; DOS function read from standard in
        mov     bx,0                    ; file handle for standard in
        mov     cx,255                  ; number of characters to read
        lea     dx,gbuff                ; point to where to put the data
        mov     gbptr,dx                ; save pointer to first character
        int     21h                     ; call DOS function
        mov     glen,ax                 ; save the number of characters read
        mov     dx,ax                   ; put data count in dx
        pop     di                      ; restore the registers
        pop     cx
        or      dx,dx                   ; did we get data or eof?
        jnz     cget01                  ; data this time
        or      flag1,f1eof             ; set the end of file encountered bit
        ret                             ; and return
;
cget01: mov     bx,gbptr                ; get pointer to character and return
        mov     ah,byte ptr [bx]
        inc     gbptr                   ; get character and increment pointer
        dec     glen                    ; decrement length
        inc     cl                      ; count this character
        ret                             ; and return the character
;
cget    endp
;
;
;
;
cput    proc    near
;
        mov     byte ptr [di],ah        ; save the character
        inc     di                      ; point to next spot
        cmp     cl,255                  ; will we overreach next time
        jne     cput01                  ; no - good thing
        dec     di                      ; yes - can't let that happen
;
cput01: ret                             ; all done
;
cput    endp
;
csect   ends
        end     main
