diff options
Diffstat (limited to 'code/tools/lcc/cpp/tokens.c')
| -rw-r--r-- | code/tools/lcc/cpp/tokens.c | 371 |
1 files changed, 371 insertions, 0 deletions
diff --git a/code/tools/lcc/cpp/tokens.c b/code/tools/lcc/cpp/tokens.c new file mode 100644 index 0000000..90ea47f --- /dev/null +++ b/code/tools/lcc/cpp/tokens.c @@ -0,0 +1,371 @@ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "cpp.h" + +static char wbuf[2*OBS]; +static char *wbp = wbuf; + +/* + * 1 for tokens that don't need whitespace when they get inserted + * by macro expansion + */ +static const char wstab[] = { + 0, /* END */ + 0, /* UNCLASS */ + 0, /* NAME */ + 0, /* NUMBER */ + 0, /* STRING */ + 0, /* CCON */ + 1, /* NL */ + 0, /* WS */ + 0, /* DSHARP */ + 0, /* EQ */ + 0, /* NEQ */ + 0, /* LEQ */ + 0, /* GEQ */ + 0, /* LSH */ + 0, /* RSH */ + 0, /* LAND */ + 0, /* LOR */ + 0, /* PPLUS */ + 0, /* MMINUS */ + 0, /* ARROW */ + 1, /* SBRA */ + 1, /* SKET */ + 1, /* LP */ + 1, /* RP */ + 0, /* DOT */ + 0, /* AND */ + 0, /* STAR */ + 0, /* PLUS */ + 0, /* MINUS */ + 0, /* TILDE */ + 0, /* NOT */ + 0, /* SLASH */ + 0, /* PCT */ + 0, /* LT */ + 0, /* GT */ + 0, /* CIRC */ + 0, /* OR */ + 0, /* QUEST */ + 0, /* COLON */ + 0, /* ASGN */ + 1, /* COMMA */ + 0, /* SHARP */ + 1, /* SEMIC */ + 1, /* CBRA */ + 1, /* CKET */ + 0, /* ASPLUS */ + 0, /* ASMINUS */ + 0, /* ASSTAR */ + 0, /* ASSLASH */ + 0, /* ASPCT */ + 0, /* ASCIRC */ + 0, /* ASLSH */ + 0, /* ASRSH */ + 0, /* ASOR */ + 0, /* ASAND */ + 0, /* ELLIPS */ + 0, /* DSHARP1 */ + 0, /* NAME1 */ + 0, /* DEFINED */ + 0, /* UMINUS */ +}; + +void +maketokenrow(int size, Tokenrow *trp) +{ + trp->max = size; + if (size>0) + trp->bp = (Token *)domalloc(size*sizeof(Token)); + else + trp->bp = NULL; + trp->tp = trp->bp; + trp->lp = trp->bp; +} + +Token * +growtokenrow(Tokenrow *trp) +{ + int ncur = trp->tp - trp->bp; + int nlast = trp->lp - trp->bp; + + trp->max = 3*trp->max/2 + 1; + trp->bp = (Token *)realloc(trp->bp, trp->max*sizeof(Token)); + if (trp->bp == NULL) + error(FATAL, "Out of memory from realloc"); + trp->lp = &trp->bp[nlast]; + trp->tp = &trp->bp[ncur]; + return trp->lp; +} + +/* + * Compare a row of tokens, ignoring the content of WS; return !=0 if different + */ +int +comparetokens(Tokenrow *tr1, Tokenrow *tr2) +{ + Token *tp1, *tp2; + + tp1 = tr1->tp; + tp2 = tr2->tp; + if (tr1->lp-tp1 != tr2->lp-tp2) + return 1; + for (; tp1<tr1->lp ; tp1++, tp2++) { + if (tp1->type != tp2->type + || (tp1->wslen==0) != (tp2->wslen==0) + || tp1->len != tp2->len + || strncmp((char*)tp1->t, (char*)tp2->t, tp1->len)!=0) + return 1; + } + return 0; +} + +/* + * replace ntok tokens starting at dtr->tp with the contents of str. + * tp ends up pointing just beyond the replacement. + * Canonical whitespace is assured on each side. + */ +void +insertrow(Tokenrow *dtr, int ntok, Tokenrow *str) +{ + int nrtok = rowlen(str); + + dtr->tp += ntok; + adjustrow(dtr, nrtok-ntok); + dtr->tp -= ntok; + movetokenrow(dtr, str); + makespace(dtr); + dtr->tp += nrtok; + makespace(dtr); +} + +/* + * make sure there is WS before trp->tp, if tokens might merge in the output + */ +void +makespace(Tokenrow *trp) +{ + uchar *tt; + Token *tp = trp->tp; + + if (tp >= trp->lp) + return; + if (tp->wslen) { + if (tp->flag&XPWS + && (wstab[tp->type] || (trp->tp>trp->bp && wstab[(tp-1)->type]))) { + tp->wslen = 0; + return; + } + tp->t[-1] = ' '; + return; + } + if (wstab[tp->type] || (trp->tp>trp->bp && wstab[(tp-1)->type])) + return; + tt = newstring(tp->t, tp->len, 1); + *tt++ = ' '; + tp->t = tt; + tp->wslen = 1; + tp->flag |= XPWS; +} + +/* + * Copy an entire tokenrow into another, at tp. + * It is assumed that there is enough space. + * Not strictly conforming. + */ +void +movetokenrow(Tokenrow *dtr, Tokenrow *str) +{ + int nby; + + /* nby = sizeof(Token) * (str->lp - str->bp); */ + nby = (char *)str->lp - (char *)str->bp; + memmove(dtr->tp, str->bp, nby); +} + +/* + * Move the tokens in a row, starting at tr->tp, rightward by nt tokens; + * nt may be negative (left move). + * The row may need to be grown. + * Non-strictly conforming because of the (char *), but easily fixed + */ +void +adjustrow(Tokenrow *trp, int nt) +{ + int nby, size; + + if (nt==0) + return; + size = (trp->lp - trp->bp) + nt; + while (size > trp->max) + growtokenrow(trp); + /* nby = sizeof(Token) * (trp->lp - trp->tp); */ + nby = (char *)trp->lp - (char *)trp->tp; + if (nby) + memmove(trp->tp+nt, trp->tp, nby); + trp->lp += nt; +} + +/* + * Copy a row of tokens into the destination holder, allocating + * the space for the contents. Return the destination. + */ +Tokenrow * +copytokenrow(Tokenrow *dtr, Tokenrow *str) +{ + int len = rowlen(str); + + maketokenrow(len, dtr); + movetokenrow(dtr, str); + dtr->lp += len; + return dtr; +} + +/* + * Produce a copy of a row of tokens. Start at trp->tp. + * The value strings are copied as well. The first token + * has WS available. + */ +Tokenrow * +normtokenrow(Tokenrow *trp) +{ + Token *tp; + Tokenrow *ntrp = new(Tokenrow); + int len; + + len = trp->lp - trp->tp; + if (len<=0) + len = 1; + maketokenrow(len, ntrp); + for (tp=trp->tp; tp < trp->lp; tp++) { + *ntrp->lp = *tp; + if (tp->len) { + ntrp->lp->t = newstring(tp->t, tp->len, 1); + *ntrp->lp->t++ = ' '; + if (tp->wslen) + ntrp->lp->wslen = 1; + } + ntrp->lp++; + } + if (ntrp->lp > ntrp->bp) + ntrp->bp->wslen = 0; + return ntrp; +} + +/* + * Debugging + */ +void +peektokens(Tokenrow *trp, char *str) +{ + Token *tp; + + tp = trp->tp; + flushout(); + if (str) + fprintf(stderr, "%s ", str); + if (tp<trp->bp || tp>trp->lp) + fprintf(stderr, "(tp offset %d) ", tp-trp->bp); + for (tp=trp->bp; tp<trp->lp && tp<trp->bp+32; tp++) { + if (tp->type!=NL) { + int c = tp->t[tp->len]; + tp->t[tp->len] = 0; + fprintf(stderr, "%s", tp->t); + tp->t[tp->len] = c; + } + if (tp->type==NAME) { + fprintf(stderr, tp==trp->tp?"{*":"{"); + prhideset(tp->hideset); + fprintf(stderr, "} "); + } else + fprintf(stderr, tp==trp->tp?"{%x*} ":"{%x} ", tp->type); + } + fprintf(stderr, "\n"); + fflush(stderr); +} + +void +puttokens(Tokenrow *trp) +{ + Token *tp; + int len, wlen; + uchar *p; + + if (verbose) + peektokens(trp, ""); + tp = trp->bp; + for (; tp<trp->lp; tp++) { + len = tp->len+tp->wslen; + p = tp->t-tp->wslen; + while (tp<trp->lp-1 && p+len == (tp+1)->t - (tp+1)->wslen) { + tp++; + len += tp->wslen+tp->len; + } + if (len>OBS/2) { /* handle giant token */ + if (wbp > wbuf) + wlen = write(1, wbuf, wbp-wbuf); + wlen = write(1, (char *)p, len); + wbp = wbuf; + } else { + memcpy(wbp, p, len); + wbp += len; + } + if (wbp >= &wbuf[OBS]) { + wlen = write(1, wbuf, OBS); + if (wbp > &wbuf[OBS]) + memcpy(wbuf, wbuf+OBS, wbp - &wbuf[OBS]); + wbp -= OBS; + } + } + trp->tp = tp; + if (cursource->fd==0) + flushout(); +} + +void +flushout(void) +{ + int wlen; + if (wbp>wbuf) { + wlen = write(1, wbuf, wbp-wbuf); + wbp = wbuf; + } +} + +/* + * turn a row into just a newline + */ +void +setempty(Tokenrow *trp) +{ + trp->tp = trp->bp; + trp->lp = trp->bp+1; + *trp->bp = nltoken; +} + +/* + * generate a number + */ +char * +outnum(char *p, int n) +{ + if (n>=10) + p = outnum(p, n/10); + *p++ = n%10 + '0'; + return p; +} + +/* + * allocate and initialize a new string from s, of length l, at offset o + * Null terminated. + */ +uchar * +newstring(uchar *s, int l, int o) +{ + uchar *ns = (uchar *)domalloc(l+o+1); + + ns[l+o] = '\0'; + return (uchar*)strncpy((char*)ns+o, (char*)s, l) - o; +} |