ref: fb7dd4b3a868cb8987049c95bb32e6425a73c8b9
dir: /appl/charon/lex.b/
implement Lex;
include "common.m";
# local copies from CU
sys: Sys;
CU: CharonUtils;
S: String;
T: StringIntTab;
C: Ctype;
J: Script;
ctype: array of byte;
EOF : con -2;
EOB : con -1;
tagnames = array[] of {
" ",
"!",
"a",
"abbr",
"acronym",
"address",
"applet",
"area",
"b",
"base",
"basefont",
"bdo",
"big",
"blink",
"blockquote",
"body",
"bq",
"br",
"button",
"caption",
"center",
"cite",
"code",
"col",
"colgroup",
"dd",
"del",
"dfn",
"dir",
"div",
"dl",
"dt",
"em",
"fieldset",
"font",
"form",
"frame",
"frameset",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"head",
"hr",
"html",
"i",
"iframe",
"image",
"img",
"input",
"ins",
"isindex",
"kbd",
"label",
"legend",
"li",
"link",
"map",
"menu",
"meta",
"nobr",
"noframes",
"noscript",
"object",
"ol",
"optgroup",
"option",
"p",
"param",
"pre",
"q",
"s",
"samp",
"script",
"select",
"small",
"span",
"strike",
"strong",
"style",
"sub",
"sup",
"table",
"tbody",
"td",
"textarea",
"tfoot",
"th",
"thead",
"title",
"tr",
"tt",
"u",
"ul",
"var",
"xmp"
};
tagtable : array of T->StringInt; # initialized from tagnames
attrnames = array[] of {
"abbr",
"accept",
"accept-charset",
"accesskey",
"action",
"align",
"alink",
"alt",
"archive",
"axis",
"background",
"bgcolor",
"border",
"cellpadding",
"cellspacing",
"char",
"charoff",
"charset",
"checked",
"cite",
"class",
"classid",
"clear",
"code",
"codebase",
"codetype",
"color",
"cols",
"colspan",
"compact",
"content",
"coords",
"data",
"datafld",
"dataformatas",
"datapagesize",
"datasrc",
"datetime",
"declare",
"defer",
"dir",
"disabled",
"enctype",
"event",
"face",
"for",
"frame",
"frameborder",
"headers",
"height",
"href",
"hreflang",
"hspace",
"http-equiv",
"id",
"ismap",
"label",
"lang",
"language",
"link",
"longdesc",
"lowsrc",
"marginheight",
"marginwidth",
"maxlength",
"media",
"method",
"multiple",
"name",
"nohref",
"noresize",
"noshade",
"nowrap",
"object",
"onabort",
"onblur",
"onchange",
"onclick",
"ondblclick",
"onerror",
"onfocus",
"onkeydown",
"onkeypress",
"onkeyup",
"onload",
"onmousedown",
"onmousemove",
"onmouseout",
"onmouseover",
"onmouseup",
"onreset",
"onresize",
"onselect",
"onsubmit",
"onunload",
"profile",
"prompt",
"readonly",
"rel",
"rev",
"rows",
"rowspan",
"rules",
"scheme",
"scope",
"scrolling",
"selected",
"shape",
"size",
"span",
"src",
"standby",
"start",
"style",
"summary",
"tabindex",
"target",
"text",
"title",
"type",
"usemap",
"valign",
"value",
"valuetype",
"version",
"vlink",
"vspace",
"width"
};
attrtable : array of T->StringInt; # initialized from attrnames
chartab:= array[] of { T->StringInt
("AElig", 'Æ'),
("Aacute", 'Á'),
("Acirc", 'Â'),
("Agrave", 'À'),
("Alpha", 'Α'),
("Aring", 'Å'),
("Atilde", 'Ã'),
("Auml", 'Ä'),
("Beta", 'Β'),
("Ccedil", 'Ç'),
("Chi", 'Χ'),
("Dagger", '‡'),
("Delta", 'Δ'),
("ETH", 'Ð'),
("Eacute", 'É'),
("Ecirc", 'Ê'),
("Egrave", 'È'),
("Epsilon", 'Ε'),
("Eta", 'Η'),
("Euml", 'Ë'),
("Gamma", 'Γ'),
("Iacute", 'Í'),
("Icirc", 'Î'),
("Igrave", 'Ì'),
("Iota", 'Ι'),
("Iuml", 'Ï'),
("Kappa", 'Κ'),
("Lambda", 'Λ'),
("Mu", 'Μ'),
("Ntilde", 'Ñ'),
("Nu", 'Ν'),
("OElig", 'Œ'),
("Oacute", 'Ó'),
("Ocirc", 'Ô'),
("Ograve", 'Ò'),
("Omega", 'Ω'),
("Omicron", 'Ο'),
("Oslash", 'Ø'),
("Otilde", 'Õ'),
("Ouml", 'Ö'),
("Phi", 'Φ'),
("Pi", 'Π'),
("Prime", '″'),
("Psi", 'Ψ'),
("Rho", 'Ρ'),
("Scaron", 'Š'),
("Sigma", 'Σ'),
("THORN", 'Þ'),
("Tau", 'Τ'),
("Theta", 'Θ'),
("Uacute", 'Ú'),
("Ucirc", 'Û'),
("Ugrave", 'Ù'),
("Upsilon", 'Υ'),
("Uuml", 'Ü'),
("Xi", 'Ξ'),
("Yacute", 'Ý'),
("Yuml", 'Ÿ'),
("Zeta", 'Ζ'),
("aacute", 'á'),
("acirc", 'â'),
("acute", '´'),
("aelig", 'æ'),
("agrave", 'à'),
("alefsym", 'ℵ'),
("alpha", 'α'),
("amp", '&'),
("and", '∧'),
("ang", '∠'),
("aring", 'å'),
("asymp", '≈'),
("atilde", 'ã'),
("auml", 'ä'),
("bdquo", '„'),
("beta", 'β'),
("brvbar", '¦'),
("bull", '•'),
("cap", '∩'),
("ccedil", 'ç'),
("cdots", '⋯'),
("cedil", '¸'),
("cent", '¢'),
("chi", 'χ'),
("circ", 'ˆ'),
("clubs", '♣'),
("cong", '≅'),
("copy", '©'),
("crarr", '↵'),
("cup", '∪'),
("curren", '¤'),
("dArr", '⇓'),
("dagger", '†'),
("darr", '↓'),
("ddots", '⋱'),
("deg", '°'),
("delta", 'δ'),
("diams", '♦'),
("divide", '÷'),
("eacute", 'é'),
("ecirc", 'ê'),
("egrave", 'è'),
("emdash", '—'),
("empty", '∅'),
("emsp", ' '),
("endash", '–'),
("ensp", ' '),
("epsilon", 'ε'),
("equiv", '≡'),
("eta", 'η'),
("eth", 'ð'),
("euml", 'ë'),
("euro", '€'),
("exist", '∃'),
("fnof", 'ƒ'),
("forall", '∀'),
("frac12", '½'),
("frac14", '¼'),
("frac34", '¾'),
("frasl", '⁄'),
("gamma", 'γ'),
("ge", '≥'),
("gt", '>'),
("hArr", '⇔'),
("harr", '↔'),
("hearts", '♥'),
("hellip", '…'),
("iacute", 'í'),
("icirc", 'î'),
("iexcl", '¡'),
("igrave", 'ì'),
("image", 'ℑ'),
("infin", '∞'),
("int", '∫'),
("iota", 'ι'),
("iquest", '¿'),
("isin", '∈'),
("iuml", 'ï'),
("kappa", 'κ'),
("lArr", '⇐'),
("lambda", 'λ'),
("lang", '〈'),
("laquo", '«'),
("larr", '←'),
("lceil", '⌈'),
("ldots", '…'),
("ldquo", '“'),
("le", '≤'),
("lfloor", '⌊'),
("lowast", '∗'),
("loz", '◊'),
("lrm", ''),
("lsaquo", '‹'),
("lsquo", '‘'),
("lt", '<'),
("macr", '¯'),
("mdash", '—'),
("micro", 'µ'),
("middot", '·'),
("minus", '−'),
("mu", 'μ'),
("nabla", '∇'),
("nbsp", ' '),
("ndash", '–'),
("ne", '≠'),
("ni", '∋'),
("not", '¬'),
("notin", '∉'),
("nsub", '⊄'),
("ntilde", 'ñ'),
("nu", 'ν'),
("oacute", 'ó'),
("ocirc", 'ô'),
("oelig", 'œ'),
("ograve", 'ò'),
("oline", '‾'),
("omega", 'ω'),
("omicron", 'ο'),
("oplus", '⊕'),
("or", '∨'),
("ordf", 'ª'),
("ordm", 'º'),
("oslash", 'ø'),
("otilde", 'õ'),
("otimes", '⊗'),
("ouml", 'ö'),
("para", '¶'),
("part", '∂'),
("permil", '‰'),
("perp", '⊥'),
("phi", 'φ'),
("pi", 'π'),
("piv", 'ϖ'),
("plusmn", '±'),
("pound", '£'),
("prime", '′'),
("prod", '∏'),
("prop", '∝'),
("psi", 'ψ'),
("quad", ' '),
("quot", '"'),
("quot", '"'),
("rArr", '⇒'),
("radic", '√'),
("rang", '〉'),
("raquo", '»'),
("rarr", '→'),
("rceil", '⌉'),
("rdquo", '”'),
("real", 'ℜ'),
("reg", '®'),
("rfloor", '⌋'),
("rho", 'ρ'),
("rlm", ''),
("rsaquo", '›'),
("rsquo", '’'),
("sbquo", '‚'),
("scaron", 'š'),
("sdot", '⋅'),
("sect", '§'),
("shy", ''),
("sigma", 'σ'),
("sigmaf", 'ς'),
("sim", '∼'),
("sp", ' '),
("spades", '♠'),
("sub", '⊂'),
("sube", '⊆'),
("sum", '∑'),
("sup", '⊃'),
("sup1", '¹'),
("sup2", '²'),
("sup3", '³'),
("supe", '⊇'),
("szlig", 'ß'),
("tau", 'τ'),
("there4", '∴'),
("theta", 'θ'),
("thetasym", 'ϑ'),
("thinsp", ' '),
("thorn", 'þ'),
("tilde", '˜'),
("times", '×'),
("trade", '™'),
("uArr", '⇑'),
("uacute", 'ú'),
("uarr", '↑'),
("ucirc", 'û'),
("ugrave", 'ù'),
("uml", '¨'),
("upsih", 'ϒ'),
("upsilon", 'υ'),
("uuml", 'ü'),
("varepsilon", '∈'),
("varphi", 'ϕ'),
("varpi", 'ϖ'),
("varrho", 'ϱ'),
("vdots", '⋮'),
("vsigma", 'ς'),
("vtheta", 'ϑ'),
("weierp", '℘'),
("xi", 'ξ'),
("yacute", 'ý'),
("yen", '¥'),
("yuml", 'ÿ'),
("zeta", 'ζ'),
("zwj", ''),
("zwnj", ''),
};
# Characters Winstart..Winend are those that Windows
# uses interpolated into the Latin1 set.
# They aren't supposed to appear in HTML, but they do....
Winstart : con 16r7f;
Winend: con 16r9f;
winchars := array[] of { '•',
'•', '•', '‚', 'ƒ', '„', '…', '†', '‡',
'ˆ', '‰', 'Š', '‹', 'Œ', '•', '•', '•',
'•', '‘', '’', '“', '”', '•', '–', '—',
'˜', '™', 'š', '›', 'œ', '•', '•', 'Ÿ'
};
NAMCHAR : con (C->L|C->U|C->D|C->N);
LETTER : con (C->L|C->U);
dbg := 0;
warn := 0;
init(cu: CharonUtils)
{
CU = cu;
sys = load Sys Sys->PATH;
S = load String String->PATH;
C = cu->C;
J = cu->J;
T = load StringIntTab StringIntTab->PATH;
tagtable = CU->makestrinttab(tagnames);
attrtable = CU->makestrinttab(attrnames);
ctype = C->ctype;
}
TokenSource.new(b: ref CU->ByteSource, chset : Btos, mtype: int) : ref TokenSource
{
ts := ref TSstate (
0, # bi
0, # prevbi
"", # s
0, # si
Convcs->Startstate, # state
Convcs->Startstate # prevstate
);
ans := ref TokenSource(
b, # b
chset, # chset
ts, # state
mtype, # mtype
0 # inxmp
);
dbg = int (CU->config).dbg['x'];
warn = (int (CU->config).dbg['w']) || dbg;
return ans;
}
TokenSource.gettoks(ts: self ref TokenSource): array of ref Token
{
ToksMax : con 500; # max chunk of tokens returned
a := array[ToksMax] of ref Token;
ai := 0;
pcdai := 0;
lim := 0;
# put some dbg output in here
if(ts.mtype == CU->TextHtml) {
pcdstate : ref TSstate;
gather:
while(ai < ToksMax-1) { # always allow space for a Data token
state := getstate(ts);
c := getchar(ts);
if (c < ' ') {
c = eatctls(c, ts);
if (c < 0)
break;
}
tok : ref Token;
if(c == '<') {
tok = gettag(ts);
if (tok != nil && ts.inxmp && tok.tag != Txmp+RBRA) {
rewind(ts, state);
getchar(ts); # consume the '<'
tok = ref Token(Data, "<", nil);
}
if(tok != nil && tok.tag != Comment) {
a[ai++] = tok;
case (tok.tag) {
Tselect or Ttitle or Toption=>
# Several tags expect PCDATA after them.
# Capture state so we can rewind if necessary
pcdstate = state;
pcdai = ai-1;
Ttextarea =>
pcdstate = state;
pcdai = ai-1;
# not sure if we should parse entity references
tok = gettagdata(ts, tok.tag, 1);
if(tok != nil) {
pcdstate = nil;
a[ai++] = tok;
}
Tscript =>
pcdstate = state;
pcdai = ai-1;
# special rules for getting Data
tok = getscriptdata(ts);
if(tok != nil) {
pcdstate = nil;
a[ai++] = tok;
}
Txmp =>
pcdstate = nil;
ts.inxmp = 1;
Txmp+RBRA =>
pcdstate = nil;
ts.inxmp = 0;
Data =>
;
Tmeta =>
pcdstate = nil;
break gather;
* =>
pcdstate = nil;
}
}
} else {
tok = getdata(ts, c);
if(tok != nil)
a[ai++] = tok;
}
if(tok == nil && !eof(ts)) {
# we need more input to complete the token
lim = ts.state.bi;
rewind(ts, state);
break gather;
} else
if(dbg > 1)
sys->print("lex: got token %s\n", tok.tostring());
}
# Several tags expect PCDATA after them.
# which means that build needs to see another tag or eof
# after any data in order to know that PCDATA is ended.
# Rewind if we haven't got to the following tag yet.
if (pcdstate != nil && !eof(ts)) {
rewind(ts, pcdstate);
ai = pcdai;
}
}
else {
# plain text (non-html) tokens
while(ai < ToksMax) {
tok := getplaindata(ts);
if(tok == nil)
break;
else
a[ai++] = tok;
if(dbg > 1)
sys->print("lex: got token %s\n", tok.tostring());
}
}
if(dbg)
sys->print("lex: returning %d tokens\n", ai);
if (lim > ts.b.lim)
ts.b.lim = lim;
else
ts.b.lim = ts.state.prevbi;
if(ai == 0)
return nil;
return a[0:ai];
}
# must not be called from within TokenSource.gettoks()
# as it will not work with rewind() and ungetchar()
#
TokenSource.setchset(ts: self ref TokenSource, chset: Btos)
{
st := ts.state;
nchars := st.si;
if (nchars > 0 && nchars < len st.s) {
# align bi to the current input char
bs := ts.b;
(nil, nil, n) := ts.chset->btos(st.prevcsstate, bs.data[st.prevbi:st.bi], nchars);
st.bi = st.prevbi + n;
st.prevbi = st.bi;
}
ts.chset = chset;
st.csstate = st.prevcsstate = Convcs->Startstate;
st.s = nil;
st.si = 0;
}
eof(ts : ref TokenSource) : int
{
st := ts.state;
bs := ts.b;
return (st.s == nil && bs.eof && st.prevbi == bs.edata);
}
# For case where source isn't HTML.
# Just make data tokens, one per line (or partial line,
# at end of buffer), ignoring non-whitespace control
# characters and dumping \r's
getplaindata(ts: ref TokenSource): ref Token
{
s := "";
j := 0;
for(c := getchar(ts); c >= 0; c = getchar(ts)) {
if(c < ' ') {
if(ctype[c] == C->W) {
if(c == '\r') {
# ignore it unless no following '\n',
# in which case treat it like '\n'
c = getchar(ts);
if(c != '\n') {
if(c >= 0)
ungetchar(ts);
c = '\n';
}
}
}
else
c = 0; # ignore
}
if(c != 0)
s[j++] = c;
if(c == '\n')
break;
}
if(s == "")
return nil;
return ref Token(Data, s, nil);
}
eatctls(c: int, ts: ref TokenSource): int
{
while (c >= 0) {
if (c >= ' ')
return c;
if(ctype[c] == C->W) {
if(c == '\r') {
c = getchar(ts);
if (c != '\n' && c >= 0) {
ungetchar(ts);
c = '\n';
}
}
return c;
}
c = getchar(ts);
}
return -1;
}
# Gather data up to next start-of-tag or end-of-buffer.
# Translate entity references (&) if not in <XMP> section.
# Ignore non-whitespace control characters and get rid of \r's.
getdata(ts: ref TokenSource, firstc : int): ref Token
{
s := "";
j := 0;
c := firstc;
while(c >= 0) {
if (c < ' ')
c = eatctls(c, ts);
if (c < 0)
break;
if(c == '&' && !ts.inxmp) {
ok : int;
(c, ok) = ampersand(ts);
if(!ok) {
ungetchar(ts);
break; # incomplete entity reference (ts backed up by ampersand)
}
}
else if(c == '<') {
ungetchar(ts);
break;
}
if(c != 0)
s[j++] = c;
c = getchar(ts);
}
if(s == "")
return nil;
return ref Token(Data, s, nil);
}
# The rules for lexing scripts are different (ugh).
# Gather up everything until see a </SCRIPT>.
getscriptdata(ts: ref TokenSource): ref Token
{
tok := gettagdata(ts, Tscript, 0);
if (tok != nil)
tok.text = CU->stripscript(tok.text);
return tok;
}
gettagdata(ts: ref TokenSource, tag, doentities: int): ref Token
{
s := "";
j := 0;
c := getchar(ts);
while(c >= 0) {
if (c == '<') {
tstate := getstate(ts);
tok := gettag(ts);
rewind(ts, tstate);
if (tok != nil && tok.tag == tag+RBRA) {
ungetchar(ts);
return ref Token(Data, s, nil);
}
# tag was not </tag>, take as regular data
}
if (doentities && c == '&')
(c, nil) = ampersand(ts);
if(c < 0)
break;
if(c != 0)
s[j++] = c;
c = getchar(ts);
}
if(eof(ts))
return ref Token(Data, s, nil);
return nil;
}
# We've just seen a '<'. Gather up stuff to closing '>' (if buffer
# ends before then, return nil).
# If it's a tag, look up the name, gather the attributes, and return
# the appropriate token.
# Else it's either just plain data or some kind of ignorable stuff:
# return a Data or Comment token as appropriate.
gettag(ts: ref TokenSource): ref Token
{
rbra := 0;
ans : ref Token = nil;
al: list of Attr;
start := getstate(ts);
c := getchar(ts);
# dummy loop: break out of this when hit end of buffer
eob:
for(;;) {
if(c == '/') {
rbra = RBRA;
c = getchar(ts);
}
if(c < 0)
break eob;
if(c>=C->NCTYPE || !int (ctype[c]&LETTER)) {
# not a tag
if(c == '!') {
ans = comment(ts);
if(ans != nil)
return ans;
break eob;
}
else {
rewind(ts, start);
return ref Token(Data, "<", nil);
}
}
# c starts a tagname
ans = ref Token(Notfound, nil, nil);
name := "";
name[0] = lowerc(c);
i := 1;
for(;;) {
c = getchar(ts);
if(c < 0)
break eob;
if(c>=C->NCTYPE || !int (ctype[c]&NAMCHAR))
break;
name[i++] = lowerc(c);
}
(fnd, tag) := T->lookup(tagtable, name);
if(fnd)
ans.tag = tag+rbra;
else
ans.text = name; # for warning print, in build
attrloop:
for(;;) {
# look for "ws name" or "ws name ws = ws val" (ws=whitespace)
# skip whitespace
while(c < C->NCTYPE && ctype[c] == C->W) {
c = getchar(ts);
if(c < 0)
break eob;
}
if(c == '>')
break attrloop;
if(c == '<') {
if(warn)
sys->print("warning: unclosed tag; last name=%s\n", name);
ungetchar(ts);
break attrloop;
}
if(c >= C->NCTYPE || !int (ctype[c]&LETTER)) {
if(warn)
sys->print("warning: expected attribute name; last name=%s\n", name);
# skip to next attribute name
for(;;) {
c = getchar(ts);
if(c < 0)
break eob;
if(c < C->NCTYPE && int (ctype[c]&LETTER))
continue attrloop;
if(c == '<') {
if(warn)
sys->print("warning: unclosed tag; last name=%s\n", name);
ungetchar(ts);
break attrloop;
}
if(c == '>')
break attrloop;
}
}
# gather attribute name
name = "";
name[0] = lowerc(c);
i = 1;
for(;;) {
c = getchar(ts);
if(c < 0)
break eob;
if(c >= C->NCTYPE || !int (ctype[c]&NAMCHAR))
break;
name[i++] = lowerc(c);
}
(afnd, attid) := T->lookup(attrtable, name);
if(warn && !afnd)
sys->print("warning: unknown attribute name %s\n", name);
# skip whitespace
while(c < C->NCTYPE && ctype[c] == C->W) {
c = getchar(ts);
if(c < 0)
break eob;
}
if(c != '=') {
# no value for this attr
if(afnd)
al = (attid, "") :: al;
continue attrloop;
}
# c is '=' here; skip whitespace
for(;;) {
c = getchar(ts);
if(c < 0)
break eob;
if(c >= C->NCTYPE || ctype[c] != C->W)
break;
}
# gather value
quote := 0;
if(c == '\'' || c == '"') {
quote = c;
c = getchar(ts);
if(c < 0)
break eob;
}
val := "";
nv := 0;
valloop:
for(;;) {
if(c < 0)
break eob;
# other browsers allow value strings to be broken across lines
# especially the case for Javascript event handlers / URLs
if (c == '>' && !quote)
break valloop;
# old code otherwise ok - keep for now for reference
# if(c == '>') {
# if(quote) {
# # c might be part of string (though not good style)
# # but if line ends before close quote, assume
# # there was an unmatched quote
# ti := ts.i;
# for(;;) {
# c = getchar(ts);
# if(c < 0)
# break eob;
# if(c == quote) {
# backup(ts, ti);
# val[nv++] = '>';
# c = getchar(ts);
# continue valloop;
# }
# if(c == '\n') {
# if(warn)
# sys->print("warning: apparent unmatched quote\n");
# backup(ts, ti);
# quote = 0;
# c = '>';
# break valloop;
# }
# }
# }
# else
# break valloop;
# }
if(quote) {
if(c == quote) {
c = getchar(ts);
if(c < 0)
break eob;
break valloop;
}
if(c == '\r') {
c = getchar(ts);
continue valloop;
}
if(c == '\t' || c == '\n')
c = ' ';
}
else {
if(c < C->NCTYPE && ctype[c]==C->W)
break valloop;
}
if(c == '&') {
ok : int;
(c, ok) = ampersand(ts);
if(!ok)
break eob;
}
val[nv++] = c;
c = getchar(ts);
}
if(afnd)
al = (attid, val) :: al;
}
ans.attr = al;
return ans;
}
if(eof(ts)) {
if(warn)
sys->print("warning: incomplete tag at end of page\n");
rewind(ts, start);
return ref Token(Data, "<", nil);
}
return nil;
}
# We've just read a '<!',
# so this may be a comment or other ignored section, or it may
# be just a literal string if there is no close before end of file
# (other browsers do that).
# The accepted practice seems to be (note: contrary to SGML spec!):
# If see <!--, look for --> to close, or if none, > to close.
# If see <!(not --), look for > to close.
# If no close before end of file, leave original characters in as literal data.
#
# If we see ignorable stuff, return Comment token.
# Else return nil (caller should back up and try again when more data arrives,
# unless at end of file, in which case caller should just make '<' a data token).
comment(ts: ref TokenSource) : ref Token
{
havecomment := 0;
commentstart := 0;
c := getchar(ts);
if(c == '-') {
state := getstate(ts);
c = getchar(ts);
if(c == '-') {
commentstart = 1;
if(findstr(ts, "-->"))
havecomment = 1;
else
rewind(ts, state);
}
}
if(!havecomment) {
if(c == '>')
havecomment = 1;
else if(c >= 0) {
if(findstr(ts, ">"))
havecomment = 1;
}
}
if(havecomment)
return ref Token(Comment, nil, nil);
return nil;
}
# Look for string s in token source.
# If found, return 1, with buffer at next char after s,
# else return 0 (caller should back up).
findstr(ts: ref TokenSource, s: string) : int
{
n := len s;
eix := n-1;
buf := "";
c : int;
if (n == 1) {
while ((c = getchar(ts)) >= 0)
if (c == s[0])
return 1;
return 0;
}
for (i := 0; i < n; i++) {
c = getchar(ts);
if (c < 0)
return 0;
buf[i] = c;
}
for (;;) {
# this could be much more efficient by tracking
# the start char through buf
if (buf == s)
return 1;
c = getchar(ts);
if (c < 0)
return 0;
buf = buf[1:];
buf[eix] = c;
}
return 0; # keep the compiler quiet
}
# We've just read an '&'; look for an entity reference
# name, and if found, return (translated char, 1).
# Otherwise the input stream is rewound to just after
# the '&'
# if there is a complete entity name but it isn't known,
# ('&', 1) is returned, if an incomplete name is encountered
# (0, 0) is returned
ampersand(ts: ref TokenSource): (int, int)
{
state := getstate(ts);
c := getchar(ts);
fnd := 0;
ans := 0;
if(c == '#') {
v := 0;
c = getchar(ts);
if (c == 'x' || c == 'X') {
for (c = getchar(ts); c >= 0; c = getchar(ts)) {
if (int (ctype[c] & C->D)) {
v = v*16 + c-'0';
continue;
}
c = lowerc(c);
if (c >= 'a' && c <= 'f') {
v = v*16 + 10 + c-'a';
continue;
}
break;
}
} else {
while(c >= 0) {
if(ctype[c] != C->D)
break;
v = v*10 + c-'0';
c = getchar(ts);
}
}
if(c >= 0) {
if(!(c == ';' || c == '\n' || c == '\r' || c == '<'))
ungetchar(ts);
c = v;
if(c==160)
c = ' '; # non-breaking space
if(c >= Winstart && c <= Winend)
c = winchars[c-Winstart];
ans = c;
fnd = (v != 0);
}
}
# only US-ASCII chars can make up &charnames;
else if(c >= 0 && c < 16r80 && int (ctype[c] & LETTER)) {
s := "";
s[0] = c;
k := 1;
for(;;) {
c = getchar(ts);
if(c < 0)
break;
if(c < 16r80 && int (ctype[c]&NAMCHAR))
s[k++] = c;
else {
if(!(c == ';' || c == '\n' || c == '\r'))
ungetchar(ts);
break;
}
}
if (c < 0 || c == ' ' || c == ';' || c == '\n' || c == '\r' || c == '<')
(fnd, ans) = T->lookup(chartab, s);
}
if(!fnd) {
if(c < 0 && !eof(ts)) {
# was incomplete
rewind(ts, state);
return (0, 0);
}
else {
rewind(ts, state);
return ('&', 1);
}
}
# elide soft hyphens (­ / &xAD;)
# not suficient - need to do it for all input in getdata() which is too heavy handed
# if (ans == '')
# ans = 0;
return (ans, 1);
}
# If c is an uppercase letter, return its lowercase version,
# otherwise return c.
# Assume c is a NAMCHAR, so don't need range check on ctype[]
lowerc(c: int) : int
{
if(ctype[c] == C->U) {
# this works for accented characters in Latin1, too
return c + 16r20;
}
return c;
}
Token.aval(t: self ref Token, attid: int): (int, string)
{
attr := t.attr;
while(attr != nil) {
a := hd attr;
if(a.attid == attid)
return (1, a.value);
attr = tl attr;
}
return (0, "");
}
# for debugging
Token.tostring(t: self ref Token) : string
{
ans := "";
tag := t.tag;
if(tag == Data)
ans = ans + "'" + t.text + "'";
else {
ans = ans + "<";
if(tag >= RBRA) {
tag -= RBRA;
ans = ans + "/";
}
tname := tagnames[tag];
if(tag == Notfound)
tname = "?";
ans = ans + S->toupper(tname);
for(al := t.attr; al != nil; al = tl al) {
a := hd al;
aname := attrnames[a.attid];
ans = ans + " " + aname;
if(a.value != "")
ans = ans + "='" + a.value + "'";
}
ans = ans + ">";
}
return ans;
}
CONVBLK : con 1024; # number of characters to convert at a time
# Returns -1 if no complete character left before current end of data.
getchar(ts: ref TokenSource): int
{
st := ts.state;
if (st.s == nil || st.si >= len st.s) {
bs := ts.b;
st.si = 0;
st.s = "";
st.prevcsstate = st.csstate;
st.prevbi = st.bi;
edata := bs.edata;
if (st.bi >= edata)
return -1;
(state, s, n ) := ts.chset->btos(st.csstate, bs.data[st.bi:edata], CONVBLK);
if (s == nil) {
if (bs.eof && edata == bs.edata) {
# must have been an encoding error at eof
st.prevbi = st.bi = edata;
}
return -1;
}
st.csstate = state;
st.s = s;
st.bi += n;
}
return st.s[st.si++];
}
# back up by one input character
# NOTE: can only call this function post a successful getchar() call
ungetchar(ts : ref TokenSource)
{
st := ts.state;
# assert(len st.s >= 1 && st.si > 0)
if (st.si <= 0)
raise "EXInternal:too many backups";
st.si--;
}
rewind(ts : ref TokenSource, state : ref TSstate)
{
ts.state = state;
}
# return a copy of the TokenSource state
getstate(ts : ref TokenSource) : ref TSstate
{
return ref *ts.state;
}