www.pudn.com > tidy_src_041213.rar > lexer.h
#ifndef __LEXER_H__
#define __LEXER_H__
/* lexer.h -- Lexer for html parser
(c) 1998-2004 (W3C) MIT, ERCIM, Keio University
See tidy.h for the copyright notice.
CVS Info:
$Author: terry_teague $
$Date: 2004/07/07 01:08:23 $
$Revision: 1.20 $
*/
/*
Given an input source, it returns a sequence of tokens.
GetToken(source) gets the next token
UngetToken(source) provides one level undo
The tags include an attribute list:
- linked list of attribute/value nodes
- each node has 2 NULL-terminated strings.
- entities are replaced in attribute values
white space is compacted if not in preformatted mode
If not in preformatted mode then leading white space
is discarded and subsequent white space sequences
compacted to single space characters.
If XmlTags is no then Tag names are folded to upper
case and attribute names to lower case.
Not yet done:
- Doctype subset and marked sections
*/
#ifdef __cplusplus
extern "C" {
#endif
#include "forward.h"
/* lexer character types
*/
#define digit 1
#define letter 2
#define namechar 4
#define white 8
#define newline 16
#define lowercase 32
#define uppercase 64
/* node->type is one of these values
*/
#define RootNode 0
#define DocTypeTag 1
#define CommentTag 2
#define ProcInsTag 3
#define TextNode 4
#define StartTag 5
#define EndTag 6
#define StartEndTag 7
#define CDATATag 8
#define SectionTag 9
#define AspTag 10
#define JsteTag 11
#define PhpTag 12
#define XmlDecl 13
/* lexer GetToken states
*/
#define LEX_CONTENT 0
#define LEX_GT 1
#define LEX_ENDTAG 2
#define LEX_STARTTAG 3
#define LEX_COMMENT 4
#define LEX_DOCTYPE 5
#define LEX_PROCINSTR 6
#define LEX_ENDCOMMENT 7
#define LEX_CDATA 8
#define LEX_SECTION 9
#define LEX_ASP 10
#define LEX_JSTE 11
#define LEX_PHP 12
#define LEX_XMLDECL 13
/* ParseDocTypeDecl state constants */
#define DT_INTERMEDIATE 0
#define DT_DOCTYPENAME 1
#define DT_PUBLICSYSTEM 2
#define DT_QUOTEDSTRING 3
#define DT_INTSUBSET 4
/* content model shortcut encoding
*/
#define CM_UNKNOWN 0
#define CM_EMPTY (1 << 0)
#define CM_HTML (1 << 1)
#define CM_HEAD (1 << 2)
#define CM_BLOCK (1 << 3)
#define CM_INLINE (1 << 4)
#define CM_LIST (1 << 5)
#define CM_DEFLIST (1 << 6)
#define CM_TABLE (1 << 7)
#define CM_ROWGRP (1 << 8)
#define CM_ROW (1 << 9)
#define CM_FIELD (1 << 10)
#define CM_OBJECT (1 << 11)
#define CM_PARAM (1 << 12)
#define CM_FRAMES (1 << 13)
#define CM_HEADING (1 << 14)
#define CM_OPT (1 << 15)
#define CM_IMG (1 << 16)
#define CM_MIXED (1 << 17)
#define CM_NO_INDENT (1 << 18)
#define CM_OBSOLETE (1 << 19)
#define CM_NEW (1 << 20)
#define CM_OMITST (1 << 21)
/* If the document uses just HTML 2.0 tags and attributes described
** it as HTML 2.0 Similarly for HTML 3.2 and the 3 flavors of HTML 4.0.
** If there are proprietary tags and attributes then describe it as
** HTML Proprietary. If it includes the xml-lang or xmlns attributes
** but is otherwise HTML 2.0, 3.2 or 4.0 then describe it as one of the
** flavors of Voyager (strict, loose or frameset).
*/
/* unknown */
#define xxxx 0u
/* W3C defined HTML/XHTML family document types */
#define HT20 1u
#define HT32 2u
#define H40S 4u
#define H40T 8u
#define H40F 16u
#define H41S 32u
#define H41T 64u
#define H41F 128u
#define X10S 256u
#define X10T 512u
#define X10F 1024u
#define XH11 2048u
#define XB10 4096u
/* proprietary stuff */
#define VERS_SUN 8192u
#define VERS_NETSCAPE 16384u
#define VERS_MICROSOFT 32768u
/* special flag */
#define VERS_XML 65536u
/* compatibility symbols */
#define VERS_UNKNOWN (xxxx)
#define VERS_HTML20 (HT20)
#define VERS_HTML32 (HT32)
#define VERS_HTML40_STRICT (H40S|H41S|X10S)
#define VERS_HTML40_LOOSE (H40T|H41T|X10T)
#define VERS_FRAMESET (H40F|H41F|X10F)
#define VERS_XHTML11 (XH11)
#define VERS_BASIC (XB10)
/* meta symbols */
#define VERS_HTML40 (VERS_HTML40_STRICT|VERS_HTML40_LOOSE|VERS_FRAMESET)
#define VERS_IFRAME (VERS_HTML40_LOOSE|VERS_FRAMESET)
#define VERS_LOOSE (VERS_HTML20|VERS_HTML32|VERS_IFRAME)
#define VERS_EVENTS (VERS_HTML40|VERS_XHTML11)
#define VERS_FROM32 (VERS_HTML32|VERS_HTML40)
#define VERS_FROM40 (VERS_HTML40|VERS_XHTML11|VERS_BASIC)
#define VERS_XHTML (X10S|X10T|X10F|XH11|XB10)
/* all W3C defined document types */
#define VERS_ALL (VERS_HTML20|VERS_HTML32|VERS_FROM40)
/* all proprietary types */
#define VERS_PROPRIETARY (VERS_NETSCAPE|VERS_MICROSOFT|VERS_SUN)
/* Linked list of class names and styles
*/
struct _Style;
typedef struct _Style TagStyle;
struct _Style
{
tmbstr tag;
tmbstr tag_class;
tmbstr properties;
TagStyle *next;
};
/* Linked list of style properties
*/
struct _StyleProp;
typedef struct _StyleProp StyleProp;
struct _StyleProp
{
tmbstr name;
tmbstr value;
StyleProp *next;
};
/* Attribute/Value linked list node
*/
struct _AttVal
{
AttVal* next;
const Attribute* dict;
Node* asp;
Node* php;
int delim;
tmbstr attribute;
tmbstr value;
};
/*
Mosaic handles inlines via a separate stack from other elements
We duplicate this to recover from inline markup errors such as:
italic text
more italic text normal text
which for compatibility with Mosaic is mapped to:
italic text
more italic text normal text
Note that any inline end tag pop's the effect of the current
inline start tag, so that pop's in the above example.
*/
struct _IStack
{
IStack* next;
const Dict* tag; /* tag's dictionary definition */
tmbstr element; /* name (NULL for text nodes) */
AttVal* attributes;
};
/* HTML/XHTML/XML Element, Comment, PI, DOCTYPE, XML Decl,
** etc. etc.
*/
struct _Node
{
Node* parent; /* tree structure */
Node* prev;
Node* next;
Node* content;
Node* last;
AttVal* attributes;
const Dict* was; /* old tag when it was changed */
const Dict* tag; /* tag's dictionary definition */
tmbstr element; /* name (NULL for text nodes) */
uint start; /* start of span onto text array */
uint end; /* end of span onto text array */
uint type; /* TextNode, StartTag, EndTag etc. */
uint line; /* current line of document */
uint column; /* current column of document */
Bool closed; /* true if closed by explicit end tag */
Bool implicit; /* true if inferred */
Bool linebreak; /* true if followed by a line break */
#ifdef TIDY_STORE_ORIGINAL_TEXT
tmbstr otext;
#endif
};
/*
The following are private to the lexer
Use NewLexer() to create a lexer, and
FreeLexer() to free it.
*/
struct _Lexer
{
#if 0 /* Move to TidyDocImpl */
StreamIn* in; /* document content input */
StreamOut* errout; /* error output stream */
uint badAccess; /* for accessibility errors */
uint badLayout; /* for bad style errors */
uint badChars; /* for bad character encodings */
uint badForm; /* for mismatched/mispositioned form tags */
uint warnings; /* count of warnings in this document */
uint errors; /* count of errors */
#endif
uint lines; /* lines seen */
uint columns; /* at start of current token */
Bool waswhite; /* used to collapse contiguous white space */
Bool pushed; /* true after token has been pushed back */
Bool insertspace; /* when space is moved after end tag */
Bool excludeBlocks; /* Netscape compatibility */
Bool exiled; /* true if moved out of table */
Bool isvoyager; /* true if xmlns attribute on html element */
uint versions; /* bit vector of HTML versions */
int doctype; /* version as given by doctype (if any) */
Bool bad_doctype; /* e.g. if html or PUBLIC is missing */
uint txtstart; /* start of current node */
uint txtend; /* end of current node */
uint state; /* state of lexer's finite state machine */
Node* token; /* current parse point */
Node* root; /* remember root node of the document */
Node* parent; /* remember parent node for CDATA elements */
Bool seenEndBody; /* true if a