/[suikacvs]/markup/html/whatpm/Whatpm/HTML.pm.src
Suika

Diff of /markup/html/whatpm/Whatpm/HTML.pm.src

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.55 by wakaba, Sat Aug 11 06:53:38 2007 UTC revision 1.57 by wakaba, Sat Aug 11 08:08:12 2007 UTC
# Line 159  sub CDATA_CONTENT_MODEL () { CM_LIMITED_ Line 159  sub CDATA_CONTENT_MODEL () { CM_LIMITED_
159  sub RCDATA_CONTENT_MODEL () { CM_ENTITY | CM_LIMITED_MARKUP }  sub RCDATA_CONTENT_MODEL () { CM_ENTITY | CM_LIMITED_MARKUP }
160  sub PCDATA_CONTENT_MODEL () { CM_ENTITY | CM_FULL_MARKUP }  sub PCDATA_CONTENT_MODEL () { CM_ENTITY | CM_FULL_MARKUP }
161    
162    sub DATA_STATE () { 0 }
163    sub ENTITY_DATA_STATE () { 1 }
164    sub TAG_OPEN_STATE () { 2 }
165    sub CLOSE_TAG_OPEN_STATE () { 3 }
166    sub TAG_NAME_STATE () { 4 }
167    sub BEFORE_ATTRIBUTE_NAME_STATE () { 5 }
168    sub ATTRIBUTE_NAME_STATE () { 6 }
169    sub AFTER_ATTRIBUTE_NAME_STATE () { 7 }
170    sub BEFORE_ATTRIBUTE_VALUE_STATE () { 8 }
171    sub ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE () { 9 }
172    sub ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE () { 10 }
173    sub ATTRIBUTE_VALUE_UNQUOTED_STATE () { 11 }
174    sub ENTITY_IN_ATTRIBUTE_VALUE_STATE () { 12 }
175    sub MARKUP_DECLARATION_OPEN_STATE () { 13 }
176    sub COMMENT_START_STATE () { 14 }
177    sub COMMENT_START_DASH_STATE () { 15 }
178    sub COMMENT_STATE () { 16 }
179    sub COMMENT_END_STATE () { 17 }
180    sub COMMENT_END_DASH_STATE () { 18 }
181    sub BOGUS_COMMENT_STATE () { 19 }
182    sub DOCTYPE_STATE () { 20 }
183    sub BEFORE_DOCTYPE_NAME_STATE () { 21 }
184    sub DOCTYPE_NAME_STATE () { 22 }
185    sub AFTER_DOCTYPE_NAME_STATE () { 23 }
186    sub BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE () { 24 }
187    sub DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE () { 25 }
188    sub DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE () { 26 }
189    sub AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE () { 27 }
190    sub BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE () { 28 }
191    sub DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE () { 29 }
192    sub DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE () { 30 }
193    sub AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE () { 31 }
194    sub BOGUS_DOCTYPE_STATE () { 32 }
195    
196  sub DOCTYPE_TOKEN () { 1 }  sub DOCTYPE_TOKEN () { 1 }
197  sub COMMENT_TOKEN () { 2 }  sub COMMENT_TOKEN () { 2 }
198  sub START_TAG_TOKEN () { 3 }  sub START_TAG_TOKEN () { 3 }
# Line 169  sub CHARACTER_TOKEN () { 6 } Line 203  sub CHARACTER_TOKEN () { 6 }
203  sub AFTER_HTML_IMS () { 0b100 }  sub AFTER_HTML_IMS () { 0b100 }
204  sub HEAD_IMS ()       { 0b1000 }  sub HEAD_IMS ()       { 0b1000 }
205  sub BODY_IMS ()       { 0b10000 }  sub BODY_IMS ()       { 0b10000 }
206  sub BODY_TABLE_IMS () { 0b100000 | BODY_IMS }  sub BODY_TABLE_IMS () { 0b100000 }
207  sub TABLE_IMS ()      { 0b1000000 }  sub TABLE_IMS ()      { 0b1000000 }
208  sub ROW_IMS ()        { 0b10000000 | TABLE_IMS }  sub ROW_IMS ()        { 0b10000000 }
209  sub BODY_AFTER_IMS () { 0b100000000 }  sub BODY_AFTER_IMS () { 0b100000000 }
210  sub FRAME_IMS ()      { 0b1000000000 }  sub FRAME_IMS ()      { 0b1000000000 }
211    
# Line 182  sub IN_HEAD_NOSCRIPT_IM () { HEAD_IMS | Line 216  sub IN_HEAD_NOSCRIPT_IM () { HEAD_IMS |
216  sub AFTER_HEAD_IM () { HEAD_IMS | 0b10 }  sub AFTER_HEAD_IM () { HEAD_IMS | 0b10 }
217  sub BEFORE_HEAD_IM () { HEAD_IMS | 0b11 }  sub BEFORE_HEAD_IM () { HEAD_IMS | 0b11 }
218  sub IN_BODY_IM () { BODY_IMS }  sub IN_BODY_IM () { BODY_IMS }
219  sub IN_CELL_IM () { BODY_TABLE_IMS | 0b01 }  sub IN_CELL_IM () { BODY_IMS | BODY_TABLE_IMS | 0b01 }
220  sub IN_CAPTION_IM () { BODY_TABLE_IMS | 0b10 }  sub IN_CAPTION_IM () { BODY_IMS | BODY_TABLE_IMS | 0b10 }
221  sub IN_ROW_IM () { ROW_IMS | 0b01 }  sub IN_ROW_IM () { TABLE_IMS | ROW_IMS | 0b01 }
222  sub IN_TABLE_BODY_IM () { ROW_IMS | 0b10 }  sub IN_TABLE_BODY_IM () { TABLE_IMS | ROW_IMS | 0b10 }
223  sub IN_TABLE_IM () { TABLE_IMS }  sub IN_TABLE_IM () { TABLE_IMS }
224  sub AFTER_BODY_IM () { BODY_AFTER_IMS }  sub AFTER_BODY_IM () { BODY_AFTER_IMS }
225  sub IN_FRAMESET_IM () { FRAME_IMS | 0b01 }  sub IN_FRAMESET_IM () { FRAME_IMS | 0b01 }
# Line 197  sub IN_COLUMN_GROUP_IM () { 0b10 } Line 231  sub IN_COLUMN_GROUP_IM () { 0b10 }
231    
232  sub _initialize_tokenizer ($) {  sub _initialize_tokenizer ($) {
233    my $self = shift;    my $self = shift;
234    $self->{state} = 'data'; # MUST    $self->{state} = DATA_STATE; # MUST
235    $self->{content_model} = PCDATA_CONTENT_MODEL; # be    $self->{content_model} = PCDATA_CONTENT_MODEL; # be
236    undef $self->{current_token}; # start tag, end tag, comment, or DOCTYPE    undef $self->{current_token}; # start tag, end tag, comment, or DOCTYPE
237    undef $self->{current_attribute};    undef $self->{current_attribute};
# Line 236  sub _get_next_token ($) { Line 270  sub _get_next_token ($) {
270    }    }
271    
272    A: {    A: {
273      if ($self->{state} eq 'data') {      if ($self->{state} == DATA_STATE) {
274        if ($self->{next_input_character} == 0x0026) { # &        if ($self->{next_input_character} == 0x0026) { # &
275          if ($self->{content_model} & CM_ENTITY) { # PCDATA | RCDATA          if ($self->{content_model} & CM_ENTITY) { # PCDATA | RCDATA
276            $self->{state} = 'entity data';            $self->{state} = ENTITY_DATA_STATE;
277            !!!next-input-character;            !!!next-input-character;
278            redo A;            redo A;
279          } else {          } else {
# Line 261  sub _get_next_token ($) { Line 295  sub _get_next_token ($) {
295          if ($self->{content_model} & CM_FULL_MARKUP or # PCDATA          if ($self->{content_model} & CM_FULL_MARKUP or # PCDATA
296              (($self->{content_model} & CM_LIMITED_MARKUP) and # CDATA | RCDATA              (($self->{content_model} & CM_LIMITED_MARKUP) and # CDATA | RCDATA
297               not $self->{escape})) {               not $self->{escape})) {
298            $self->{state} = 'tag open';            $self->{state} = TAG_OPEN_STATE;
299            !!!next-input-character;            !!!next-input-character;
300            redo A;            redo A;
301          } else {          } else {
# Line 290  sub _get_next_token ($) { Line 324  sub _get_next_token ($) {
324        !!!emit ($token);        !!!emit ($token);
325    
326        redo A;        redo A;
327      } elsif ($self->{state} eq 'entity data') {      } elsif ($self->{state} == ENTITY_DATA_STATE) {
328        ## (cannot happen in CDATA state)        ## (cannot happen in CDATA state)
329                
330        my $token = $self->_tokenize_attempt_to_consume_an_entity (0);        my $token = $self->_tokenize_attempt_to_consume_an_entity (0);
331    
332        $self->{state} = 'data';        $self->{state} = DATA_STATE;
333        # next-input-character is already done        # next-input-character is already done
334    
335        unless (defined $token) {        unless (defined $token) {
# Line 305  sub _get_next_token ($) { Line 339  sub _get_next_token ($) {
339        }        }
340    
341        redo A;        redo A;
342      } elsif ($self->{state} eq 'tag open') {      } elsif ($self->{state} == TAG_OPEN_STATE) {
343        if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA        if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
344          if ($self->{next_input_character} == 0x002F) { # /          if ($self->{next_input_character} == 0x002F) { # /
345            !!!next-input-character;            !!!next-input-character;
346            $self->{state} = 'close tag open';            $self->{state} = CLOSE_TAG_OPEN_STATE;
347            redo A;            redo A;
348          } else {          } else {
349            ## reconsume            ## reconsume
350            $self->{state} = 'data';            $self->{state} = DATA_STATE;
351    
352            !!!emit ({type => CHARACTER_TOKEN, data => '<'});            !!!emit ({type => CHARACTER_TOKEN, data => '<'});
353    
# Line 321  sub _get_next_token ($) { Line 355  sub _get_next_token ($) {
355          }          }
356        } elsif ($self->{content_model} & CM_FULL_MARKUP) { # PCDATA        } elsif ($self->{content_model} & CM_FULL_MARKUP) { # PCDATA
357          if ($self->{next_input_character} == 0x0021) { # !          if ($self->{next_input_character} == 0x0021) { # !
358            $self->{state} = 'markup declaration open';            $self->{state} = MARKUP_DECLARATION_OPEN_STATE;
359            !!!next-input-character;            !!!next-input-character;
360            redo A;            redo A;
361          } elsif ($self->{next_input_character} == 0x002F) { # /          } elsif ($self->{next_input_character} == 0x002F) { # /
362            $self->{state} = 'close tag open';            $self->{state} = CLOSE_TAG_OPEN_STATE;
363            !!!next-input-character;            !!!next-input-character;
364            redo A;            redo A;
365          } elsif (0x0041 <= $self->{next_input_character} and          } elsif (0x0041 <= $self->{next_input_character} and
# Line 333  sub _get_next_token ($) { Line 367  sub _get_next_token ($) {
367            $self->{current_token}            $self->{current_token}
368              = {type => START_TAG_TOKEN,              = {type => START_TAG_TOKEN,
369                 tag_name => chr ($self->{next_input_character} + 0x0020)};                 tag_name => chr ($self->{next_input_character} + 0x0020)};
370            $self->{state} = 'tag name';            $self->{state} = TAG_NAME_STATE;
371            !!!next-input-character;            !!!next-input-character;
372            redo A;            redo A;
373          } elsif (0x0061 <= $self->{next_input_character} and          } elsif (0x0061 <= $self->{next_input_character} and
374                   $self->{next_input_character} <= 0x007A) { # a..z                   $self->{next_input_character} <= 0x007A) { # a..z
375            $self->{current_token} = {type => START_TAG_TOKEN,            $self->{current_token} = {type => START_TAG_TOKEN,
376                              tag_name => chr ($self->{next_input_character})};                              tag_name => chr ($self->{next_input_character})};
377            $self->{state} = 'tag name';            $self->{state} = TAG_NAME_STATE;
378            !!!next-input-character;            !!!next-input-character;
379            redo A;            redo A;
380          } elsif ($self->{next_input_character} == 0x003E) { # >          } elsif ($self->{next_input_character} == 0x003E) { # >
381            !!!parse-error (type => 'empty start tag');            !!!parse-error (type => 'empty start tag');
382            $self->{state} = 'data';            $self->{state} = DATA_STATE;
383            !!!next-input-character;            !!!next-input-character;
384    
385            !!!emit ({type => CHARACTER_TOKEN, data => '<>'});            !!!emit ({type => CHARACTER_TOKEN, data => '<>'});
# Line 353  sub _get_next_token ($) { Line 387  sub _get_next_token ($) {
387            redo A;            redo A;
388          } elsif ($self->{next_input_character} == 0x003F) { # ?          } elsif ($self->{next_input_character} == 0x003F) { # ?
389            !!!parse-error (type => 'pio');            !!!parse-error (type => 'pio');
390            $self->{state} = 'bogus comment';            $self->{state} = BOGUS_COMMENT_STATE;
391            ## $self->{next_input_character} is intentionally left as is            ## $self->{next_input_character} is intentionally left as is
392            redo A;            redo A;
393          } else {          } else {
394            !!!parse-error (type => 'bare stago');            !!!parse-error (type => 'bare stago');
395            $self->{state} = 'data';            $self->{state} = DATA_STATE;
396            ## reconsume            ## reconsume
397    
398            !!!emit ({type => CHARACTER_TOKEN, data => '<'});            !!!emit ({type => CHARACTER_TOKEN, data => '<'});
# Line 368  sub _get_next_token ($) { Line 402  sub _get_next_token ($) {
402        } else {        } else {
403          die "$0: $self->{content_model} in tag open";          die "$0: $self->{content_model} in tag open";
404        }        }
405      } elsif ($self->{state} eq 'close tag open') {      } elsif ($self->{state} == CLOSE_TAG_OPEN_STATE) {
406        if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA        if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
407          if (defined $self->{last_emitted_start_tag_name}) {          if (defined $self->{last_emitted_start_tag_name}) {
408            ## NOTE: <http://krijnhoetmer.nl/irc-logs/whatwg/20070626#l-564>            ## NOTE: <http://krijnhoetmer.nl/irc-logs/whatwg/20070626#l-564>
# Line 383  sub _get_next_token ($) { Line 417  sub _get_next_token ($) {
417              } else {              } else {
418                $self->{next_input_character} = shift @next_char; # reconsume                $self->{next_input_character} = shift @next_char; # reconsume
419                !!!back-next-input-character (@next_char);                !!!back-next-input-character (@next_char);
420                $self->{state} = 'data';                $self->{state} = DATA_STATE;
421    
422                !!!emit ({type => CHARACTER_TOKEN, data => '</'});                !!!emit ({type => CHARACTER_TOKEN, data => '</'});
423        
# Line 402  sub _get_next_token ($) { Line 436  sub _get_next_token ($) {
436                    $self->{next_input_character} == -1) {                    $self->{next_input_character} == -1) {
437              $self->{next_input_character} = shift @next_char; # reconsume              $self->{next_input_character} = shift @next_char; # reconsume
438              !!!back-next-input-character (@next_char);              !!!back-next-input-character (@next_char);
439              $self->{state} = 'data';              $self->{state} = DATA_STATE;
440              !!!emit ({type => CHARACTER_TOKEN, data => '</'});              !!!emit ({type => CHARACTER_TOKEN, data => '</'});
441              redo A;              redo A;
442            } else {            } else {
# Line 413  sub _get_next_token ($) { Line 447  sub _get_next_token ($) {
447          } else {          } else {
448            ## No start tag token has ever been emitted            ## No start tag token has ever been emitted
449            # next-input-character is already done            # next-input-character is already done
450            $self->{state} = 'data';            $self->{state} = DATA_STATE;
451            !!!emit ({type => CHARACTER_TOKEN, data => '</'});            !!!emit ({type => CHARACTER_TOKEN, data => '</'});
452            redo A;            redo A;
453          }          }
# Line 423  sub _get_next_token ($) { Line 457  sub _get_next_token ($) {
457            $self->{next_input_character} <= 0x005A) { # A..Z            $self->{next_input_character} <= 0x005A) { # A..Z
458          $self->{current_token} = {type => END_TAG_TOKEN,          $self->{current_token} = {type => END_TAG_TOKEN,
459                            tag_name => chr ($self->{next_input_character} + 0x0020)};                            tag_name => chr ($self->{next_input_character} + 0x0020)};
460          $self->{state} = 'tag name';          $self->{state} = TAG_NAME_STATE;
461          !!!next-input-character;          !!!next-input-character;
462          redo A;          redo A;
463        } elsif (0x0061 <= $self->{next_input_character} and        } elsif (0x0061 <= $self->{next_input_character} and
464                 $self->{next_input_character} <= 0x007A) { # a..z                 $self->{next_input_character} <= 0x007A) { # a..z
465          $self->{current_token} = {type => END_TAG_TOKEN,          $self->{current_token} = {type => END_TAG_TOKEN,
466                            tag_name => chr ($self->{next_input_character})};                            tag_name => chr ($self->{next_input_character})};
467          $self->{state} = 'tag name';          $self->{state} = TAG_NAME_STATE;
468          !!!next-input-character;          !!!next-input-character;
469          redo A;          redo A;
470        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
471          !!!parse-error (type => 'empty end tag');          !!!parse-error (type => 'empty end tag');
472          $self->{state} = 'data';          $self->{state} = DATA_STATE;
473          !!!next-input-character;          !!!next-input-character;
474          redo A;          redo A;
475        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
476          !!!parse-error (type => 'bare etago');          !!!parse-error (type => 'bare etago');
477          $self->{state} = 'data';          $self->{state} = DATA_STATE;
478          # reconsume          # reconsume
479    
480          !!!emit ({type => CHARACTER_TOKEN, data => '</'});          !!!emit ({type => CHARACTER_TOKEN, data => '</'});
# Line 448  sub _get_next_token ($) { Line 482  sub _get_next_token ($) {
482          redo A;          redo A;
483        } else {        } else {
484          !!!parse-error (type => 'bogus end tag');          !!!parse-error (type => 'bogus end tag');
485          $self->{state} = 'bogus comment';          $self->{state} = BOGUS_COMMENT_STATE;
486          ## $self->{next_input_character} is intentionally left as is          ## $self->{next_input_character} is intentionally left as is
487          redo A;          redo A;
488        }        }
489      } elsif ($self->{state} eq 'tag name') {      } elsif ($self->{state} == TAG_NAME_STATE) {
490        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
491            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
492            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
493            $self->{next_input_character} == 0x000C or # FF            $self->{next_input_character} == 0x000C or # FF
494            $self->{next_input_character} == 0x0020) { # SP            $self->{next_input_character} == 0x0020) { # SP
495          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
496          !!!next-input-character;          !!!next-input-character;
497          redo A;          redo A;
498        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
# Line 474  sub _get_next_token ($) { Line 508  sub _get_next_token ($) {
508          } else {          } else {
509            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
510          }          }
511          $self->{state} = 'data';          $self->{state} = DATA_STATE;
512          !!!next-input-character;          !!!next-input-character;
513    
514          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 501  sub _get_next_token ($) { Line 535  sub _get_next_token ($) {
535          } else {          } else {
536            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
537          }          }
538          $self->{state} = 'data';          $self->{state} = DATA_STATE;
539          # reconsume          # reconsume
540    
541          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 517  sub _get_next_token ($) { Line 551  sub _get_next_token ($) {
551          } else {          } else {
552            !!!parse-error (type => 'nestc');            !!!parse-error (type => 'nestc');
553          }          }
554          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
555          # next-input-character is already done          # next-input-character is already done
556          redo A;          redo A;
557        } else {        } else {
# Line 527  sub _get_next_token ($) { Line 561  sub _get_next_token ($) {
561          !!!next-input-character;          !!!next-input-character;
562          redo A;          redo A;
563        }        }
564      } elsif ($self->{state} eq 'before attribute name') {      } elsif ($self->{state} == BEFORE_ATTRIBUTE_NAME_STATE) {
565        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
566            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
567            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
# Line 549  sub _get_next_token ($) { Line 583  sub _get_next_token ($) {
583          } else {          } else {
584            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
585          }          }
586          $self->{state} = 'data';          $self->{state} = DATA_STATE;
587          !!!next-input-character;          !!!next-input-character;
588    
589          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 559  sub _get_next_token ($) { Line 593  sub _get_next_token ($) {
593                 $self->{next_input_character} <= 0x005A) { # A..Z                 $self->{next_input_character} <= 0x005A) { # A..Z
594          $self->{current_attribute} = {name => chr ($self->{next_input_character} + 0x0020),          $self->{current_attribute} = {name => chr ($self->{next_input_character} + 0x0020),
595                                value => ''};                                value => ''};
596          $self->{state} = 'attribute name';          $self->{state} = ATTRIBUTE_NAME_STATE;
597          !!!next-input-character;          !!!next-input-character;
598          redo A;          redo A;
599        } elsif ($self->{next_input_character} == 0x002F) { # /        } elsif ($self->{next_input_character} == 0x002F) { # /
# Line 589  sub _get_next_token ($) { Line 623  sub _get_next_token ($) {
623          } else {          } else {
624            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
625          }          }
626          $self->{state} = 'data';          $self->{state} = DATA_STATE;
627          # reconsume          # reconsume
628    
629          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 598  sub _get_next_token ($) { Line 632  sub _get_next_token ($) {
632        } else {        } else {
633          $self->{current_attribute} = {name => chr ($self->{next_input_character}),          $self->{current_attribute} = {name => chr ($self->{next_input_character}),
634                                value => ''};                                value => ''};
635          $self->{state} = 'attribute name';          $self->{state} = ATTRIBUTE_NAME_STATE;
636          !!!next-input-character;          !!!next-input-character;
637          redo A;          redo A;
638        }        }
639      } elsif ($self->{state} eq 'attribute name') {      } elsif ($self->{state} == ATTRIBUTE_NAME_STATE) {
640        my $before_leave = sub {        my $before_leave = sub {
641          if (exists $self->{current_token}->{attributes} # start tag or end tag          if (exists $self->{current_token}->{attributes} # start tag or end tag
642              ->{$self->{current_attribute}->{name}}) { # MUST              ->{$self->{current_attribute}->{name}}) { # MUST
# Line 620  sub _get_next_token ($) { Line 654  sub _get_next_token ($) {
654            $self->{next_input_character} == 0x000C or # FF            $self->{next_input_character} == 0x000C or # FF
655            $self->{next_input_character} == 0x0020) { # SP            $self->{next_input_character} == 0x0020) { # SP
656          $before_leave->();          $before_leave->();
657          $self->{state} = 'after attribute name';          $self->{state} = AFTER_ATTRIBUTE_NAME_STATE;
658          !!!next-input-character;          !!!next-input-character;
659          redo A;          redo A;
660        } elsif ($self->{next_input_character} == 0x003D) { # =        } elsif ($self->{next_input_character} == 0x003D) { # =
661          $before_leave->();          $before_leave->();
662          $self->{state} = 'before attribute value';          $self->{state} = BEFORE_ATTRIBUTE_VALUE_STATE;
663          !!!next-input-character;          !!!next-input-character;
664          redo A;          redo A;
665        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
# Line 642  sub _get_next_token ($) { Line 676  sub _get_next_token ($) {
676          } else {          } else {
677            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
678          }          }
679          $self->{state} = 'data';          $self->{state} = DATA_STATE;
680          !!!next-input-character;          !!!next-input-character;
681    
682          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 665  sub _get_next_token ($) { Line 699  sub _get_next_token ($) {
699          } else {          } else {
700            !!!parse-error (type => 'nestc');            !!!parse-error (type => 'nestc');
701          }          }
702          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
703          # next-input-character is already done          # next-input-character is already done
704          redo A;          redo A;
705        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
# Line 683  sub _get_next_token ($) { Line 717  sub _get_next_token ($) {
717          } else {          } else {
718            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
719          }          }
720          $self->{state} = 'data';          $self->{state} = DATA_STATE;
721          # reconsume          # reconsume
722    
723          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 695  sub _get_next_token ($) { Line 729  sub _get_next_token ($) {
729          !!!next-input-character;          !!!next-input-character;
730          redo A;          redo A;
731        }        }
732      } elsif ($self->{state} eq 'after attribute name') {      } elsif ($self->{state} == AFTER_ATTRIBUTE_NAME_STATE) {
733        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
734            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
735            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
# Line 705  sub _get_next_token ($) { Line 739  sub _get_next_token ($) {
739          !!!next-input-character;          !!!next-input-character;
740          redo A;          redo A;
741        } elsif ($self->{next_input_character} == 0x003D) { # =        } elsif ($self->{next_input_character} == 0x003D) { # =
742          $self->{state} = 'before attribute value';          $self->{state} = BEFORE_ATTRIBUTE_VALUE_STATE;
743          !!!next-input-character;          !!!next-input-character;
744          redo A;          redo A;
745        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
# Line 721  sub _get_next_token ($) { Line 755  sub _get_next_token ($) {
755          } else {          } else {
756            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
757          }          }
758          $self->{state} = 'data';          $self->{state} = DATA_STATE;
759          !!!next-input-character;          !!!next-input-character;
760    
761          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 731  sub _get_next_token ($) { Line 765  sub _get_next_token ($) {
765                 $self->{next_input_character} <= 0x005A) { # A..Z                 $self->{next_input_character} <= 0x005A) { # A..Z
766          $self->{current_attribute} = {name => chr ($self->{next_input_character} + 0x0020),          $self->{current_attribute} = {name => chr ($self->{next_input_character} + 0x0020),
767                                value => ''};                                value => ''};
768          $self->{state} = 'attribute name';          $self->{state} = ATTRIBUTE_NAME_STATE;
769          !!!next-input-character;          !!!next-input-character;
770          redo A;          redo A;
771        } elsif ($self->{next_input_character} == 0x002F) { # /        } elsif ($self->{next_input_character} == 0x002F) { # /
# Line 745  sub _get_next_token ($) { Line 779  sub _get_next_token ($) {
779            !!!parse-error (type => 'nestc');            !!!parse-error (type => 'nestc');
780            ## TODO: Different error type for <aa / bb> than <aa/>            ## TODO: Different error type for <aa / bb> than <aa/>
781          }          }
782          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
783          # next-input-character is already done          # next-input-character is already done
784          redo A;          redo A;
785        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
# Line 762  sub _get_next_token ($) { Line 796  sub _get_next_token ($) {
796          } else {          } else {
797            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
798          }          }
799          $self->{state} = 'data';          $self->{state} = DATA_STATE;
800          # reconsume          # reconsume
801    
802          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 771  sub _get_next_token ($) { Line 805  sub _get_next_token ($) {
805        } else {        } else {
806          $self->{current_attribute} = {name => chr ($self->{next_input_character}),          $self->{current_attribute} = {name => chr ($self->{next_input_character}),
807                                value => ''};                                value => ''};
808          $self->{state} = 'attribute name';          $self->{state} = ATTRIBUTE_NAME_STATE;
809          !!!next-input-character;          !!!next-input-character;
810          redo A;                  redo A;        
811        }        }
812      } elsif ($self->{state} eq 'before attribute value') {      } elsif ($self->{state} == BEFORE_ATTRIBUTE_VALUE_STATE) {
813        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
814            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
815            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
# Line 785  sub _get_next_token ($) { Line 819  sub _get_next_token ($) {
819          !!!next-input-character;          !!!next-input-character;
820          redo A;          redo A;
821        } elsif ($self->{next_input_character} == 0x0022) { # "        } elsif ($self->{next_input_character} == 0x0022) { # "
822          $self->{state} = 'attribute value (double-quoted)';          $self->{state} = ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE;
823          !!!next-input-character;          !!!next-input-character;
824          redo A;          redo A;
825        } elsif ($self->{next_input_character} == 0x0026) { # &        } elsif ($self->{next_input_character} == 0x0026) { # &
826          $self->{state} = 'attribute value (unquoted)';          $self->{state} = ATTRIBUTE_VALUE_UNQUOTED_STATE;
827          ## reconsume          ## reconsume
828          redo A;          redo A;
829        } elsif ($self->{next_input_character} == 0x0027) { # '        } elsif ($self->{next_input_character} == 0x0027) { # '
830          $self->{state} = 'attribute value (single-quoted)';          $self->{state} = ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE;
831          !!!next-input-character;          !!!next-input-character;
832          redo A;          redo A;
833        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
# Line 809  sub _get_next_token ($) { Line 843  sub _get_next_token ($) {
843          } else {          } else {
844            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
845          }          }
846          $self->{state} = 'data';          $self->{state} = DATA_STATE;
847          !!!next-input-character;          !!!next-input-character;
848    
849          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 829  sub _get_next_token ($) { Line 863  sub _get_next_token ($) {
863          } else {          } else {
864            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
865          }          }
866          $self->{state} = 'data';          $self->{state} = DATA_STATE;
867          ## reconsume          ## reconsume
868    
869          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 837  sub _get_next_token ($) { Line 871  sub _get_next_token ($) {
871          redo A;          redo A;
872        } else {        } else {
873          $self->{current_attribute}->{value} .= chr ($self->{next_input_character});          $self->{current_attribute}->{value} .= chr ($self->{next_input_character});
874          $self->{state} = 'attribute value (unquoted)';          $self->{state} = ATTRIBUTE_VALUE_UNQUOTED_STATE;
875          !!!next-input-character;          !!!next-input-character;
876          redo A;          redo A;
877        }        }
878      } elsif ($self->{state} eq 'attribute value (double-quoted)') {      } elsif ($self->{state} == ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE) {
879        if ($self->{next_input_character} == 0x0022) { # "        if ($self->{next_input_character} == 0x0022) { # "
880          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
881          !!!next-input-character;          !!!next-input-character;
882          redo A;          redo A;
883        } elsif ($self->{next_input_character} == 0x0026) { # &        } elsif ($self->{next_input_character} == 0x0026) { # &
884          $self->{last_attribute_value_state} = 'attribute value (double-quoted)';          $self->{last_attribute_value_state} = $self->{state};
885          $self->{state} = 'entity in attribute value';          $self->{state} = ENTITY_IN_ATTRIBUTE_VALUE_STATE;
886          !!!next-input-character;          !!!next-input-character;
887          redo A;          redo A;
888        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
# Line 865  sub _get_next_token ($) { Line 899  sub _get_next_token ($) {
899          } else {          } else {
900            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
901          }          }
902          $self->{state} = 'data';          $self->{state} = DATA_STATE;
903          ## reconsume          ## reconsume
904    
905          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 877  sub _get_next_token ($) { Line 911  sub _get_next_token ($) {
911          !!!next-input-character;          !!!next-input-character;
912          redo A;          redo A;
913        }        }
914      } elsif ($self->{state} eq 'attribute value (single-quoted)') {      } elsif ($self->{state} == ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE) {
915        if ($self->{next_input_character} == 0x0027) { # '        if ($self->{next_input_character} == 0x0027) { # '
916          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
917          !!!next-input-character;          !!!next-input-character;
918          redo A;          redo A;
919        } elsif ($self->{next_input_character} == 0x0026) { # &        } elsif ($self->{next_input_character} == 0x0026) { # &
920          $self->{last_attribute_value_state} = 'attribute value (single-quoted)';          $self->{last_attribute_value_state} = $self->{state};
921          $self->{state} = 'entity in attribute value';          $self->{state} = ENTITY_IN_ATTRIBUTE_VALUE_STATE;
922          !!!next-input-character;          !!!next-input-character;
923          redo A;          redo A;
924        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
# Line 901  sub _get_next_token ($) { Line 935  sub _get_next_token ($) {
935          } else {          } else {
936            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
937          }          }
938          $self->{state} = 'data';          $self->{state} = DATA_STATE;
939          ## reconsume          ## reconsume
940    
941          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 913  sub _get_next_token ($) { Line 947  sub _get_next_token ($) {
947          !!!next-input-character;          !!!next-input-character;
948          redo A;          redo A;
949        }        }
950      } elsif ($self->{state} eq 'attribute value (unquoted)') {      } elsif ($self->{state} == ATTRIBUTE_VALUE_UNQUOTED_STATE) {
951        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
952            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
953            $self->{next_input_character} == 0x000B or # HT            $self->{next_input_character} == 0x000B or # HT
954            $self->{next_input_character} == 0x000C or # FF            $self->{next_input_character} == 0x000C or # FF
955            $self->{next_input_character} == 0x0020) { # SP            $self->{next_input_character} == 0x0020) { # SP
956          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
957          !!!next-input-character;          !!!next-input-character;
958          redo A;          redo A;
959        } elsif ($self->{next_input_character} == 0x0026) { # &        } elsif ($self->{next_input_character} == 0x0026) { # &
960          $self->{last_attribute_value_state} = 'attribute value (unquoted)';          $self->{last_attribute_value_state} = $self->{state};
961          $self->{state} = 'entity in attribute value';          $self->{state} = ENTITY_IN_ATTRIBUTE_VALUE_STATE;
962          !!!next-input-character;          !!!next-input-character;
963          redo A;          redo A;
964        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
# Line 940  sub _get_next_token ($) { Line 974  sub _get_next_token ($) {
974          } else {          } else {
975            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
976          }          }
977          $self->{state} = 'data';          $self->{state} = DATA_STATE;
978          !!!next-input-character;          !!!next-input-character;
979    
980          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 960  sub _get_next_token ($) { Line 994  sub _get_next_token ($) {
994          } else {          } else {
995            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
996          }          }
997          $self->{state} = 'data';          $self->{state} = DATA_STATE;
998          ## reconsume          ## reconsume
999    
1000          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 972  sub _get_next_token ($) { Line 1006  sub _get_next_token ($) {
1006          !!!next-input-character;          !!!next-input-character;
1007          redo A;          redo A;
1008        }        }
1009      } elsif ($self->{state} eq 'entity in attribute value') {      } elsif ($self->{state} == ENTITY_IN_ATTRIBUTE_VALUE_STATE) {
1010        my $token = $self->_tokenize_attempt_to_consume_an_entity (1);        my $token = $self->_tokenize_attempt_to_consume_an_entity (1);
1011    
1012        unless (defined $token) {        unless (defined $token) {
# Line 985  sub _get_next_token ($) { Line 1019  sub _get_next_token ($) {
1019        $self->{state} = $self->{last_attribute_value_state};        $self->{state} = $self->{last_attribute_value_state};
1020        # next-input-character is already done        # next-input-character is already done
1021        redo A;        redo A;
1022      } elsif ($self->{state} eq 'bogus comment') {      } elsif ($self->{state} == BOGUS_COMMENT_STATE) {
1023        ## (only happen if PCDATA state)        ## (only happen if PCDATA state)
1024                
1025        my $token = {type => COMMENT_TOKEN, data => ''};        my $token = {type => COMMENT_TOKEN, data => ''};
1026    
1027        BC: {        BC: {
1028          if ($self->{next_input_character} == 0x003E) { # >          if ($self->{next_input_character} == 0x003E) { # >
1029            $self->{state} = 'data';            $self->{state} = DATA_STATE;
1030            !!!next-input-character;            !!!next-input-character;
1031    
1032            !!!emit ($token);            !!!emit ($token);
1033    
1034            redo A;            redo A;
1035          } elsif ($self->{next_input_character} == -1) {          } elsif ($self->{next_input_character} == -1) {
1036            $self->{state} = 'data';            $self->{state} = DATA_STATE;
1037            ## reconsume            ## reconsume
1038    
1039            !!!emit ($token);            !!!emit ($token);
# Line 1011  sub _get_next_token ($) { Line 1045  sub _get_next_token ($) {
1045            redo BC;            redo BC;
1046          }          }
1047        } # BC        } # BC
1048      } elsif ($self->{state} eq 'markup declaration open') {      } elsif ($self->{state} == MARKUP_DECLARATION_OPEN_STATE) {
1049        ## (only happen if PCDATA state)        ## (only happen if PCDATA state)
1050    
1051        my @next_char;        my @next_char;
# Line 1022  sub _get_next_token ($) { Line 1056  sub _get_next_token ($) {
1056          push @next_char, $self->{next_input_character};          push @next_char, $self->{next_input_character};
1057          if ($self->{next_input_character} == 0x002D) { # -          if ($self->{next_input_character} == 0x002D) { # -
1058            $self->{current_token} = {type => COMMENT_TOKEN, data => ''};            $self->{current_token} = {type => COMMENT_TOKEN, data => ''};
1059            $self->{state} = 'comment start';            $self->{state} = COMMENT_START_STATE;
1060            !!!next-input-character;            !!!next-input-character;
1061            redo A;            redo A;
1062          }          }
# Line 1053  sub _get_next_token ($) { Line 1087  sub _get_next_token ($) {
1087                    if ($self->{next_input_character} == 0x0045 or # E                    if ($self->{next_input_character} == 0x0045 or # E
1088                        $self->{next_input_character} == 0x0065) { # e                        $self->{next_input_character} == 0x0065) { # e
1089                      ## ISSUE: What a stupid code this is!                      ## ISSUE: What a stupid code this is!
1090                      $self->{state} = 'DOCTYPE';                      $self->{state} = DOCTYPE_STATE;
1091                      !!!next-input-character;                      !!!next-input-character;
1092                      redo A;                      redo A;
1093                    }                    }
# Line 1067  sub _get_next_token ($) { Line 1101  sub _get_next_token ($) {
1101        !!!parse-error (type => 'bogus comment');        !!!parse-error (type => 'bogus comment');
1102        $self->{next_input_character} = shift @next_char;        $self->{next_input_character} = shift @next_char;
1103        !!!back-next-input-character (@next_char);        !!!back-next-input-character (@next_char);
1104        $self->{state} = 'bogus comment';        $self->{state} = BOGUS_COMMENT_STATE;
1105        redo A;        redo A;
1106                
1107        ## ISSUE: typos in spec: chacacters, is is a parse error        ## ISSUE: typos in spec: chacacters, is is a parse error
1108        ## ISSUE: spec is somewhat unclear on "is the first character that will be in the comment"; what is "that will be in the comment" is what the algorithm defines, isn't it?        ## ISSUE: spec is somewhat unclear on "is the first character that will be in the comment"; what is "that will be in the comment" is what the algorithm defines, isn't it?
1109      } elsif ($self->{state} eq 'comment start') {      } elsif ($self->{state} == COMMENT_START_STATE) {
1110        if ($self->{next_input_character} == 0x002D) { # -        if ($self->{next_input_character} == 0x002D) { # -
1111          $self->{state} = 'comment start dash';          $self->{state} = COMMENT_START_DASH_STATE;
1112          !!!next-input-character;          !!!next-input-character;
1113          redo A;          redo A;
1114        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1115          !!!parse-error (type => 'bogus comment');          !!!parse-error (type => 'bogus comment');
1116          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1117          !!!next-input-character;          !!!next-input-character;
1118    
1119          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1087  sub _get_next_token ($) { Line 1121  sub _get_next_token ($) {
1121          redo A;          redo A;
1122        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1123          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
1124          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1125          ## reconsume          ## reconsume
1126    
1127          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1096  sub _get_next_token ($) { Line 1130  sub _get_next_token ($) {
1130        } else {        } else {
1131          $self->{current_token}->{data} # comment          $self->{current_token}->{data} # comment
1132              .= chr ($self->{next_input_character});              .= chr ($self->{next_input_character});
1133          $self->{state} = 'comment';          $self->{state} = COMMENT_STATE;
1134          !!!next-input-character;          !!!next-input-character;
1135          redo A;          redo A;
1136        }        }
1137      } elsif ($self->{state} eq 'comment start dash') {      } elsif ($self->{state} == COMMENT_START_DASH_STATE) {
1138        if ($self->{next_input_character} == 0x002D) { # -        if ($self->{next_input_character} == 0x002D) { # -
1139          $self->{state} = 'comment end';          $self->{state} = COMMENT_END_STATE;
1140          !!!next-input-character;          !!!next-input-character;
1141          redo A;          redo A;
1142        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1143          !!!parse-error (type => 'bogus comment');          !!!parse-error (type => 'bogus comment');
1144          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1145          !!!next-input-character;          !!!next-input-character;
1146    
1147          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1115  sub _get_next_token ($) { Line 1149  sub _get_next_token ($) {
1149          redo A;          redo A;
1150        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1151          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
1152          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1153          ## reconsume          ## reconsume
1154    
1155          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1124  sub _get_next_token ($) { Line 1158  sub _get_next_token ($) {
1158        } else {        } else {
1159          $self->{current_token}->{data} # comment          $self->{current_token}->{data} # comment
1160              .= '-' . chr ($self->{next_input_character});              .= '-' . chr ($self->{next_input_character});
1161          $self->{state} = 'comment';          $self->{state} = COMMENT_STATE;
1162          !!!next-input-character;          !!!next-input-character;
1163          redo A;          redo A;
1164        }        }
1165      } elsif ($self->{state} eq 'comment') {      } elsif ($self->{state} == COMMENT_STATE) {
1166        if ($self->{next_input_character} == 0x002D) { # -        if ($self->{next_input_character} == 0x002D) { # -
1167          $self->{state} = 'comment end dash';          $self->{state} = COMMENT_END_DASH_STATE;
1168          !!!next-input-character;          !!!next-input-character;
1169          redo A;          redo A;
1170        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1171          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
1172          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1173          ## reconsume          ## reconsume
1174    
1175          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1147  sub _get_next_token ($) { Line 1181  sub _get_next_token ($) {
1181          !!!next-input-character;          !!!next-input-character;
1182          redo A;          redo A;
1183        }        }
1184      } elsif ($self->{state} eq 'comment end dash') {      } elsif ($self->{state} == COMMENT_END_DASH_STATE) {
1185        if ($self->{next_input_character} == 0x002D) { # -        if ($self->{next_input_character} == 0x002D) { # -
1186          $self->{state} = 'comment end';          $self->{state} = COMMENT_END_STATE;
1187          !!!next-input-character;          !!!next-input-character;
1188          redo A;          redo A;
1189        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1190          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
1191          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1192          ## reconsume          ## reconsume
1193    
1194          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1162  sub _get_next_token ($) { Line 1196  sub _get_next_token ($) {
1196          redo A;          redo A;
1197        } else {        } else {
1198          $self->{current_token}->{data} .= '-' . chr ($self->{next_input_character}); # comment          $self->{current_token}->{data} .= '-' . chr ($self->{next_input_character}); # comment
1199          $self->{state} = 'comment';          $self->{state} = COMMENT_STATE;
1200          !!!next-input-character;          !!!next-input-character;
1201          redo A;          redo A;
1202        }        }
1203      } elsif ($self->{state} eq 'comment end') {      } elsif ($self->{state} == COMMENT_END_STATE) {
1204        if ($self->{next_input_character} == 0x003E) { # >        if ($self->{next_input_character} == 0x003E) { # >
1205          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1206          !!!next-input-character;          !!!next-input-character;
1207    
1208          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1182  sub _get_next_token ($) { Line 1216  sub _get_next_token ($) {
1216          redo A;          redo A;
1217        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1218          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
1219          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1220          ## reconsume          ## reconsume
1221    
1222          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1191  sub _get_next_token ($) { Line 1225  sub _get_next_token ($) {
1225        } else {        } else {
1226          !!!parse-error (type => 'dash in comment');          !!!parse-error (type => 'dash in comment');
1227          $self->{current_token}->{data} .= '--' . chr ($self->{next_input_character}); # comment          $self->{current_token}->{data} .= '--' . chr ($self->{next_input_character}); # comment
1228          $self->{state} = 'comment';          $self->{state} = COMMENT_STATE;
1229          !!!next-input-character;          !!!next-input-character;
1230          redo A;          redo A;
1231        }        }
1232      } elsif ($self->{state} eq 'DOCTYPE') {      } elsif ($self->{state} == DOCTYPE_STATE) {
1233        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
1234            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
1235            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
1236            $self->{next_input_character} == 0x000C or # FF            $self->{next_input_character} == 0x000C or # FF
1237            $self->{next_input_character} == 0x0020) { # SP            $self->{next_input_character} == 0x0020) { # SP
1238          $self->{state} = 'before DOCTYPE name';          $self->{state} = BEFORE_DOCTYPE_NAME_STATE;
1239          !!!next-input-character;          !!!next-input-character;
1240          redo A;          redo A;
1241        } else {        } else {
1242          !!!parse-error (type => 'no space before DOCTYPE name');          !!!parse-error (type => 'no space before DOCTYPE name');
1243          $self->{state} = 'before DOCTYPE name';          $self->{state} = BEFORE_DOCTYPE_NAME_STATE;
1244          ## reconsume          ## reconsume
1245          redo A;          redo A;
1246        }        }
1247      } elsif ($self->{state} eq 'before DOCTYPE name') {      } elsif ($self->{state} == BEFORE_DOCTYPE_NAME_STATE) {
1248        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
1249            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
1250            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
# Line 1221  sub _get_next_token ($) { Line 1255  sub _get_next_token ($) {
1255          redo A;          redo A;
1256        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1257          !!!parse-error (type => 'no DOCTYPE name');          !!!parse-error (type => 'no DOCTYPE name');
1258          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1259          !!!next-input-character;          !!!next-input-character;
1260    
1261          !!!emit ({type => DOCTYPE_TOKEN}); # incorrect          !!!emit ({type => DOCTYPE_TOKEN}); # incorrect
# Line 1229  sub _get_next_token ($) { Line 1263  sub _get_next_token ($) {
1263          redo A;          redo A;
1264        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1265          !!!parse-error (type => 'no DOCTYPE name');          !!!parse-error (type => 'no DOCTYPE name');
1266          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1267          ## reconsume          ## reconsume
1268    
1269          !!!emit ({type => DOCTYPE_TOKEN}); # incorrect          !!!emit ({type => DOCTYPE_TOKEN}); # incorrect
# Line 1241  sub _get_next_token ($) { Line 1275  sub _get_next_token ($) {
1275                 name => chr ($self->{next_input_character}),                 name => chr ($self->{next_input_character}),
1276                 correct => 1};                 correct => 1};
1277  ## ISSUE: "Set the token's name name to the" in the spec  ## ISSUE: "Set the token's name name to the" in the spec
1278          $self->{state} = 'DOCTYPE name';          $self->{state} = DOCTYPE_NAME_STATE;
1279          !!!next-input-character;          !!!next-input-character;
1280          redo A;          redo A;
1281        }        }
1282      } elsif ($self->{state} eq 'DOCTYPE name') {      } elsif ($self->{state} == DOCTYPE_NAME_STATE) {
1283  ## ISSUE: Redundant "First," in the spec.  ## ISSUE: Redundant "First," in the spec.
1284        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
1285            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
1286            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
1287            $self->{next_input_character} == 0x000C or # FF            $self->{next_input_character} == 0x000C or # FF
1288            $self->{next_input_character} == 0x0020) { # SP            $self->{next_input_character} == 0x0020) { # SP
1289          $self->{state} = 'after DOCTYPE name';          $self->{state} = AFTER_DOCTYPE_NAME_STATE;
1290          !!!next-input-character;          !!!next-input-character;
1291          redo A;          redo A;
1292        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1293          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1294          !!!next-input-character;          !!!next-input-character;
1295    
1296          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{current_token}); # DOCTYPE
# Line 1264  sub _get_next_token ($) { Line 1298  sub _get_next_token ($) {
1298          redo A;          redo A;
1299        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1300          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1301          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1302          ## reconsume          ## reconsume
1303    
1304          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1278  sub _get_next_token ($) { Line 1312  sub _get_next_token ($) {
1312          !!!next-input-character;          !!!next-input-character;
1313          redo A;          redo A;
1314        }        }
1315      } elsif ($self->{state} eq 'after DOCTYPE name') {      } elsif ($self->{state} == AFTER_DOCTYPE_NAME_STATE) {
1316        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
1317            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
1318            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
# Line 1288  sub _get_next_token ($) { Line 1322  sub _get_next_token ($) {
1322          !!!next-input-character;          !!!next-input-character;
1323          redo A;          redo A;
1324        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1325          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1326          !!!next-input-character;          !!!next-input-character;
1327    
1328          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{current_token}); # DOCTYPE
# Line 1296  sub _get_next_token ($) { Line 1330  sub _get_next_token ($) {
1330          redo A;          redo A;
1331        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1332          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1333          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1334          ## reconsume          ## reconsume
1335    
1336          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1320  sub _get_next_token ($) { Line 1354  sub _get_next_token ($) {
1354                  !!!next-input-character;                  !!!next-input-character;
1355                  if ($self->{next_input_character} == 0x0043 or # C                  if ($self->{next_input_character} == 0x0043 or # C
1356                      $self->{next_input_character} == 0x0063) { # c                      $self->{next_input_character} == 0x0063) { # c
1357                    $self->{state} = 'before DOCTYPE public identifier';                    $self->{state} = BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
1358                    !!!next-input-character;                    !!!next-input-character;
1359                    redo A;                    redo A;
1360                  }                  }
# Line 1347  sub _get_next_token ($) { Line 1381  sub _get_next_token ($) {
1381                  !!!next-input-character;                  !!!next-input-character;
1382                  if ($self->{next_input_character} == 0x004D or # M                  if ($self->{next_input_character} == 0x004D or # M
1383                      $self->{next_input_character} == 0x006D) { # m                      $self->{next_input_character} == 0x006D) { # m
1384                    $self->{state} = 'before DOCTYPE system identifier';                    $self->{state} = BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
1385                    !!!next-input-character;                    !!!next-input-character;
1386                    redo A;                    redo A;
1387                  }                  }
# Line 1363  sub _get_next_token ($) { Line 1397  sub _get_next_token ($) {
1397        }        }
1398    
1399        !!!parse-error (type => 'string after DOCTYPE name');        !!!parse-error (type => 'string after DOCTYPE name');
1400        $self->{state} = 'bogus DOCTYPE';        $self->{state} = BOGUS_DOCTYPE_STATE;
1401        # next-input-character is already done        # next-input-character is already done
1402        redo A;        redo A;
1403      } elsif ($self->{state} eq 'before DOCTYPE public identifier') {      } elsif ($self->{state} == BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE) {
1404        if ({        if ({
1405              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,
1406              #0x000D => 1, # HT, LF, VT, FF, SP, CR              #0x000D => 1, # HT, LF, VT, FF, SP, CR
# Line 1376  sub _get_next_token ($) { Line 1410  sub _get_next_token ($) {
1410          redo A;          redo A;
1411        } elsif ($self->{next_input_character} eq 0x0022) { # "        } elsif ($self->{next_input_character} eq 0x0022) { # "
1412          $self->{current_token}->{public_identifier} = ''; # DOCTYPE          $self->{current_token}->{public_identifier} = ''; # DOCTYPE
1413          $self->{state} = 'DOCTYPE public identifier (double-quoted)';          $self->{state} = DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE;
1414          !!!next-input-character;          !!!next-input-character;
1415          redo A;          redo A;
1416        } elsif ($self->{next_input_character} eq 0x0027) { # '        } elsif ($self->{next_input_character} eq 0x0027) { # '
1417          $self->{current_token}->{public_identifier} = ''; # DOCTYPE          $self->{current_token}->{public_identifier} = ''; # DOCTYPE
1418          $self->{state} = 'DOCTYPE public identifier (single-quoted)';          $self->{state} = DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE;
1419          !!!next-input-character;          !!!next-input-character;
1420          redo A;          redo A;
1421        } elsif ($self->{next_input_character} eq 0x003E) { # >        } elsif ($self->{next_input_character} eq 0x003E) { # >
1422          !!!parse-error (type => 'no PUBLIC literal');          !!!parse-error (type => 'no PUBLIC literal');
1423    
1424          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1425          !!!next-input-character;          !!!next-input-character;
1426    
1427          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1397  sub _get_next_token ($) { Line 1431  sub _get_next_token ($) {
1431        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1432          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1433    
1434          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1435          ## reconsume          ## reconsume
1436    
1437          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1406  sub _get_next_token ($) { Line 1440  sub _get_next_token ($) {
1440          redo A;          redo A;
1441        } else {        } else {
1442          !!!parse-error (type => 'string after PUBLIC');          !!!parse-error (type => 'string after PUBLIC');
1443          $self->{state} = 'bogus DOCTYPE';          $self->{state} = BOGUS_DOCTYPE_STATE;
1444          !!!next-input-character;          !!!next-input-character;
1445          redo A;          redo A;
1446        }        }
1447      } elsif ($self->{state} eq 'DOCTYPE public identifier (double-quoted)') {      } elsif ($self->{state} == DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE) {
1448        if ($self->{next_input_character} == 0x0022) { # "        if ($self->{next_input_character} == 0x0022) { # "
1449          $self->{state} = 'after DOCTYPE public identifier';          $self->{state} = AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
1450          !!!next-input-character;          !!!next-input-character;
1451          redo A;          redo A;
1452        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1453          !!!parse-error (type => 'unclosed PUBLIC literal');          !!!parse-error (type => 'unclosed PUBLIC literal');
1454    
1455          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1456          ## reconsume          ## reconsume
1457    
1458          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1432  sub _get_next_token ($) { Line 1466  sub _get_next_token ($) {
1466          !!!next-input-character;          !!!next-input-character;
1467          redo A;          redo A;
1468        }        }
1469      } elsif ($self->{state} eq 'DOCTYPE public identifier (single-quoted)') {      } elsif ($self->{state} == DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE) {
1470        if ($self->{next_input_character} == 0x0027) { # '        if ($self->{next_input_character} == 0x0027) { # '
1471          $self->{state} = 'after DOCTYPE public identifier';          $self->{state} = AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
1472          !!!next-input-character;          !!!next-input-character;
1473          redo A;          redo A;
1474        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1475          !!!parse-error (type => 'unclosed PUBLIC literal');          !!!parse-error (type => 'unclosed PUBLIC literal');
1476    
1477          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1478          ## reconsume          ## reconsume
1479    
1480          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1454  sub _get_next_token ($) { Line 1488  sub _get_next_token ($) {
1488          !!!next-input-character;          !!!next-input-character;
1489          redo A;          redo A;
1490        }        }
1491      } elsif ($self->{state} eq 'after DOCTYPE public identifier') {      } elsif ($self->{state} == AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE) {
1492        if ({        if ({
1493              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,
1494              #0x000D => 1, # HT, LF, VT, FF, SP, CR              #0x000D => 1, # HT, LF, VT, FF, SP, CR
# Line 1464  sub _get_next_token ($) { Line 1498  sub _get_next_token ($) {
1498          redo A;          redo A;
1499        } elsif ($self->{next_input_character} == 0x0022) { # "        } elsif ($self->{next_input_character} == 0x0022) { # "
1500          $self->{current_token}->{system_identifier} = ''; # DOCTYPE          $self->{current_token}->{system_identifier} = ''; # DOCTYPE
1501          $self->{state} = 'DOCTYPE system identifier (double-quoted)';          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE;
1502          !!!next-input-character;          !!!next-input-character;
1503          redo A;          redo A;
1504        } elsif ($self->{next_input_character} == 0x0027) { # '        } elsif ($self->{next_input_character} == 0x0027) { # '
1505          $self->{current_token}->{system_identifier} = ''; # DOCTYPE          $self->{current_token}->{system_identifier} = ''; # DOCTYPE
1506          $self->{state} = 'DOCTYPE system identifier (single-quoted)';          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE;
1507          !!!next-input-character;          !!!next-input-character;
1508          redo A;          redo A;
1509        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1510          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1511          !!!next-input-character;          !!!next-input-character;
1512    
1513          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{current_token}); # DOCTYPE
# Line 1482  sub _get_next_token ($) { Line 1516  sub _get_next_token ($) {
1516        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1517          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1518    
1519          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1520          ## reconsume          ## reconsume
1521    
1522          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1491  sub _get_next_token ($) { Line 1525  sub _get_next_token ($) {
1525          redo A;          redo A;
1526        } else {        } else {
1527          !!!parse-error (type => 'string after PUBLIC literal');          !!!parse-error (type => 'string after PUBLIC literal');
1528          $self->{state} = 'bogus DOCTYPE';          $self->{state} = BOGUS_DOCTYPE_STATE;
1529          !!!next-input-character;          !!!next-input-character;
1530          redo A;          redo A;
1531        }        }
1532      } elsif ($self->{state} eq 'before DOCTYPE system identifier') {      } elsif ($self->{state} == BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE) {
1533        if ({        if ({
1534              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,
1535              #0x000D => 1, # HT, LF, VT, FF, SP, CR              #0x000D => 1, # HT, LF, VT, FF, SP, CR
# Line 1505  sub _get_next_token ($) { Line 1539  sub _get_next_token ($) {
1539          redo A;          redo A;
1540        } elsif ($self->{next_input_character} == 0x0022) { # "        } elsif ($self->{next_input_character} == 0x0022) { # "
1541          $self->{current_token}->{system_identifier} = ''; # DOCTYPE          $self->{current_token}->{system_identifier} = ''; # DOCTYPE
1542          $self->{state} = 'DOCTYPE system identifier (double-quoted)';          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE;
1543          !!!next-input-character;          !!!next-input-character;
1544          redo A;          redo A;
1545        } elsif ($self->{next_input_character} == 0x0027) { # '        } elsif ($self->{next_input_character} == 0x0027) { # '
1546          $self->{current_token}->{system_identifier} = ''; # DOCTYPE          $self->{current_token}->{system_identifier} = ''; # DOCTYPE
1547          $self->{state} = 'DOCTYPE system identifier (single-quoted)';          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE;
1548          !!!next-input-character;          !!!next-input-character;
1549          redo A;          redo A;
1550        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1551          !!!parse-error (type => 'no SYSTEM literal');          !!!parse-error (type => 'no SYSTEM literal');
1552          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1553          !!!next-input-character;          !!!next-input-character;
1554    
1555          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1525  sub _get_next_token ($) { Line 1559  sub _get_next_token ($) {
1559        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1560          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1561    
1562          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1563          ## reconsume          ## reconsume
1564    
1565          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1534  sub _get_next_token ($) { Line 1568  sub _get_next_token ($) {
1568          redo A;          redo A;
1569        } else {        } else {
1570          !!!parse-error (type => 'string after SYSTEM');          !!!parse-error (type => 'string after SYSTEM');
1571          $self->{state} = 'bogus DOCTYPE';          $self->{state} = BOGUS_DOCTYPE_STATE;
1572          !!!next-input-character;          !!!next-input-character;
1573          redo A;          redo A;
1574        }        }
1575      } elsif ($self->{state} eq 'DOCTYPE system identifier (double-quoted)') {      } elsif ($self->{state} == DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE) {
1576        if ($self->{next_input_character} == 0x0022) { # "        if ($self->{next_input_character} == 0x0022) { # "
1577          $self->{state} = 'after DOCTYPE system identifier';          $self->{state} = AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
1578          !!!next-input-character;          !!!next-input-character;
1579          redo A;          redo A;
1580        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1581          !!!parse-error (type => 'unclosed SYSTEM literal');          !!!parse-error (type => 'unclosed SYSTEM literal');
1582    
1583          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1584          ## reconsume          ## reconsume
1585    
1586          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1560  sub _get_next_token ($) { Line 1594  sub _get_next_token ($) {
1594          !!!next-input-character;          !!!next-input-character;
1595          redo A;          redo A;
1596        }        }
1597      } elsif ($self->{state} eq 'DOCTYPE system identifier (single-quoted)') {      } elsif ($self->{state} == DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE) {
1598        if ($self->{next_input_character} == 0x0027) { # '        if ($self->{next_input_character} == 0x0027) { # '
1599          $self->{state} = 'after DOCTYPE system identifier';          $self->{state} = AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
1600          !!!next-input-character;          !!!next-input-character;
1601          redo A;          redo A;
1602        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1603          !!!parse-error (type => 'unclosed SYSTEM literal');          !!!parse-error (type => 'unclosed SYSTEM literal');
1604    
1605          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1606          ## reconsume          ## reconsume
1607    
1608          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1582  sub _get_next_token ($) { Line 1616  sub _get_next_token ($) {
1616          !!!next-input-character;          !!!next-input-character;
1617          redo A;          redo A;
1618        }        }
1619      } elsif ($self->{state} eq 'after DOCTYPE system identifier') {      } elsif ($self->{state} == AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE) {
1620        if ({        if ({
1621              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,
1622              #0x000D => 1, # HT, LF, VT, FF, SP, CR              #0x000D => 1, # HT, LF, VT, FF, SP, CR
# Line 1591  sub _get_next_token ($) { Line 1625  sub _get_next_token ($) {
1625          !!!next-input-character;          !!!next-input-character;
1626          redo A;          redo A;
1627        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1628          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1629          !!!next-input-character;          !!!next-input-character;
1630    
1631          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{current_token}); # DOCTYPE
# Line 1600  sub _get_next_token ($) { Line 1634  sub _get_next_token ($) {
1634        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1635          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1636    
1637          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1638          ## reconsume          ## reconsume
1639    
1640          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1609  sub _get_next_token ($) { Line 1643  sub _get_next_token ($) {
1643          redo A;          redo A;
1644        } else {        } else {
1645          !!!parse-error (type => 'string after SYSTEM literal');          !!!parse-error (type => 'string after SYSTEM literal');
1646          $self->{state} = 'bogus DOCTYPE';          $self->{state} = BOGUS_DOCTYPE_STATE;
1647          !!!next-input-character;          !!!next-input-character;
1648          redo A;          redo A;
1649        }        }
1650      } elsif ($self->{state} eq 'bogus DOCTYPE') {      } elsif ($self->{state} == BOGUS_DOCTYPE_STATE) {
1651        if ($self->{next_input_character} == 0x003E) { # >        if ($self->{next_input_character} == 0x003E) { # >
1652          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1653          !!!next-input-character;          !!!next-input-character;
1654    
1655          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1624  sub _get_next_token ($) { Line 1658  sub _get_next_token ($) {
1658          redo A;          redo A;
1659        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1660          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1661          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1662          ## reconsume          ## reconsume
1663    
1664          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 2539  sub _tree_construction_main ($) { Line 2573  sub _tree_construction_main ($) {
2573        !!!next-token;        !!!next-token;
2574        redo B;        redo B;
2575      } elsif ($token->{type} == END_OF_FILE_TOKEN) {      } elsif ($token->{type} == END_OF_FILE_TOKEN) {
2576        if ($self->{insertion_mode} == AFTER_HTML_BODY_IM or        if ($self->{insertion_mode} & AFTER_HTML_IMS) {
           $self->{insertion_mode} == AFTER_HTML_FRAMESET_IM) {  
2577          #          #
2578        } else {        } else {
2579          ## Generate implied end tags          ## Generate implied end tags
# Line 2596  sub _tree_construction_main ($) { Line 2629  sub _tree_construction_main ($) {
2629        redo B;        redo B;
2630      } elsif ($token->{type} == COMMENT_TOKEN) {      } elsif ($token->{type} == COMMENT_TOKEN) {
2631        my $comment = $self->{document}->create_comment ($token->{data});        my $comment = $self->{document}->create_comment ($token->{data});
2632        if ($self->{insertion_mode} == AFTER_HTML_BODY_IM or        if ($self->{insertion_mode} & AFTER_HTML_IMS) {
           $self->{insertion_mode} == AFTER_HTML_FRAMESET_IM) {  
2633          $self->{document}->append_child ($comment);          $self->{document}->append_child ($comment);
2634        } elsif ($self->{insertion_mode} == AFTER_BODY_IM) {        } elsif ($self->{insertion_mode} == AFTER_BODY_IM) {
2635          $self->{open_elements}->[0]->[0]->append_child ($comment);          $self->{open_elements}->[0]->[0]->append_child ($comment);
# Line 2606  sub _tree_construction_main ($) { Line 2638  sub _tree_construction_main ($) {
2638        }        }
2639        !!!next-token;        !!!next-token;
2640        redo B;        redo B;
2641      } elsif ($self->{insertion_mode} == IN_HEAD_IM or      } elsif ($self->{insertion_mode} & HEAD_IMS) {
              $self->{insertion_mode} == IN_HEAD_NOSCRIPT_IM or  
              $self->{insertion_mode} == AFTER_HEAD_IM or  
              $self->{insertion_mode} == BEFORE_HEAD_IM) {  
2642        if ($token->{type} == CHARACTER_TOKEN) {        if ($token->{type} == CHARACTER_TOKEN) {
2643          if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {          if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {
2644            $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);            $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);
# Line 2986  sub _tree_construction_main ($) { Line 3015  sub _tree_construction_main ($) {
3015            }            }
3016    
3017            ## ISSUE: An issue in the spec.            ## ISSUE: An issue in the spec.
3018      } elsif ($self->{insertion_mode} == IN_BODY_IM or      } elsif ($self->{insertion_mode} & BODY_IMS) {
              $self->{insertion_mode} == IN_CELL_IM or  
              $self->{insertion_mode} == IN_CAPTION_IM) {  
3019            if ($token->{type} == CHARACTER_TOKEN) {            if ($token->{type} == CHARACTER_TOKEN) {
3020              ## NOTE: There is a code clone of "character in body".              ## NOTE: There is a code clone of "character in body".
3021              $reconstruct_active_formatting_elements->($insert_to_current);              $reconstruct_active_formatting_elements->($insert_to_current);
# Line 3282  sub _tree_construction_main ($) { Line 3309  sub _tree_construction_main ($) {
3309              } elsif ({              } elsif ({
3310                        body => 1, col => 1, colgroup => 1, html => 1,                        body => 1, col => 1, colgroup => 1, html => 1,
3311                       }->{$token->{tag_name}}) {                       }->{$token->{tag_name}}) {
3312                if ($self->{insertion_mode} == IN_CELL_IM or                if ($self->{insertion_mode} & BODY_TABLE_IMS) {
                   $self->{insertion_mode} == IN_CAPTION_IM) {  
3313                  !!!parse-error (type => 'unmatched end tag:'.$token->{tag_name});                  !!!parse-error (type => 'unmatched end tag:'.$token->{tag_name});
3314                  ## Ignore the token                  ## Ignore the token
3315                  !!!next-token;                  !!!next-token;
# Line 3309  sub _tree_construction_main ($) { Line 3335  sub _tree_construction_main ($) {
3335    
3336        $insert = $insert_to_current;        $insert = $insert_to_current;
3337        #        #
3338      } elsif ($self->{insertion_mode} == IN_ROW_IM or      } elsif ($self->{insertion_mode} & TABLE_IMS) {
              $self->{insertion_mode} == IN_TABLE_BODY_IM or  
              $self->{insertion_mode} == IN_TABLE_IM) {  
3339            if ($token->{type} == CHARACTER_TOKEN) {            if ($token->{type} == CHARACTER_TOKEN) {
3340              ## NOTE: There are "character in table" code clones.              ## NOTE: There are "character in table" code clones.
3341              if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {              if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {
# Line 3766  sub _tree_construction_main ($) { Line 3790  sub _tree_construction_main ($) {
3790              } elsif ({              } elsif ({
3791                        tbody => 1, tfoot => 1, thead => 1,                        tbody => 1, tfoot => 1, thead => 1,
3792                       }->{$token->{tag_name}} and                       }->{$token->{tag_name}} and
3793                       ($self->{insertion_mode} == IN_ROW_IM or                       $self->{insertion_mode} & ROW_IMS) {
                       $self->{insertion_mode} == IN_TABLE_BODY_IM)) {  
3794                if ($self->{insertion_mode} == IN_ROW_IM) {                if ($self->{insertion_mode} == IN_ROW_IM) {
3795                  ## have an element in table scope                  ## have an element in table scope
3796                  my $i;                  my $i;
# Line 4107  sub _tree_construction_main ($) { Line 4130  sub _tree_construction_main ($) {
4130            ## Ignore the token            ## Ignore the token
4131            !!!next-token;            !!!next-token;
4132            redo B;            redo B;
4133      } elsif ($self->{insertion_mode} == AFTER_BODY_IM or      } elsif ($self->{insertion_mode} & BODY_AFTER_IMS) {
              $self->{insertion_mode} == AFTER_HTML_BODY_IM) {  
4134        if ($token->{type} == CHARACTER_TOKEN) {        if ($token->{type} == CHARACTER_TOKEN) {
4135          if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {          if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {
4136            my $data = $1;            my $data = $1;
# Line 4178  sub _tree_construction_main ($) { Line 4200  sub _tree_construction_main ($) {
4200        } else {        } else {
4201          die "$0: $token->{type}: Unknown token type";          die "$0: $token->{type}: Unknown token type";
4202        }        }
4203      } elsif ($self->{insertion_mode} == IN_FRAMESET_IM or      } elsif ($self->{insertion_mode} & FRAME_IMS) {
              $self->{insertion_mode} == AFTER_FRAMESET_IM or  
              $self->{insertion_mode} == AFTER_HTML_FRAMESET_IM) {  
4204        if ($token->{type} == CHARACTER_TOKEN) {        if ($token->{type} == CHARACTER_TOKEN) {
4205          if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {          if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {
4206            $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);            $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);
Legend:
Removed from v.1.55  
changed lines
  Added in v.1.57
[email protected]
 ViewVC Help
Powered by ViewVC 1.1.24