/[suikacvs]/markup/html/whatpm/Whatpm/HTML.pm.src
Suika

Diff of /markup/html/whatpm/Whatpm/HTML.pm.src

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.56 by wakaba, Sat Aug 11 07:19:18 2007 UTC revision 1.65 by wakaba, Mon Nov 19 12:18:26 2007 UTC
# Line 1  Line 1 
1  package Whatpm::HTML;  package Whatpm::HTML;
2  use strict;  use strict;
3  our $VERSION=do{my @r=(q$Revision$=~/\d+/g);sprintf "%d."."%02d" x $#r,@r};  our $VERSION=do{my @r=(q$Revision$=~/\d+/g);sprintf "%d."."%02d" x $#r,@r};
4    use Error qw(:try);
5    
6  ## ISSUE:  ## ISSUE:
7  ## var doc = implementation.createDocument (null, null, null);  ## var doc = implementation.createDocument (null, null, null);
# Line 84  my $formatting_category = { Line 85  my $formatting_category = {
85  };  };
86  # $phrasing_category: all other elements  # $phrasing_category: all other elements
87    
88    sub parse_byte_string ($$$$;$) {
89      my $self = ref $_[0] ? shift : shift->new;
90      my $charset = shift;
91      my $bytes_s = ref $_[0] ? $_[0] : \($_[0]);
92      my $s;
93      
94      if (defined $charset) {
95        require Encode; ## TODO: decode(utf8) don't delete BOM
96        $s = \ (Encode::decode ($charset, $$bytes_s));
97        $self->{input_encoding} = lc $charset; ## TODO: normalize name
98        $self->{confident} = 1;
99      } else {
100        ## TODO: Implement HTML5 detection algorithm
101        require Whatpm::Charset::UniversalCharDet;
102        $charset = Whatpm::Charset::UniversalCharDet->detect_byte_string
103            (substr ($$bytes_s, 0, 1024));
104        $charset ||= 'windows-1252';
105        $s = \ (Encode::decode ($charset, $$bytes_s));
106        $self->{input_encoding} = $charset;
107        $self->{confident} = 0;
108      }
109    
110      $self->{change_encoding} = sub {
111        my $self = shift;
112        my $charset = lc shift;
113        ## TODO: if $charset is supported
114        ## TODO: normalize charset name
115    
116        ## "Change the encoding" algorithm:
117    
118        ## Step 1    
119        if ($charset eq 'utf-16') { ## ISSUE: UTF-16BE -> UTF-8? UTF-16LE -> UTF-8?
120          $charset = 'utf-8';
121        }
122    
123        ## Step 2
124        if (defined $self->{input_encoding} and
125            $self->{input_encoding} eq $charset) {
126          $self->{confident} = 1;
127          return;
128        }
129    
130        !!!parse-error (type => 'charset label detected:'.$self->{input_encoding}.
131            ':'.$charset, level => 'w');
132    
133        ## Step 3
134        # if (can) {
135          ## change the encoding on the fly.
136          #$self->{confident} = 1;
137          #return;
138        # }
139    
140        ## Step 4
141        throw Whatpm::HTML::RestartParser (charset => $charset);
142      }; # $self->{change_encoding}
143    
144      my @args = @_; shift @args; # $s
145      my $return;
146      try {
147        $return = $self->parse_char_string ($s, @args);  
148      } catch Whatpm::HTML::RestartParser with {
149        my $charset = shift->{charset};
150        $s = \ (Encode::decode ($charset, $$bytes_s));    
151        $self->{input_encoding} = $charset; ## TODO: normalize
152        $self->{confident} = 1;
153        $return = $self->parse_char_string ($s, @args);
154      };
155      return $return;
156    } # parse_byte_string
157    
158    *parse_char_string = \&parse_string;
159    
160  sub parse_string ($$$;$) {  sub parse_string ($$$;$) {
161    my $self = shift->new;    my $self = ref $_[0] ? shift : shift->new;
162    my $s = \$_[0];    my $s = ref $_[0] ? $_[0] : \($_[0]);
163    $self->{document} = $_[1];    $self->{document} = $_[1];
164      @{$self->{document}->child_nodes} = ();
165    
166    ## NOTE: |set_inner_html| copies most of this method's code    ## NOTE: |set_inner_html| copies most of this method's code
167    
168      $self->{confident} = 1 unless exists $self->{confident};
169      $self->{document}->input_encoding ($self->{input_encoding})
170          if defined $self->{input_encoding};
171    
172    my $i = 0;    my $i = 0;
173    my $line = 1;    my $line = 1;
174    my $column = 0;    my $column = 0;
# Line 147  sub new ($) { Line 225  sub new ($) {
225    $self->{parse_error} = sub {    $self->{parse_error} = sub {
226      #      #
227    };    };
228      $self->{change_encoding} = sub {
229        # if ($_[0] is a supported encoding) {
230        #   run "change the encoding" algorithm;
231        #   throw Whatpm::HTML::RestartParser (charset => $new_encoding);
232        # }
233      };
234      $self->{application_cache_selection} = sub {
235        #
236      };
237    return $self;    return $self;
238  } # new  } # new
239    
# Line 159  sub CDATA_CONTENT_MODEL () { CM_LIMITED_ Line 246  sub CDATA_CONTENT_MODEL () { CM_LIMITED_
246  sub RCDATA_CONTENT_MODEL () { CM_ENTITY | CM_LIMITED_MARKUP }  sub RCDATA_CONTENT_MODEL () { CM_ENTITY | CM_LIMITED_MARKUP }
247  sub PCDATA_CONTENT_MODEL () { CM_ENTITY | CM_FULL_MARKUP }  sub PCDATA_CONTENT_MODEL () { CM_ENTITY | CM_FULL_MARKUP }
248    
249    sub DATA_STATE () { 0 }
250    sub ENTITY_DATA_STATE () { 1 }
251    sub TAG_OPEN_STATE () { 2 }
252    sub CLOSE_TAG_OPEN_STATE () { 3 }
253    sub TAG_NAME_STATE () { 4 }
254    sub BEFORE_ATTRIBUTE_NAME_STATE () { 5 }
255    sub ATTRIBUTE_NAME_STATE () { 6 }
256    sub AFTER_ATTRIBUTE_NAME_STATE () { 7 }
257    sub BEFORE_ATTRIBUTE_VALUE_STATE () { 8 }
258    sub ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE () { 9 }
259    sub ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE () { 10 }
260    sub ATTRIBUTE_VALUE_UNQUOTED_STATE () { 11 }
261    sub ENTITY_IN_ATTRIBUTE_VALUE_STATE () { 12 }
262    sub MARKUP_DECLARATION_OPEN_STATE () { 13 }
263    sub COMMENT_START_STATE () { 14 }
264    sub COMMENT_START_DASH_STATE () { 15 }
265    sub COMMENT_STATE () { 16 }
266    sub COMMENT_END_STATE () { 17 }
267    sub COMMENT_END_DASH_STATE () { 18 }
268    sub BOGUS_COMMENT_STATE () { 19 }
269    sub DOCTYPE_STATE () { 20 }
270    sub BEFORE_DOCTYPE_NAME_STATE () { 21 }
271    sub DOCTYPE_NAME_STATE () { 22 }
272    sub AFTER_DOCTYPE_NAME_STATE () { 23 }
273    sub BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE () { 24 }
274    sub DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE () { 25 }
275    sub DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE () { 26 }
276    sub AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE () { 27 }
277    sub BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE () { 28 }
278    sub DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE () { 29 }
279    sub DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE () { 30 }
280    sub AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE () { 31 }
281    sub BOGUS_DOCTYPE_STATE () { 32 }
282    
283  sub DOCTYPE_TOKEN () { 1 }  sub DOCTYPE_TOKEN () { 1 }
284  sub COMMENT_TOKEN () { 2 }  sub COMMENT_TOKEN () { 2 }
285  sub START_TAG_TOKEN () { 3 }  sub START_TAG_TOKEN () { 3 }
# Line 197  sub IN_COLUMN_GROUP_IM () { 0b10 } Line 318  sub IN_COLUMN_GROUP_IM () { 0b10 }
318    
319  sub _initialize_tokenizer ($) {  sub _initialize_tokenizer ($) {
320    my $self = shift;    my $self = shift;
321    $self->{state} = 'data'; # MUST    $self->{state} = DATA_STATE; # MUST
322    $self->{content_model} = PCDATA_CONTENT_MODEL; # be    $self->{content_model} = PCDATA_CONTENT_MODEL; # be
323    undef $self->{current_token}; # start tag, end tag, comment, or DOCTYPE    undef $self->{current_token}; # start tag, end tag, comment, or DOCTYPE
324    undef $self->{current_attribute};    undef $self->{current_attribute};
# Line 229  sub _initialize_tokenizer ($) { Line 350  sub _initialize_tokenizer ($) {
350  ## has completed loading.  If one has, then it MUST be executed  ## has completed loading.  If one has, then it MUST be executed
351  ## and removed from the list.  ## and removed from the list.
352    
353    ## NOTE: HTML5 "Writing HTML documents" section, applied to
354    ## documents and not to user agents and conformance checkers,
355    ## contains some requirements that are not detected by the
356    ## parsing algorithm:
357    ## - Some requirements on character encoding declarations. ## TODO
358    ## - "Elements MUST NOT contain content that their content model disallows."
359    ##   ... Some are parse error, some are not (will be reported by c.c.).
360    ## - Polytheistic slash SHOULD NOT be used. (Applied only to atheists.) ## TODO
361    ## - Text (in elements, attributes, and comments) SHOULD NOT contain
362    ##   control characters other than space characters. ## TODO: (what is control character? C0, C1 and DEL?  Unicode control character?)
363    
364    ## TODO: HTML5 poses authors two SHOULD-level requirements that cannot
365    ## be detected by the HTML5 parsing algorithm:
366    ## - Text,
367    
368  sub _get_next_token ($) {  sub _get_next_token ($) {
369    my $self = shift;    my $self = shift;
370    if (@{$self->{token}}) {    if (@{$self->{token}}) {
# Line 236  sub _get_next_token ($) { Line 372  sub _get_next_token ($) {
372    }    }
373    
374    A: {    A: {
375      if ($self->{state} eq 'data') {      if ($self->{state} == DATA_STATE) {
376        if ($self->{next_input_character} == 0x0026) { # &        if ($self->{next_input_character} == 0x0026) { # &
377          if ($self->{content_model} & CM_ENTITY) { # PCDATA | RCDATA          if ($self->{content_model} & CM_ENTITY) { # PCDATA | RCDATA
378            $self->{state} = 'entity data';            $self->{state} = ENTITY_DATA_STATE;
379            !!!next-input-character;            !!!next-input-character;
380            redo A;            redo A;
381          } else {          } else {
# Line 261  sub _get_next_token ($) { Line 397  sub _get_next_token ($) {
397          if ($self->{content_model} & CM_FULL_MARKUP or # PCDATA          if ($self->{content_model} & CM_FULL_MARKUP or # PCDATA
398              (($self->{content_model} & CM_LIMITED_MARKUP) and # CDATA | RCDATA              (($self->{content_model} & CM_LIMITED_MARKUP) and # CDATA | RCDATA
399               not $self->{escape})) {               not $self->{escape})) {
400            $self->{state} = 'tag open';            $self->{state} = TAG_OPEN_STATE;
401            !!!next-input-character;            !!!next-input-character;
402            redo A;            redo A;
403          } else {          } else {
# Line 290  sub _get_next_token ($) { Line 426  sub _get_next_token ($) {
426        !!!emit ($token);        !!!emit ($token);
427    
428        redo A;        redo A;
429      } elsif ($self->{state} eq 'entity data') {      } elsif ($self->{state} == ENTITY_DATA_STATE) {
430        ## (cannot happen in CDATA state)        ## (cannot happen in CDATA state)
431                
432        my $token = $self->_tokenize_attempt_to_consume_an_entity (0);        my $token = $self->_tokenize_attempt_to_consume_an_entity (0);
433    
434        $self->{state} = 'data';        $self->{state} = DATA_STATE;
435        # next-input-character is already done        # next-input-character is already done
436    
437        unless (defined $token) {        unless (defined $token) {
# Line 305  sub _get_next_token ($) { Line 441  sub _get_next_token ($) {
441        }        }
442    
443        redo A;        redo A;
444      } elsif ($self->{state} eq 'tag open') {      } elsif ($self->{state} == TAG_OPEN_STATE) {
445        if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA        if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
446          if ($self->{next_input_character} == 0x002F) { # /          if ($self->{next_input_character} == 0x002F) { # /
447            !!!next-input-character;            !!!next-input-character;
448            $self->{state} = 'close tag open';            $self->{state} = CLOSE_TAG_OPEN_STATE;
449            redo A;            redo A;
450          } else {          } else {
451            ## reconsume            ## reconsume
452            $self->{state} = 'data';            $self->{state} = DATA_STATE;
453    
454            !!!emit ({type => CHARACTER_TOKEN, data => '<'});            !!!emit ({type => CHARACTER_TOKEN, data => '<'});
455    
# Line 321  sub _get_next_token ($) { Line 457  sub _get_next_token ($) {
457          }          }
458        } elsif ($self->{content_model} & CM_FULL_MARKUP) { # PCDATA        } elsif ($self->{content_model} & CM_FULL_MARKUP) { # PCDATA
459          if ($self->{next_input_character} == 0x0021) { # !          if ($self->{next_input_character} == 0x0021) { # !
460            $self->{state} = 'markup declaration open';            $self->{state} = MARKUP_DECLARATION_OPEN_STATE;
461            !!!next-input-character;            !!!next-input-character;
462            redo A;            redo A;
463          } elsif ($self->{next_input_character} == 0x002F) { # /          } elsif ($self->{next_input_character} == 0x002F) { # /
464            $self->{state} = 'close tag open';            $self->{state} = CLOSE_TAG_OPEN_STATE;
465            !!!next-input-character;            !!!next-input-character;
466            redo A;            redo A;
467          } elsif (0x0041 <= $self->{next_input_character} and          } elsif (0x0041 <= $self->{next_input_character} and
# Line 333  sub _get_next_token ($) { Line 469  sub _get_next_token ($) {
469            $self->{current_token}            $self->{current_token}
470              = {type => START_TAG_TOKEN,              = {type => START_TAG_TOKEN,
471                 tag_name => chr ($self->{next_input_character} + 0x0020)};                 tag_name => chr ($self->{next_input_character} + 0x0020)};
472            $self->{state} = 'tag name';            $self->{state} = TAG_NAME_STATE;
473            !!!next-input-character;            !!!next-input-character;
474            redo A;            redo A;
475          } elsif (0x0061 <= $self->{next_input_character} and          } elsif (0x0061 <= $self->{next_input_character} and
476                   $self->{next_input_character} <= 0x007A) { # a..z                   $self->{next_input_character} <= 0x007A) { # a..z
477            $self->{current_token} = {type => START_TAG_TOKEN,            $self->{current_token} = {type => START_TAG_TOKEN,
478                              tag_name => chr ($self->{next_input_character})};                              tag_name => chr ($self->{next_input_character})};
479            $self->{state} = 'tag name';            $self->{state} = TAG_NAME_STATE;
480            !!!next-input-character;            !!!next-input-character;
481            redo A;            redo A;
482          } elsif ($self->{next_input_character} == 0x003E) { # >          } elsif ($self->{next_input_character} == 0x003E) { # >
483            !!!parse-error (type => 'empty start tag');            !!!parse-error (type => 'empty start tag');
484            $self->{state} = 'data';            $self->{state} = DATA_STATE;
485            !!!next-input-character;            !!!next-input-character;
486    
487            !!!emit ({type => CHARACTER_TOKEN, data => '<>'});            !!!emit ({type => CHARACTER_TOKEN, data => '<>'});
# Line 353  sub _get_next_token ($) { Line 489  sub _get_next_token ($) {
489            redo A;            redo A;
490          } elsif ($self->{next_input_character} == 0x003F) { # ?          } elsif ($self->{next_input_character} == 0x003F) { # ?
491            !!!parse-error (type => 'pio');            !!!parse-error (type => 'pio');
492            $self->{state} = 'bogus comment';            $self->{state} = BOGUS_COMMENT_STATE;
493            ## $self->{next_input_character} is intentionally left as is            ## $self->{next_input_character} is intentionally left as is
494            redo A;            redo A;
495          } else {          } else {
496            !!!parse-error (type => 'bare stago');            !!!parse-error (type => 'bare stago');
497            $self->{state} = 'data';            $self->{state} = DATA_STATE;
498            ## reconsume            ## reconsume
499    
500            !!!emit ({type => CHARACTER_TOKEN, data => '<'});            !!!emit ({type => CHARACTER_TOKEN, data => '<'});
# Line 368  sub _get_next_token ($) { Line 504  sub _get_next_token ($) {
504        } else {        } else {
505          die "$0: $self->{content_model} in tag open";          die "$0: $self->{content_model} in tag open";
506        }        }
507      } elsif ($self->{state} eq 'close tag open') {      } elsif ($self->{state} == CLOSE_TAG_OPEN_STATE) {
508        if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA        if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
509          if (defined $self->{last_emitted_start_tag_name}) {          if (defined $self->{last_emitted_start_tag_name}) {
510            ## NOTE: <http://krijnhoetmer.nl/irc-logs/whatwg/20070626#l-564>            ## NOTE: <http://krijnhoetmer.nl/irc-logs/whatwg/20070626#l-564>
# Line 383  sub _get_next_token ($) { Line 519  sub _get_next_token ($) {
519              } else {              } else {
520                $self->{next_input_character} = shift @next_char; # reconsume                $self->{next_input_character} = shift @next_char; # reconsume
521                !!!back-next-input-character (@next_char);                !!!back-next-input-character (@next_char);
522                $self->{state} = 'data';                $self->{state} = DATA_STATE;
523    
524                !!!emit ({type => CHARACTER_TOKEN, data => '</'});                !!!emit ({type => CHARACTER_TOKEN, data => '</'});
525        
# Line 402  sub _get_next_token ($) { Line 538  sub _get_next_token ($) {
538                    $self->{next_input_character} == -1) {                    $self->{next_input_character} == -1) {
539              $self->{next_input_character} = shift @next_char; # reconsume              $self->{next_input_character} = shift @next_char; # reconsume
540              !!!back-next-input-character (@next_char);              !!!back-next-input-character (@next_char);
541              $self->{state} = 'data';              $self->{state} = DATA_STATE;
542              !!!emit ({type => CHARACTER_TOKEN, data => '</'});              !!!emit ({type => CHARACTER_TOKEN, data => '</'});
543              redo A;              redo A;
544            } else {            } else {
# Line 413  sub _get_next_token ($) { Line 549  sub _get_next_token ($) {
549          } else {          } else {
550            ## No start tag token has ever been emitted            ## No start tag token has ever been emitted
551            # next-input-character is already done            # next-input-character is already done
552            $self->{state} = 'data';            $self->{state} = DATA_STATE;
553            !!!emit ({type => CHARACTER_TOKEN, data => '</'});            !!!emit ({type => CHARACTER_TOKEN, data => '</'});
554            redo A;            redo A;
555          }          }
# Line 423  sub _get_next_token ($) { Line 559  sub _get_next_token ($) {
559            $self->{next_input_character} <= 0x005A) { # A..Z            $self->{next_input_character} <= 0x005A) { # A..Z
560          $self->{current_token} = {type => END_TAG_TOKEN,          $self->{current_token} = {type => END_TAG_TOKEN,
561                            tag_name => chr ($self->{next_input_character} + 0x0020)};                            tag_name => chr ($self->{next_input_character} + 0x0020)};
562          $self->{state} = 'tag name';          $self->{state} = TAG_NAME_STATE;
563          !!!next-input-character;          !!!next-input-character;
564          redo A;          redo A;
565        } elsif (0x0061 <= $self->{next_input_character} and        } elsif (0x0061 <= $self->{next_input_character} and
566                 $self->{next_input_character} <= 0x007A) { # a..z                 $self->{next_input_character} <= 0x007A) { # a..z
567          $self->{current_token} = {type => END_TAG_TOKEN,          $self->{current_token} = {type => END_TAG_TOKEN,
568                            tag_name => chr ($self->{next_input_character})};                            tag_name => chr ($self->{next_input_character})};
569          $self->{state} = 'tag name';          $self->{state} = TAG_NAME_STATE;
570          !!!next-input-character;          !!!next-input-character;
571          redo A;          redo A;
572        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
573          !!!parse-error (type => 'empty end tag');          !!!parse-error (type => 'empty end tag');
574          $self->{state} = 'data';          $self->{state} = DATA_STATE;
575          !!!next-input-character;          !!!next-input-character;
576          redo A;          redo A;
577        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
578          !!!parse-error (type => 'bare etago');          !!!parse-error (type => 'bare etago');
579          $self->{state} = 'data';          $self->{state} = DATA_STATE;
580          # reconsume          # reconsume
581    
582          !!!emit ({type => CHARACTER_TOKEN, data => '</'});          !!!emit ({type => CHARACTER_TOKEN, data => '</'});
# Line 448  sub _get_next_token ($) { Line 584  sub _get_next_token ($) {
584          redo A;          redo A;
585        } else {        } else {
586          !!!parse-error (type => 'bogus end tag');          !!!parse-error (type => 'bogus end tag');
587          $self->{state} = 'bogus comment';          $self->{state} = BOGUS_COMMENT_STATE;
588          ## $self->{next_input_character} is intentionally left as is          ## $self->{next_input_character} is intentionally left as is
589          redo A;          redo A;
590        }        }
591      } elsif ($self->{state} eq 'tag name') {      } elsif ($self->{state} == TAG_NAME_STATE) {
592        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
593            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
594            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
595            $self->{next_input_character} == 0x000C or # FF            $self->{next_input_character} == 0x000C or # FF
596            $self->{next_input_character} == 0x0020) { # SP            $self->{next_input_character} == 0x0020) { # SP
597          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
598          !!!next-input-character;          !!!next-input-character;
599          redo A;          redo A;
600        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
# Line 474  sub _get_next_token ($) { Line 610  sub _get_next_token ($) {
610          } else {          } else {
611            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
612          }          }
613          $self->{state} = 'data';          $self->{state} = DATA_STATE;
614          !!!next-input-character;          !!!next-input-character;
615    
616          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 501  sub _get_next_token ($) { Line 637  sub _get_next_token ($) {
637          } else {          } else {
638            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
639          }          }
640          $self->{state} = 'data';          $self->{state} = DATA_STATE;
641          # reconsume          # reconsume
642    
643          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 517  sub _get_next_token ($) { Line 653  sub _get_next_token ($) {
653          } else {          } else {
654            !!!parse-error (type => 'nestc');            !!!parse-error (type => 'nestc');
655          }          }
656          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
657          # next-input-character is already done          # next-input-character is already done
658          redo A;          redo A;
659        } else {        } else {
# Line 527  sub _get_next_token ($) { Line 663  sub _get_next_token ($) {
663          !!!next-input-character;          !!!next-input-character;
664          redo A;          redo A;
665        }        }
666      } elsif ($self->{state} eq 'before attribute name') {      } elsif ($self->{state} == BEFORE_ATTRIBUTE_NAME_STATE) {
667        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
668            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
669            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
# Line 549  sub _get_next_token ($) { Line 685  sub _get_next_token ($) {
685          } else {          } else {
686            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
687          }          }
688          $self->{state} = 'data';          $self->{state} = DATA_STATE;
689          !!!next-input-character;          !!!next-input-character;
690    
691          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 559  sub _get_next_token ($) { Line 695  sub _get_next_token ($) {
695                 $self->{next_input_character} <= 0x005A) { # A..Z                 $self->{next_input_character} <= 0x005A) { # A..Z
696          $self->{current_attribute} = {name => chr ($self->{next_input_character} + 0x0020),          $self->{current_attribute} = {name => chr ($self->{next_input_character} + 0x0020),
697                                value => ''};                                value => ''};
698          $self->{state} = 'attribute name';          $self->{state} = ATTRIBUTE_NAME_STATE;
699          !!!next-input-character;          !!!next-input-character;
700          redo A;          redo A;
701        } elsif ($self->{next_input_character} == 0x002F) { # /        } elsif ($self->{next_input_character} == 0x002F) { # /
# Line 589  sub _get_next_token ($) { Line 725  sub _get_next_token ($) {
725          } else {          } else {
726            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
727          }          }
728          $self->{state} = 'data';          $self->{state} = DATA_STATE;
729          # reconsume          # reconsume
730    
731          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 598  sub _get_next_token ($) { Line 734  sub _get_next_token ($) {
734        } else {        } else {
735          $self->{current_attribute} = {name => chr ($self->{next_input_character}),          $self->{current_attribute} = {name => chr ($self->{next_input_character}),
736                                value => ''};                                value => ''};
737          $self->{state} = 'attribute name';          $self->{state} = ATTRIBUTE_NAME_STATE;
738          !!!next-input-character;          !!!next-input-character;
739          redo A;          redo A;
740        }        }
741      } elsif ($self->{state} eq 'attribute name') {      } elsif ($self->{state} == ATTRIBUTE_NAME_STATE) {
742        my $before_leave = sub {        my $before_leave = sub {
743          if (exists $self->{current_token}->{attributes} # start tag or end tag          if (exists $self->{current_token}->{attributes} # start tag or end tag
744              ->{$self->{current_attribute}->{name}}) { # MUST              ->{$self->{current_attribute}->{name}}) { # MUST
# Line 620  sub _get_next_token ($) { Line 756  sub _get_next_token ($) {
756            $self->{next_input_character} == 0x000C or # FF            $self->{next_input_character} == 0x000C or # FF
757            $self->{next_input_character} == 0x0020) { # SP            $self->{next_input_character} == 0x0020) { # SP
758          $before_leave->();          $before_leave->();
759          $self->{state} = 'after attribute name';          $self->{state} = AFTER_ATTRIBUTE_NAME_STATE;
760          !!!next-input-character;          !!!next-input-character;
761          redo A;          redo A;
762        } elsif ($self->{next_input_character} == 0x003D) { # =        } elsif ($self->{next_input_character} == 0x003D) { # =
763          $before_leave->();          $before_leave->();
764          $self->{state} = 'before attribute value';          $self->{state} = BEFORE_ATTRIBUTE_VALUE_STATE;
765          !!!next-input-character;          !!!next-input-character;
766          redo A;          redo A;
767        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
# Line 642  sub _get_next_token ($) { Line 778  sub _get_next_token ($) {
778          } else {          } else {
779            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
780          }          }
781          $self->{state} = 'data';          $self->{state} = DATA_STATE;
782          !!!next-input-character;          !!!next-input-character;
783    
784          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 665  sub _get_next_token ($) { Line 801  sub _get_next_token ($) {
801          } else {          } else {
802            !!!parse-error (type => 'nestc');            !!!parse-error (type => 'nestc');
803          }          }
804          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
805          # next-input-character is already done          # next-input-character is already done
806          redo A;          redo A;
807        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
# Line 683  sub _get_next_token ($) { Line 819  sub _get_next_token ($) {
819          } else {          } else {
820            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
821          }          }
822          $self->{state} = 'data';          $self->{state} = DATA_STATE;
823          # reconsume          # reconsume
824    
825          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 695  sub _get_next_token ($) { Line 831  sub _get_next_token ($) {
831          !!!next-input-character;          !!!next-input-character;
832          redo A;          redo A;
833        }        }
834      } elsif ($self->{state} eq 'after attribute name') {      } elsif ($self->{state} == AFTER_ATTRIBUTE_NAME_STATE) {
835        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
836            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
837            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
# Line 705  sub _get_next_token ($) { Line 841  sub _get_next_token ($) {
841          !!!next-input-character;          !!!next-input-character;
842          redo A;          redo A;
843        } elsif ($self->{next_input_character} == 0x003D) { # =        } elsif ($self->{next_input_character} == 0x003D) { # =
844          $self->{state} = 'before attribute value';          $self->{state} = BEFORE_ATTRIBUTE_VALUE_STATE;
845          !!!next-input-character;          !!!next-input-character;
846          redo A;          redo A;
847        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
# Line 721  sub _get_next_token ($) { Line 857  sub _get_next_token ($) {
857          } else {          } else {
858            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
859          }          }
860          $self->{state} = 'data';          $self->{state} = DATA_STATE;
861          !!!next-input-character;          !!!next-input-character;
862    
863          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 731  sub _get_next_token ($) { Line 867  sub _get_next_token ($) {
867                 $self->{next_input_character} <= 0x005A) { # A..Z                 $self->{next_input_character} <= 0x005A) { # A..Z
868          $self->{current_attribute} = {name => chr ($self->{next_input_character} + 0x0020),          $self->{current_attribute} = {name => chr ($self->{next_input_character} + 0x0020),
869                                value => ''};                                value => ''};
870          $self->{state} = 'attribute name';          $self->{state} = ATTRIBUTE_NAME_STATE;
871          !!!next-input-character;          !!!next-input-character;
872          redo A;          redo A;
873        } elsif ($self->{next_input_character} == 0x002F) { # /        } elsif ($self->{next_input_character} == 0x002F) { # /
# Line 745  sub _get_next_token ($) { Line 881  sub _get_next_token ($) {
881            !!!parse-error (type => 'nestc');            !!!parse-error (type => 'nestc');
882            ## TODO: Different error type for <aa / bb> than <aa/>            ## TODO: Different error type for <aa / bb> than <aa/>
883          }          }
884          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
885          # next-input-character is already done          # next-input-character is already done
886          redo A;          redo A;
887        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
# Line 762  sub _get_next_token ($) { Line 898  sub _get_next_token ($) {
898          } else {          } else {
899            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
900          }          }
901          $self->{state} = 'data';          $self->{state} = DATA_STATE;
902          # reconsume          # reconsume
903    
904          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 771  sub _get_next_token ($) { Line 907  sub _get_next_token ($) {
907        } else {        } else {
908          $self->{current_attribute} = {name => chr ($self->{next_input_character}),          $self->{current_attribute} = {name => chr ($self->{next_input_character}),
909                                value => ''};                                value => ''};
910          $self->{state} = 'attribute name';          $self->{state} = ATTRIBUTE_NAME_STATE;
911          !!!next-input-character;          !!!next-input-character;
912          redo A;                  redo A;        
913        }        }
914      } elsif ($self->{state} eq 'before attribute value') {      } elsif ($self->{state} == BEFORE_ATTRIBUTE_VALUE_STATE) {
915        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
916            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
917            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
# Line 785  sub _get_next_token ($) { Line 921  sub _get_next_token ($) {
921          !!!next-input-character;          !!!next-input-character;
922          redo A;          redo A;
923        } elsif ($self->{next_input_character} == 0x0022) { # "        } elsif ($self->{next_input_character} == 0x0022) { # "
924          $self->{state} = 'attribute value (double-quoted)';          $self->{state} = ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE;
925          !!!next-input-character;          !!!next-input-character;
926          redo A;          redo A;
927        } elsif ($self->{next_input_character} == 0x0026) { # &        } elsif ($self->{next_input_character} == 0x0026) { # &
928          $self->{state} = 'attribute value (unquoted)';          $self->{state} = ATTRIBUTE_VALUE_UNQUOTED_STATE;
929          ## reconsume          ## reconsume
930          redo A;          redo A;
931        } elsif ($self->{next_input_character} == 0x0027) { # '        } elsif ($self->{next_input_character} == 0x0027) { # '
932          $self->{state} = 'attribute value (single-quoted)';          $self->{state} = ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE;
933          !!!next-input-character;          !!!next-input-character;
934          redo A;          redo A;
935        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
# Line 809  sub _get_next_token ($) { Line 945  sub _get_next_token ($) {
945          } else {          } else {
946            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
947          }          }
948          $self->{state} = 'data';          $self->{state} = DATA_STATE;
949          !!!next-input-character;          !!!next-input-character;
950    
951          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 829  sub _get_next_token ($) { Line 965  sub _get_next_token ($) {
965          } else {          } else {
966            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
967          }          }
968          $self->{state} = 'data';          $self->{state} = DATA_STATE;
969          ## reconsume          ## reconsume
970    
971          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 837  sub _get_next_token ($) { Line 973  sub _get_next_token ($) {
973          redo A;          redo A;
974        } else {        } else {
975          $self->{current_attribute}->{value} .= chr ($self->{next_input_character});          $self->{current_attribute}->{value} .= chr ($self->{next_input_character});
976          $self->{state} = 'attribute value (unquoted)';          $self->{state} = ATTRIBUTE_VALUE_UNQUOTED_STATE;
977          !!!next-input-character;          !!!next-input-character;
978          redo A;          redo A;
979        }        }
980      } elsif ($self->{state} eq 'attribute value (double-quoted)') {      } elsif ($self->{state} == ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE) {
981        if ($self->{next_input_character} == 0x0022) { # "        if ($self->{next_input_character} == 0x0022) { # "
982          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
983          !!!next-input-character;          !!!next-input-character;
984          redo A;          redo A;
985        } elsif ($self->{next_input_character} == 0x0026) { # &        } elsif ($self->{next_input_character} == 0x0026) { # &
986          $self->{last_attribute_value_state} = 'attribute value (double-quoted)';          $self->{last_attribute_value_state} = $self->{state};
987          $self->{state} = 'entity in attribute value';          $self->{state} = ENTITY_IN_ATTRIBUTE_VALUE_STATE;
988          !!!next-input-character;          !!!next-input-character;
989          redo A;          redo A;
990        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
# Line 865  sub _get_next_token ($) { Line 1001  sub _get_next_token ($) {
1001          } else {          } else {
1002            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
1003          }          }
1004          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1005          ## reconsume          ## reconsume
1006    
1007          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 877  sub _get_next_token ($) { Line 1013  sub _get_next_token ($) {
1013          !!!next-input-character;          !!!next-input-character;
1014          redo A;          redo A;
1015        }        }
1016      } elsif ($self->{state} eq 'attribute value (single-quoted)') {      } elsif ($self->{state} == ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE) {
1017        if ($self->{next_input_character} == 0x0027) { # '        if ($self->{next_input_character} == 0x0027) { # '
1018          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1019          !!!next-input-character;          !!!next-input-character;
1020          redo A;          redo A;
1021        } elsif ($self->{next_input_character} == 0x0026) { # &        } elsif ($self->{next_input_character} == 0x0026) { # &
1022          $self->{last_attribute_value_state} = 'attribute value (single-quoted)';          $self->{last_attribute_value_state} = $self->{state};
1023          $self->{state} = 'entity in attribute value';          $self->{state} = ENTITY_IN_ATTRIBUTE_VALUE_STATE;
1024          !!!next-input-character;          !!!next-input-character;
1025          redo A;          redo A;
1026        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
# Line 901  sub _get_next_token ($) { Line 1037  sub _get_next_token ($) {
1037          } else {          } else {
1038            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
1039          }          }
1040          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1041          ## reconsume          ## reconsume
1042    
1043          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 913  sub _get_next_token ($) { Line 1049  sub _get_next_token ($) {
1049          !!!next-input-character;          !!!next-input-character;
1050          redo A;          redo A;
1051        }        }
1052      } elsif ($self->{state} eq 'attribute value (unquoted)') {      } elsif ($self->{state} == ATTRIBUTE_VALUE_UNQUOTED_STATE) {
1053        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
1054            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
1055            $self->{next_input_character} == 0x000B or # HT            $self->{next_input_character} == 0x000B or # HT
1056            $self->{next_input_character} == 0x000C or # FF            $self->{next_input_character} == 0x000C or # FF
1057            $self->{next_input_character} == 0x0020) { # SP            $self->{next_input_character} == 0x0020) { # SP
1058          $self->{state} = 'before attribute name';          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1059          !!!next-input-character;          !!!next-input-character;
1060          redo A;          redo A;
1061        } elsif ($self->{next_input_character} == 0x0026) { # &        } elsif ($self->{next_input_character} == 0x0026) { # &
1062          $self->{last_attribute_value_state} = 'attribute value (unquoted)';          $self->{last_attribute_value_state} = $self->{state};
1063          $self->{state} = 'entity in attribute value';          $self->{state} = ENTITY_IN_ATTRIBUTE_VALUE_STATE;
1064          !!!next-input-character;          !!!next-input-character;
1065          redo A;          redo A;
1066        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
# Line 940  sub _get_next_token ($) { Line 1076  sub _get_next_token ($) {
1076          } else {          } else {
1077            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
1078          }          }
1079          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1080          !!!next-input-character;          !!!next-input-character;
1081    
1082          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 960  sub _get_next_token ($) { Line 1096  sub _get_next_token ($) {
1096          } else {          } else {
1097            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{current_token}->{type}: Unknown token type";
1098          }          }
1099          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1100          ## reconsume          ## reconsume
1101    
1102          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{current_token}); # start tag or end tag
# Line 972  sub _get_next_token ($) { Line 1108  sub _get_next_token ($) {
1108          !!!next-input-character;          !!!next-input-character;
1109          redo A;          redo A;
1110        }        }
1111      } elsif ($self->{state} eq 'entity in attribute value') {      } elsif ($self->{state} == ENTITY_IN_ATTRIBUTE_VALUE_STATE) {
1112        my $token = $self->_tokenize_attempt_to_consume_an_entity (1);        my $token = $self->_tokenize_attempt_to_consume_an_entity (1);
1113    
1114        unless (defined $token) {        unless (defined $token) {
# Line 985  sub _get_next_token ($) { Line 1121  sub _get_next_token ($) {
1121        $self->{state} = $self->{last_attribute_value_state};        $self->{state} = $self->{last_attribute_value_state};
1122        # next-input-character is already done        # next-input-character is already done
1123        redo A;        redo A;
1124      } elsif ($self->{state} eq 'bogus comment') {      } elsif ($self->{state} == BOGUS_COMMENT_STATE) {
1125        ## (only happen if PCDATA state)        ## (only happen if PCDATA state)
1126                
1127        my $token = {type => COMMENT_TOKEN, data => ''};        my $token = {type => COMMENT_TOKEN, data => ''};
1128    
1129        BC: {        BC: {
1130          if ($self->{next_input_character} == 0x003E) { # >          if ($self->{next_input_character} == 0x003E) { # >
1131            $self->{state} = 'data';            $self->{state} = DATA_STATE;
1132            !!!next-input-character;            !!!next-input-character;
1133    
1134            !!!emit ($token);            !!!emit ($token);
1135    
1136            redo A;            redo A;
1137          } elsif ($self->{next_input_character} == -1) {          } elsif ($self->{next_input_character} == -1) {
1138            $self->{state} = 'data';            $self->{state} = DATA_STATE;
1139            ## reconsume            ## reconsume
1140    
1141            !!!emit ($token);            !!!emit ($token);
# Line 1011  sub _get_next_token ($) { Line 1147  sub _get_next_token ($) {
1147            redo BC;            redo BC;
1148          }          }
1149        } # BC        } # BC
1150      } elsif ($self->{state} eq 'markup declaration open') {      } elsif ($self->{state} == MARKUP_DECLARATION_OPEN_STATE) {
1151        ## (only happen if PCDATA state)        ## (only happen if PCDATA state)
1152    
1153        my @next_char;        my @next_char;
# Line 1022  sub _get_next_token ($) { Line 1158  sub _get_next_token ($) {
1158          push @next_char, $self->{next_input_character};          push @next_char, $self->{next_input_character};
1159          if ($self->{next_input_character} == 0x002D) { # -          if ($self->{next_input_character} == 0x002D) { # -
1160            $self->{current_token} = {type => COMMENT_TOKEN, data => ''};            $self->{current_token} = {type => COMMENT_TOKEN, data => ''};
1161            $self->{state} = 'comment start';            $self->{state} = COMMENT_START_STATE;
1162            !!!next-input-character;            !!!next-input-character;
1163            redo A;            redo A;
1164          }          }
# Line 1053  sub _get_next_token ($) { Line 1189  sub _get_next_token ($) {
1189                    if ($self->{next_input_character} == 0x0045 or # E                    if ($self->{next_input_character} == 0x0045 or # E
1190                        $self->{next_input_character} == 0x0065) { # e                        $self->{next_input_character} == 0x0065) { # e
1191                      ## ISSUE: What a stupid code this is!                      ## ISSUE: What a stupid code this is!
1192                      $self->{state} = 'DOCTYPE';                      $self->{state} = DOCTYPE_STATE;
1193                      !!!next-input-character;                      !!!next-input-character;
1194                      redo A;                      redo A;
1195                    }                    }
# Line 1067  sub _get_next_token ($) { Line 1203  sub _get_next_token ($) {
1203        !!!parse-error (type => 'bogus comment');        !!!parse-error (type => 'bogus comment');
1204        $self->{next_input_character} = shift @next_char;        $self->{next_input_character} = shift @next_char;
1205        !!!back-next-input-character (@next_char);        !!!back-next-input-character (@next_char);
1206        $self->{state} = 'bogus comment';        $self->{state} = BOGUS_COMMENT_STATE;
1207        redo A;        redo A;
1208                
1209        ## ISSUE: typos in spec: chacacters, is is a parse error        ## ISSUE: typos in spec: chacacters, is is a parse error
1210        ## ISSUE: spec is somewhat unclear on "is the first character that will be in the comment"; what is "that will be in the comment" is what the algorithm defines, isn't it?        ## ISSUE: spec is somewhat unclear on "is the first character that will be in the comment"; what is "that will be in the comment" is what the algorithm defines, isn't it?
1211      } elsif ($self->{state} eq 'comment start') {      } elsif ($self->{state} == COMMENT_START_STATE) {
1212        if ($self->{next_input_character} == 0x002D) { # -        if ($self->{next_input_character} == 0x002D) { # -
1213          $self->{state} = 'comment start dash';          $self->{state} = COMMENT_START_DASH_STATE;
1214          !!!next-input-character;          !!!next-input-character;
1215          redo A;          redo A;
1216        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1217          !!!parse-error (type => 'bogus comment');          !!!parse-error (type => 'bogus comment');
1218          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1219          !!!next-input-character;          !!!next-input-character;
1220    
1221          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1087  sub _get_next_token ($) { Line 1223  sub _get_next_token ($) {
1223          redo A;          redo A;
1224        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1225          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
1226          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1227          ## reconsume          ## reconsume
1228    
1229          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1096  sub _get_next_token ($) { Line 1232  sub _get_next_token ($) {
1232        } else {        } else {
1233          $self->{current_token}->{data} # comment          $self->{current_token}->{data} # comment
1234              .= chr ($self->{next_input_character});              .= chr ($self->{next_input_character});
1235          $self->{state} = 'comment';          $self->{state} = COMMENT_STATE;
1236          !!!next-input-character;          !!!next-input-character;
1237          redo A;          redo A;
1238        }        }
1239      } elsif ($self->{state} eq 'comment start dash') {      } elsif ($self->{state} == COMMENT_START_DASH_STATE) {
1240        if ($self->{next_input_character} == 0x002D) { # -        if ($self->{next_input_character} == 0x002D) { # -
1241          $self->{state} = 'comment end';          $self->{state} = COMMENT_END_STATE;
1242          !!!next-input-character;          !!!next-input-character;
1243          redo A;          redo A;
1244        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1245          !!!parse-error (type => 'bogus comment');          !!!parse-error (type => 'bogus comment');
1246          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1247          !!!next-input-character;          !!!next-input-character;
1248    
1249          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1115  sub _get_next_token ($) { Line 1251  sub _get_next_token ($) {
1251          redo A;          redo A;
1252        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1253          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
1254          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1255          ## reconsume          ## reconsume
1256    
1257          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1124  sub _get_next_token ($) { Line 1260  sub _get_next_token ($) {
1260        } else {        } else {
1261          $self->{current_token}->{data} # comment          $self->{current_token}->{data} # comment
1262              .= '-' . chr ($self->{next_input_character});              .= '-' . chr ($self->{next_input_character});
1263          $self->{state} = 'comment';          $self->{state} = COMMENT_STATE;
1264          !!!next-input-character;          !!!next-input-character;
1265          redo A;          redo A;
1266        }        }
1267      } elsif ($self->{state} eq 'comment') {      } elsif ($self->{state} == COMMENT_STATE) {
1268        if ($self->{next_input_character} == 0x002D) { # -        if ($self->{next_input_character} == 0x002D) { # -
1269          $self->{state} = 'comment end dash';          $self->{state} = COMMENT_END_DASH_STATE;
1270          !!!next-input-character;          !!!next-input-character;
1271          redo A;          redo A;
1272        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1273          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
1274          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1275          ## reconsume          ## reconsume
1276    
1277          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1147  sub _get_next_token ($) { Line 1283  sub _get_next_token ($) {
1283          !!!next-input-character;          !!!next-input-character;
1284          redo A;          redo A;
1285        }        }
1286      } elsif ($self->{state} eq 'comment end dash') {      } elsif ($self->{state} == COMMENT_END_DASH_STATE) {
1287        if ($self->{next_input_character} == 0x002D) { # -        if ($self->{next_input_character} == 0x002D) { # -
1288          $self->{state} = 'comment end';          $self->{state} = COMMENT_END_STATE;
1289          !!!next-input-character;          !!!next-input-character;
1290          redo A;          redo A;
1291        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1292          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
1293          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1294          ## reconsume          ## reconsume
1295    
1296          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1162  sub _get_next_token ($) { Line 1298  sub _get_next_token ($) {
1298          redo A;          redo A;
1299        } else {        } else {
1300          $self->{current_token}->{data} .= '-' . chr ($self->{next_input_character}); # comment          $self->{current_token}->{data} .= '-' . chr ($self->{next_input_character}); # comment
1301          $self->{state} = 'comment';          $self->{state} = COMMENT_STATE;
1302          !!!next-input-character;          !!!next-input-character;
1303          redo A;          redo A;
1304        }        }
1305      } elsif ($self->{state} eq 'comment end') {      } elsif ($self->{state} == COMMENT_END_STATE) {
1306        if ($self->{next_input_character} == 0x003E) { # >        if ($self->{next_input_character} == 0x003E) { # >
1307          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1308          !!!next-input-character;          !!!next-input-character;
1309    
1310          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1182  sub _get_next_token ($) { Line 1318  sub _get_next_token ($) {
1318          redo A;          redo A;
1319        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1320          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
1321          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1322          ## reconsume          ## reconsume
1323    
1324          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{current_token}); # comment
# Line 1191  sub _get_next_token ($) { Line 1327  sub _get_next_token ($) {
1327        } else {        } else {
1328          !!!parse-error (type => 'dash in comment');          !!!parse-error (type => 'dash in comment');
1329          $self->{current_token}->{data} .= '--' . chr ($self->{next_input_character}); # comment          $self->{current_token}->{data} .= '--' . chr ($self->{next_input_character}); # comment
1330          $self->{state} = 'comment';          $self->{state} = COMMENT_STATE;
1331          !!!next-input-character;          !!!next-input-character;
1332          redo A;          redo A;
1333        }        }
1334      } elsif ($self->{state} eq 'DOCTYPE') {      } elsif ($self->{state} == DOCTYPE_STATE) {
1335        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
1336            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
1337            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
1338            $self->{next_input_character} == 0x000C or # FF            $self->{next_input_character} == 0x000C or # FF
1339            $self->{next_input_character} == 0x0020) { # SP            $self->{next_input_character} == 0x0020) { # SP
1340          $self->{state} = 'before DOCTYPE name';          $self->{state} = BEFORE_DOCTYPE_NAME_STATE;
1341          !!!next-input-character;          !!!next-input-character;
1342          redo A;          redo A;
1343        } else {        } else {
1344          !!!parse-error (type => 'no space before DOCTYPE name');          !!!parse-error (type => 'no space before DOCTYPE name');
1345          $self->{state} = 'before DOCTYPE name';          $self->{state} = BEFORE_DOCTYPE_NAME_STATE;
1346          ## reconsume          ## reconsume
1347          redo A;          redo A;
1348        }        }
1349      } elsif ($self->{state} eq 'before DOCTYPE name') {      } elsif ($self->{state} == BEFORE_DOCTYPE_NAME_STATE) {
1350        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
1351            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
1352            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
# Line 1221  sub _get_next_token ($) { Line 1357  sub _get_next_token ($) {
1357          redo A;          redo A;
1358        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1359          !!!parse-error (type => 'no DOCTYPE name');          !!!parse-error (type => 'no DOCTYPE name');
1360          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1361          !!!next-input-character;          !!!next-input-character;
1362    
1363          !!!emit ({type => DOCTYPE_TOKEN}); # incorrect          !!!emit ({type => DOCTYPE_TOKEN}); # incorrect
# Line 1229  sub _get_next_token ($) { Line 1365  sub _get_next_token ($) {
1365          redo A;          redo A;
1366        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1367          !!!parse-error (type => 'no DOCTYPE name');          !!!parse-error (type => 'no DOCTYPE name');
1368          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1369          ## reconsume          ## reconsume
1370    
1371          !!!emit ({type => DOCTYPE_TOKEN}); # incorrect          !!!emit ({type => DOCTYPE_TOKEN}); # incorrect
# Line 1241  sub _get_next_token ($) { Line 1377  sub _get_next_token ($) {
1377                 name => chr ($self->{next_input_character}),                 name => chr ($self->{next_input_character}),
1378                 correct => 1};                 correct => 1};
1379  ## ISSUE: "Set the token's name name to the" in the spec  ## ISSUE: "Set the token's name name to the" in the spec
1380          $self->{state} = 'DOCTYPE name';          $self->{state} = DOCTYPE_NAME_STATE;
1381          !!!next-input-character;          !!!next-input-character;
1382          redo A;          redo A;
1383        }        }
1384      } elsif ($self->{state} eq 'DOCTYPE name') {      } elsif ($self->{state} == DOCTYPE_NAME_STATE) {
1385  ## ISSUE: Redundant "First," in the spec.  ## ISSUE: Redundant "First," in the spec.
1386        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
1387            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
1388            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
1389            $self->{next_input_character} == 0x000C or # FF            $self->{next_input_character} == 0x000C or # FF
1390            $self->{next_input_character} == 0x0020) { # SP            $self->{next_input_character} == 0x0020) { # SP
1391          $self->{state} = 'after DOCTYPE name';          $self->{state} = AFTER_DOCTYPE_NAME_STATE;
1392          !!!next-input-character;          !!!next-input-character;
1393          redo A;          redo A;
1394        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1395          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1396          !!!next-input-character;          !!!next-input-character;
1397    
1398          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{current_token}); # DOCTYPE
# Line 1264  sub _get_next_token ($) { Line 1400  sub _get_next_token ($) {
1400          redo A;          redo A;
1401        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1402          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1403          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1404          ## reconsume          ## reconsume
1405    
1406          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1278  sub _get_next_token ($) { Line 1414  sub _get_next_token ($) {
1414          !!!next-input-character;          !!!next-input-character;
1415          redo A;          redo A;
1416        }        }
1417      } elsif ($self->{state} eq 'after DOCTYPE name') {      } elsif ($self->{state} == AFTER_DOCTYPE_NAME_STATE) {
1418        if ($self->{next_input_character} == 0x0009 or # HT        if ($self->{next_input_character} == 0x0009 or # HT
1419            $self->{next_input_character} == 0x000A or # LF            $self->{next_input_character} == 0x000A or # LF
1420            $self->{next_input_character} == 0x000B or # VT            $self->{next_input_character} == 0x000B or # VT
# Line 1288  sub _get_next_token ($) { Line 1424  sub _get_next_token ($) {
1424          !!!next-input-character;          !!!next-input-character;
1425          redo A;          redo A;
1426        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1427          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1428          !!!next-input-character;          !!!next-input-character;
1429    
1430          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{current_token}); # DOCTYPE
# Line 1296  sub _get_next_token ($) { Line 1432  sub _get_next_token ($) {
1432          redo A;          redo A;
1433        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1434          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1435          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1436          ## reconsume          ## reconsume
1437    
1438          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1320  sub _get_next_token ($) { Line 1456  sub _get_next_token ($) {
1456                  !!!next-input-character;                  !!!next-input-character;
1457                  if ($self->{next_input_character} == 0x0043 or # C                  if ($self->{next_input_character} == 0x0043 or # C
1458                      $self->{next_input_character} == 0x0063) { # c                      $self->{next_input_character} == 0x0063) { # c
1459                    $self->{state} = 'before DOCTYPE public identifier';                    $self->{state} = BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
1460                    !!!next-input-character;                    !!!next-input-character;
1461                    redo A;                    redo A;
1462                  }                  }
# Line 1347  sub _get_next_token ($) { Line 1483  sub _get_next_token ($) {
1483                  !!!next-input-character;                  !!!next-input-character;
1484                  if ($self->{next_input_character} == 0x004D or # M                  if ($self->{next_input_character} == 0x004D or # M
1485                      $self->{next_input_character} == 0x006D) { # m                      $self->{next_input_character} == 0x006D) { # m
1486                    $self->{state} = 'before DOCTYPE system identifier';                    $self->{state} = BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
1487                    !!!next-input-character;                    !!!next-input-character;
1488                    redo A;                    redo A;
1489                  }                  }
# Line 1363  sub _get_next_token ($) { Line 1499  sub _get_next_token ($) {
1499        }        }
1500    
1501        !!!parse-error (type => 'string after DOCTYPE name');        !!!parse-error (type => 'string after DOCTYPE name');
1502        $self->{state} = 'bogus DOCTYPE';        $self->{state} = BOGUS_DOCTYPE_STATE;
1503        # next-input-character is already done        # next-input-character is already done
1504        redo A;        redo A;
1505      } elsif ($self->{state} eq 'before DOCTYPE public identifier') {      } elsif ($self->{state} == BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE) {
1506        if ({        if ({
1507              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,
1508              #0x000D => 1, # HT, LF, VT, FF, SP, CR              #0x000D => 1, # HT, LF, VT, FF, SP, CR
# Line 1376  sub _get_next_token ($) { Line 1512  sub _get_next_token ($) {
1512          redo A;          redo A;
1513        } elsif ($self->{next_input_character} eq 0x0022) { # "        } elsif ($self->{next_input_character} eq 0x0022) { # "
1514          $self->{current_token}->{public_identifier} = ''; # DOCTYPE          $self->{current_token}->{public_identifier} = ''; # DOCTYPE
1515          $self->{state} = 'DOCTYPE public identifier (double-quoted)';          $self->{state} = DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE;
1516          !!!next-input-character;          !!!next-input-character;
1517          redo A;          redo A;
1518        } elsif ($self->{next_input_character} eq 0x0027) { # '        } elsif ($self->{next_input_character} eq 0x0027) { # '
1519          $self->{current_token}->{public_identifier} = ''; # DOCTYPE          $self->{current_token}->{public_identifier} = ''; # DOCTYPE
1520          $self->{state} = 'DOCTYPE public identifier (single-quoted)';          $self->{state} = DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE;
1521          !!!next-input-character;          !!!next-input-character;
1522          redo A;          redo A;
1523        } elsif ($self->{next_input_character} eq 0x003E) { # >        } elsif ($self->{next_input_character} eq 0x003E) { # >
1524          !!!parse-error (type => 'no PUBLIC literal');          !!!parse-error (type => 'no PUBLIC literal');
1525    
1526          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1527          !!!next-input-character;          !!!next-input-character;
1528    
1529          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1397  sub _get_next_token ($) { Line 1533  sub _get_next_token ($) {
1533        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1534          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1535    
1536          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1537          ## reconsume          ## reconsume
1538    
1539          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1406  sub _get_next_token ($) { Line 1542  sub _get_next_token ($) {
1542          redo A;          redo A;
1543        } else {        } else {
1544          !!!parse-error (type => 'string after PUBLIC');          !!!parse-error (type => 'string after PUBLIC');
1545          $self->{state} = 'bogus DOCTYPE';          $self->{state} = BOGUS_DOCTYPE_STATE;
1546          !!!next-input-character;          !!!next-input-character;
1547          redo A;          redo A;
1548        }        }
1549      } elsif ($self->{state} eq 'DOCTYPE public identifier (double-quoted)') {      } elsif ($self->{state} == DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE) {
1550        if ($self->{next_input_character} == 0x0022) { # "        if ($self->{next_input_character} == 0x0022) { # "
1551          $self->{state} = 'after DOCTYPE public identifier';          $self->{state} = AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
1552          !!!next-input-character;          !!!next-input-character;
1553          redo A;          redo A;
1554        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1555          !!!parse-error (type => 'unclosed PUBLIC literal');          !!!parse-error (type => 'unclosed PUBLIC literal');
1556    
1557          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1558          ## reconsume          ## reconsume
1559    
1560          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1432  sub _get_next_token ($) { Line 1568  sub _get_next_token ($) {
1568          !!!next-input-character;          !!!next-input-character;
1569          redo A;          redo A;
1570        }        }
1571      } elsif ($self->{state} eq 'DOCTYPE public identifier (single-quoted)') {      } elsif ($self->{state} == DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE) {
1572        if ($self->{next_input_character} == 0x0027) { # '        if ($self->{next_input_character} == 0x0027) { # '
1573          $self->{state} = 'after DOCTYPE public identifier';          $self->{state} = AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
1574          !!!next-input-character;          !!!next-input-character;
1575          redo A;          redo A;
1576        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1577          !!!parse-error (type => 'unclosed PUBLIC literal');          !!!parse-error (type => 'unclosed PUBLIC literal');
1578    
1579          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1580          ## reconsume          ## reconsume
1581    
1582          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1454  sub _get_next_token ($) { Line 1590  sub _get_next_token ($) {
1590          !!!next-input-character;          !!!next-input-character;
1591          redo A;          redo A;
1592        }        }
1593      } elsif ($self->{state} eq 'after DOCTYPE public identifier') {      } elsif ($self->{state} == AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE) {
1594        if ({        if ({
1595              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,
1596              #0x000D => 1, # HT, LF, VT, FF, SP, CR              #0x000D => 1, # HT, LF, VT, FF, SP, CR
# Line 1464  sub _get_next_token ($) { Line 1600  sub _get_next_token ($) {
1600          redo A;          redo A;
1601        } elsif ($self->{next_input_character} == 0x0022) { # "        } elsif ($self->{next_input_character} == 0x0022) { # "
1602          $self->{current_token}->{system_identifier} = ''; # DOCTYPE          $self->{current_token}->{system_identifier} = ''; # DOCTYPE
1603          $self->{state} = 'DOCTYPE system identifier (double-quoted)';          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE;
1604          !!!next-input-character;          !!!next-input-character;
1605          redo A;          redo A;
1606        } elsif ($self->{next_input_character} == 0x0027) { # '        } elsif ($self->{next_input_character} == 0x0027) { # '
1607          $self->{current_token}->{system_identifier} = ''; # DOCTYPE          $self->{current_token}->{system_identifier} = ''; # DOCTYPE
1608          $self->{state} = 'DOCTYPE system identifier (single-quoted)';          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE;
1609          !!!next-input-character;          !!!next-input-character;
1610          redo A;          redo A;
1611        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1612          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1613          !!!next-input-character;          !!!next-input-character;
1614    
1615          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{current_token}); # DOCTYPE
# Line 1482  sub _get_next_token ($) { Line 1618  sub _get_next_token ($) {
1618        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1619          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1620    
1621          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1622          ## reconsume          ## reconsume
1623    
1624          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1491  sub _get_next_token ($) { Line 1627  sub _get_next_token ($) {
1627          redo A;          redo A;
1628        } else {        } else {
1629          !!!parse-error (type => 'string after PUBLIC literal');          !!!parse-error (type => 'string after PUBLIC literal');
1630          $self->{state} = 'bogus DOCTYPE';          $self->{state} = BOGUS_DOCTYPE_STATE;
1631          !!!next-input-character;          !!!next-input-character;
1632          redo A;          redo A;
1633        }        }
1634      } elsif ($self->{state} eq 'before DOCTYPE system identifier') {      } elsif ($self->{state} == BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE) {
1635        if ({        if ({
1636              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,
1637              #0x000D => 1, # HT, LF, VT, FF, SP, CR              #0x000D => 1, # HT, LF, VT, FF, SP, CR
# Line 1505  sub _get_next_token ($) { Line 1641  sub _get_next_token ($) {
1641          redo A;          redo A;
1642        } elsif ($self->{next_input_character} == 0x0022) { # "        } elsif ($self->{next_input_character} == 0x0022) { # "
1643          $self->{current_token}->{system_identifier} = ''; # DOCTYPE          $self->{current_token}->{system_identifier} = ''; # DOCTYPE
1644          $self->{state} = 'DOCTYPE system identifier (double-quoted)';          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE;
1645          !!!next-input-character;          !!!next-input-character;
1646          redo A;          redo A;
1647        } elsif ($self->{next_input_character} == 0x0027) { # '        } elsif ($self->{next_input_character} == 0x0027) { # '
1648          $self->{current_token}->{system_identifier} = ''; # DOCTYPE          $self->{current_token}->{system_identifier} = ''; # DOCTYPE
1649          $self->{state} = 'DOCTYPE system identifier (single-quoted)';          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE;
1650          !!!next-input-character;          !!!next-input-character;
1651          redo A;          redo A;
1652        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1653          !!!parse-error (type => 'no SYSTEM literal');          !!!parse-error (type => 'no SYSTEM literal');
1654          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1655          !!!next-input-character;          !!!next-input-character;
1656    
1657          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1525  sub _get_next_token ($) { Line 1661  sub _get_next_token ($) {
1661        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1662          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1663    
1664          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1665          ## reconsume          ## reconsume
1666    
1667          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1534  sub _get_next_token ($) { Line 1670  sub _get_next_token ($) {
1670          redo A;          redo A;
1671        } else {        } else {
1672          !!!parse-error (type => 'string after SYSTEM');          !!!parse-error (type => 'string after SYSTEM');
1673          $self->{state} = 'bogus DOCTYPE';          $self->{state} = BOGUS_DOCTYPE_STATE;
1674          !!!next-input-character;          !!!next-input-character;
1675          redo A;          redo A;
1676        }        }
1677      } elsif ($self->{state} eq 'DOCTYPE system identifier (double-quoted)') {      } elsif ($self->{state} == DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE) {
1678        if ($self->{next_input_character} == 0x0022) { # "        if ($self->{next_input_character} == 0x0022) { # "
1679          $self->{state} = 'after DOCTYPE system identifier';          $self->{state} = AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
1680          !!!next-input-character;          !!!next-input-character;
1681          redo A;          redo A;
1682        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1683          !!!parse-error (type => 'unclosed SYSTEM literal');          !!!parse-error (type => 'unclosed SYSTEM literal');
1684    
1685          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1686          ## reconsume          ## reconsume
1687    
1688          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1560  sub _get_next_token ($) { Line 1696  sub _get_next_token ($) {
1696          !!!next-input-character;          !!!next-input-character;
1697          redo A;          redo A;
1698        }        }
1699      } elsif ($self->{state} eq 'DOCTYPE system identifier (single-quoted)') {      } elsif ($self->{state} == DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE) {
1700        if ($self->{next_input_character} == 0x0027) { # '        if ($self->{next_input_character} == 0x0027) { # '
1701          $self->{state} = 'after DOCTYPE system identifier';          $self->{state} = AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
1702          !!!next-input-character;          !!!next-input-character;
1703          redo A;          redo A;
1704        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1705          !!!parse-error (type => 'unclosed SYSTEM literal');          !!!parse-error (type => 'unclosed SYSTEM literal');
1706    
1707          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1708          ## reconsume          ## reconsume
1709    
1710          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1582  sub _get_next_token ($) { Line 1718  sub _get_next_token ($) {
1718          !!!next-input-character;          !!!next-input-character;
1719          redo A;          redo A;
1720        }        }
1721      } elsif ($self->{state} eq 'after DOCTYPE system identifier') {      } elsif ($self->{state} == AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE) {
1722        if ({        if ({
1723              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,              0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,
1724              #0x000D => 1, # HT, LF, VT, FF, SP, CR              #0x000D => 1, # HT, LF, VT, FF, SP, CR
# Line 1591  sub _get_next_token ($) { Line 1727  sub _get_next_token ($) {
1727          !!!next-input-character;          !!!next-input-character;
1728          redo A;          redo A;
1729        } elsif ($self->{next_input_character} == 0x003E) { # >        } elsif ($self->{next_input_character} == 0x003E) { # >
1730          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1731          !!!next-input-character;          !!!next-input-character;
1732    
1733          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{current_token}); # DOCTYPE
# Line 1600  sub _get_next_token ($) { Line 1736  sub _get_next_token ($) {
1736        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1737          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1738    
1739          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1740          ## reconsume          ## reconsume
1741    
1742          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1609  sub _get_next_token ($) { Line 1745  sub _get_next_token ($) {
1745          redo A;          redo A;
1746        } else {        } else {
1747          !!!parse-error (type => 'string after SYSTEM literal');          !!!parse-error (type => 'string after SYSTEM literal');
1748          $self->{state} = 'bogus DOCTYPE';          $self->{state} = BOGUS_DOCTYPE_STATE;
1749          !!!next-input-character;          !!!next-input-character;
1750          redo A;          redo A;
1751        }        }
1752      } elsif ($self->{state} eq 'bogus DOCTYPE') {      } elsif ($self->{state} == BOGUS_DOCTYPE_STATE) {
1753        if ($self->{next_input_character} == 0x003E) { # >        if ($self->{next_input_character} == 0x003E) { # >
1754          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1755          !!!next-input-character;          !!!next-input-character;
1756    
1757          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 1624  sub _get_next_token ($) { Line 1760  sub _get_next_token ($) {
1760          redo A;          redo A;
1761        } elsif ($self->{next_input_character} == -1) {        } elsif ($self->{next_input_character} == -1) {
1762          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
1763          $self->{state} = 'data';          $self->{state} = DATA_STATE;
1764          ## reconsume          ## reconsume
1765    
1766          delete $self->{current_token}->{correct};          delete $self->{current_token}->{correct};
# Line 2046  sub _tree_construction_root_element ($) Line 2182  sub _tree_construction_root_element ($)
2182              redo B;              redo B;
2183            }            }
2184          }          }
2185    
2186            $self->{application_cache_selection}->(undef);
2187    
2188            #
2189          } elsif ($token->{type} == START_TAG_TOKEN) {
2190            if ($token->{tag_name} eq 'html' and
2191                $token->{attributes}->{manifest}) { ## ISSUE: Spec spells as "application"
2192              $self->{application_cache_selection}
2193                   ->($token->{attributes}->{manifest}->{value});
2194              ## ISSUE: No relative reference resolution?
2195            } else {
2196              $self->{application_cache_selection}->(undef);
2197            }
2198    
2199            ## ISSUE: There is an issue in the spec
2200          #          #
2201        } elsif ({        } elsif ({
                 START_TAG_TOKEN, 1,  
2202                  END_TAG_TOKEN, 1,                  END_TAG_TOKEN, 1,
2203                  END_OF_FILE_TOKEN, 1,                  END_OF_FILE_TOKEN, 1,
2204                 }->{$token->{type}}) {                 }->{$token->{type}}) {
2205            $self->{application_cache_selection}->(undef);
2206    
2207          ## ISSUE: There is an issue in the spec          ## ISSUE: There is an issue in the spec
2208          #          #
2209        } else {        } else {
2210          die "$0: $token->{type}: Unknown token type";          die "$0: $token->{type}: Unknown token type";
2211        }        }
2212    
2213        my $root_element; !!!create-element ($root_element, 'html');        my $root_element; !!!create-element ($root_element, 'html');
2214        $self->{document}->append_child ($root_element);        $self->{document}->append_child ($root_element);
2215        push @{$self->{open_elements}}, [$root_element, 'html'];        push @{$self->{open_elements}}, [$root_element, 'html'];
# Line 2716  sub _tree_construction_main ($) { Line 2869  sub _tree_construction_main ($) {
2869                pop @{$self->{open_elements}}; ## ISSUE: This step is missing in the spec.                pop @{$self->{open_elements}}; ## ISSUE: This step is missing in the spec.
2870    
2871                unless ($self->{confident}) {                unless ($self->{confident}) {
                 my $charset;  
2872                  if ($token->{attributes}->{charset}) { ## TODO: And if supported                  if ($token->{attributes}->{charset}) { ## TODO: And if supported
2873                    $charset = $token->{attributes}->{charset}->{value};                    $self->{change_encoding}
2874                  }                        ->($self, $token->{attributes}->{charset}->{value});
2875                  if ($token->{attributes}->{'http-equiv'}) {                  } elsif ($token->{attributes}->{content}) {
2876                    ## ISSUE: Algorithm name in the spec was incorrect so that not linked to the definition.                    ## ISSUE: Algorithm name in the spec was incorrect so that not linked to the definition.
2877                    if ($token->{attributes}->{'http-equiv'}->{value}                    if ($token->{attributes}->{content}->{value}
2878                        =~ /\A[^;]*;[\x09-\x0D\x20]*charset[\x09-\x0D\x20]*=                        =~ /\A[^;]*;[\x09-\x0D\x20]*charset[\x09-\x0D\x20]*=
2879                            [\x09-\x0D\x20]*(?>"([^"]*)"|'([^']*)'|                            [\x09-\x0D\x20]*(?>"([^"]*)"|'([^']*)'|
2880                            ([^"'\x09-\x0D\x20][^\x09-\x0D\x20]*))/x) {                            ([^"'\x09-\x0D\x20][^\x09-\x0D\x20]*))/x) {
2881                      $charset = defined $1 ? $1 : defined $2 ? $2 : $3;                      $self->{change_encoding}
2882                    } ## TODO: And if supported                          ->($self, defined $1 ? $1 : defined $2 ? $2 : $3);
2883                      }
2884                  }                  }
                 ## TODO: Change the encoding  
2885                }                }
2886    
               ## TODO: Extracting |charset| from |meta|.  
2887                pop @{$self->{open_elements}}                pop @{$self->{open_elements}}
2888                    if $self->{insertion_mode} == AFTER_HEAD_IM;                    if $self->{insertion_mode} == AFTER_HEAD_IM;
2889                !!!next-token;                !!!next-token;
# Line 3302  sub _tree_construction_main ($) { Line 3453  sub _tree_construction_main ($) {
3453        $insert = $insert_to_current;        $insert = $insert_to_current;
3454        #        #
3455      } elsif ($self->{insertion_mode} & TABLE_IMS) {      } elsif ($self->{insertion_mode} & TABLE_IMS) {
3456            if ($token->{type} == CHARACTER_TOKEN) {        if ($token->{type} == CHARACTER_TOKEN) {
             ## NOTE: There are "character in table" code clones.  
3457              if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {              if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {
3458                $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);                $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);
3459                                
# Line 3360  sub _tree_construction_main ($) { Line 3510  sub _tree_construction_main ($) {
3510                            
3511              !!!next-token;              !!!next-token;
3512              redo B;              redo B;
3513            } elsif ($token->{type} == START_TAG_TOKEN) {        } elsif ($token->{type} == START_TAG_TOKEN) {
3514              if ({              if ({
3515                   tr => ($self->{insertion_mode} != IN_ROW_IM),                   tr => ($self->{insertion_mode} != IN_ROW_IM),
3516                   th => 1, td => 1,                   th => 1, td => 1,
# Line 3546  sub _tree_construction_main ($) { Line 3696  sub _tree_construction_main ($) {
3696                  die "$0: in table: <>: $token->{tag_name}";                  die "$0: in table: <>: $token->{tag_name}";
3697                }                }
3698              } elsif ($token->{tag_name} eq 'table') {              } elsif ($token->{tag_name} eq 'table') {
               ## NOTE: There are code clones for this "table in table"  
3699                !!!parse-error (type => 'not closed:'.$self->{open_elements}->[-1]->[1]);                !!!parse-error (type => 'not closed:'.$self->{open_elements}->[-1]->[1]);
3700    
3701                ## As if </table>                ## As if </table>
# Line 3594  sub _tree_construction_main ($) { Line 3743  sub _tree_construction_main ($) {
3743    
3744                ## reprocess                ## reprocess
3745                redo B;                redo B;
3746              } else {          } else {
3747                #            !!!parse-error (type => 'in table:'.$token->{tag_name});
3748              }  
3749            } elsif ($token->{type} == END_TAG_TOKEN) {            $insert = $insert_to_foster;
3750              #
3751            }
3752          } elsif ($token->{type} == END_TAG_TOKEN) {
3753              if ($token->{tag_name} eq 'tr' and              if ($token->{tag_name} eq 'tr' and
3754                  $self->{insertion_mode} == IN_ROW_IM) {                  $self->{insertion_mode} == IN_ROW_IM) {
3755                ## have an element in table scope                ## have an element in table scope
# Line 3854  sub _tree_construction_main ($) { Line 4006  sub _tree_construction_main ($) {
4006                ## Ignore the token                ## Ignore the token
4007                !!!next-token;                !!!next-token;
4008                redo B;                redo B;
4009              } else {          } else {
4010                #            !!!parse-error (type => 'in table:/'.$token->{tag_name});
             }  
           } else {  
             die "$0: $token->{type}: Unknown token type";  
           }  
   
       !!!parse-error (type => 'in table:'.$token->{tag_name});  
4011    
4012        $insert = $insert_to_foster;            $insert = $insert_to_foster;
4013        #            #
4014            }
4015          } else {
4016            die "$0: $token->{type}: Unknown token type";
4017          }
4018      } elsif ($self->{insertion_mode} == IN_COLUMN_GROUP_IM) {      } elsif ($self->{insertion_mode} == IN_COLUMN_GROUP_IM) {
4019            if ($token->{type} == CHARACTER_TOKEN) {            if ($token->{type} == CHARACTER_TOKEN) {
4020              if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {              if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {
# Line 3923  sub _tree_construction_main ($) { Line 4073  sub _tree_construction_main ($) {
4073              redo B;              redo B;
4074            }            }
4075      } elsif ($self->{insertion_mode} == IN_SELECT_IM) {      } elsif ($self->{insertion_mode} == IN_SELECT_IM) {
4076            if ($token->{type} == CHARACTER_TOKEN) {        if ($token->{type} == CHARACTER_TOKEN) {
4077              $self->{open_elements}->[-1]->[0]->manakai_append_text ($token->{data});          $self->{open_elements}->[-1]->[0]->manakai_append_text ($token->{data});
4078              !!!next-token;          !!!next-token;
4079              redo B;          redo B;
4080            } elsif ($token->{type} == START_TAG_TOKEN) {        } elsif ($token->{type} == START_TAG_TOKEN) {
4081              if ($token->{tag_name} eq 'option') {              if ($token->{tag_name} eq 'option') {
4082                if ($self->{open_elements}->[-1]->[1] eq 'option') {                if ($self->{open_elements}->[-1]->[1] eq 'option') {
4083                  ## As if </option>                  ## As if </option>
# Line 3980  sub _tree_construction_main ($) { Line 4130  sub _tree_construction_main ($) {
4130    
4131                !!!next-token;                !!!next-token;
4132                redo B;                redo B;
4133              } else {          } else {
4134                #            !!!parse-error (type => 'in select:'.$token->{tag_name});
4135              }            ## Ignore the token
4136            } elsif ($token->{type} == END_TAG_TOKEN) {            !!!next-token;
4137              redo B;
4138            }
4139          } elsif ($token->{type} == END_TAG_TOKEN) {
4140              if ($token->{tag_name} eq 'optgroup') {              if ($token->{tag_name} eq 'optgroup') {
4141                if ($self->{open_elements}->[-1]->[1] eq 'option' and                if ($self->{open_elements}->[-1]->[1] eq 'option' and
4142                    $self->{open_elements}->[-2]->[1] eq 'optgroup') {                    $self->{open_elements}->[-2]->[1] eq 'optgroup') {
# Line 4085  sub _tree_construction_main ($) { Line 4238  sub _tree_construction_main ($) {
4238    
4239                ## reprocess                ## reprocess
4240                redo B;                redo B;
4241              } else {          } else {
4242                #            !!!parse-error (type => 'in select:/'.$token->{tag_name});
             }  
           } else {  
             #  
           }  
   
           !!!parse-error (type => 'in select:'.$token->{tag_name});  
4243            ## Ignore the token            ## Ignore the token
4244            !!!next-token;            !!!next-token;
4245            redo B;            redo B;
4246            }
4247          } else {
4248            die "$0: $token->{type}: Unknown token type";
4249          }
4250      } elsif ($self->{insertion_mode} & BODY_AFTER_IMS) {      } elsif ($self->{insertion_mode} & BODY_AFTER_IMS) {
4251        if ($token->{type} == CHARACTER_TOKEN) {        if ($token->{type} == CHARACTER_TOKEN) {
4252          if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {          if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {
# Line 4306  sub _tree_construction_main ($) { Line 4457  sub _tree_construction_main ($) {
4457          pop @{$self->{open_elements}}; ## ISSUE: This step is missing in the spec.          pop @{$self->{open_elements}}; ## ISSUE: This step is missing in the spec.
4458    
4459          unless ($self->{confident}) {          unless ($self->{confident}) {
           my $charset;  
4460            if ($token->{attributes}->{charset}) { ## TODO: And if supported            if ($token->{attributes}->{charset}) { ## TODO: And if supported
4461              $charset = $token->{attributes}->{charset}->{value};              $self->{change_encoding}
4462            }                  ->($self, $token->{attributes}->{charset}->{value});
4463            if ($token->{attributes}->{'http-equiv'}) {            } elsif ($token->{attributes}->{content}) {
4464              ## ISSUE: Algorithm name in the spec was incorrect so that not linked to the definition.              ## ISSUE: Algorithm name in the spec was incorrect so that not linked to the definition.
4465              if ($token->{attributes}->{'http-equiv'}->{value}              if ($token->{attributes}->{content}->{value}
4466                  =~ /\A[^;]*;[\x09-\x0D\x20]*charset[\x09-\x0D\x20]*=                  =~ /\A[^;]*;[\x09-\x0D\x20]*charset[\x09-\x0D\x20]*=
4467                      [\x09-\x0D\x20]*(?>"([^"]*)"|'([^']*)'|                      [\x09-\x0D\x20]*(?>"([^"]*)"|'([^']*)'|
4468                      ([^"'\x09-\x0D\x20][^\x09-\x0D\x20]*))/x) {                      ([^"'\x09-\x0D\x20][^\x09-\x0D\x20]*))/x) {
4469                $charset = defined $1 ? $1 : defined $2 ? $2 : $3;                $self->{change_encoding}
4470              } ## TODO: And if supported                    ->($self, defined $1 ? $1 : defined $2 ? $2 : $3);
4471                }
4472            }            }
           ## TODO: Change the encoding  
4473          }          }
4474    
4475          !!!next-token;          !!!next-token;
# Line 4626  sub _tree_construction_main ($) { Line 4776  sub _tree_construction_main ($) {
4776          INSCOPE: for (reverse 0..$#{$self->{open_elements}}) {          INSCOPE: for (reverse 0..$#{$self->{open_elements}}) {
4777            my $node = $self->{open_elements}->[$_];            my $node = $self->{open_elements}->[$_];
4778            if ($node->[1] eq 'nobr') {            if ($node->[1] eq 'nobr') {
4779              !!!parse-error (type => 'not closed:nobr');              !!!parse-error (type => 'in nobr:nobr');
4780              !!!back-token;              !!!back-token;
4781              $token = {type => END_TAG_TOKEN, tag_name => 'nobr'};              $token = {type => END_TAG_TOKEN, tag_name => 'nobr'};
4782              redo B;              redo B;
# Line 4831  sub _tree_construction_main ($) { Line 4981  sub _tree_construction_main ($) {
4981                  noframes => 1,                  noframes => 1,
4982                  noscript => 0, ## TODO: 1 if scripting is enabled                  noscript => 0, ## TODO: 1 if scripting is enabled
4983                 }->{$token->{tag_name}}) {                 }->{$token->{tag_name}}) {
4984          ## NOTE: There are two "as if in body" code clones.          ## NOTE: There is an "as if in body" code clone.
4985          $parse_rcdata->(CDATA_CONTENT_MODEL, $insert);          $parse_rcdata->(CDATA_CONTENT_MODEL, $insert);
4986          redo B;          redo B;
4987        } elsif ($token->{tag_name} eq 'select') {        } elsif ($token->{tag_name} eq 'select') {
# Line 4987  sub _tree_construction_main ($) { Line 5137  sub _tree_construction_main ($) {
5137          if ($self->{open_elements}->[-1]->[1] eq $token->{tag_name}) {          if ($self->{open_elements}->[-1]->[1] eq $token->{tag_name}) {
5138            pop @{$self->{open_elements}};            pop @{$self->{open_elements}};
5139          } else {          } else {
5140            !!!parse-error (type => 'not closed:'.$self->{open_elements}->[-1]->[1]);            !!!parse-error (type => 'unmatched end tag:'.$token->{tag_name});
5141          }          }
5142    
5143          undef $self->{form_element};          undef $self->{form_element};
# Line 5025  sub _tree_construction_main ($) { Line 5175  sub _tree_construction_main ($) {
5175          } # INSCOPE          } # INSCOPE
5176                    
5177          if ($self->{open_elements}->[-1]->[1] ne $token->{tag_name}) {          if ($self->{open_elements}->[-1]->[1] ne $token->{tag_name}) {
5178            !!!parse-error (type => 'not closed:'.$self->{open_elements}->[-1]->[1]);            !!!parse-error (type => 'unmatched end tag:'.$token->{tag_name});
5179          }          }
5180                    
5181          splice @{$self->{open_elements}}, $i if defined $i;          splice @{$self->{open_elements}}, $i if defined $i;
# Line 5094  sub _tree_construction_main ($) { Line 5244  sub _tree_construction_main ($) {
5244                    
5245              ## Step 2              ## Step 2
5246              if ($token->{tag_name} ne $self->{open_elements}->[-1]->[1]) {              if ($token->{tag_name} ne $self->{open_elements}->[-1]->[1]) {
5247                  ## NOTE: <x><y></x>
5248                !!!parse-error (type => 'not closed:'.$self->{open_elements}->[-1]->[1]);                !!!parse-error (type => 'not closed:'.$self->{open_elements}->[-1]->[1]);
5249              }              }
5250                            
# Line 5144  sub set_inner_html ($$$) { Line 5295  sub set_inner_html ($$$) {
5295    my $s = \$_[0];    my $s = \$_[0];
5296    my $onerror = $_[1];    my $onerror = $_[1];
5297    
5298      ## ISSUE: Should {confident} be true?
5299    
5300    my $nt = $node->node_type;    my $nt = $node->node_type;
5301    if ($nt == 9) {    if ($nt == 9) {
5302      # MUST      # MUST
# Line 5296  sub set_inner_html ($$$) { Line 5449  sub set_inner_html ($$$) {
5449    
5450  } # tree construction stage  } # tree construction stage
5451    
5452  sub get_inner_html ($$$) {  package Whatpm::HTML::RestartParser;
5453    my (undef, $node, $on_error) = @_;  push our @ISA, 'Error';
   
   ## Step 1  
   my $s = '';  
   
   my $in_cdata;  
   my $parent = $node;  
   while (defined $parent) {  
     if ($parent->node_type == 1 and  
         $parent->namespace_uri eq 'http://www.w3.org/1999/xhtml' and  
         {  
           style => 1, script => 1, xmp => 1, iframe => 1,  
           noembed => 1, noframes => 1, noscript => 1,  
         }->{$parent->local_name}) { ## TODO: case thingy  
       $in_cdata = 1;  
     }  
     $parent = $parent->parent_node;  
   }  
   
   ## Step 2  
   my @node = @{$node->child_nodes};  
   C: while (@node) {  
     my $child = shift @node;  
     unless (ref $child) {  
       if ($child eq 'cdata-out') {  
         $in_cdata = 0;  
       } else {  
         $s .= $child; # end tag  
       }  
       next C;  
     }  
       
     my $nt = $child->node_type;  
     if ($nt == 1) { # Element  
       my $tag_name = $child->tag_name; ## TODO: manakai_tag_name  
       $s .= '<' . $tag_name;  
       ## NOTE: Non-HTML case:  
       ## <http://permalink.gmane.org/gmane.org.w3c.whatwg.discuss/11191>  
   
       my @attrs = @{$child->attributes}; # sort order MUST be stable  
       for my $attr (@attrs) { # order is implementation dependent  
         my $attr_name = $attr->name; ## TODO: manakai_name  
         $s .= ' ' . $attr_name . '="';  
         my $attr_value = $attr->value;  
         ## escape  
         $attr_value =~ s/&/&amp;/g;  
         $attr_value =~ s/</&lt;/g;  
         $attr_value =~ s/>/&gt;/g;  
         $attr_value =~ s/"/&quot;/g;  
         $s .= $attr_value . '"';  
       }  
       $s .= '>';  
         
       next C if {  
         area => 1, base => 1, basefont => 1, bgsound => 1,  
         br => 1, col => 1, embed => 1, frame => 1, hr => 1,  
         img => 1, input => 1, link => 1, meta => 1, param => 1,  
         spacer => 1, wbr => 1,  
       }->{$tag_name};  
   
       $s .= "\x0A" if $tag_name eq 'pre' or $tag_name eq 'textarea';  
   
       if (not $in_cdata and {  
         style => 1, script => 1, xmp => 1, iframe => 1,  
         noembed => 1, noframes => 1, noscript => 1,  
         plaintext => 1,  
       }->{$tag_name}) {  
         unshift @node, 'cdata-out';  
         $in_cdata = 1;  
       }  
   
       unshift @node, @{$child->child_nodes}, '</' . $tag_name . '>';  
     } elsif ($nt == 3 or $nt == 4) {  
       if ($in_cdata) {  
         $s .= $child->data;  
       } else {  
         my $value = $child->data;  
         $value =~ s/&/&amp;/g;  
         $value =~ s/</&lt;/g;  
         $value =~ s/>/&gt;/g;  
         $value =~ s/"/&quot;/g;  
         $s .= $value;  
       }  
     } elsif ($nt == 8) {  
       $s .= '<!--' . $child->data . '-->';  
     } elsif ($nt == 10) {  
       $s .= '<!DOCTYPE ' . $child->name . '>';  
     } elsif ($nt == 5) { # entrefs  
       push @node, @{$child->child_nodes};  
     } else {  
       $on_error->($child) if defined $on_error;  
     }  
     ## ISSUE: This code does not support PIs.  
   } # C  
     
   ## Step 3  
   return \$s;  
 } # get_inner_html  
5454    
5455  1;  1;
5456  # $Date$  # $Date$

Legend:
Removed from v.1.56  
changed lines
  Added in v.1.65

[email protected]
ViewVC Help
Powered by ViewVC 1.1.24