/[suikacvs]/markup/html/whatpm/Whatpm/HTML.pm.src
Suika

Diff of /markup/html/whatpm/Whatpm/HTML.pm.src

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.167 by wakaba, Sat Sep 13 09:02:28 2008 UTC revision 1.191 by wakaba, Mon Sep 22 06:04:29 2008 UTC
# Line 3  use strict; Line 3  use strict;
3  our $VERSION=do{my @r=(q$Revision$=~/\d+/g);sprintf "%d."."%02d" x $#r,@r};  our $VERSION=do{my @r=(q$Revision$=~/\d+/g);sprintf "%d."."%02d" x $#r,@r};
4  use Error qw(:try);  use Error qw(:try);
5    
6    ## NOTE: This module don't check all HTML5 parse errors; character
7    ## encoding related parse errors are expected to be handled by relevant
8    ## modules.
9    ## Parse errors for control characters that are not allowed in HTML5
10    ## documents, for surrogate code points, and for noncharacter code
11    ## points, as well as U+FFFD substitions for characters whose code points
12    ## is higher than U+10FFFF may be detected by combining the parser with
13    ## the checker implemented by Whatpm::Charset::UnicodeChecker (for its
14    ## usage example, see |t/HTML-tree.t| in the Whatpm package or the
15    ## WebHACC::Language::HTML module in the WebHACC package).
16    
17  ## ISSUE:  ## ISSUE:
18  ## var doc = implementation.createDocument (null, null, null);  ## var doc = implementation.createDocument (null, null, null);
19  ## doc.write ('');  ## doc.write ('');
# Line 312  my $foreign_attr_xname = { Line 323  my $foreign_attr_xname = {
323    
324  ## ISSUE: xmlns:xlink="non-xlink-ns" is not an error.  ## ISSUE: xmlns:xlink="non-xlink-ns" is not an error.
325    
326  my $c1_entity_char = {  my $charref_map = {
327      0x0D => 0x000A,
328    0x80 => 0x20AC,    0x80 => 0x20AC,
329    0x81 => 0xFFFD,    0x81 => 0xFFFD,
330    0x82 => 0x201A,    0x82 => 0x201A,
# Line 345  my $c1_entity_char = { Line 357  my $c1_entity_char = {
357    0x9D => 0xFFFD,    0x9D => 0xFFFD,
358    0x9E => 0x017E,    0x9E => 0x017E,
359    0x9F => 0x0178,    0x9F => 0x0178,
360  }; # $c1_entity_char  }; # $charref_map
361    $charref_map->{$_} = 0xFFFD
362        for 0x0000..0x0008, 0x000B, 0x000E..0x001F, 0x007F,
363            0xD800..0xDFFF, 0xFDD0..0xFDDF, ## ISSUE: 0xFDEF
364            0xFFFE, 0xFFFF, 0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, 0x3FFFF,
365            0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF, 0x6FFFE, 0x6FFFF, 0x7FFFE,
366            0x7FFFF, 0x8FFFE, 0x8FFFF, 0x9FFFE, 0x9FFFF, 0xAFFFE, 0xAFFFF,
367            0xBFFFE, 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE, 0xDFFFF, 0xEFFFE,
368            0xEFFFF, 0xFFFFE, 0xFFFFF, 0x10FFFE, 0x10FFFF;
369    
370  sub parse_byte_string ($$$$;$) {  sub parse_byte_string ($$$$;$) {
371    my $self = shift;    my $self = shift;
# Line 390  sub parse_byte_stream ($$$$;$$) { Line 410  sub parse_byte_stream ($$$$;$$) {
410            ## TODO: Is this ok?  Transfer protocol's parameter should be            ## TODO: Is this ok?  Transfer protocol's parameter should be
411            ## interpreted in its semantics?            ## interpreted in its semantics?
412    
       ## ISSUE: Unsupported encoding is not ignored according to the spec.  
413        ($char_stream, $e_status) = $charset->get_decode_handle        ($char_stream, $e_status) = $charset->get_decode_handle
414            ($byte_stream, allow_error_reporting => 1,            ($byte_stream, allow_error_reporting => 1,
415             allow_fallback => 1);             allow_fallback => 1);
# Line 398  sub parse_byte_stream ($$$$;$$) { Line 417  sub parse_byte_stream ($$$$;$$) {
417          $self->{confident} = 1;          $self->{confident} = 1;
418          last SNIFFING;          last SNIFFING;
419        } else {        } else {
420          ## TODO: unsupported error          !!!parse-error (type => 'charset:not supported',
421                            layer => 'encode',
422                            line => 1, column => 1,
423                            value => $charset_name,
424                            level => $self->{level}->{uncertain});
425        }        }
426      }      }
427    
# Line 496  sub parse_byte_stream ($$$$;$$) { Line 519  sub parse_byte_stream ($$$$;$$) {
519                      line => 1, column => 1,                      line => 1, column => 1,
520                      layer => 'encode');                      layer => 'encode');
521    } elsif (not ($e_status &    } elsif (not ($e_status &
522                  Message::Charset::Info::ERROR_REPORTING_ENCODING_IMPL())) {                  Message::Charset::Info::ERROR_REPORTING_ENCODING_IMPL ())) {
523      $self->{input_encoding} = $charset->get_iana_name;      $self->{input_encoding} = $charset->get_iana_name;
524      !!!parse-error (type => 'chardecode:no error',      !!!parse-error (type => 'chardecode:no error',
525                      text => $self->{input_encoding},                      text => $self->{input_encoding},
# Line 561  sub parse_byte_stream ($$$$;$$) { Line 584  sub parse_byte_stream ($$$$;$$) {
584    my $char_onerror = sub {    my $char_onerror = sub {
585      my (undef, $type, %opt) = @_;      my (undef, $type, %opt) = @_;
586      !!!parse-error (layer => 'encode',      !!!parse-error (layer => 'encode',
587                      %opt, type => $type,                      line => $self->{line}, column => $self->{column} + 1,
588                      line => $self->{line}, column => $self->{column} + 1);                      %opt, type => $type);
589      if ($opt{octets}) {      if ($opt{octets}) {
590        ${$opt{octets}} = "\x{FFFD}"; # relacement character        ${$opt{octets}} = "\x{FFFD}"; # relacement character
591      }      }
# Line 571  sub parse_byte_stream ($$$$;$$) { Line 594  sub parse_byte_stream ($$$$;$$) {
594    my $wrapped_char_stream = $get_wrapper->($char_stream);    my $wrapped_char_stream = $get_wrapper->($char_stream);
595    $wrapped_char_stream->onerror ($char_onerror);    $wrapped_char_stream->onerror ($char_onerror);
596    
597    my @args = @_; shift @args; # $s    my @args = ($_[1], $_[2]); # $doc, $onerror - $get_wrapper = undef;
598    my $return;    my $return;
599    try {    try {
600      $return = $self->parse_char_stream ($wrapped_char_stream, @args);        $return = $self->parse_char_stream ($wrapped_char_stream, @args);  
# Line 586  sub parse_byte_stream ($$$$;$$) { Line 609  sub parse_byte_stream ($$$$;$$) {
609                        line => 1, column => 1,                        line => 1, column => 1,
610                        layer => 'encode');                        layer => 'encode');
611      } elsif (not ($e_status &      } elsif (not ($e_status &
612                    Message::Charset::Info::ERROR_REPORTING_ENCODING_IMPL())) {                    Message::Charset::Info::ERROR_REPORTING_ENCODING_IMPL ())) {
613        $self->{input_encoding} = $charset->get_iana_name;        $self->{input_encoding} = $charset->get_iana_name;
614        !!!parse-error (type => 'chardecode:no error',        !!!parse-error (type => 'chardecode:no error',
615                        text => $self->{input_encoding},                        text => $self->{input_encoding},
# Line 618  sub parse_byte_stream ($$$$;$$) { Line 641  sub parse_byte_stream ($$$$;$$) {
641  sub parse_char_string ($$$;$$) {  sub parse_char_string ($$$;$$) {
642    #my ($self, $s, $doc, $onerror, $get_wrapper) = @_;    #my ($self, $s, $doc, $onerror, $get_wrapper) = @_;
643    my $self = shift;    my $self = shift;
   require utf8;  
644    my $s = ref $_[0] ? $_[0] : \($_[0]);    my $s = ref $_[0] ? $_[0] : \($_[0]);
645    open my $input, '<' . (utf8::is_utf8 ($$s) ? ':utf8' : ''), $s;    require Whatpm::Charset::DecodeHandle;
646    if ($_[3]) {    my $input = Whatpm::Charset::DecodeHandle::CharString->new ($s);
     $input = $_[3]->($input);  
   }  
647    return $self->parse_char_stream ($input, @_[1..$#_]);    return $self->parse_char_stream ($input, @_[1..$#_]);
648  } # parse_char_string  } # parse_char_string
649  *parse_string = \&parse_char_string; ## NOTE: Alias for backward compatibility.  *parse_string = \&parse_char_string; ## NOTE: Alias for backward compatibility.
650    
651  sub parse_char_stream ($$$;$) {  sub parse_char_stream ($$$;$$) {
652    my $self = ref $_[0] ? shift : shift->new;    my $self = ref $_[0] ? shift : shift->new;
653    my $input = $_[0];    my $input = $_[0];
654    $self->{document} = $_[1];    $self->{document} = $_[1];
# Line 639  sub parse_char_stream ($$$;$) { Line 659  sub parse_char_stream ($$$;$) {
659    $self->{confident} = 1 unless exists $self->{confident};    $self->{confident} = 1 unless exists $self->{confident};
660    $self->{document}->input_encoding ($self->{input_encoding})    $self->{document}->input_encoding ($self->{input_encoding})
661        if defined $self->{input_encoding};        if defined $self->{input_encoding};
662    ## TODO: |{input_encoding}| is needless?
663    
   my $i = 0;  
664    $self->{line_prev} = $self->{line} = 1;    $self->{line_prev} = $self->{line} = 1;
665    $self->{column_prev} = $self->{column} = 0;    $self->{column_prev} = -1;
666    $self->{set_next_char} = sub {    $self->{column} = 0;
667      $self->{set_nc} = sub {
668      my $self = shift;      my $self = shift;
669    
670      pop @{$self->{prev_char}};      my $char = '';
671      unshift @{$self->{prev_char}}, $self->{next_char};      if (defined $self->{next_nc}) {
672          $char = $self->{next_nc};
673      my $char;        delete $self->{next_nc};
674      if (defined $self->{next_next_char}) {        $self->{nc} = ord $char;
       $char = $self->{next_next_char};  
       delete $self->{next_next_char};  
675      } else {      } else {
676        $char = $input->getc;        $self->{char_buffer} = '';
677          $self->{char_buffer_pos} = 0;
678    
679          my $count = $input->manakai_read_until
680             ($self->{char_buffer}, qr/[^\x00\x0A\x0D]/, $self->{char_buffer_pos});
681          if ($count) {
682            $self->{line_prev} = $self->{line};
683            $self->{column_prev} = $self->{column};
684            $self->{column}++;
685            $self->{nc}
686                = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
687            return;
688          }
689    
690          if ($input->read ($char, 1)) {
691            $self->{nc} = ord $char;
692          } else {
693            $self->{nc} = -1;
694            return;
695          }
696      }      }
     $self->{next_char} = -1 and return unless defined $char;  
     $self->{next_char} = ord $char;  
697    
698      ($self->{line_prev}, $self->{column_prev})      ($self->{line_prev}, $self->{column_prev})
699          = ($self->{line}, $self->{column});          = ($self->{line}, $self->{column});
700      $self->{column}++;      $self->{column}++;
701            
702      if ($self->{next_char} == 0x000A) { # LF      if ($self->{nc} == 0x000A) { # LF
703        !!!cp ('j1');        !!!cp ('j1');
704        $self->{line}++;        $self->{line}++;
705        $self->{column} = 0;        $self->{column} = 0;
706      } elsif ($self->{next_char} == 0x000D) { # CR      } elsif ($self->{nc} == 0x000D) { # CR
707        !!!cp ('j2');        !!!cp ('j2');
708        my $next = $input->getc;  ## TODO: support for abort/streaming
709        if (defined $next and $next ne "\x0A") {        my $next = '';
710          $self->{next_next_char} = $next;        if ($input->read ($next, 1) and $next ne "\x0A") {
711            $self->{next_nc} = $next;
712        }        }
713        $self->{next_char} = 0x000A; # LF # MUST        $self->{nc} = 0x000A; # LF # MUST
714        $self->{line}++;        $self->{line}++;
715        $self->{column} = 0;        $self->{column} = 0;
716      } elsif ($self->{next_char} > 0x10FFFF) {      } elsif ($self->{nc} == 0x0000) { # NULL
       !!!cp ('j3');  
       $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST  
     } elsif ($self->{next_char} == 0x0000) { # NULL  
717        !!!cp ('j4');        !!!cp ('j4');
718        !!!parse-error (type => 'NULL');        !!!parse-error (type => 'NULL');
719        $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST        $self->{nc} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
720      } elsif ($self->{next_char} <= 0x0008 or      }
721               (0x000E <= $self->{next_char} and $self->{next_char} <= 0x001F) or    };
722               (0x007F <= $self->{next_char} and $self->{next_char} <= 0x009F) or  
723               (0xD800 <= $self->{next_char} and $self->{next_char} <= 0xDFFF) or    $self->{read_until} = sub {
724               (0xFDD0 <= $self->{next_char} and $self->{next_char} <= 0xFDDF) or      #my ($scalar, $specials_range, $offset) = @_;
725               {      return 0 if defined $self->{next_nc};
726                0xFFFE => 1, 0xFFFF => 1, 0x1FFFE => 1, 0x1FFFF => 1,  
727                0x2FFFE => 1, 0x2FFFF => 1, 0x3FFFE => 1, 0x3FFFF => 1,      my $pattern = qr/[^$_[1]\x00\x0A\x0D]/;
728                0x4FFFE => 1, 0x4FFFF => 1, 0x5FFFE => 1, 0x5FFFF => 1,      my $offset = $_[2] || 0;
729                0x6FFFE => 1, 0x6FFFF => 1, 0x7FFFE => 1, 0x7FFFF => 1,  
730                0x8FFFE => 1, 0x8FFFF => 1, 0x9FFFE => 1, 0x9FFFF => 1,      if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
731                0xAFFFE => 1, 0xAFFFF => 1, 0xBFFFE => 1, 0xBFFFF => 1,        pos ($self->{char_buffer}) = $self->{char_buffer_pos};
732                0xCFFFE => 1, 0xCFFFF => 1, 0xDFFFE => 1, 0xDFFFF => 1,        if ($self->{char_buffer} =~ /\G(?>$pattern)+/) {
733                0xEFFFE => 1, 0xEFFFF => 1, 0xFFFFE => 1, 0xFFFFF => 1,          substr ($_[0], $offset)
734                0x10FFFE => 1, 0x10FFFF => 1,              = substr ($self->{char_buffer}, $-[0], $+[0] - $-[0]);
735               }->{$self->{next_char}}) {          my $count = $+[0] - $-[0];
736        !!!cp ('j5');          if ($count) {
737        if ($self->{next_char} < 0x10000) {            $self->{column} += $count;
738          !!!parse-error (type => 'control char',            $self->{char_buffer_pos} += $count;
739                          text => (sprintf 'U+%04X', $self->{next_char}));            $self->{line_prev} = $self->{line};
740              $self->{column_prev} = $self->{column} - 1;
741              $self->{nc} = -1;
742            }
743            return $count;
744        } else {        } else {
745          !!!parse-error (type => 'control char',          return 0;
                         text => (sprintf 'U-%08X', $self->{next_char}));  
746        }        }
747        } else {
748          my $count = $input->manakai_read_until ($_[0], $pattern, $_[2]);
749          if ($count) {
750            $self->{column} += $count;
751            $self->{line_prev} = $self->{line};
752            $self->{column_prev} = $self->{column} - 1;
753            $self->{nc} = -1;
754          }
755          return $count;
756      }      }
757    };    }; # $self->{read_until}
   $self->{prev_char} = [-1, -1, -1];  
   $self->{next_char} = -1;  
758    
759    my $onerror = $_[2] || sub {    my $onerror = $_[2] || sub {
760      my (%opt) = @_;      my (%opt) = @_;
# Line 722  sub parse_char_stream ($$$;$) { Line 766  sub parse_char_stream ($$$;$) {
766      $onerror->(line => $self->{line}, column => $self->{column}, @_);      $onerror->(line => $self->{line}, column => $self->{column}, @_);
767    };    };
768    
769      my $char_onerror = sub {
770        my (undef, $type, %opt) = @_;
771        !!!parse-error (layer => 'encode',
772                        line => $self->{line}, column => $self->{column} + 1,
773                        %opt, type => $type);
774      }; # $char_onerror
775    
776      if ($_[3]) {
777        $input = $_[3]->($input);
778        $input->onerror ($char_onerror);
779      } else {
780        $input->onerror ($char_onerror) unless defined $input->onerror;
781      }
782    
783    $self->_initialize_tokenizer;    $self->_initialize_tokenizer;
784    $self->_initialize_tree_constructor;    $self->_initialize_tree_constructor;
785    $self->_construct_tree;    $self->_construct_tree;
# Line 741  sub new ($) { Line 799  sub new ($) {
799                info => 'i',                info => 'i',
800                uncertain => 'u'},                uncertain => 'u'},
801    }, $class;    }, $class;
802    $self->{set_next_char} = sub {    $self->{set_nc} = sub {
803      $self->{next_char} = -1;      $self->{nc} = -1;
804    };    };
805    $self->{parse_error} = sub {    $self->{parse_error} = sub {
806      #      #
# Line 769  sub RCDATA_CONTENT_MODEL () { CM_ENTITY Line 827  sub RCDATA_CONTENT_MODEL () { CM_ENTITY
827  sub PCDATA_CONTENT_MODEL () { CM_ENTITY | CM_FULL_MARKUP }  sub PCDATA_CONTENT_MODEL () { CM_ENTITY | CM_FULL_MARKUP }
828    
829  sub DATA_STATE () { 0 }  sub DATA_STATE () { 0 }
830  sub ENTITY_DATA_STATE () { 1 }  #sub ENTITY_DATA_STATE () { 1 }
831  sub TAG_OPEN_STATE () { 2 }  sub TAG_OPEN_STATE () { 2 }
832  sub CLOSE_TAG_OPEN_STATE () { 3 }  sub CLOSE_TAG_OPEN_STATE () { 3 }
833  sub TAG_NAME_STATE () { 4 }  sub TAG_NAME_STATE () { 4 }
# Line 780  sub BEFORE_ATTRIBUTE_VALUE_STATE () { 8 Line 838  sub BEFORE_ATTRIBUTE_VALUE_STATE () { 8
838  sub ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE () { 9 }  sub ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE () { 9 }
839  sub ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE () { 10 }  sub ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE () { 10 }
840  sub ATTRIBUTE_VALUE_UNQUOTED_STATE () { 11 }  sub ATTRIBUTE_VALUE_UNQUOTED_STATE () { 11 }
841  sub ENTITY_IN_ATTRIBUTE_VALUE_STATE () { 12 }  #sub ENTITY_IN_ATTRIBUTE_VALUE_STATE () { 12 }
842  sub MARKUP_DECLARATION_OPEN_STATE () { 13 }  sub MARKUP_DECLARATION_OPEN_STATE () { 13 }
843  sub COMMENT_START_STATE () { 14 }  sub COMMENT_START_STATE () { 14 }
844  sub COMMENT_START_DASH_STATE () { 15 }  sub COMMENT_START_DASH_STATE () { 15 }
# Line 807  sub CDATA_SECTION_STATE () { 35 } Line 865  sub CDATA_SECTION_STATE () { 35 }
865  sub MD_HYPHEN_STATE () { 36 } # "markup declaration open state" in the spec  sub MD_HYPHEN_STATE () { 36 } # "markup declaration open state" in the spec
866  sub MD_DOCTYPE_STATE () { 37 } # "markup declaration open state" in the spec  sub MD_DOCTYPE_STATE () { 37 } # "markup declaration open state" in the spec
867  sub MD_CDATA_STATE () { 38 } # "markup declaration open state" in the spec  sub MD_CDATA_STATE () { 38 } # "markup declaration open state" in the spec
868  sub CDATA_PCDATA_CLOSE_TAG_STATE () { 39 } # "close tag open state" in the spec  sub CDATA_RCDATA_CLOSE_TAG_STATE () { 39 } # "close tag open state" in the spec
869  sub CDATA_SECTION_MSE1_STATE () { 40 } # "CDATA section state" in the spec  sub CDATA_SECTION_MSE1_STATE () { 40 } # "CDATA section state" in the spec
870  sub CDATA_SECTION_MSE2_STATE () { 41 } # "CDATA section state" in the spec  sub CDATA_SECTION_MSE2_STATE () { 41 } # "CDATA section state" in the spec
871  sub PUBLIC_STATE () { 42 } # "after DOCTYPE name state" in the spec  sub PUBLIC_STATE () { 42 } # "after DOCTYPE name state" in the spec
872  sub SYSTEM_STATE () { 43 } # "after DOCTYPE name state" in the spec  sub SYSTEM_STATE () { 43 } # "after DOCTYPE name state" in the spec
873  sub ENTITY_STATE () { 44 } # "consume a character reference" in the spec  ## NOTE: "Entity data state", "entity in attribute value state", and
874    ## "consume a character reference" algorithm are jointly implemented
875    ## using the following six states:
876    sub ENTITY_STATE () { 44 }
877    sub ENTITY_HASH_STATE () { 45 }
878    sub NCR_NUM_STATE () { 46 }
879    sub HEXREF_X_STATE () { 47 }
880    sub HEXREF_HEX_STATE () { 48 }
881    sub ENTITY_NAME_STATE () { 49 }
882    sub PCDATA_STATE () { 50 } # "data state" in the spec
883    
884  sub DOCTYPE_TOKEN () { 1 }  sub DOCTYPE_TOKEN () { 1 }
885  sub COMMENT_TOKEN () { 2 }  sub COMMENT_TOKEN () { 2 }
# Line 865  sub IN_COLUMN_GROUP_IM () { 0b10 } Line 932  sub IN_COLUMN_GROUP_IM () { 0b10 }
932  sub _initialize_tokenizer ($) {  sub _initialize_tokenizer ($) {
933    my $self = shift;    my $self = shift;
934    $self->{state} = DATA_STATE; # MUST    $self->{state} = DATA_STATE; # MUST
935    #$self->{state_keyword}; # initialized when used    #$self->{s_kwd}; # state keyword - initialized when used
936      #$self->{entity__value}; # initialized when used
937      #$self->{entity__match}; # initialized when used
938    $self->{content_model} = PCDATA_CONTENT_MODEL; # be    $self->{content_model} = PCDATA_CONTENT_MODEL; # be
939    undef $self->{current_token};    undef $self->{ct}; # current token
940    undef $self->{current_attribute};    undef $self->{ca}; # current attribute
941    undef $self->{last_emitted_start_tag_name};    undef $self->{last_stag_name}; # last emitted start tag name
942    undef $self->{last_attribute_value_state};    #$self->{prev_state}; # initialized when used
943    delete $self->{self_closing};    delete $self->{self_closing};
944    $self->{char} = [];    $self->{char_buffer} = '';
945    # $self->{next_char}    $self->{char_buffer_pos} = 0;
946      $self->{nc} = -1; # next input character
947      #$self->{next_nc}
948    !!!next-input-character;    !!!next-input-character;
949    $self->{token} = [];    $self->{token} = [];
950    # $self->{escape}    # $self->{escape}
# Line 884  sub _initialize_tokenizer ($) { Line 955  sub _initialize_tokenizer ($) {
955  ##       CHARACTER_TOKEN, or END_OF_FILE_TOKEN  ##       CHARACTER_TOKEN, or END_OF_FILE_TOKEN
956  ##   ->{name} (DOCTYPE_TOKEN)  ##   ->{name} (DOCTYPE_TOKEN)
957  ##   ->{tag_name} (START_TAG_TOKEN, END_TAG_TOKEN)  ##   ->{tag_name} (START_TAG_TOKEN, END_TAG_TOKEN)
958  ##   ->{public_identifier} (DOCTYPE_TOKEN)  ##   ->{pubid} (DOCTYPE_TOKEN)
959  ##   ->{system_identifier} (DOCTYPE_TOKEN)  ##   ->{sysid} (DOCTYPE_TOKEN)
960  ##   ->{quirks} == 1 or 0 (DOCTYPE_TOKEN): "force-quirks" flag  ##   ->{quirks} == 1 or 0 (DOCTYPE_TOKEN): "force-quirks" flag
961  ##   ->{attributes} isa HASH (START_TAG_TOKEN, END_TAG_TOKEN)  ##   ->{attributes} isa HASH (START_TAG_TOKEN, END_TAG_TOKEN)
962  ##        ->{name}  ##        ->{name}
# Line 904  sub _initialize_tokenizer ($) { Line 975  sub _initialize_tokenizer ($) {
975  ## has completed loading.  If one has, then it MUST be executed  ## has completed loading.  If one has, then it MUST be executed
976  ## and removed from the list.  ## and removed from the list.
977    
978  ## NOTE: HTML5 "Writing HTML documents" section, applied to  ## TODO: Polytheistic slash SHOULD NOT be used. (Applied only to atheists.)
979  ## documents and not to user agents and conformance checkers,  ## (This requirement was dropped from HTML5 spec, unfortunately.)
980  ## contains some requirements that are not detected by the  
981  ## parsing algorithm:  my $is_space = {
982  ## - Some requirements on character encoding declarations. ## TODO    0x0009 => 1, # CHARACTER TABULATION (HT)
983  ## - "Elements MUST NOT contain content that their content model disallows."    0x000A => 1, # LINE FEED (LF)
984  ##   ... Some are parse error, some are not (will be reported by c.c.).    #0x000B => 0, # LINE TABULATION (VT)
985  ## - Polytheistic slash SHOULD NOT be used. (Applied only to atheists.) ## TODO    0x000C => 1, # FORM FEED (FF)
986  ## - Text (in elements, attributes, and comments) SHOULD NOT contain    #0x000D => 1, # CARRIAGE RETURN (CR)
987  ##   control characters other than space characters. ## TODO: (what is control character? C0, C1 and DEL?  Unicode control character?)    0x0020 => 1, # SPACE (SP)
988    };
 ## TODO: HTML5 poses authors two SHOULD-level requirements that cannot  
 ## be detected by the HTML5 parsing algorithm:  
 ## - Text,  
989    
990  sub _get_next_token ($) {  sub _get_next_token ($) {
991    my $self = shift;    my $self = shift;
992    
993    if ($self->{self_closing}) {    if ($self->{self_closing}) {
994      !!!parse-error (type => 'nestc', token => $self->{current_token});      !!!parse-error (type => 'nestc', token => $self->{ct});
995      ## NOTE: The |self_closing| flag is only set by start tag token.      ## NOTE: The |self_closing| flag is only set by start tag token.
996      ## In addition, when a start tag token is emitted, it is always set to      ## In addition, when a start tag token is emitted, it is always set to
997      ## |current_token|.      ## |ct|.
998      delete $self->{self_closing};      delete $self->{self_closing};
999    }    }
1000    
# Line 936  sub _get_next_token ($) { Line 1004  sub _get_next_token ($) {
1004    }    }
1005    
1006    A: {    A: {
1007      if ($self->{state} == DATA_STATE) {      if ($self->{state} == PCDATA_STATE) {
1008        if ($self->{next_char} == 0x0026) { # &        ## NOTE: Same as |DATA_STATE|, but only for |PCDATA| content model.
1009    
1010          if ($self->{nc} == 0x0026) { # &
1011            !!!cp (0.1);
1012            ## NOTE: In the spec, the tokenizer is switched to the
1013            ## "entity data state".  In this implementation, the tokenizer
1014            ## is switched to the |ENTITY_STATE|, which is an implementation
1015            ## of the "consume a character reference" algorithm.
1016            $self->{entity_add} = -1;
1017            $self->{prev_state} = DATA_STATE;
1018            $self->{state} = ENTITY_STATE;
1019            !!!next-input-character;
1020            redo A;
1021          } elsif ($self->{nc} == 0x003C) { # <
1022            !!!cp (0.2);
1023            $self->{state} = TAG_OPEN_STATE;
1024            !!!next-input-character;
1025            redo A;
1026          } elsif ($self->{nc} == -1) {
1027            !!!cp (0.3);
1028            !!!emit ({type => END_OF_FILE_TOKEN,
1029                      line => $self->{line}, column => $self->{column}});
1030            last A; ## TODO: ok?
1031          } else {
1032            !!!cp (0.4);
1033            #
1034          }
1035    
1036          # Anything else
1037          my $token = {type => CHARACTER_TOKEN,
1038                       data => chr $self->{nc},
1039                       line => $self->{line}, column => $self->{column},
1040                      };
1041          $self->{read_until}->($token->{data}, q[<&], length $token->{data});
1042    
1043          ## Stay in the state.
1044          !!!next-input-character;
1045          !!!emit ($token);
1046          redo A;
1047        } elsif ($self->{state} == DATA_STATE) {
1048          $self->{s_kwd} = '' unless defined $self->{s_kwd};
1049          if ($self->{nc} == 0x0026) { # &
1050            $self->{s_kwd} = '';
1051          if ($self->{content_model} & CM_ENTITY and # PCDATA | RCDATA          if ($self->{content_model} & CM_ENTITY and # PCDATA | RCDATA
1052              not $self->{escape}) {              not $self->{escape}) {
1053            !!!cp (1);            !!!cp (1);
# Line 945  sub _get_next_token ($) { Line 1055  sub _get_next_token ($) {
1055            ## "entity data state".  In this implementation, the tokenizer            ## "entity data state".  In this implementation, the tokenizer
1056            ## is switched to the |ENTITY_STATE|, which is an implementation            ## is switched to the |ENTITY_STATE|, which is an implementation
1057            ## of the "consume a character reference" algorithm.            ## of the "consume a character reference" algorithm.
1058            #$self->{state} = ENTITY_DATA_STATE;            $self->{entity_add} = -1;
1059            $self->{entity_in_attr} = 0;            $self->{prev_state} = DATA_STATE;
           $self->{entity_additional} = -1;  
1060            $self->{state} = ENTITY_STATE;            $self->{state} = ENTITY_STATE;
1061            !!!next-input-character;            !!!next-input-character;
1062            redo A;            redo A;
# Line 955  sub _get_next_token ($) { Line 1064  sub _get_next_token ($) {
1064            !!!cp (2);            !!!cp (2);
1065            #            #
1066          }          }
1067        } elsif ($self->{next_char} == 0x002D) { # -        } elsif ($self->{nc} == 0x002D) { # -
1068          if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA          if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
1069            unless ($self->{escape}) {            $self->{s_kwd} .= '-';
1070              if ($self->{prev_char}->[0] == 0x002D and # -            
1071                  $self->{prev_char}->[1] == 0x0021 and # !            if ($self->{s_kwd} eq '<!--') {
1072                  $self->{prev_char}->[2] == 0x003C) { # <              !!!cp (3);
1073                !!!cp (3);              $self->{escape} = 1; # unless $self->{escape};
1074                $self->{escape} = 1;              $self->{s_kwd} = '--';
1075              } else {              #
1076                !!!cp (4);            } elsif ($self->{s_kwd} eq '---') {
1077              }              !!!cp (4);
1078                $self->{s_kwd} = '--';
1079                #
1080            } else {            } else {
1081              !!!cp (5);              !!!cp (5);
1082                #
1083            }            }
1084          }          }
1085                    
1086          #          #
1087        } elsif ($self->{next_char} == 0x003C) { # <        } elsif ($self->{nc} == 0x0021) { # !
1088            if (length $self->{s_kwd}) {
1089              !!!cp (5.1);
1090              $self->{s_kwd} .= '!';
1091              #
1092            } else {
1093              !!!cp (5.2);
1094              #$self->{s_kwd} = '';
1095              #
1096            }
1097            #
1098          } elsif ($self->{nc} == 0x003C) { # <
1099          if ($self->{content_model} & CM_FULL_MARKUP or # PCDATA          if ($self->{content_model} & CM_FULL_MARKUP or # PCDATA
1100              (($self->{content_model} & CM_LIMITED_MARKUP) and # CDATA | RCDATA              (($self->{content_model} & CM_LIMITED_MARKUP) and # CDATA | RCDATA
1101               not $self->{escape})) {               not $self->{escape})) {
# Line 982  sub _get_next_token ($) { Line 1105  sub _get_next_token ($) {
1105            redo A;            redo A;
1106          } else {          } else {
1107            !!!cp (7);            !!!cp (7);
1108              $self->{s_kwd} = '';
1109            #            #
1110          }          }
1111        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
1112          if ($self->{escape} and          if ($self->{escape} and
1113              ($self->{content_model} & CM_LIMITED_MARKUP)) { # RCDATA | CDATA              ($self->{content_model} & CM_LIMITED_MARKUP)) { # RCDATA | CDATA
1114            if ($self->{prev_char}->[0] == 0x002D and # -            if ($self->{s_kwd} eq '--') {
               $self->{prev_char}->[1] == 0x002D) { # -  
1115              !!!cp (8);              !!!cp (8);
1116              delete $self->{escape};              delete $self->{escape};
1117            } else {            } else {
# Line 998  sub _get_next_token ($) { Line 1121  sub _get_next_token ($) {
1121            !!!cp (10);            !!!cp (10);
1122          }          }
1123                    
1124            $self->{s_kwd} = '';
1125          #          #
1126        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
1127          !!!cp (11);          !!!cp (11);
1128            $self->{s_kwd} = '';
1129          !!!emit ({type => END_OF_FILE_TOKEN,          !!!emit ({type => END_OF_FILE_TOKEN,
1130                    line => $self->{line}, column => $self->{column}});                    line => $self->{line}, column => $self->{column}});
1131          last A; ## TODO: ok?          last A; ## TODO: ok?
1132        } else {        } else {
1133          !!!cp (12);          !!!cp (12);
1134            $self->{s_kwd} = '';
1135            #
1136        }        }
1137    
1138        # Anything else        # Anything else
1139        my $token = {type => CHARACTER_TOKEN,        my $token = {type => CHARACTER_TOKEN,
1140                     data => chr $self->{next_char},                     data => chr $self->{nc},
1141                     line => $self->{line}, column => $self->{column},                     line => $self->{line}, column => $self->{column},
1142                    };                    };
1143        ## Stay in the data state        if ($self->{read_until}->($token->{data}, q[-!<>&],
1144        !!!next-input-character;                                  length $token->{data})) {
1145            $self->{s_kwd} = '';
1146        !!!emit ($token);        }
   
       redo A;  
     } elsif ($self->{state} == ENTITY_DATA_STATE) {  
       my ($l, $c) = ($self->{line_prev}, $self->{column_prev});  
   
       my $token = $self->{entity_return};  
   
       $self->{state} = DATA_STATE;  
       # next-input-character is already done  
1147    
1148        unless (defined $token) {        ## Stay in the data state.
1149          if ($self->{content_model} == PCDATA_CONTENT_MODEL) {
1150          !!!cp (13);          !!!cp (13);
1151          !!!emit ({type => CHARACTER_TOKEN, data => '&',          $self->{state} = PCDATA_STATE;
                   line => $l, column => $c,  
                  });  
1152        } else {        } else {
1153          !!!cp (14);          !!!cp (14);
1154          !!!emit ($token);          ## Stay in the state.
1155        }        }
1156          !!!next-input-character;
1157          !!!emit ($token);
1158        redo A;        redo A;
1159      } elsif ($self->{state} == TAG_OPEN_STATE) {      } elsif ($self->{state} == TAG_OPEN_STATE) {
1160        if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA        if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
1161          if ($self->{next_char} == 0x002F) { # /          if ($self->{nc} == 0x002F) { # /
1162            !!!cp (15);            !!!cp (15);
1163            !!!next-input-character;            !!!next-input-character;
1164            $self->{state} = CLOSE_TAG_OPEN_STATE;            $self->{state} = CLOSE_TAG_OPEN_STATE;
1165            redo A;            redo A;
1166            } elsif ($self->{nc} == 0x0021) { # !
1167              !!!cp (15.1);
1168              $self->{s_kwd} = '<' unless $self->{escape};
1169              #
1170          } else {          } else {
1171            !!!cp (16);            !!!cp (16);
1172            ## reconsume            #
           $self->{state} = DATA_STATE;  
   
           !!!emit ({type => CHARACTER_TOKEN, data => '<',  
                     line => $self->{line_prev},  
                     column => $self->{column_prev},  
                    });  
   
           redo A;  
1173          }          }
1174    
1175            ## reconsume
1176            $self->{state} = DATA_STATE;
1177            !!!emit ({type => CHARACTER_TOKEN, data => '<',
1178                      line => $self->{line_prev},
1179                      column => $self->{column_prev},
1180                     });
1181            redo A;
1182        } elsif ($self->{content_model} & CM_FULL_MARKUP) { # PCDATA        } elsif ($self->{content_model} & CM_FULL_MARKUP) { # PCDATA
1183          if ($self->{next_char} == 0x0021) { # !          if ($self->{nc} == 0x0021) { # !
1184            !!!cp (17);            !!!cp (17);
1185            $self->{state} = MARKUP_DECLARATION_OPEN_STATE;            $self->{state} = MARKUP_DECLARATION_OPEN_STATE;
1186            !!!next-input-character;            !!!next-input-character;
1187            redo A;            redo A;
1188          } elsif ($self->{next_char} == 0x002F) { # /          } elsif ($self->{nc} == 0x002F) { # /
1189            !!!cp (18);            !!!cp (18);
1190            $self->{state} = CLOSE_TAG_OPEN_STATE;            $self->{state} = CLOSE_TAG_OPEN_STATE;
1191            !!!next-input-character;            !!!next-input-character;
1192            redo A;            redo A;
1193          } elsif (0x0041 <= $self->{next_char} and          } elsif (0x0041 <= $self->{nc} and
1194                   $self->{next_char} <= 0x005A) { # A..Z                   $self->{nc} <= 0x005A) { # A..Z
1195            !!!cp (19);            !!!cp (19);
1196            $self->{current_token}            $self->{ct}
1197              = {type => START_TAG_TOKEN,              = {type => START_TAG_TOKEN,
1198                 tag_name => chr ($self->{next_char} + 0x0020),                 tag_name => chr ($self->{nc} + 0x0020),
1199                 line => $self->{line_prev},                 line => $self->{line_prev},
1200                 column => $self->{column_prev}};                 column => $self->{column_prev}};
1201            $self->{state} = TAG_NAME_STATE;            $self->{state} = TAG_NAME_STATE;
1202            !!!next-input-character;            !!!next-input-character;
1203            redo A;            redo A;
1204          } elsif (0x0061 <= $self->{next_char} and          } elsif (0x0061 <= $self->{nc} and
1205                   $self->{next_char} <= 0x007A) { # a..z                   $self->{nc} <= 0x007A) { # a..z
1206            !!!cp (20);            !!!cp (20);
1207            $self->{current_token} = {type => START_TAG_TOKEN,            $self->{ct} = {type => START_TAG_TOKEN,
1208                                      tag_name => chr ($self->{next_char}),                                      tag_name => chr ($self->{nc}),
1209                                      line => $self->{line_prev},                                      line => $self->{line_prev},
1210                                      column => $self->{column_prev}};                                      column => $self->{column_prev}};
1211            $self->{state} = TAG_NAME_STATE;            $self->{state} = TAG_NAME_STATE;
1212            !!!next-input-character;            !!!next-input-character;
1213            redo A;            redo A;
1214          } elsif ($self->{next_char} == 0x003E) { # >          } elsif ($self->{nc} == 0x003E) { # >
1215            !!!cp (21);            !!!cp (21);
1216            !!!parse-error (type => 'empty start tag',            !!!parse-error (type => 'empty start tag',
1217                            line => $self->{line_prev},                            line => $self->{line_prev},
# Line 1102  sub _get_next_token ($) { Line 1225  sub _get_next_token ($) {
1225                     });                     });
1226    
1227            redo A;            redo A;
1228          } elsif ($self->{next_char} == 0x003F) { # ?          } elsif ($self->{nc} == 0x003F) { # ?
1229            !!!cp (22);            !!!cp (22);
1230            !!!parse-error (type => 'pio',            !!!parse-error (type => 'pio',
1231                            line => $self->{line_prev},                            line => $self->{line_prev},
1232                            column => $self->{column_prev});                            column => $self->{column_prev});
1233            $self->{state} = BOGUS_COMMENT_STATE;            $self->{state} = BOGUS_COMMENT_STATE;
1234            $self->{current_token} = {type => COMMENT_TOKEN, data => '',            $self->{ct} = {type => COMMENT_TOKEN, data => '',
1235                                      line => $self->{line_prev},                                      line => $self->{line_prev},
1236                                      column => $self->{column_prev},                                      column => $self->{column_prev},
1237                                     };                                     };
1238            ## $self->{next_char} is intentionally left as is            ## $self->{nc} is intentionally left as is
1239            redo A;            redo A;
1240          } else {          } else {
1241            !!!cp (23);            !!!cp (23);
# Line 1134  sub _get_next_token ($) { Line 1257  sub _get_next_token ($) {
1257        }        }
1258      } elsif ($self->{state} == CLOSE_TAG_OPEN_STATE) {      } elsif ($self->{state} == CLOSE_TAG_OPEN_STATE) {
1259        ## NOTE: The "close tag open state" in the spec is implemented as        ## NOTE: The "close tag open state" in the spec is implemented as
1260        ## |CLOSE_TAG_OPEN_STATE| and |CDATA_PCDATA_CLOSE_TAG_STATE|.        ## |CLOSE_TAG_OPEN_STATE| and |CDATA_RCDATA_CLOSE_TAG_STATE|.
1261    
1262        my ($l, $c) = ($self->{line_prev}, $self->{column_prev} - 1); # "<"of"</"        my ($l, $c) = ($self->{line_prev}, $self->{column_prev} - 1); # "<"of"</"
1263        if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA        if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
1264          if (defined $self->{last_emitted_start_tag_name}) {          if (defined $self->{last_stag_name}) {
1265            $self->{state} = CDATA_PCDATA_CLOSE_TAG_STATE;            $self->{state} = CDATA_RCDATA_CLOSE_TAG_STATE;
1266            $self->{state_keyword} = '';            $self->{s_kwd} = '';
1267            ## Reconsume.            ## Reconsume.
1268            redo A;            redo A;
1269          } else {          } else {
# Line 1156  sub _get_next_token ($) { Line 1279  sub _get_next_token ($) {
1279          }          }
1280        }        }
1281    
1282        if (0x0041 <= $self->{next_char} and        if (0x0041 <= $self->{nc} and
1283            $self->{next_char} <= 0x005A) { # A..Z            $self->{nc} <= 0x005A) { # A..Z
1284          !!!cp (29);          !!!cp (29);
1285          $self->{current_token}          $self->{ct}
1286              = {type => END_TAG_TOKEN,              = {type => END_TAG_TOKEN,
1287                 tag_name => chr ($self->{next_char} + 0x0020),                 tag_name => chr ($self->{nc} + 0x0020),
1288                 line => $l, column => $c};                 line => $l, column => $c};
1289          $self->{state} = TAG_NAME_STATE;          $self->{state} = TAG_NAME_STATE;
1290          !!!next-input-character;          !!!next-input-character;
1291          redo A;          redo A;
1292        } elsif (0x0061 <= $self->{next_char} and        } elsif (0x0061 <= $self->{nc} and
1293                 $self->{next_char} <= 0x007A) { # a..z                 $self->{nc} <= 0x007A) { # a..z
1294          !!!cp (30);          !!!cp (30);
1295          $self->{current_token} = {type => END_TAG_TOKEN,          $self->{ct} = {type => END_TAG_TOKEN,
1296                                    tag_name => chr ($self->{next_char}),                                    tag_name => chr ($self->{nc}),
1297                                    line => $l, column => $c};                                    line => $l, column => $c};
1298          $self->{state} = TAG_NAME_STATE;          $self->{state} = TAG_NAME_STATE;
1299          !!!next-input-character;          !!!next-input-character;
1300          redo A;          redo A;
1301        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
1302          !!!cp (31);          !!!cp (31);
1303          !!!parse-error (type => 'empty end tag',          !!!parse-error (type => 'empty end tag',
1304                          line => $self->{line_prev}, ## "<" in "</>"                          line => $self->{line_prev}, ## "<" in "</>"
# Line 1183  sub _get_next_token ($) { Line 1306  sub _get_next_token ($) {
1306          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1307          !!!next-input-character;          !!!next-input-character;
1308          redo A;          redo A;
1309        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
1310          !!!cp (32);          !!!cp (32);
1311          !!!parse-error (type => 'bare etago');          !!!parse-error (type => 'bare etago');
1312          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
# Line 1198  sub _get_next_token ($) { Line 1321  sub _get_next_token ($) {
1321          !!!cp (33);          !!!cp (33);
1322          !!!parse-error (type => 'bogus end tag');          !!!parse-error (type => 'bogus end tag');
1323          $self->{state} = BOGUS_COMMENT_STATE;          $self->{state} = BOGUS_COMMENT_STATE;
1324          $self->{current_token} = {type => COMMENT_TOKEN, data => '',          $self->{ct} = {type => COMMENT_TOKEN, data => '',
1325                                    line => $self->{line_prev}, # "<" of "</"                                    line => $self->{line_prev}, # "<" of "</"
1326                                    column => $self->{column_prev} - 1,                                    column => $self->{column_prev} - 1,
1327                                   };                                   };
1328          ## NOTE: $self->{next_char} is intentionally left as is.          ## NOTE: $self->{nc} is intentionally left as is.
1329          ## Although the "anything else" case of the spec not explicitly          ## Although the "anything else" case of the spec not explicitly
1330          ## states that the next input character is to be reconsumed,          ## states that the next input character is to be reconsumed,
1331          ## it will be included to the |data| of the comment token          ## it will be included to the |data| of the comment token
# Line 1210  sub _get_next_token ($) { Line 1333  sub _get_next_token ($) {
1333          ## "bogus comment state" entry.          ## "bogus comment state" entry.
1334          redo A;          redo A;
1335        }        }
1336      } elsif ($self->{state} == CDATA_PCDATA_CLOSE_TAG_STATE) {      } elsif ($self->{state} == CDATA_RCDATA_CLOSE_TAG_STATE) {
1337        my $ch = substr $self->{last_emitted_start_tag_name}, length $self->{state_keyword}, 1;        my $ch = substr $self->{last_stag_name}, length $self->{s_kwd}, 1;
1338        if (length $ch) {        if (length $ch) {
1339          my $CH = $ch;          my $CH = $ch;
1340          $ch =~ tr/a-z/A-Z/;          $ch =~ tr/a-z/A-Z/;
1341          my $nch = chr $self->{next_char};          my $nch = chr $self->{nc};
1342          if ($nch eq $ch or $nch eq $CH) {          if ($nch eq $ch or $nch eq $CH) {
1343            !!!cp (24);            !!!cp (24);
1344            ## Stay in the state.            ## Stay in the state.
1345            $self->{state_keyword} .= $nch;            $self->{s_kwd} .= $nch;
1346            !!!next-input-character;            !!!next-input-character;
1347            redo A;            redo A;
1348          } else {          } else {
# Line 1227  sub _get_next_token ($) { Line 1350  sub _get_next_token ($) {
1350            $self->{state} = DATA_STATE;            $self->{state} = DATA_STATE;
1351            ## Reconsume.            ## Reconsume.
1352            !!!emit ({type => CHARACTER_TOKEN,            !!!emit ({type => CHARACTER_TOKEN,
1353                      data => '</' . $self->{state_keyword},                      data => '</' . $self->{s_kwd},
1354                      line => $self->{line_prev},                      line => $self->{line_prev},
1355                      column => $self->{column_prev} - 1 - length $self->{state_keyword},                      column => $self->{column_prev} - 1 - length $self->{s_kwd},
1356                     });                     });
1357            redo A;            redo A;
1358          }          }
1359        } else { # after "<{tag-name}"        } else { # after "<{tag-name}"
1360          unless ({          unless ($is_space->{$self->{nc}} or
1361                   0x0009 => 1, # HT                  {
                  0x000A => 1, # LF  
                  0x000B => 1, # VT  
                  0x000C => 1, # FF  
                  0x0020 => 1, # SP  
1362                   0x003E => 1, # >                   0x003E => 1, # >
1363                   0x002F => 1, # /                   0x002F => 1, # /
1364                   -1 => 1, # EOF                   -1 => 1, # EOF
1365                  }->{$self->{next_char}}) {                  }->{$self->{nc}}) {
1366            !!!cp (26);            !!!cp (26);
1367            ## Reconsume.            ## Reconsume.
1368            $self->{state} = DATA_STATE;            $self->{state} = DATA_STATE;
1369            !!!emit ({type => CHARACTER_TOKEN,            !!!emit ({type => CHARACTER_TOKEN,
1370                      data => '</' . $self->{state_keyword},                      data => '</' . $self->{s_kwd},
1371                      line => $self->{line_prev},                      line => $self->{line_prev},
1372                      column => $self->{column_prev} - 1 - length $self->{state_keyword},                      column => $self->{column_prev} - 1 - length $self->{s_kwd},
1373                     });                     });
1374            redo A;            redo A;
1375          } else {          } else {
1376            !!!cp (27);            !!!cp (27);
1377            $self->{current_token}            $self->{ct}
1378                = {type => END_TAG_TOKEN,                = {type => END_TAG_TOKEN,
1379                   tag_name => $self->{last_emitted_start_tag_name},                   tag_name => $self->{last_stag_name},
1380                   line => $self->{line_prev},                   line => $self->{line_prev},
1381                   column => $self->{column_prev} - 1 - length $self->{state_keyword}};                   column => $self->{column_prev} - 1 - length $self->{s_kwd}};
1382            $self->{state} = TAG_NAME_STATE;            $self->{state} = TAG_NAME_STATE;
1383            ## Reconsume.            ## Reconsume.
1384            redo A;            redo A;
1385          }          }
1386        }        }
1387      } elsif ($self->{state} == TAG_NAME_STATE) {      } elsif ($self->{state} == TAG_NAME_STATE) {
1388        if ($self->{next_char} == 0x0009 or # HT        if ($is_space->{$self->{nc}}) {
           $self->{next_char} == 0x000A or # LF  
           $self->{next_char} == 0x000B or # VT  
           $self->{next_char} == 0x000C or # FF  
           $self->{next_char} == 0x0020) { # SP  
1389          !!!cp (34);          !!!cp (34);
1390          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1391          !!!next-input-character;          !!!next-input-character;
1392          redo A;          redo A;
1393        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
1394          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1395            !!!cp (35);            !!!cp (35);
1396            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1397          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1398            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1399            #if ($self->{current_token}->{attributes}) {            #if ($self->{ct}->{attributes}) {
1400            #  ## NOTE: This should never be reached.            #  ## NOTE: This should never be reached.
1401            #  !!! cp (36);            #  !!! cp (36);
1402            #  !!! parse-error (type => 'end tag attribute');            #  !!! parse-error (type => 'end tag attribute');
# Line 1289  sub _get_next_token ($) { Line 1404  sub _get_next_token ($) {
1404              !!!cp (37);              !!!cp (37);
1405            #}            #}
1406          } else {          } else {
1407            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1408          }          }
1409          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1410          !!!next-input-character;          !!!next-input-character;
1411    
1412          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1413    
1414          redo A;          redo A;
1415        } elsif (0x0041 <= $self->{next_char} and        } elsif (0x0041 <= $self->{nc} and
1416                 $self->{next_char} <= 0x005A) { # A..Z                 $self->{nc} <= 0x005A) { # A..Z
1417          !!!cp (38);          !!!cp (38);
1418          $self->{current_token}->{tag_name} .= chr ($self->{next_char} + 0x0020);          $self->{ct}->{tag_name} .= chr ($self->{nc} + 0x0020);
1419            # start tag or end tag            # start tag or end tag
1420          ## Stay in this state          ## Stay in this state
1421          !!!next-input-character;          !!!next-input-character;
1422          redo A;          redo A;
1423        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
1424          !!!parse-error (type => 'unclosed tag');          !!!parse-error (type => 'unclosed tag');
1425          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1426            !!!cp (39);            !!!cp (39);
1427            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1428          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1429            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1430            #if ($self->{current_token}->{attributes}) {            #if ($self->{ct}->{attributes}) {
1431            #  ## NOTE: This state should never be reached.            #  ## NOTE: This state should never be reached.
1432            #  !!! cp (40);            #  !!! cp (40);
1433            #  !!! parse-error (type => 'end tag attribute');            #  !!! parse-error (type => 'end tag attribute');
# Line 1320  sub _get_next_token ($) { Line 1435  sub _get_next_token ($) {
1435              !!!cp (41);              !!!cp (41);
1436            #}            #}
1437          } else {          } else {
1438            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1439          }          }
1440          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1441          # reconsume          # reconsume
1442    
1443          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1444    
1445          redo A;          redo A;
1446        } elsif ($self->{next_char} == 0x002F) { # /        } elsif ($self->{nc} == 0x002F) { # /
1447          !!!cp (42);          !!!cp (42);
1448          $self->{state} = SELF_CLOSING_START_TAG_STATE;          $self->{state} = SELF_CLOSING_START_TAG_STATE;
1449          !!!next-input-character;          !!!next-input-character;
1450          redo A;          redo A;
1451        } else {        } else {
1452          !!!cp (44);          !!!cp (44);
1453          $self->{current_token}->{tag_name} .= chr $self->{next_char};          $self->{ct}->{tag_name} .= chr $self->{nc};
1454            # start tag or end tag            # start tag or end tag
1455          ## Stay in the state          ## Stay in the state
1456          !!!next-input-character;          !!!next-input-character;
1457          redo A;          redo A;
1458        }        }
1459      } elsif ($self->{state} == BEFORE_ATTRIBUTE_NAME_STATE) {      } elsif ($self->{state} == BEFORE_ATTRIBUTE_NAME_STATE) {
1460        if ($self->{next_char} == 0x0009 or # HT        if ($is_space->{$self->{nc}}) {
           $self->{next_char} == 0x000A or # LF  
           $self->{next_char} == 0x000B or # VT  
           $self->{next_char} == 0x000C or # FF  
           $self->{next_char} == 0x0020) { # SP  
1461          !!!cp (45);          !!!cp (45);
1462          ## Stay in the state          ## Stay in the state
1463          !!!next-input-character;          !!!next-input-character;
1464          redo A;          redo A;
1465        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
1466          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1467            !!!cp (46);            !!!cp (46);
1468            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1469          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1470            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1471            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
1472              !!!cp (47);              !!!cp (47);
1473              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
1474            } else {            } else {
1475              !!!cp (48);              !!!cp (48);
1476            }            }
1477          } else {          } else {
1478            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1479          }          }
1480          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1481          !!!next-input-character;          !!!next-input-character;
1482    
1483          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1484    
1485          redo A;          redo A;
1486        } elsif (0x0041 <= $self->{next_char} and        } elsif (0x0041 <= $self->{nc} and
1487                 $self->{next_char} <= 0x005A) { # A..Z                 $self->{nc} <= 0x005A) { # A..Z
1488          !!!cp (49);          !!!cp (49);
1489          $self->{current_attribute}          $self->{ca}
1490              = {name => chr ($self->{next_char} + 0x0020),              = {name => chr ($self->{nc} + 0x0020),
1491                 value => '',                 value => '',
1492                 line => $self->{line}, column => $self->{column}};                 line => $self->{line}, column => $self->{column}};
1493          $self->{state} = ATTRIBUTE_NAME_STATE;          $self->{state} = ATTRIBUTE_NAME_STATE;
1494          !!!next-input-character;          !!!next-input-character;
1495          redo A;          redo A;
1496        } elsif ($self->{next_char} == 0x002F) { # /        } elsif ($self->{nc} == 0x002F) { # /
1497          !!!cp (50);          !!!cp (50);
1498          $self->{state} = SELF_CLOSING_START_TAG_STATE;          $self->{state} = SELF_CLOSING_START_TAG_STATE;
1499          !!!next-input-character;          !!!next-input-character;
1500          redo A;          redo A;
1501        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
1502          !!!parse-error (type => 'unclosed tag');          !!!parse-error (type => 'unclosed tag');
1503          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1504            !!!cp (52);            !!!cp (52);
1505            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1506          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1507            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1508            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
1509              !!!cp (53);              !!!cp (53);
1510              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
1511            } else {            } else {
1512              !!!cp (54);              !!!cp (54);
1513            }            }
1514          } else {          } else {
1515            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1516          }          }
1517          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1518          # reconsume          # reconsume
1519    
1520          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1521    
1522          redo A;          redo A;
1523        } else {        } else {
# Line 1414  sub _get_next_token ($) { Line 1525  sub _get_next_token ($) {
1525               0x0022 => 1, # "               0x0022 => 1, # "
1526               0x0027 => 1, # '               0x0027 => 1, # '
1527               0x003D => 1, # =               0x003D => 1, # =
1528              }->{$self->{next_char}}) {              }->{$self->{nc}}) {
1529            !!!cp (55);            !!!cp (55);
1530            !!!parse-error (type => 'bad attribute name');            !!!parse-error (type => 'bad attribute name');
1531          } else {          } else {
1532            !!!cp (56);            !!!cp (56);
1533          }          }
1534          $self->{current_attribute}          $self->{ca}
1535              = {name => chr ($self->{next_char}),              = {name => chr ($self->{nc}),
1536                 value => '',                 value => '',
1537                 line => $self->{line}, column => $self->{column}};                 line => $self->{line}, column => $self->{column}};
1538          $self->{state} = ATTRIBUTE_NAME_STATE;          $self->{state} = ATTRIBUTE_NAME_STATE;
# Line 1430  sub _get_next_token ($) { Line 1541  sub _get_next_token ($) {
1541        }        }
1542      } elsif ($self->{state} == ATTRIBUTE_NAME_STATE) {      } elsif ($self->{state} == ATTRIBUTE_NAME_STATE) {
1543        my $before_leave = sub {        my $before_leave = sub {
1544          if (exists $self->{current_token}->{attributes} # start tag or end tag          if (exists $self->{ct}->{attributes} # start tag or end tag
1545              ->{$self->{current_attribute}->{name}}) { # MUST              ->{$self->{ca}->{name}}) { # MUST
1546            !!!cp (57);            !!!cp (57);
1547            !!!parse-error (type => 'duplicate attribute', text => $self->{current_attribute}->{name}, line => $self->{current_attribute}->{line}, column => $self->{current_attribute}->{column});            !!!parse-error (type => 'duplicate attribute', text => $self->{ca}->{name}, line => $self->{ca}->{line}, column => $self->{ca}->{column});
1548            ## Discard $self->{current_attribute} # MUST            ## Discard $self->{ca} # MUST
1549          } else {          } else {
1550            !!!cp (58);            !!!cp (58);
1551            $self->{current_token}->{attributes}->{$self->{current_attribute}->{name}}            $self->{ct}->{attributes}->{$self->{ca}->{name}}
1552              = $self->{current_attribute};              = $self->{ca};
1553          }          }
1554        }; # $before_leave        }; # $before_leave
1555    
1556        if ($self->{next_char} == 0x0009 or # HT        if ($is_space->{$self->{nc}}) {
           $self->{next_char} == 0x000A or # LF  
           $self->{next_char} == 0x000B or # VT  
           $self->{next_char} == 0x000C or # FF  
           $self->{next_char} == 0x0020) { # SP  
1557          !!!cp (59);          !!!cp (59);
1558          $before_leave->();          $before_leave->();
1559          $self->{state} = AFTER_ATTRIBUTE_NAME_STATE;          $self->{state} = AFTER_ATTRIBUTE_NAME_STATE;
1560          !!!next-input-character;          !!!next-input-character;
1561          redo A;          redo A;
1562        } elsif ($self->{next_char} == 0x003D) { # =        } elsif ($self->{nc} == 0x003D) { # =
1563          !!!cp (60);          !!!cp (60);
1564          $before_leave->();          $before_leave->();
1565          $self->{state} = BEFORE_ATTRIBUTE_VALUE_STATE;          $self->{state} = BEFORE_ATTRIBUTE_VALUE_STATE;
1566          !!!next-input-character;          !!!next-input-character;
1567          redo A;          redo A;
1568        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
1569          $before_leave->();          $before_leave->();
1570          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1571            !!!cp (61);            !!!cp (61);
1572            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1573          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1574            !!!cp (62);            !!!cp (62);
1575            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1576            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
1577              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
1578            }            }
1579          } else {          } else {
1580            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1581          }          }
1582          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1583          !!!next-input-character;          !!!next-input-character;
1584    
1585          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1586    
1587          redo A;          redo A;
1588        } elsif (0x0041 <= $self->{next_char} and        } elsif (0x0041 <= $self->{nc} and
1589                 $self->{next_char} <= 0x005A) { # A..Z                 $self->{nc} <= 0x005A) { # A..Z
1590          !!!cp (63);          !!!cp (63);
1591          $self->{current_attribute}->{name} .= chr ($self->{next_char} + 0x0020);          $self->{ca}->{name} .= chr ($self->{nc} + 0x0020);
1592          ## Stay in the state          ## Stay in the state
1593          !!!next-input-character;          !!!next-input-character;
1594          redo A;          redo A;
1595        } elsif ($self->{next_char} == 0x002F) { # /        } elsif ($self->{nc} == 0x002F) { # /
1596          !!!cp (64);          !!!cp (64);
1597          $before_leave->();          $before_leave->();
1598          $self->{state} = SELF_CLOSING_START_TAG_STATE;          $self->{state} = SELF_CLOSING_START_TAG_STATE;
1599          !!!next-input-character;          !!!next-input-character;
1600          redo A;          redo A;
1601        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
1602          !!!parse-error (type => 'unclosed tag');          !!!parse-error (type => 'unclosed tag');
1603          $before_leave->();          $before_leave->();
1604          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1605            !!!cp (66);            !!!cp (66);
1606            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1607          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1608            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1609            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
1610              !!!cp (67);              !!!cp (67);
1611              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
1612            } else {            } else {
# Line 1507  sub _get_next_token ($) { Line 1614  sub _get_next_token ($) {
1614              !!!cp (68);              !!!cp (68);
1615            }            }
1616          } else {          } else {
1617            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1618          }          }
1619          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1620          # reconsume          # reconsume
1621    
1622          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1623    
1624          redo A;          redo A;
1625        } else {        } else {
1626          if ($self->{next_char} == 0x0022 or # "          if ($self->{nc} == 0x0022 or # "
1627              $self->{next_char} == 0x0027) { # '              $self->{nc} == 0x0027) { # '
1628            !!!cp (69);            !!!cp (69);
1629            !!!parse-error (type => 'bad attribute name');            !!!parse-error (type => 'bad attribute name');
1630          } else {          } else {
1631            !!!cp (70);            !!!cp (70);
1632          }          }
1633          $self->{current_attribute}->{name} .= chr ($self->{next_char});          $self->{ca}->{name} .= chr ($self->{nc});
1634          ## Stay in the state          ## Stay in the state
1635          !!!next-input-character;          !!!next-input-character;
1636          redo A;          redo A;
1637        }        }
1638      } elsif ($self->{state} == AFTER_ATTRIBUTE_NAME_STATE) {      } elsif ($self->{state} == AFTER_ATTRIBUTE_NAME_STATE) {
1639        if ($self->{next_char} == 0x0009 or # HT        if ($is_space->{$self->{nc}}) {
           $self->{next_char} == 0x000A or # LF  
           $self->{next_char} == 0x000B or # VT  
           $self->{next_char} == 0x000C or # FF  
           $self->{next_char} == 0x0020) { # SP  
1640          !!!cp (71);          !!!cp (71);
1641          ## Stay in the state          ## Stay in the state
1642          !!!next-input-character;          !!!next-input-character;
1643          redo A;          redo A;
1644        } elsif ($self->{next_char} == 0x003D) { # =        } elsif ($self->{nc} == 0x003D) { # =
1645          !!!cp (72);          !!!cp (72);
1646          $self->{state} = BEFORE_ATTRIBUTE_VALUE_STATE;          $self->{state} = BEFORE_ATTRIBUTE_VALUE_STATE;
1647          !!!next-input-character;          !!!next-input-character;
1648          redo A;          redo A;
1649        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
1650          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1651            !!!cp (73);            !!!cp (73);
1652            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1653          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1654            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1655            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
1656              !!!cp (74);              !!!cp (74);
1657              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
1658            } else {            } else {
# Line 1557  sub _get_next_token ($) { Line 1660  sub _get_next_token ($) {
1660              !!!cp (75);              !!!cp (75);
1661            }            }
1662          } else {          } else {
1663            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1664          }          }
1665          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1666          !!!next-input-character;          !!!next-input-character;
1667    
1668          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1669    
1670          redo A;          redo A;
1671        } elsif (0x0041 <= $self->{next_char} and        } elsif (0x0041 <= $self->{nc} and
1672                 $self->{next_char} <= 0x005A) { # A..Z                 $self->{nc} <= 0x005A) { # A..Z
1673          !!!cp (76);          !!!cp (76);
1674          $self->{current_attribute}          $self->{ca}
1675              = {name => chr ($self->{next_char} + 0x0020),              = {name => chr ($self->{nc} + 0x0020),
1676                 value => '',                 value => '',
1677                 line => $self->{line}, column => $self->{column}};                 line => $self->{line}, column => $self->{column}};
1678          $self->{state} = ATTRIBUTE_NAME_STATE;          $self->{state} = ATTRIBUTE_NAME_STATE;
1679          !!!next-input-character;          !!!next-input-character;
1680          redo A;          redo A;
1681        } elsif ($self->{next_char} == 0x002F) { # /        } elsif ($self->{nc} == 0x002F) { # /
1682          !!!cp (77);          !!!cp (77);
1683          $self->{state} = SELF_CLOSING_START_TAG_STATE;          $self->{state} = SELF_CLOSING_START_TAG_STATE;
1684          !!!next-input-character;          !!!next-input-character;
1685          redo A;          redo A;
1686        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
1687          !!!parse-error (type => 'unclosed tag');          !!!parse-error (type => 'unclosed tag');
1688          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1689            !!!cp (79);            !!!cp (79);
1690            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1691          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1692            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1693            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
1694              !!!cp (80);              !!!cp (80);
1695              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
1696            } else {            } else {
# Line 1595  sub _get_next_token ($) { Line 1698  sub _get_next_token ($) {
1698              !!!cp (81);              !!!cp (81);
1699            }            }
1700          } else {          } else {
1701            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1702          }          }
1703          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1704          # reconsume          # reconsume
1705    
1706          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1707    
1708          redo A;          redo A;
1709        } else {        } else {
1710          if ($self->{next_char} == 0x0022 or # "          if ($self->{nc} == 0x0022 or # "
1711              $self->{next_char} == 0x0027) { # '              $self->{nc} == 0x0027) { # '
1712            !!!cp (78);            !!!cp (78);
1713            !!!parse-error (type => 'bad attribute name');            !!!parse-error (type => 'bad attribute name');
1714          } else {          } else {
1715            !!!cp (82);            !!!cp (82);
1716          }          }
1717          $self->{current_attribute}          $self->{ca}
1718              = {name => chr ($self->{next_char}),              = {name => chr ($self->{nc}),
1719                 value => '',                 value => '',
1720                 line => $self->{line}, column => $self->{column}};                 line => $self->{line}, column => $self->{column}};
1721          $self->{state} = ATTRIBUTE_NAME_STATE;          $self->{state} = ATTRIBUTE_NAME_STATE;
# Line 1620  sub _get_next_token ($) { Line 1723  sub _get_next_token ($) {
1723          redo A;                  redo A;        
1724        }        }
1725      } elsif ($self->{state} == BEFORE_ATTRIBUTE_VALUE_STATE) {      } elsif ($self->{state} == BEFORE_ATTRIBUTE_VALUE_STATE) {
1726        if ($self->{next_char} == 0x0009 or # HT        if ($is_space->{$self->{nc}}) {
           $self->{next_char} == 0x000A or # LF  
           $self->{next_char} == 0x000B or # VT  
           $self->{next_char} == 0x000C or # FF  
           $self->{next_char} == 0x0020) { # SP        
1727          !!!cp (83);          !!!cp (83);
1728          ## Stay in the state          ## Stay in the state
1729          !!!next-input-character;          !!!next-input-character;
1730          redo A;          redo A;
1731        } elsif ($self->{next_char} == 0x0022) { # "        } elsif ($self->{nc} == 0x0022) { # "
1732          !!!cp (84);          !!!cp (84);
1733          $self->{state} = ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE;          $self->{state} = ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE;
1734          !!!next-input-character;          !!!next-input-character;
1735          redo A;          redo A;
1736        } elsif ($self->{next_char} == 0x0026) { # &        } elsif ($self->{nc} == 0x0026) { # &
1737          !!!cp (85);          !!!cp (85);
1738          $self->{state} = ATTRIBUTE_VALUE_UNQUOTED_STATE;          $self->{state} = ATTRIBUTE_VALUE_UNQUOTED_STATE;
1739          ## reconsume          ## reconsume
1740          redo A;          redo A;
1741        } elsif ($self->{next_char} == 0x0027) { # '        } elsif ($self->{nc} == 0x0027) { # '
1742          !!!cp (86);          !!!cp (86);
1743          $self->{state} = ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE;          $self->{state} = ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE;
1744          !!!next-input-character;          !!!next-input-character;
1745          redo A;          redo A;
1746        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
1747          !!!parse-error (type => 'empty unquoted attribute value');          !!!parse-error (type => 'empty unquoted attribute value');
1748          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1749            !!!cp (87);            !!!cp (87);
1750            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1751          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1752            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1753            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
1754              !!!cp (88);              !!!cp (88);
1755              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
1756            } else {            } else {
# Line 1659  sub _get_next_token ($) { Line 1758  sub _get_next_token ($) {
1758              !!!cp (89);              !!!cp (89);
1759            }            }
1760          } else {          } else {
1761            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1762          }          }
1763          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1764          !!!next-input-character;          !!!next-input-character;
1765    
1766          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1767    
1768          redo A;          redo A;
1769        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
1770          !!!parse-error (type => 'unclosed tag');          !!!parse-error (type => 'unclosed tag');
1771          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1772            !!!cp (90);            !!!cp (90);
1773            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1774          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1775            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1776            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
1777              !!!cp (91);              !!!cp (91);
1778              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
1779            } else {            } else {
# Line 1682  sub _get_next_token ($) { Line 1781  sub _get_next_token ($) {
1781              !!!cp (92);              !!!cp (92);
1782            }            }
1783          } else {          } else {
1784            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1785          }          }
1786          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1787          ## reconsume          ## reconsume
1788    
1789          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1790    
1791          redo A;          redo A;
1792        } else {        } else {
1793          if ($self->{next_char} == 0x003D) { # =          if ($self->{nc} == 0x003D) { # =
1794            !!!cp (93);            !!!cp (93);
1795            !!!parse-error (type => 'bad attribute value');            !!!parse-error (type => 'bad attribute value');
1796          } else {          } else {
1797            !!!cp (94);            !!!cp (94);
1798          }          }
1799          $self->{current_attribute}->{value} .= chr ($self->{next_char});          $self->{ca}->{value} .= chr ($self->{nc});
1800          $self->{state} = ATTRIBUTE_VALUE_UNQUOTED_STATE;          $self->{state} = ATTRIBUTE_VALUE_UNQUOTED_STATE;
1801          !!!next-input-character;          !!!next-input-character;
1802          redo A;          redo A;
1803        }        }
1804      } elsif ($self->{state} == ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE) {      } elsif ($self->{state} == ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE) {
1805        if ($self->{next_char} == 0x0022) { # "        if ($self->{nc} == 0x0022) { # "
1806          !!!cp (95);          !!!cp (95);
1807          $self->{state} = AFTER_ATTRIBUTE_VALUE_QUOTED_STATE;          $self->{state} = AFTER_ATTRIBUTE_VALUE_QUOTED_STATE;
1808          !!!next-input-character;          !!!next-input-character;
1809          redo A;          redo A;
1810        } elsif ($self->{next_char} == 0x0026) { # &        } elsif ($self->{nc} == 0x0026) { # &
1811          !!!cp (96);          !!!cp (96);
         $self->{last_attribute_value_state} = $self->{state};  
1812          ## NOTE: In the spec, the tokenizer is switched to the          ## NOTE: In the spec, the tokenizer is switched to the
1813          ## "entity in attribute value state".  In this implementation, the          ## "entity in attribute value state".  In this implementation, the
1814          ## tokenizer is switched to the |ENTITY_STATE|, which is an          ## tokenizer is switched to the |ENTITY_STATE|, which is an
1815          ## implementation of the "consume a character reference" algorithm.          ## implementation of the "consume a character reference" algorithm.
1816          #$self->{state} = ENTITY_IN_ATTRIBUTE_VALUE_STATE;          $self->{prev_state} = $self->{state};
1817          $self->{entity_in_attr} = 1;          $self->{entity_add} = 0x0022; # "
         $self->{entity_additional} = 0x0022; # "  
1818          $self->{state} = ENTITY_STATE;          $self->{state} = ENTITY_STATE;
1819          !!!next-input-character;          !!!next-input-character;
1820          redo A;          redo A;
1821        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
1822          !!!parse-error (type => 'unclosed attribute value');          !!!parse-error (type => 'unclosed attribute value');
1823          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1824            !!!cp (97);            !!!cp (97);
1825            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1826          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1827            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1828            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
1829              !!!cp (98);              !!!cp (98);
1830              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
1831            } else {            } else {
# Line 1736  sub _get_next_token ($) { Line 1833  sub _get_next_token ($) {
1833              !!!cp (99);              !!!cp (99);
1834            }            }
1835          } else {          } else {
1836            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1837          }          }
1838          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1839          ## reconsume          ## reconsume
1840    
1841          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1842    
1843          redo A;          redo A;
1844        } else {        } else {
1845          !!!cp (100);          !!!cp (100);
1846          $self->{current_attribute}->{value} .= chr ($self->{next_char});          $self->{ca}->{value} .= chr ($self->{nc});
1847            $self->{read_until}->($self->{ca}->{value},
1848                                  q["&],
1849                                  length $self->{ca}->{value});
1850    
1851          ## Stay in the state          ## Stay in the state
1852          !!!next-input-character;          !!!next-input-character;
1853          redo A;          redo A;
1854        }        }
1855      } elsif ($self->{state} == ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE) {      } elsif ($self->{state} == ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE) {
1856        if ($self->{next_char} == 0x0027) { # '        if ($self->{nc} == 0x0027) { # '
1857          !!!cp (101);          !!!cp (101);
1858          $self->{state} = AFTER_ATTRIBUTE_VALUE_QUOTED_STATE;          $self->{state} = AFTER_ATTRIBUTE_VALUE_QUOTED_STATE;
1859          !!!next-input-character;          !!!next-input-character;
1860          redo A;          redo A;
1861        } elsif ($self->{next_char} == 0x0026) { # &        } elsif ($self->{nc} == 0x0026) { # &
1862          !!!cp (102);          !!!cp (102);
         $self->{last_attribute_value_state} = $self->{state};  
1863          ## NOTE: In the spec, the tokenizer is switched to the          ## NOTE: In the spec, the tokenizer is switched to the
1864          ## "entity in attribute value state".  In this implementation, the          ## "entity in attribute value state".  In this implementation, the
1865          ## tokenizer is switched to the |ENTITY_STATE|, which is an          ## tokenizer is switched to the |ENTITY_STATE|, which is an
1866          ## implementation of the "consume a character reference" algorithm.          ## implementation of the "consume a character reference" algorithm.
1867          #$self->{state} = ENTITY_IN_ATTRIBUTE_VALUE_STATE;          $self->{entity_add} = 0x0027; # '
1868          $self->{entity_in_attr} = 1;          $self->{prev_state} = $self->{state};
         $self->{entity_additional} = 0x0027; # '  
1869          $self->{state} = ENTITY_STATE;          $self->{state} = ENTITY_STATE;
1870          !!!next-input-character;          !!!next-input-character;
1871          redo A;          redo A;
1872        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
1873          !!!parse-error (type => 'unclosed attribute value');          !!!parse-error (type => 'unclosed attribute value');
1874          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1875            !!!cp (103);            !!!cp (103);
1876            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1877          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1878            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1879            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
1880              !!!cp (104);              !!!cp (104);
1881              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
1882            } else {            } else {
# Line 1785  sub _get_next_token ($) { Line 1884  sub _get_next_token ($) {
1884              !!!cp (105);              !!!cp (105);
1885            }            }
1886          } else {          } else {
1887            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1888          }          }
1889          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1890          ## reconsume          ## reconsume
1891    
1892          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1893    
1894          redo A;          redo A;
1895        } else {        } else {
1896          !!!cp (106);          !!!cp (106);
1897          $self->{current_attribute}->{value} .= chr ($self->{next_char});          $self->{ca}->{value} .= chr ($self->{nc});
1898            $self->{read_until}->($self->{ca}->{value},
1899                                  q['&],
1900                                  length $self->{ca}->{value});
1901    
1902          ## Stay in the state          ## Stay in the state
1903          !!!next-input-character;          !!!next-input-character;
1904          redo A;          redo A;
1905        }        }
1906      } elsif ($self->{state} == ATTRIBUTE_VALUE_UNQUOTED_STATE) {      } elsif ($self->{state} == ATTRIBUTE_VALUE_UNQUOTED_STATE) {
1907        if ($self->{next_char} == 0x0009 or # HT        if ($is_space->{$self->{nc}}) {
           $self->{next_char} == 0x000A or # LF  
           $self->{next_char} == 0x000B or # HT  
           $self->{next_char} == 0x000C or # FF  
           $self->{next_char} == 0x0020) { # SP  
1908          !!!cp (107);          !!!cp (107);
1909          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1910          !!!next-input-character;          !!!next-input-character;
1911          redo A;          redo A;
1912        } elsif ($self->{next_char} == 0x0026) { # &        } elsif ($self->{nc} == 0x0026) { # &
1913          !!!cp (108);          !!!cp (108);
         $self->{last_attribute_value_state} = $self->{state};  
1914          ## NOTE: In the spec, the tokenizer is switched to the          ## NOTE: In the spec, the tokenizer is switched to the
1915          ## "entity in attribute value state".  In this implementation, the          ## "entity in attribute value state".  In this implementation, the
1916          ## tokenizer is switched to the |ENTITY_STATE|, which is an          ## tokenizer is switched to the |ENTITY_STATE|, which is an
1917          ## implementation of the "consume a character reference" algorithm.          ## implementation of the "consume a character reference" algorithm.
1918          #$self->{state} = ENTITY_IN_ATTRIBUTE_VALUE_STATE;          $self->{entity_add} = -1;
1919          $self->{entity_in_attr} = 1;          $self->{prev_state} = $self->{state};
         $self->{entity_additional} = -1;  
1920          $self->{state} = ENTITY_STATE;          $self->{state} = ENTITY_STATE;
1921          !!!next-input-character;          !!!next-input-character;
1922          redo A;          redo A;
1923        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
1924          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1925            !!!cp (109);            !!!cp (109);
1926            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1927          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1928            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1929            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
1930              !!!cp (110);              !!!cp (110);
1931              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
1932            } else {            } else {
# Line 1837  sub _get_next_token ($) { Line 1934  sub _get_next_token ($) {
1934              !!!cp (111);              !!!cp (111);
1935            }            }
1936          } else {          } else {
1937            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1938          }          }
1939          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1940          !!!next-input-character;          !!!next-input-character;
1941    
1942          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1943    
1944          redo A;          redo A;
1945        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
1946          !!!parse-error (type => 'unclosed tag');          !!!parse-error (type => 'unclosed tag');
1947          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1948            !!!cp (112);            !!!cp (112);
1949            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1950          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1951            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1952            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
1953              !!!cp (113);              !!!cp (113);
1954              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
1955            } else {            } else {
# Line 1860  sub _get_next_token ($) { Line 1957  sub _get_next_token ($) {
1957              !!!cp (114);              !!!cp (114);
1958            }            }
1959          } else {          } else {
1960            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
1961          }          }
1962          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
1963          ## reconsume          ## reconsume
1964    
1965          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
1966    
1967          redo A;          redo A;
1968        } else {        } else {
# Line 1873  sub _get_next_token ($) { Line 1970  sub _get_next_token ($) {
1970               0x0022 => 1, # "               0x0022 => 1, # "
1971               0x0027 => 1, # '               0x0027 => 1, # '
1972               0x003D => 1, # =               0x003D => 1, # =
1973              }->{$self->{next_char}}) {              }->{$self->{nc}}) {
1974            !!!cp (115);            !!!cp (115);
1975            !!!parse-error (type => 'bad attribute value');            !!!parse-error (type => 'bad attribute value');
1976          } else {          } else {
1977            !!!cp (116);            !!!cp (116);
1978          }          }
1979          $self->{current_attribute}->{value} .= chr ($self->{next_char});          $self->{ca}->{value} .= chr ($self->{nc});
1980            $self->{read_until}->($self->{ca}->{value},
1981                                  q["'=& >],
1982                                  length $self->{ca}->{value});
1983    
1984          ## Stay in the state          ## Stay in the state
1985          !!!next-input-character;          !!!next-input-character;
1986          redo A;          redo A;
1987        }        }
     } elsif ($self->{state} == ENTITY_IN_ATTRIBUTE_VALUE_STATE) {  
       my $token = $self->{entity_return};  
   
       unless (defined $token) {  
         !!!cp (117);  
         $self->{current_attribute}->{value} .= '&';  
       } else {  
         !!!cp (118);  
         $self->{current_attribute}->{value} .= $token->{data};  
         $self->{current_attribute}->{has_reference} = $token->{has_reference};  
         ## ISSUE: spec says "append the returned character token to the current attribute's value"  
       }  
   
       $self->{state} = $self->{last_attribute_value_state};  
       # next-input-character is already done  
       redo A;  
1988      } elsif ($self->{state} == AFTER_ATTRIBUTE_VALUE_QUOTED_STATE) {      } elsif ($self->{state} == AFTER_ATTRIBUTE_VALUE_QUOTED_STATE) {
1989        if ($self->{next_char} == 0x0009 or # HT        if ($is_space->{$self->{nc}}) {
           $self->{next_char} == 0x000A or # LF  
           $self->{next_char} == 0x000B or # VT  
           $self->{next_char} == 0x000C or # FF  
           $self->{next_char} == 0x0020) { # SP  
1990          !!!cp (118);          !!!cp (118);
1991          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;          $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1992          !!!next-input-character;          !!!next-input-character;
1993          redo A;          redo A;
1994        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
1995          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
1996            !!!cp (119);            !!!cp (119);
1997            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
1998          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1999            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
2000            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
2001              !!!cp (120);              !!!cp (120);
2002              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
2003            } else {            } else {
# Line 1924  sub _get_next_token ($) { Line 2005  sub _get_next_token ($) {
2005              !!!cp (121);              !!!cp (121);
2006            }            }
2007          } else {          } else {
2008            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
2009          }          }
2010          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2011          !!!next-input-character;          !!!next-input-character;
2012    
2013          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
2014    
2015          redo A;          redo A;
2016        } elsif ($self->{next_char} == 0x002F) { # /        } elsif ($self->{nc} == 0x002F) { # /
2017          !!!cp (122);          !!!cp (122);
2018          $self->{state} = SELF_CLOSING_START_TAG_STATE;          $self->{state} = SELF_CLOSING_START_TAG_STATE;
2019          !!!next-input-character;          !!!next-input-character;
2020          redo A;          redo A;
2021        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2022          !!!parse-error (type => 'unclosed tag');          !!!parse-error (type => 'unclosed tag');
2023          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
2024            !!!cp (122.3);            !!!cp (122.3);
2025            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
2026          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
2027            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
2028              !!!cp (122.1);              !!!cp (122.1);
2029              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
2030            } else {            } else {
# Line 1951  sub _get_next_token ($) { Line 2032  sub _get_next_token ($) {
2032              !!!cp (122.2);              !!!cp (122.2);
2033            }            }
2034          } else {          } else {
2035            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
2036          }          }
2037          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2038          ## Reconsume.          ## Reconsume.
2039          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
2040          redo A;          redo A;
2041        } else {        } else {
2042          !!!cp ('124.1');          !!!cp ('124.1');
# Line 1965  sub _get_next_token ($) { Line 2046  sub _get_next_token ($) {
2046          redo A;          redo A;
2047        }        }
2048      } elsif ($self->{state} == SELF_CLOSING_START_TAG_STATE) {      } elsif ($self->{state} == SELF_CLOSING_START_TAG_STATE) {
2049        if ($self->{next_char} == 0x003E) { # >        if ($self->{nc} == 0x003E) { # >
2050          if ($self->{current_token}->{type} == END_TAG_TOKEN) {          if ($self->{ct}->{type} == END_TAG_TOKEN) {
2051            !!!cp ('124.2');            !!!cp ('124.2');
2052            !!!parse-error (type => 'nestc', token => $self->{current_token});            !!!parse-error (type => 'nestc', token => $self->{ct});
2053            ## TODO: Different type than slash in start tag            ## TODO: Different type than slash in start tag
2054            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST            $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
2055            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
2056              !!!cp ('124.4');              !!!cp ('124.4');
2057              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
2058            } else {            } else {
# Line 1986  sub _get_next_token ($) { Line 2067  sub _get_next_token ($) {
2067          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2068          !!!next-input-character;          !!!next-input-character;
2069    
2070          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
2071    
2072          redo A;          redo A;
2073        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2074          !!!parse-error (type => 'unclosed tag');          !!!parse-error (type => 'unclosed tag');
2075          if ($self->{current_token}->{type} == START_TAG_TOKEN) {          if ($self->{ct}->{type} == START_TAG_TOKEN) {
2076            !!!cp (124.7);            !!!cp (124.7);
2077            $self->{last_emitted_start_tag_name} = $self->{current_token}->{tag_name};            $self->{last_stag_name} = $self->{ct}->{tag_name};
2078          } elsif ($self->{current_token}->{type} == END_TAG_TOKEN) {          } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
2079            if ($self->{current_token}->{attributes}) {            if ($self->{ct}->{attributes}) {
2080              !!!cp (124.5);              !!!cp (124.5);
2081              !!!parse-error (type => 'end tag attribute');              !!!parse-error (type => 'end tag attribute');
2082            } else {            } else {
# Line 2003  sub _get_next_token ($) { Line 2084  sub _get_next_token ($) {
2084              !!!cp (124.6);              !!!cp (124.6);
2085            }            }
2086          } else {          } else {
2087            die "$0: $self->{current_token}->{type}: Unknown token type";            die "$0: $self->{ct}->{type}: Unknown token type";
2088          }          }
2089          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2090          ## Reconsume.          ## Reconsume.
2091          !!!emit ($self->{current_token}); # start tag or end tag          !!!emit ($self->{ct}); # start tag or end tag
2092          redo A;          redo A;
2093        } else {        } else {
2094          !!!cp ('124.4');          !!!cp ('124.4');
# Line 2023  sub _get_next_token ($) { Line 2104  sub _get_next_token ($) {
2104        ## NOTE: Unlike spec's "bogus comment state", this implementation        ## NOTE: Unlike spec's "bogus comment state", this implementation
2105        ## consumes characters one-by-one basis.        ## consumes characters one-by-one basis.
2106                
2107        if ($self->{next_char} == 0x003E) { # >        if ($self->{nc} == 0x003E) { # >
2108          !!!cp (124);          !!!cp (124);
2109          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2110          !!!next-input-character;          !!!next-input-character;
2111    
2112          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{ct}); # comment
2113          redo A;          redo A;
2114        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2115          !!!cp (125);          !!!cp (125);
2116          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2117          ## reconsume          ## reconsume
2118    
2119          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{ct}); # comment
2120          redo A;          redo A;
2121        } else {        } else {
2122          !!!cp (126);          !!!cp (126);
2123          $self->{current_token}->{data} .= chr ($self->{next_char}); # comment          $self->{ct}->{data} .= chr ($self->{nc}); # comment
2124            $self->{read_until}->($self->{ct}->{data},
2125                                  q[>],
2126                                  length $self->{ct}->{data});
2127    
2128          ## Stay in the state.          ## Stay in the state.
2129          !!!next-input-character;          !!!next-input-character;
2130          redo A;          redo A;
# Line 2047  sub _get_next_token ($) { Line 2132  sub _get_next_token ($) {
2132      } elsif ($self->{state} == MARKUP_DECLARATION_OPEN_STATE) {      } elsif ($self->{state} == MARKUP_DECLARATION_OPEN_STATE) {
2133        ## (only happen if PCDATA state)        ## (only happen if PCDATA state)
2134                
2135        if ($self->{next_char} == 0x002D) { # -        if ($self->{nc} == 0x002D) { # -
2136          !!!cp (133);          !!!cp (133);
2137          $self->{state} = MD_HYPHEN_STATE;          $self->{state} = MD_HYPHEN_STATE;
2138          !!!next-input-character;          !!!next-input-character;
2139          redo A;          redo A;
2140        } elsif ($self->{next_char} == 0x0044 or # D        } elsif ($self->{nc} == 0x0044 or # D
2141                 $self->{next_char} == 0x0064) { # d                 $self->{nc} == 0x0064) { # d
2142          ## ASCII case-insensitive.          ## ASCII case-insensitive.
2143          !!!cp (130);          !!!cp (130);
2144          $self->{state} = MD_DOCTYPE_STATE;          $self->{state} = MD_DOCTYPE_STATE;
2145          $self->{state_keyword} = chr $self->{next_char};          $self->{s_kwd} = chr $self->{nc};
2146          !!!next-input-character;          !!!next-input-character;
2147          redo A;          redo A;
2148        } elsif ($self->{insertion_mode} & IN_FOREIGN_CONTENT_IM and        } elsif ($self->{insertion_mode} & IN_FOREIGN_CONTENT_IM and
2149                 $self->{open_elements}->[-1]->[1] & FOREIGN_EL and                 $self->{open_elements}->[-1]->[1] & FOREIGN_EL and
2150                 $self->{next_char} == 0x005B) { # [                 $self->{nc} == 0x005B) { # [
2151          !!!cp (135.4);                          !!!cp (135.4);                
2152          $self->{state} = MD_CDATA_STATE;          $self->{state} = MD_CDATA_STATE;
2153          $self->{state_keyword} = '[';          $self->{s_kwd} = '[';
2154          !!!next-input-character;          !!!next-input-character;
2155          redo A;          redo A;
2156        } else {        } else {
# Line 2077  sub _get_next_token ($) { Line 2162  sub _get_next_token ($) {
2162                        column => $self->{column_prev} - 1);                        column => $self->{column_prev} - 1);
2163        ## Reconsume.        ## Reconsume.
2164        $self->{state} = BOGUS_COMMENT_STATE;        $self->{state} = BOGUS_COMMENT_STATE;
2165        $self->{current_token} = {type => COMMENT_TOKEN, data => '',        $self->{ct} = {type => COMMENT_TOKEN, data => '',
2166                                  line => $self->{line_prev},                                  line => $self->{line_prev},
2167                                  column => $self->{column_prev} - 1,                                  column => $self->{column_prev} - 1,
2168                                 };                                 };
2169        redo A;        redo A;
2170      } elsif ($self->{state} == MD_HYPHEN_STATE) {      } elsif ($self->{state} == MD_HYPHEN_STATE) {
2171        if ($self->{next_char} == 0x002D) { # -        if ($self->{nc} == 0x002D) { # -
2172          !!!cp (127);          !!!cp (127);
2173          $self->{current_token} = {type => COMMENT_TOKEN, data => '',          $self->{ct} = {type => COMMENT_TOKEN, data => '',
2174                                    line => $self->{line_prev},                                    line => $self->{line_prev},
2175                                    column => $self->{column_prev} - 2,                                    column => $self->{column_prev} - 2,
2176                                   };                                   };
# Line 2099  sub _get_next_token ($) { Line 2184  sub _get_next_token ($) {
2184                          column => $self->{column_prev} - 2);                          column => $self->{column_prev} - 2);
2185          $self->{state} = BOGUS_COMMENT_STATE;          $self->{state} = BOGUS_COMMENT_STATE;
2186          ## Reconsume.          ## Reconsume.
2187          $self->{current_token} = {type => COMMENT_TOKEN,          $self->{ct} = {type => COMMENT_TOKEN,
2188                                    data => '-',                                    data => '-',
2189                                    line => $self->{line_prev},                                    line => $self->{line_prev},
2190                                    column => $self->{column_prev} - 2,                                    column => $self->{column_prev} - 2,
# Line 2108  sub _get_next_token ($) { Line 2193  sub _get_next_token ($) {
2193        }        }
2194      } elsif ($self->{state} == MD_DOCTYPE_STATE) {      } elsif ($self->{state} == MD_DOCTYPE_STATE) {
2195        ## ASCII case-insensitive.        ## ASCII case-insensitive.
2196        if ($self->{next_char} == [        if ($self->{nc} == [
2197              undef,              undef,
2198              0x004F, # O              0x004F, # O
2199              0x0043, # C              0x0043, # C
2200              0x0054, # T              0x0054, # T
2201              0x0059, # Y              0x0059, # Y
2202              0x0050, # P              0x0050, # P
2203            ]->[length $self->{state_keyword}] or            ]->[length $self->{s_kwd}] or
2204            $self->{next_char} == [            $self->{nc} == [
2205              undef,              undef,
2206              0x006F, # o              0x006F, # o
2207              0x0063, # c              0x0063, # c
2208              0x0074, # t              0x0074, # t
2209              0x0079, # y              0x0079, # y
2210              0x0070, # p              0x0070, # p
2211            ]->[length $self->{state_keyword}]) {            ]->[length $self->{s_kwd}]) {
2212          !!!cp (131);          !!!cp (131);
2213          ## Stay in the state.          ## Stay in the state.
2214          $self->{state_keyword} .= chr $self->{next_char};          $self->{s_kwd} .= chr $self->{nc};
2215          !!!next-input-character;          !!!next-input-character;
2216          redo A;          redo A;
2217        } elsif ((length $self->{state_keyword}) == 6 and        } elsif ((length $self->{s_kwd}) == 6 and
2218                 ($self->{next_char} == 0x0045 or # E                 ($self->{nc} == 0x0045 or # E
2219                  $self->{next_char} == 0x0065)) { # e                  $self->{nc} == 0x0065)) { # e
2220          !!!cp (129);          !!!cp (129);
2221          $self->{state} = DOCTYPE_STATE;          $self->{state} = DOCTYPE_STATE;
2222          $self->{current_token} = {type => DOCTYPE_TOKEN,          $self->{ct} = {type => DOCTYPE_TOKEN,
2223                                    quirks => 1,                                    quirks => 1,
2224                                    line => $self->{line_prev},                                    line => $self->{line_prev},
2225                                    column => $self->{column_prev} - 7,                                    column => $self->{column_prev} - 7,
# Line 2145  sub _get_next_token ($) { Line 2230  sub _get_next_token ($) {
2230          !!!cp (132);                  !!!cp (132);        
2231          !!!parse-error (type => 'bogus comment',          !!!parse-error (type => 'bogus comment',
2232                          line => $self->{line_prev},                          line => $self->{line_prev},
2233                          column => $self->{column_prev} - 1 - length $self->{state_keyword});                          column => $self->{column_prev} - 1 - length $self->{s_kwd});
2234          $self->{state} = BOGUS_COMMENT_STATE;          $self->{state} = BOGUS_COMMENT_STATE;
2235          ## Reconsume.          ## Reconsume.
2236          $self->{current_token} = {type => COMMENT_TOKEN,          $self->{ct} = {type => COMMENT_TOKEN,
2237                                    data => $self->{state_keyword},                                    data => $self->{s_kwd},
2238                                    line => $self->{line_prev},                                    line => $self->{line_prev},
2239                                    column => $self->{column_prev} - 1 - length $self->{state_keyword},                                    column => $self->{column_prev} - 1 - length $self->{s_kwd},
2240                                   };                                   };
2241          redo A;          redo A;
2242        }        }
2243      } elsif ($self->{state} == MD_CDATA_STATE) {      } elsif ($self->{state} == MD_CDATA_STATE) {
2244        if ($self->{next_char} == {        if ($self->{nc} == {
2245              '[' => 0x0043, # C              '[' => 0x0043, # C
2246              '[C' => 0x0044, # D              '[C' => 0x0044, # D
2247              '[CD' => 0x0041, # A              '[CD' => 0x0041, # A
2248              '[CDA' => 0x0054, # T              '[CDA' => 0x0054, # T
2249              '[CDAT' => 0x0041, # A              '[CDAT' => 0x0041, # A
2250            }->{$self->{state_keyword}}) {            }->{$self->{s_kwd}}) {
2251          !!!cp (135.1);          !!!cp (135.1);
2252          ## Stay in the state.          ## Stay in the state.
2253          $self->{state_keyword} .= chr $self->{next_char};          $self->{s_kwd} .= chr $self->{nc};
2254          !!!next-input-character;          !!!next-input-character;
2255          redo A;          redo A;
2256        } elsif ($self->{state_keyword} eq '[CDATA' and        } elsif ($self->{s_kwd} eq '[CDATA' and
2257                 $self->{next_char} == 0x005B) { # [                 $self->{nc} == 0x005B) { # [
2258          !!!cp (135.2);          !!!cp (135.2);
2259          $self->{current_token} = {type => CHARACTER_TOKEN,          $self->{ct} = {type => CHARACTER_TOKEN,
2260                                    data => '',                                    data => '',
2261                                    line => $self->{line_prev},                                    line => $self->{line_prev},
2262                                    column => $self->{column_prev} - 7};                                    column => $self->{column_prev} - 7};
# Line 2182  sub _get_next_token ($) { Line 2267  sub _get_next_token ($) {
2267          !!!cp (135.3);          !!!cp (135.3);
2268          !!!parse-error (type => 'bogus comment',          !!!parse-error (type => 'bogus comment',
2269                          line => $self->{line_prev},                          line => $self->{line_prev},
2270                          column => $self->{column_prev} - 1 - length $self->{state_keyword});                          column => $self->{column_prev} - 1 - length $self->{s_kwd});
2271          $self->{state} = BOGUS_COMMENT_STATE;          $self->{state} = BOGUS_COMMENT_STATE;
2272          ## Reconsume.          ## Reconsume.
2273          $self->{current_token} = {type => COMMENT_TOKEN,          $self->{ct} = {type => COMMENT_TOKEN,
2274                                    data => $self->{state_keyword},                                    data => $self->{s_kwd},
2275                                    line => $self->{line_prev},                                    line => $self->{line_prev},
2276                                    column => $self->{column_prev} - 1 - length $self->{state_keyword},                                    column => $self->{column_prev} - 1 - length $self->{s_kwd},
2277                                   };                                   };
2278          redo A;          redo A;
2279        }        }
2280      } elsif ($self->{state} == COMMENT_START_STATE) {      } elsif ($self->{state} == COMMENT_START_STATE) {
2281        if ($self->{next_char} == 0x002D) { # -        if ($self->{nc} == 0x002D) { # -
2282          !!!cp (137);          !!!cp (137);
2283          $self->{state} = COMMENT_START_DASH_STATE;          $self->{state} = COMMENT_START_DASH_STATE;
2284          !!!next-input-character;          !!!next-input-character;
2285          redo A;          redo A;
2286        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
2287          !!!cp (138);          !!!cp (138);
2288          !!!parse-error (type => 'bogus comment');          !!!parse-error (type => 'bogus comment');
2289          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2290          !!!next-input-character;          !!!next-input-character;
2291    
2292          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{ct}); # comment
2293    
2294          redo A;          redo A;
2295        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2296          !!!cp (139);          !!!cp (139);
2297          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
2298          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2299          ## reconsume          ## reconsume
2300    
2301          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{ct}); # comment
2302    
2303          redo A;          redo A;
2304        } else {        } else {
2305          !!!cp (140);          !!!cp (140);
2306          $self->{current_token}->{data} # comment          $self->{ct}->{data} # comment
2307              .= chr ($self->{next_char});              .= chr ($self->{nc});
2308          $self->{state} = COMMENT_STATE;          $self->{state} = COMMENT_STATE;
2309          !!!next-input-character;          !!!next-input-character;
2310          redo A;          redo A;
2311        }        }
2312      } elsif ($self->{state} == COMMENT_START_DASH_STATE) {      } elsif ($self->{state} == COMMENT_START_DASH_STATE) {
2313        if ($self->{next_char} == 0x002D) { # -        if ($self->{nc} == 0x002D) { # -
2314          !!!cp (141);          !!!cp (141);
2315          $self->{state} = COMMENT_END_STATE;          $self->{state} = COMMENT_END_STATE;
2316          !!!next-input-character;          !!!next-input-character;
2317          redo A;          redo A;
2318        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
2319          !!!cp (142);          !!!cp (142);
2320          !!!parse-error (type => 'bogus comment');          !!!parse-error (type => 'bogus comment');
2321          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2322          !!!next-input-character;          !!!next-input-character;
2323    
2324          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{ct}); # comment
2325    
2326          redo A;          redo A;
2327        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2328          !!!cp (143);          !!!cp (143);
2329          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
2330          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2331          ## reconsume          ## reconsume
2332    
2333          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{ct}); # comment
2334    
2335          redo A;          redo A;
2336        } else {        } else {
2337          !!!cp (144);          !!!cp (144);
2338          $self->{current_token}->{data} # comment          $self->{ct}->{data} # comment
2339              .= '-' . chr ($self->{next_char});              .= '-' . chr ($self->{nc});
2340          $self->{state} = COMMENT_STATE;          $self->{state} = COMMENT_STATE;
2341          !!!next-input-character;          !!!next-input-character;
2342          redo A;          redo A;
2343        }        }
2344      } elsif ($self->{state} == COMMENT_STATE) {      } elsif ($self->{state} == COMMENT_STATE) {
2345        if ($self->{next_char} == 0x002D) { # -        if ($self->{nc} == 0x002D) { # -
2346          !!!cp (145);          !!!cp (145);
2347          $self->{state} = COMMENT_END_DASH_STATE;          $self->{state} = COMMENT_END_DASH_STATE;
2348          !!!next-input-character;          !!!next-input-character;
2349          redo A;          redo A;
2350        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2351          !!!cp (146);          !!!cp (146);
2352          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
2353          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2354          ## reconsume          ## reconsume
2355    
2356          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{ct}); # comment
2357    
2358          redo A;          redo A;
2359        } else {        } else {
2360          !!!cp (147);          !!!cp (147);
2361          $self->{current_token}->{data} .= chr ($self->{next_char}); # comment          $self->{ct}->{data} .= chr ($self->{nc}); # comment
2362            $self->{read_until}->($self->{ct}->{data},
2363                                  q[-],
2364                                  length $self->{ct}->{data});
2365    
2366          ## Stay in the state          ## Stay in the state
2367          !!!next-input-character;          !!!next-input-character;
2368          redo A;          redo A;
2369        }        }
2370      } elsif ($self->{state} == COMMENT_END_DASH_STATE) {      } elsif ($self->{state} == COMMENT_END_DASH_STATE) {
2371        if ($self->{next_char} == 0x002D) { # -        if ($self->{nc} == 0x002D) { # -
2372          !!!cp (148);          !!!cp (148);
2373          $self->{state} = COMMENT_END_STATE;          $self->{state} = COMMENT_END_STATE;
2374          !!!next-input-character;          !!!next-input-character;
2375          redo A;          redo A;
2376        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2377          !!!cp (149);          !!!cp (149);
2378          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
2379          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2380          ## reconsume          ## reconsume
2381    
2382          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{ct}); # comment
2383    
2384          redo A;          redo A;
2385        } else {        } else {
2386          !!!cp (150);          !!!cp (150);
2387          $self->{current_token}->{data} .= '-' . chr ($self->{next_char}); # comment          $self->{ct}->{data} .= '-' . chr ($self->{nc}); # comment
2388          $self->{state} = COMMENT_STATE;          $self->{state} = COMMENT_STATE;
2389          !!!next-input-character;          !!!next-input-character;
2390          redo A;          redo A;
2391        }        }
2392      } elsif ($self->{state} == COMMENT_END_STATE) {      } elsif ($self->{state} == COMMENT_END_STATE) {
2393        if ($self->{next_char} == 0x003E) { # >        if ($self->{nc} == 0x003E) { # >
2394          !!!cp (151);          !!!cp (151);
2395          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2396          !!!next-input-character;          !!!next-input-character;
2397    
2398          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{ct}); # comment
2399    
2400          redo A;          redo A;
2401        } elsif ($self->{next_char} == 0x002D) { # -        } elsif ($self->{nc} == 0x002D) { # -
2402          !!!cp (152);          !!!cp (152);
2403          !!!parse-error (type => 'dash in comment',          !!!parse-error (type => 'dash in comment',
2404                          line => $self->{line_prev},                          line => $self->{line_prev},
2405                          column => $self->{column_prev});                          column => $self->{column_prev});
2406          $self->{current_token}->{data} .= '-'; # comment          $self->{ct}->{data} .= '-'; # comment
2407          ## Stay in the state          ## Stay in the state
2408          !!!next-input-character;          !!!next-input-character;
2409          redo A;          redo A;
2410        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2411          !!!cp (153);          !!!cp (153);
2412          !!!parse-error (type => 'unclosed comment');          !!!parse-error (type => 'unclosed comment');
2413          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2414          ## reconsume          ## reconsume
2415    
2416          !!!emit ($self->{current_token}); # comment          !!!emit ($self->{ct}); # comment
2417    
2418          redo A;          redo A;
2419        } else {        } else {
# Line 2332  sub _get_next_token ($) { Line 2421  sub _get_next_token ($) {
2421          !!!parse-error (type => 'dash in comment',          !!!parse-error (type => 'dash in comment',
2422                          line => $self->{line_prev},                          line => $self->{line_prev},
2423                          column => $self->{column_prev});                          column => $self->{column_prev});
2424          $self->{current_token}->{data} .= '--' . chr ($self->{next_char}); # comment          $self->{ct}->{data} .= '--' . chr ($self->{nc}); # comment
2425          $self->{state} = COMMENT_STATE;          $self->{state} = COMMENT_STATE;
2426          !!!next-input-character;          !!!next-input-character;
2427          redo A;          redo A;
2428        }        }
2429      } elsif ($self->{state} == DOCTYPE_STATE) {      } elsif ($self->{state} == DOCTYPE_STATE) {
2430        if ($self->{next_char} == 0x0009 or # HT        if ($is_space->{$self->{nc}}) {
           $self->{next_char} == 0x000A or # LF  
           $self->{next_char} == 0x000B or # VT  
           $self->{next_char} == 0x000C or # FF  
           $self->{next_char} == 0x0020) { # SP  
2431          !!!cp (155);          !!!cp (155);
2432          $self->{state} = BEFORE_DOCTYPE_NAME_STATE;          $self->{state} = BEFORE_DOCTYPE_NAME_STATE;
2433          !!!next-input-character;          !!!next-input-character;
# Line 2355  sub _get_next_token ($) { Line 2440  sub _get_next_token ($) {
2440          redo A;          redo A;
2441        }        }
2442      } elsif ($self->{state} == BEFORE_DOCTYPE_NAME_STATE) {      } elsif ($self->{state} == BEFORE_DOCTYPE_NAME_STATE) {
2443        if ($self->{next_char} == 0x0009 or # HT        if ($is_space->{$self->{nc}}) {
           $self->{next_char} == 0x000A or # LF  
           $self->{next_char} == 0x000B or # VT  
           $self->{next_char} == 0x000C or # FF  
           $self->{next_char} == 0x0020) { # SP  
2444          !!!cp (157);          !!!cp (157);
2445          ## Stay in the state          ## Stay in the state
2446          !!!next-input-character;          !!!next-input-character;
2447          redo A;          redo A;
2448        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
2449          !!!cp (158);          !!!cp (158);
2450          !!!parse-error (type => 'no DOCTYPE name');          !!!parse-error (type => 'no DOCTYPE name');
2451          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2452          !!!next-input-character;          !!!next-input-character;
2453    
2454          !!!emit ($self->{current_token}); # DOCTYPE (quirks)          !!!emit ($self->{ct}); # DOCTYPE (quirks)
2455    
2456          redo A;          redo A;
2457        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2458          !!!cp (159);          !!!cp (159);
2459          !!!parse-error (type => 'no DOCTYPE name');          !!!parse-error (type => 'no DOCTYPE name');
2460          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2461          ## reconsume          ## reconsume
2462    
2463          !!!emit ($self->{current_token}); # DOCTYPE (quirks)          !!!emit ($self->{ct}); # DOCTYPE (quirks)
2464    
2465          redo A;          redo A;
2466        } else {        } else {
2467          !!!cp (160);          !!!cp (160);
2468          $self->{current_token}->{name} = chr $self->{next_char};          $self->{ct}->{name} = chr $self->{nc};
2469          delete $self->{current_token}->{quirks};          delete $self->{ct}->{quirks};
2470  ## ISSUE: "Set the token's name name to the" in the spec  ## ISSUE: "Set the token's name name to the" in the spec
2471          $self->{state} = DOCTYPE_NAME_STATE;          $self->{state} = DOCTYPE_NAME_STATE;
2472          !!!next-input-character;          !!!next-input-character;
# Line 2393  sub _get_next_token ($) { Line 2474  sub _get_next_token ($) {
2474        }        }
2475      } elsif ($self->{state} == DOCTYPE_NAME_STATE) {      } elsif ($self->{state} == DOCTYPE_NAME_STATE) {
2476  ## ISSUE: Redundant "First," in the spec.  ## ISSUE: Redundant "First," in the spec.
2477        if ($self->{next_char} == 0x0009 or # HT        if ($is_space->{$self->{nc}}) {
           $self->{next_char} == 0x000A or # LF  
           $self->{next_char} == 0x000B or # VT  
           $self->{next_char} == 0x000C or # FF  
           $self->{next_char} == 0x0020) { # SP  
2478          !!!cp (161);          !!!cp (161);
2479          $self->{state} = AFTER_DOCTYPE_NAME_STATE;          $self->{state} = AFTER_DOCTYPE_NAME_STATE;
2480          !!!next-input-character;          !!!next-input-character;
2481          redo A;          redo A;
2482        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
2483          !!!cp (162);          !!!cp (162);
2484          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2485          !!!next-input-character;          !!!next-input-character;
2486    
2487          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2488    
2489          redo A;          redo A;
2490        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2491          !!!cp (163);          !!!cp (163);
2492          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
2493          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2494          ## reconsume          ## reconsume
2495    
2496          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2497          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2498    
2499          redo A;          redo A;
2500        } else {        } else {
2501          !!!cp (164);          !!!cp (164);
2502          $self->{current_token}->{name}          $self->{ct}->{name}
2503            .= chr ($self->{next_char}); # DOCTYPE            .= chr ($self->{nc}); # DOCTYPE
2504          ## Stay in the state          ## Stay in the state
2505          !!!next-input-character;          !!!next-input-character;
2506          redo A;          redo A;
2507        }        }
2508      } elsif ($self->{state} == AFTER_DOCTYPE_NAME_STATE) {      } elsif ($self->{state} == AFTER_DOCTYPE_NAME_STATE) {
2509        if ($self->{next_char} == 0x0009 or # HT        if ($is_space->{$self->{nc}}) {
           $self->{next_char} == 0x000A or # LF  
           $self->{next_char} == 0x000B or # VT  
           $self->{next_char} == 0x000C or # FF  
           $self->{next_char} == 0x0020) { # SP  
2510          !!!cp (165);          !!!cp (165);
2511          ## Stay in the state          ## Stay in the state
2512          !!!next-input-character;          !!!next-input-character;
2513          redo A;          redo A;
2514        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
2515          !!!cp (166);          !!!cp (166);
2516          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2517          !!!next-input-character;          !!!next-input-character;
2518    
2519          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2520    
2521          redo A;          redo A;
2522        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2523          !!!cp (167);          !!!cp (167);
2524          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
2525          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2526          ## reconsume          ## reconsume
2527    
2528          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2529          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2530    
2531          redo A;          redo A;
2532        } elsif ($self->{next_char} == 0x0050 or # P        } elsif ($self->{nc} == 0x0050 or # P
2533                 $self->{next_char} == 0x0070) { # p                 $self->{nc} == 0x0070) { # p
2534          $self->{state} = PUBLIC_STATE;          $self->{state} = PUBLIC_STATE;
2535          $self->{state_keyword} = chr $self->{next_char};          $self->{s_kwd} = chr $self->{nc};
2536          !!!next-input-character;          !!!next-input-character;
2537          redo A;          redo A;
2538        } elsif ($self->{next_char} == 0x0053 or # S        } elsif ($self->{nc} == 0x0053 or # S
2539                 $self->{next_char} == 0x0073) { # s                 $self->{nc} == 0x0073) { # s
2540          $self->{state} = SYSTEM_STATE;          $self->{state} = SYSTEM_STATE;
2541          $self->{state_keyword} = chr $self->{next_char};          $self->{s_kwd} = chr $self->{nc};
2542          !!!next-input-character;          !!!next-input-character;
2543          redo A;          redo A;
2544        } else {        } else {
2545          !!!cp (180);          !!!cp (180);
2546          !!!parse-error (type => 'string after DOCTYPE name');          !!!parse-error (type => 'string after DOCTYPE name');
2547          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2548    
2549          $self->{state} = BOGUS_DOCTYPE_STATE;          $self->{state} = BOGUS_DOCTYPE_STATE;
2550          !!!next-input-character;          !!!next-input-character;
# Line 2479  sub _get_next_token ($) { Line 2552  sub _get_next_token ($) {
2552        }        }
2553      } elsif ($self->{state} == PUBLIC_STATE) {      } elsif ($self->{state} == PUBLIC_STATE) {
2554        ## ASCII case-insensitive        ## ASCII case-insensitive
2555        if ($self->{next_char} == [        if ($self->{nc} == [
2556              undef,              undef,
2557              0x0055, # U              0x0055, # U
2558              0x0042, # B              0x0042, # B
2559              0x004C, # L              0x004C, # L
2560              0x0049, # I              0x0049, # I
2561            ]->[length $self->{state_keyword}] or            ]->[length $self->{s_kwd}] or
2562            $self->{next_char} == [            $self->{nc} == [
2563              undef,              undef,
2564              0x0075, # u              0x0075, # u
2565              0x0062, # b              0x0062, # b
2566              0x006C, # l              0x006C, # l
2567              0x0069, # i              0x0069, # i
2568            ]->[length $self->{state_keyword}]) {            ]->[length $self->{s_kwd}]) {
2569          !!!cp (175);          !!!cp (175);
2570          ## Stay in the state.          ## Stay in the state.
2571          $self->{state_keyword} .= chr $self->{next_char};          $self->{s_kwd} .= chr $self->{nc};
2572          !!!next-input-character;          !!!next-input-character;
2573          redo A;          redo A;
2574        } elsif ((length $self->{state_keyword}) == 5 and        } elsif ((length $self->{s_kwd}) == 5 and
2575                 ($self->{next_char} == 0x0043 or # C                 ($self->{nc} == 0x0043 or # C
2576                  $self->{next_char} == 0x0063)) { # c                  $self->{nc} == 0x0063)) { # c
2577          !!!cp (168);          !!!cp (168);
2578          $self->{state} = BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE;          $self->{state} = BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
2579          !!!next-input-character;          !!!next-input-character;
# Line 2509  sub _get_next_token ($) { Line 2582  sub _get_next_token ($) {
2582          !!!cp (169);          !!!cp (169);
2583          !!!parse-error (type => 'string after DOCTYPE name',          !!!parse-error (type => 'string after DOCTYPE name',
2584                          line => $self->{line_prev},                          line => $self->{line_prev},
2585                          column => $self->{column_prev} + 1 - length $self->{state_keyword});                          column => $self->{column_prev} + 1 - length $self->{s_kwd});
2586          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2587    
2588          $self->{state} = BOGUS_DOCTYPE_STATE;          $self->{state} = BOGUS_DOCTYPE_STATE;
2589          ## Reconsume.          ## Reconsume.
# Line 2518  sub _get_next_token ($) { Line 2591  sub _get_next_token ($) {
2591        }        }
2592      } elsif ($self->{state} == SYSTEM_STATE) {      } elsif ($self->{state} == SYSTEM_STATE) {
2593        ## ASCII case-insensitive        ## ASCII case-insensitive
2594        if ($self->{next_char} == [        if ($self->{nc} == [
2595              undef,              undef,
2596              0x0059, # Y              0x0059, # Y
2597              0x0053, # S              0x0053, # S
2598              0x0054, # T              0x0054, # T
2599              0x0045, # E              0x0045, # E
2600            ]->[length $self->{state_keyword}] or            ]->[length $self->{s_kwd}] or
2601            $self->{next_char} == [            $self->{nc} == [
2602              undef,              undef,
2603              0x0079, # y              0x0079, # y
2604              0x0073, # s              0x0073, # s
2605              0x0074, # t              0x0074, # t
2606              0x0065, # e              0x0065, # e
2607            ]->[length $self->{state_keyword}]) {            ]->[length $self->{s_kwd}]) {
2608          !!!cp (170);          !!!cp (170);
2609          ## Stay in the state.          ## Stay in the state.
2610          $self->{state_keyword} .= chr $self->{next_char};          $self->{s_kwd} .= chr $self->{nc};
2611          !!!next-input-character;          !!!next-input-character;
2612          redo A;          redo A;
2613        } elsif ((length $self->{state_keyword}) == 5 and        } elsif ((length $self->{s_kwd}) == 5 and
2614                 ($self->{next_char} == 0x004D or # M                 ($self->{nc} == 0x004D or # M
2615                  $self->{next_char} == 0x006D)) { # m                  $self->{nc} == 0x006D)) { # m
2616          !!!cp (171);          !!!cp (171);
2617          $self->{state} = BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE;          $self->{state} = BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
2618          !!!next-input-character;          !!!next-input-character;
# Line 2548  sub _get_next_token ($) { Line 2621  sub _get_next_token ($) {
2621          !!!cp (172);          !!!cp (172);
2622          !!!parse-error (type => 'string after DOCTYPE name',          !!!parse-error (type => 'string after DOCTYPE name',
2623                          line => $self->{line_prev},                          line => $self->{line_prev},
2624                          column => $self->{column_prev} + 1 - length $self->{state_keyword});                          column => $self->{column_prev} + 1 - length $self->{s_kwd});
2625          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2626    
2627          $self->{state} = BOGUS_DOCTYPE_STATE;          $self->{state} = BOGUS_DOCTYPE_STATE;
2628          ## Reconsume.          ## Reconsume.
2629          redo A;          redo A;
2630        }        }
2631      } elsif ($self->{state} == BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE) {      } elsif ($self->{state} == BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE) {
2632        if ({        if ($is_space->{$self->{nc}}) {
             0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,  
             #0x000D => 1, # HT, LF, VT, FF, SP, CR  
           }->{$self->{next_char}}) {  
2633          !!!cp (181);          !!!cp (181);
2634          ## Stay in the state          ## Stay in the state
2635          !!!next-input-character;          !!!next-input-character;
2636          redo A;          redo A;
2637        } elsif ($self->{next_char} eq 0x0022) { # "        } elsif ($self->{nc} eq 0x0022) { # "
2638          !!!cp (182);          !!!cp (182);
2639          $self->{current_token}->{public_identifier} = ''; # DOCTYPE          $self->{ct}->{pubid} = ''; # DOCTYPE
2640          $self->{state} = DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE;          $self->{state} = DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE;
2641          !!!next-input-character;          !!!next-input-character;
2642          redo A;          redo A;
2643        } elsif ($self->{next_char} eq 0x0027) { # '        } elsif ($self->{nc} eq 0x0027) { # '
2644          !!!cp (183);          !!!cp (183);
2645          $self->{current_token}->{public_identifier} = ''; # DOCTYPE          $self->{ct}->{pubid} = ''; # DOCTYPE
2646          $self->{state} = DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE;          $self->{state} = DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE;
2647          !!!next-input-character;          !!!next-input-character;
2648          redo A;          redo A;
2649        } elsif ($self->{next_char} eq 0x003E) { # >        } elsif ($self->{nc} eq 0x003E) { # >
2650          !!!cp (184);          !!!cp (184);
2651          !!!parse-error (type => 'no PUBLIC literal');          !!!parse-error (type => 'no PUBLIC literal');
2652    
2653          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2654          !!!next-input-character;          !!!next-input-character;
2655    
2656          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2657          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2658    
2659          redo A;          redo A;
2660        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2661          !!!cp (185);          !!!cp (185);
2662          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
2663    
2664          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2665          ## reconsume          ## reconsume
2666    
2667          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2668          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2669    
2670          redo A;          redo A;
2671        } else {        } else {
2672          !!!cp (186);          !!!cp (186);
2673          !!!parse-error (type => 'string after PUBLIC');          !!!parse-error (type => 'string after PUBLIC');
2674          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2675    
2676          $self->{state} = BOGUS_DOCTYPE_STATE;          $self->{state} = BOGUS_DOCTYPE_STATE;
2677          !!!next-input-character;          !!!next-input-character;
2678          redo A;          redo A;
2679        }        }
2680      } elsif ($self->{state} == DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE) {      } elsif ($self->{state} == DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE) {
2681        if ($self->{next_char} == 0x0022) { # "        if ($self->{nc} == 0x0022) { # "
2682          !!!cp (187);          !!!cp (187);
2683          $self->{state} = AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE;          $self->{state} = AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
2684          !!!next-input-character;          !!!next-input-character;
2685          redo A;          redo A;
2686        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
2687          !!!cp (188);          !!!cp (188);
2688          !!!parse-error (type => 'unclosed PUBLIC literal');          !!!parse-error (type => 'unclosed PUBLIC literal');
2689    
2690          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2691          !!!next-input-character;          !!!next-input-character;
2692    
2693          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2694          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2695    
2696          redo A;          redo A;
2697        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2698          !!!cp (189);          !!!cp (189);
2699          !!!parse-error (type => 'unclosed PUBLIC literal');          !!!parse-error (type => 'unclosed PUBLIC literal');
2700    
2701          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2702          ## reconsume          ## reconsume
2703    
2704          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2705          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2706    
2707          redo A;          redo A;
2708        } else {        } else {
2709          !!!cp (190);          !!!cp (190);
2710          $self->{current_token}->{public_identifier} # DOCTYPE          $self->{ct}->{pubid} # DOCTYPE
2711              .= chr $self->{next_char};              .= chr $self->{nc};
2712            $self->{read_until}->($self->{ct}->{pubid}, q[">],
2713                                  length $self->{ct}->{pubid});
2714    
2715          ## Stay in the state          ## Stay in the state
2716          !!!next-input-character;          !!!next-input-character;
2717          redo A;          redo A;
2718        }        }
2719      } elsif ($self->{state} == DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE) {      } elsif ($self->{state} == DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE) {
2720        if ($self->{next_char} == 0x0027) { # '        if ($self->{nc} == 0x0027) { # '
2721          !!!cp (191);          !!!cp (191);
2722          $self->{state} = AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE;          $self->{state} = AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
2723          !!!next-input-character;          !!!next-input-character;
2724          redo A;          redo A;
2725        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
2726          !!!cp (192);          !!!cp (192);
2727          !!!parse-error (type => 'unclosed PUBLIC literal');          !!!parse-error (type => 'unclosed PUBLIC literal');
2728    
2729          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2730          !!!next-input-character;          !!!next-input-character;
2731    
2732          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2733          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2734    
2735          redo A;          redo A;
2736        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2737          !!!cp (193);          !!!cp (193);
2738          !!!parse-error (type => 'unclosed PUBLIC literal');          !!!parse-error (type => 'unclosed PUBLIC literal');
2739    
2740          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2741          ## reconsume          ## reconsume
2742    
2743          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2744          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2745    
2746          redo A;          redo A;
2747        } else {        } else {
2748          !!!cp (194);          !!!cp (194);
2749          $self->{current_token}->{public_identifier} # DOCTYPE          $self->{ct}->{pubid} # DOCTYPE
2750              .= chr $self->{next_char};              .= chr $self->{nc};
2751            $self->{read_until}->($self->{ct}->{pubid}, q['>],
2752                                  length $self->{ct}->{pubid});
2753    
2754          ## Stay in the state          ## Stay in the state
2755          !!!next-input-character;          !!!next-input-character;
2756          redo A;          redo A;
2757        }        }
2758      } elsif ($self->{state} == AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE) {      } elsif ($self->{state} == AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE) {
2759        if ({        if ($is_space->{$self->{nc}}) {
             0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,  
             #0x000D => 1, # HT, LF, VT, FF, SP, CR  
           }->{$self->{next_char}}) {  
2760          !!!cp (195);          !!!cp (195);
2761          ## Stay in the state          ## Stay in the state
2762          !!!next-input-character;          !!!next-input-character;
2763          redo A;          redo A;
2764        } elsif ($self->{next_char} == 0x0022) { # "        } elsif ($self->{nc} == 0x0022) { # "
2765          !!!cp (196);          !!!cp (196);
2766          $self->{current_token}->{system_identifier} = ''; # DOCTYPE          $self->{ct}->{sysid} = ''; # DOCTYPE
2767          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE;          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE;
2768          !!!next-input-character;          !!!next-input-character;
2769          redo A;          redo A;
2770        } elsif ($self->{next_char} == 0x0027) { # '        } elsif ($self->{nc} == 0x0027) { # '
2771          !!!cp (197);          !!!cp (197);
2772          $self->{current_token}->{system_identifier} = ''; # DOCTYPE          $self->{ct}->{sysid} = ''; # DOCTYPE
2773          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE;          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE;
2774          !!!next-input-character;          !!!next-input-character;
2775          redo A;          redo A;
2776        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
2777          !!!cp (198);          !!!cp (198);
2778          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2779          !!!next-input-character;          !!!next-input-character;
2780    
2781          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2782    
2783          redo A;          redo A;
2784        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2785          !!!cp (199);          !!!cp (199);
2786          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
2787    
2788          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2789          ## reconsume          ## reconsume
2790    
2791          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2792          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2793    
2794          redo A;          redo A;
2795        } else {        } else {
2796          !!!cp (200);          !!!cp (200);
2797          !!!parse-error (type => 'string after PUBLIC literal');          !!!parse-error (type => 'string after PUBLIC literal');
2798          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2799    
2800          $self->{state} = BOGUS_DOCTYPE_STATE;          $self->{state} = BOGUS_DOCTYPE_STATE;
2801          !!!next-input-character;          !!!next-input-character;
2802          redo A;          redo A;
2803        }        }
2804      } elsif ($self->{state} == BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE) {      } elsif ($self->{state} == BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE) {
2805        if ({        if ($is_space->{$self->{nc}}) {
             0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,  
             #0x000D => 1, # HT, LF, VT, FF, SP, CR  
           }->{$self->{next_char}}) {  
2806          !!!cp (201);          !!!cp (201);
2807          ## Stay in the state          ## Stay in the state
2808          !!!next-input-character;          !!!next-input-character;
2809          redo A;          redo A;
2810        } elsif ($self->{next_char} == 0x0022) { # "        } elsif ($self->{nc} == 0x0022) { # "
2811          !!!cp (202);          !!!cp (202);
2812          $self->{current_token}->{system_identifier} = ''; # DOCTYPE          $self->{ct}->{sysid} = ''; # DOCTYPE
2813          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE;          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE;
2814          !!!next-input-character;          !!!next-input-character;
2815          redo A;          redo A;
2816        } elsif ($self->{next_char} == 0x0027) { # '        } elsif ($self->{nc} == 0x0027) { # '
2817          !!!cp (203);          !!!cp (203);
2818          $self->{current_token}->{system_identifier} = ''; # DOCTYPE          $self->{ct}->{sysid} = ''; # DOCTYPE
2819          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE;          $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE;
2820          !!!next-input-character;          !!!next-input-character;
2821          redo A;          redo A;
2822        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
2823          !!!cp (204);          !!!cp (204);
2824          !!!parse-error (type => 'no SYSTEM literal');          !!!parse-error (type => 'no SYSTEM literal');
2825          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2826          !!!next-input-character;          !!!next-input-character;
2827    
2828          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2829          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2830    
2831          redo A;          redo A;
2832        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2833          !!!cp (205);          !!!cp (205);
2834          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
2835    
2836          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2837          ## reconsume          ## reconsume
2838    
2839          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2840          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2841    
2842          redo A;          redo A;
2843        } else {        } else {
2844          !!!cp (206);          !!!cp (206);
2845          !!!parse-error (type => 'string after SYSTEM');          !!!parse-error (type => 'string after SYSTEM');
2846          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2847    
2848          $self->{state} = BOGUS_DOCTYPE_STATE;          $self->{state} = BOGUS_DOCTYPE_STATE;
2849          !!!next-input-character;          !!!next-input-character;
2850          redo A;          redo A;
2851        }        }
2852      } elsif ($self->{state} == DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE) {      } elsif ($self->{state} == DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE) {
2853        if ($self->{next_char} == 0x0022) { # "        if ($self->{nc} == 0x0022) { # "
2854          !!!cp (207);          !!!cp (207);
2855          $self->{state} = AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE;          $self->{state} = AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
2856          !!!next-input-character;          !!!next-input-character;
2857          redo A;          redo A;
2858        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
2859          !!!cp (208);          !!!cp (208);
2860          !!!parse-error (type => 'unclosed SYSTEM literal');          !!!parse-error (type => 'unclosed SYSTEM literal');
2861    
2862          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2863          !!!next-input-character;          !!!next-input-character;
2864    
2865          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2866          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2867    
2868          redo A;          redo A;
2869        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2870          !!!cp (209);          !!!cp (209);
2871          !!!parse-error (type => 'unclosed SYSTEM literal');          !!!parse-error (type => 'unclosed SYSTEM literal');
2872    
2873          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2874          ## reconsume          ## reconsume
2875    
2876          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2877          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2878    
2879          redo A;          redo A;
2880        } else {        } else {
2881          !!!cp (210);          !!!cp (210);
2882          $self->{current_token}->{system_identifier} # DOCTYPE          $self->{ct}->{sysid} # DOCTYPE
2883              .= chr $self->{next_char};              .= chr $self->{nc};
2884            $self->{read_until}->($self->{ct}->{sysid}, q[">],
2885                                  length $self->{ct}->{sysid});
2886    
2887          ## Stay in the state          ## Stay in the state
2888          !!!next-input-character;          !!!next-input-character;
2889          redo A;          redo A;
2890        }        }
2891      } elsif ($self->{state} == DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE) {      } elsif ($self->{state} == DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE) {
2892        if ($self->{next_char} == 0x0027) { # '        if ($self->{nc} == 0x0027) { # '
2893          !!!cp (211);          !!!cp (211);
2894          $self->{state} = AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE;          $self->{state} = AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
2895          !!!next-input-character;          !!!next-input-character;
2896          redo A;          redo A;
2897        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
2898          !!!cp (212);          !!!cp (212);
2899          !!!parse-error (type => 'unclosed SYSTEM literal');          !!!parse-error (type => 'unclosed SYSTEM literal');
2900    
2901          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2902          !!!next-input-character;          !!!next-input-character;
2903    
2904          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2905          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2906    
2907          redo A;          redo A;
2908        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2909          !!!cp (213);          !!!cp (213);
2910          !!!parse-error (type => 'unclosed SYSTEM literal');          !!!parse-error (type => 'unclosed SYSTEM literal');
2911    
2912          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2913          ## reconsume          ## reconsume
2914    
2915          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2916          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2917    
2918          redo A;          redo A;
2919        } else {        } else {
2920          !!!cp (214);          !!!cp (214);
2921          $self->{current_token}->{system_identifier} # DOCTYPE          $self->{ct}->{sysid} # DOCTYPE
2922              .= chr $self->{next_char};              .= chr $self->{nc};
2923            $self->{read_until}->($self->{ct}->{sysid}, q['>],
2924                                  length $self->{ct}->{sysid});
2925    
2926          ## Stay in the state          ## Stay in the state
2927          !!!next-input-character;          !!!next-input-character;
2928          redo A;          redo A;
2929        }        }
2930      } elsif ($self->{state} == AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE) {      } elsif ($self->{state} == AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE) {
2931        if ({        if ($is_space->{$self->{nc}}) {
             0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, 0x0020 => 1,  
             #0x000D => 1, # HT, LF, VT, FF, SP, CR  
           }->{$self->{next_char}}) {  
2932          !!!cp (215);          !!!cp (215);
2933          ## Stay in the state          ## Stay in the state
2934          !!!next-input-character;          !!!next-input-character;
2935          redo A;          redo A;
2936        } elsif ($self->{next_char} == 0x003E) { # >        } elsif ($self->{nc} == 0x003E) { # >
2937          !!!cp (216);          !!!cp (216);
2938          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2939          !!!next-input-character;          !!!next-input-character;
2940    
2941          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2942    
2943          redo A;          redo A;
2944        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2945          !!!cp (217);          !!!cp (217);
2946          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
2947          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2948          ## reconsume          ## reconsume
2949    
2950          $self->{current_token}->{quirks} = 1;          $self->{ct}->{quirks} = 1;
2951          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2952    
2953          redo A;          redo A;
2954        } else {        } else {
2955          !!!cp (218);          !!!cp (218);
2956          !!!parse-error (type => 'string after SYSTEM literal');          !!!parse-error (type => 'string after SYSTEM literal');
2957          #$self->{current_token}->{quirks} = 1;          #$self->{ct}->{quirks} = 1;
2958    
2959          $self->{state} = BOGUS_DOCTYPE_STATE;          $self->{state} = BOGUS_DOCTYPE_STATE;
2960          !!!next-input-character;          !!!next-input-character;
2961          redo A;          redo A;
2962        }        }
2963      } elsif ($self->{state} == BOGUS_DOCTYPE_STATE) {      } elsif ($self->{state} == BOGUS_DOCTYPE_STATE) {
2964        if ($self->{next_char} == 0x003E) { # >        if ($self->{nc} == 0x003E) { # >
2965          !!!cp (219);          !!!cp (219);
2966          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2967          !!!next-input-character;          !!!next-input-character;
2968    
2969          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2970    
2971          redo A;          redo A;
2972        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
2973          !!!cp (220);          !!!cp (220);
2974          !!!parse-error (type => 'unclosed DOCTYPE');          !!!parse-error (type => 'unclosed DOCTYPE');
2975          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
2976          ## reconsume          ## reconsume
2977    
2978          !!!emit ($self->{current_token}); # DOCTYPE          !!!emit ($self->{ct}); # DOCTYPE
2979    
2980          redo A;          redo A;
2981        } else {        } else {
2982          !!!cp (221);          !!!cp (221);
2983            my $s = '';
2984            $self->{read_until}->($s, q[>], 0);
2985    
2986          ## Stay in the state          ## Stay in the state
2987          !!!next-input-character;          !!!next-input-character;
2988          redo A;          redo A;
# Line 2916  sub _get_next_token ($) { Line 2992  sub _get_next_token ($) {
2992        ## by three states, |CDATA_SECTION_STATE|, |CDATA_SECTION_MSE1_STATE|,        ## by three states, |CDATA_SECTION_STATE|, |CDATA_SECTION_MSE1_STATE|,
2993        ## and |CDATA_SECTION_MSE2_STATE|.        ## and |CDATA_SECTION_MSE2_STATE|.
2994                
2995        if ($self->{next_char} == 0x005D) { # ]        if ($self->{nc} == 0x005D) { # ]
2996          !!!cp (221.1);          !!!cp (221.1);
2997          $self->{state} = CDATA_SECTION_MSE1_STATE;          $self->{state} = CDATA_SECTION_MSE1_STATE;
2998          !!!next-input-character;          !!!next-input-character;
2999          redo A;          redo A;
3000        } elsif ($self->{next_char} == -1) {        } elsif ($self->{nc} == -1) {
3001          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
3002          !!!next-input-character;          !!!next-input-character;
3003          if (length $self->{current_token}->{data}) { # character          if (length $self->{ct}->{data}) { # character
3004            !!!cp (221.2);            !!!cp (221.2);
3005            !!!emit ($self->{current_token}); # character            !!!emit ($self->{ct}); # character
3006          } else {          } else {
3007            !!!cp (221.3);            !!!cp (221.3);
3008            ## No token to emit. $self->{current_token} is discarded.            ## No token to emit. $self->{ct} is discarded.
3009          }                  }        
3010          redo A;          redo A;
3011        } else {        } else {
3012          !!!cp (221.4);          !!!cp (221.4);
3013          $self->{current_token}->{data} .= chr $self->{next_char};          $self->{ct}->{data} .= chr $self->{nc};
3014            $self->{read_until}->($self->{ct}->{data},
3015                                  q<]>,
3016                                  length $self->{ct}->{data});
3017    
3018          ## Stay in the state.          ## Stay in the state.
3019          !!!next-input-character;          !!!next-input-character;
3020          redo A;          redo A;
# Line 2942  sub _get_next_token ($) { Line 3022  sub _get_next_token ($) {
3022    
3023        ## ISSUE: "text tokens" in spec.        ## ISSUE: "text tokens" in spec.
3024      } elsif ($self->{state} == CDATA_SECTION_MSE1_STATE) {      } elsif ($self->{state} == CDATA_SECTION_MSE1_STATE) {
3025        if ($self->{next_char} == 0x005D) { # ]        if ($self->{nc} == 0x005D) { # ]
3026          !!!cp (221.5);          !!!cp (221.5);
3027          $self->{state} = CDATA_SECTION_MSE2_STATE;          $self->{state} = CDATA_SECTION_MSE2_STATE;
3028          !!!next-input-character;          !!!next-input-character;
3029          redo A;          redo A;
3030        } else {        } else {
3031          !!!cp (221.6);          !!!cp (221.6);
3032          $self->{current_token}->{data} .= ']';          $self->{ct}->{data} .= ']';
3033          $self->{state} = CDATA_SECTION_STATE;          $self->{state} = CDATA_SECTION_STATE;
3034          ## Reconsume.          ## Reconsume.
3035          redo A;          redo A;
3036        }        }
3037      } elsif ($self->{state} == CDATA_SECTION_MSE2_STATE) {      } elsif ($self->{state} == CDATA_SECTION_MSE2_STATE) {
3038        if ($self->{next_char} == 0x003E) { # >        if ($self->{nc} == 0x003E) { # >
3039          $self->{state} = DATA_STATE;          $self->{state} = DATA_STATE;
3040          !!!next-input-character;          !!!next-input-character;
3041          if (length $self->{current_token}->{data}) { # character          if (length $self->{ct}->{data}) { # character
3042            !!!cp (221.7);            !!!cp (221.7);
3043            !!!emit ($self->{current_token}); # character            !!!emit ($self->{ct}); # character
3044          } else {          } else {
3045            !!!cp (221.8);            !!!cp (221.8);
3046            ## No token to emit. $self->{current_token} is discarded.            ## No token to emit. $self->{ct} is discarded.
3047          }          }
3048          redo A;          redo A;
3049        } elsif ($self->{next_char} == 0x005D) { # ]        } elsif ($self->{nc} == 0x005D) { # ]
3050          !!!cp (221.9); # character          !!!cp (221.9); # character
3051          $self->{current_token}->{data} .= ']'; ## Add first "]" of "]]]".          $self->{ct}->{data} .= ']'; ## Add first "]" of "]]]".
3052          ## Stay in the state.          ## Stay in the state.
3053          !!!next-input-character;          !!!next-input-character;
3054          redo A;          redo A;
3055        } else {        } else {
3056          !!!cp (221.11);          !!!cp (221.11);
3057          $self->{current_token}->{data} .= ']]'; # character          $self->{ct}->{data} .= ']]'; # character
3058          $self->{state} = CDATA_SECTION_STATE;          $self->{state} = CDATA_SECTION_STATE;
3059          ## Reconsume.          ## Reconsume.
3060          redo A;          redo A;
3061        }        }
   
3062      } elsif ($self->{state} == ENTITY_STATE) {      } elsif ($self->{state} == ENTITY_STATE) {
3063        my $in_attr = $self->{entity_in_attr};        if ($is_space->{$self->{nc}} or
3064        my $additional = $self->{entity_additional};            {
3065                0x003C => 1, 0x0026 => 1, -1 => 1, # <, &
3066                $self->{entity_add} => 1,
3067              }->{$self->{nc}}) {
3068            !!!cp (1001);
3069            ## Don't consume
3070            ## No error
3071            ## Return nothing.
3072            #
3073          } elsif ($self->{nc} == 0x0023) { # #
3074            !!!cp (999);
3075            $self->{state} = ENTITY_HASH_STATE;
3076            $self->{s_kwd} = '#';
3077            !!!next-input-character;
3078            redo A;
3079          } elsif ((0x0041 <= $self->{nc} and
3080                    $self->{nc} <= 0x005A) or # A..Z
3081                   (0x0061 <= $self->{nc} and
3082                    $self->{nc} <= 0x007A)) { # a..z
3083            !!!cp (998);
3084            require Whatpm::_NamedEntityList;
3085            $self->{state} = ENTITY_NAME_STATE;
3086            $self->{s_kwd} = chr $self->{nc};
3087            $self->{entity__value} = $self->{s_kwd};
3088            $self->{entity__match} = 0;
3089            !!!next-input-character;
3090            redo A;
3091          } else {
3092            !!!cp (1027);
3093            !!!parse-error (type => 'bare ero');
3094            ## Return nothing.
3095            #
3096          }
3097    
3098    my ($l, $c) = ($self->{line_prev}, $self->{column_prev});        ## NOTE: No character is consumed by the "consume a character
3099          ## reference" algorithm.  In other word, there is an "&" character
3100          ## that does not introduce a character reference, which would be
3101          ## appended to the parent element or the attribute value in later
3102          ## process of the tokenizer.
3103    
3104          if ($self->{prev_state} == DATA_STATE) {
3105            !!!cp (997);
3106            $self->{state} = $self->{prev_state};
3107            ## Reconsume.
3108            !!!emit ({type => CHARACTER_TOKEN, data => '&',
3109                      line => $self->{line_prev},
3110                      column => $self->{column_prev},
3111                     });
3112            redo A;
3113          } else {
3114            !!!cp (996);
3115            $self->{ca}->{value} .= '&';
3116            $self->{state} = $self->{prev_state};
3117            ## Reconsume.
3118            redo A;
3119          }
3120        } elsif ($self->{state} == ENTITY_HASH_STATE) {
3121          if ($self->{nc} == 0x0078 or # x
3122              $self->{nc} == 0x0058) { # X
3123            !!!cp (995);
3124            $self->{state} = HEXREF_X_STATE;
3125            $self->{s_kwd} .= chr $self->{nc};
3126            !!!next-input-character;
3127            redo A;
3128          } elsif (0x0030 <= $self->{nc} and
3129                   $self->{nc} <= 0x0039) { # 0..9
3130            !!!cp (994);
3131            $self->{state} = NCR_NUM_STATE;
3132            $self->{s_kwd} = $self->{nc} - 0x0030;
3133            !!!next-input-character;
3134            redo A;
3135          } else {
3136            !!!parse-error (type => 'bare nero',
3137                            line => $self->{line_prev},
3138                            column => $self->{column_prev} - 1);
3139    
3140    if ({          ## NOTE: According to the spec algorithm, nothing is returned,
3141         0x0009 => 1, 0x000A => 1, 0x000B => 1, 0x000C => 1, # HT, LF, VT, FF,          ## and then "&#" is appended to the parent element or the attribute
3142         0x0020 => 1, 0x003C => 1, 0x0026 => 1, -1 => 1, # SP, <, & # 0x000D # CR          ## value in the later processing.
3143         $additional => 1,  
3144        }->{$self->{next_char}}) {          if ($self->{prev_state} == DATA_STATE) {
3145      !!!cp (1001);            !!!cp (1019);
3146      ## Don't consume            $self->{state} = $self->{prev_state};
3147      ## No error            ## Reconsume.
3148      $self->{entity_return} = undef;            !!!emit ({type => CHARACTER_TOKEN,
3149      $self->{state} = $self->{entity_in_attr} ? ENTITY_IN_ATTRIBUTE_VALUE_STATE : ENTITY_DATA_STATE;                      data => '&#',
3150      redo A;                      line => $self->{line_prev},
3151    } elsif ($self->{next_char} == 0x0023) { # #                      column => $self->{column_prev} - 1,
3152      !!!next-input-character;                     });
     if ($self->{next_char} == 0x0078 or # x  
         $self->{next_char} == 0x0058) { # X  
       my $code;  
       X: {  
         my $x_char = $self->{next_char};  
         !!!next-input-character;  
         if (0x0030 <= $self->{next_char} and  
             $self->{next_char} <= 0x0039) { # 0..9  
           !!!cp (1002);  
           $code ||= 0;  
           $code *= 0x10;  
           $code += $self->{next_char} - 0x0030;  
           redo X;  
         } elsif (0x0061 <= $self->{next_char} and  
                  $self->{next_char} <= 0x0066) { # a..f  
           !!!cp (1003);  
           $code ||= 0;  
           $code *= 0x10;  
           $code += $self->{next_char} - 0x0060 + 9;  
           redo X;  
         } elsif (0x0041 <= $self->{next_char} and  
                  $self->{next_char} <= 0x0046) { # A..F  
           !!!cp (1004);  
           $code ||= 0;  
           $code *= 0x10;  
           $code += $self->{next_char} - 0x0040 + 9;  
           redo X;  
         } elsif (not defined $code) { # no hexadecimal digit  
           !!!cp (1005);  
           !!!parse-error (type => 'bare hcro', line => $l, column => $c);  
           !!!back-next-input-character ($x_char, $self->{next_char});  
           $self->{next_char} = 0x0023; # #  
           $self->{entity_return} = undef;  
           $self->{state} = $self->{entity_in_attr} ? ENTITY_IN_ATTRIBUTE_VALUE_STATE : ENTITY_DATA_STATE;  
3153            redo A;            redo A;
         } elsif ($self->{next_char} == 0x003B) { # ;  
           !!!cp (1006);  
           !!!next-input-character;  
3154          } else {          } else {
3155            !!!cp (1007);            !!!cp (993);
3156            !!!parse-error (type => 'no refc', line => $l, column => $c);            $self->{ca}->{value} .= '&#';
3157              $self->{state} = $self->{prev_state};
3158              ## Reconsume.
3159              redo A;
3160          }          }
3161          }
3162          if ($code == 0 or (0xD800 <= $code and $code <= 0xDFFF)) {      } elsif ($self->{state} == NCR_NUM_STATE) {
3163            !!!cp (1008);        if (0x0030 <= $self->{nc} and
3164            !!!parse-error (type => 'invalid character reference',            $self->{nc} <= 0x0039) { # 0..9
                           text => (sprintf 'U+%04X', $code),  
                           line => $l, column => $c);  
           $code = 0xFFFD;  
         } elsif ($code > 0x10FFFF) {  
           !!!cp (1009);  
           !!!parse-error (type => 'invalid character reference',  
                           text => (sprintf 'U-%08X', $code),  
                           line => $l, column => $c);  
           $code = 0xFFFD;  
         } elsif ($code == 0x000D) {  
           !!!cp (1010);  
           !!!parse-error (type => 'CR character reference', line => $l, column => $c);  
           $code = 0x000A;  
         } elsif (0x80 <= $code and $code <= 0x9F) {  
           !!!cp (1011);  
           !!!parse-error (type => 'C1 character reference', text => (sprintf 'U+%04X', $code), line => $l, column => $c);  
           $code = $c1_entity_char->{$code};  
         }  
   
         $self->{entity_return} = {type => CHARACTER_TOKEN, data => chr $code,  
                 has_reference => 1,  
                 line => $l, column => $c,  
                };  
         $self->{state} = $self->{entity_in_attr} ? ENTITY_IN_ATTRIBUTE_VALUE_STATE : ENTITY_DATA_STATE;  
         redo A;  
       } # X  
     } elsif (0x0030 <= $self->{next_char} and  
              $self->{next_char} <= 0x0039) { # 0..9  
       my $code = $self->{next_char} - 0x0030;  
       !!!next-input-character;  
         
       while (0x0030 <= $self->{next_char} and  
                 $self->{next_char} <= 0x0039) { # 0..9  
3165          !!!cp (1012);          !!!cp (1012);
3166          $code *= 10;          $self->{s_kwd} *= 10;
3167          $code += $self->{next_char} - 0x0030;          $self->{s_kwd} += $self->{nc} - 0x0030;
3168                    
3169            ## Stay in the state.
3170          !!!next-input-character;          !!!next-input-character;
3171        }          redo A;
3172          } elsif ($self->{nc} == 0x003B) { # ;
       if ($self->{next_char} == 0x003B) { # ;  
3173          !!!cp (1013);          !!!cp (1013);
3174          !!!next-input-character;          !!!next-input-character;
3175            #
3176        } else {        } else {
3177          !!!cp (1014);          !!!cp (1014);
3178          !!!parse-error (type => 'no refc', line => $l, column => $c);          !!!parse-error (type => 'no refc');
3179            ## Reconsume.
3180            #
3181        }        }
3182    
3183        if ($code == 0 or (0xD800 <= $code and $code <= 0xDFFF)) {        my $code = $self->{s_kwd};
3184          my $l = $self->{line_prev};
3185          my $c = $self->{column_prev};
3186          if ($charref_map->{$code}) {
3187          !!!cp (1015);          !!!cp (1015);
3188          !!!parse-error (type => 'invalid character reference',          !!!parse-error (type => 'invalid character reference',
3189                          text => (sprintf 'U+%04X', $code),                          text => (sprintf 'U+%04X', $code),
3190                          line => $l, column => $c);                          line => $l, column => $c);
3191          $code = 0xFFFD;          $code = $charref_map->{$code};
3192        } elsif ($code > 0x10FFFF) {        } elsif ($code > 0x10FFFF) {
3193          !!!cp (1016);          !!!cp (1016);
3194          !!!parse-error (type => 'invalid character reference',          !!!parse-error (type => 'invalid character reference',
3195                          text => (sprintf 'U-%08X', $code),                          text => (sprintf 'U-%08X', $code),
3196                          line => $l, column => $c);                          line => $l, column => $c);
3197          $code = 0xFFFD;          $code = 0xFFFD;
3198        } elsif ($code == 0x000D) {        }
3199          !!!cp (1017);  
3200          !!!parse-error (type => 'CR character reference',        if ($self->{prev_state} == DATA_STATE) {
3201                          line => $l, column => $c);          !!!cp (992);
3202          $code = 0x000A;          $self->{state} = $self->{prev_state};
3203        } elsif (0x80 <= $code and $code <= 0x9F) {          ## Reconsume.
3204          !!!cp (1018);          !!!emit ({type => CHARACTER_TOKEN, data => chr $code,
3205          !!!parse-error (type => 'C1 character reference',                    line => $l, column => $c,
3206                     });
3207            redo A;
3208          } else {
3209            !!!cp (991);
3210            $self->{ca}->{value} .= chr $code;
3211            $self->{ca}->{has_reference} = 1;
3212            $self->{state} = $self->{prev_state};
3213            ## Reconsume.
3214            redo A;
3215          }
3216        } elsif ($self->{state} == HEXREF_X_STATE) {
3217          if ((0x0030 <= $self->{nc} and $self->{nc} <= 0x0039) or
3218              (0x0041 <= $self->{nc} and $self->{nc} <= 0x0046) or
3219              (0x0061 <= $self->{nc} and $self->{nc} <= 0x0066)) {
3220            # 0..9, A..F, a..f
3221            !!!cp (990);
3222            $self->{state} = HEXREF_HEX_STATE;
3223            $self->{s_kwd} = 0;
3224            ## Reconsume.
3225            redo A;
3226          } else {
3227            !!!parse-error (type => 'bare hcro',
3228                            line => $self->{line_prev},
3229                            column => $self->{column_prev} - 2);
3230    
3231            ## NOTE: According to the spec algorithm, nothing is returned,
3232            ## and then "&#" followed by "X" or "x" is appended to the parent
3233            ## element or the attribute value in the later processing.
3234    
3235            if ($self->{prev_state} == DATA_STATE) {
3236              !!!cp (1005);
3237              $self->{state} = $self->{prev_state};
3238              ## Reconsume.
3239              !!!emit ({type => CHARACTER_TOKEN,
3240                        data => '&' . $self->{s_kwd},
3241                        line => $self->{line_prev},
3242                        column => $self->{column_prev} - length $self->{s_kwd},
3243                       });
3244              redo A;
3245            } else {
3246              !!!cp (989);
3247              $self->{ca}->{value} .= '&' . $self->{s_kwd};
3248              $self->{state} = $self->{prev_state};
3249              ## Reconsume.
3250              redo A;
3251            }
3252          }
3253        } elsif ($self->{state} == HEXREF_HEX_STATE) {
3254          if (0x0030 <= $self->{nc} and $self->{nc} <= 0x0039) {
3255            # 0..9
3256            !!!cp (1002);
3257            $self->{s_kwd} *= 0x10;
3258            $self->{s_kwd} += $self->{nc} - 0x0030;
3259            ## Stay in the state.
3260            !!!next-input-character;
3261            redo A;
3262          } elsif (0x0061 <= $self->{nc} and
3263                   $self->{nc} <= 0x0066) { # a..f
3264            !!!cp (1003);
3265            $self->{s_kwd} *= 0x10;
3266            $self->{s_kwd} += $self->{nc} - 0x0060 + 9;
3267            ## Stay in the state.
3268            !!!next-input-character;
3269            redo A;
3270          } elsif (0x0041 <= $self->{nc} and
3271                   $self->{nc} <= 0x0046) { # A..F
3272            !!!cp (1004);
3273            $self->{s_kwd} *= 0x10;
3274            $self->{s_kwd} += $self->{nc} - 0x0040 + 9;
3275            ## Stay in the state.
3276            !!!next-input-character;
3277            redo A;
3278          } elsif ($self->{nc} == 0x003B) { # ;
3279            !!!cp (1006);
3280            !!!next-input-character;
3281            #
3282          } else {
3283            !!!cp (1007);
3284            !!!parse-error (type => 'no refc',
3285                            line => $self->{line},
3286                            column => $self->{column});
3287            ## Reconsume.
3288            #
3289          }
3290    
3291          my $code = $self->{s_kwd};
3292          my $l = $self->{line_prev};
3293          my $c = $self->{column_prev};
3294          if ($charref_map->{$code}) {
3295            !!!cp (1008);
3296            !!!parse-error (type => 'invalid character reference',
3297                          text => (sprintf 'U+%04X', $code),                          text => (sprintf 'U+%04X', $code),
3298                          line => $l, column => $c);                          line => $l, column => $c);
3299          $code = $c1_entity_char->{$code};          $code = $charref_map->{$code};
3300          } elsif ($code > 0x10FFFF) {
3301            !!!cp (1009);
3302            !!!parse-error (type => 'invalid character reference',
3303                            text => (sprintf 'U-%08X', $code),
3304                            line => $l, column => $c);
3305            $code = 0xFFFD;
3306        }        }
3307          
3308        $self->{entity_return} = {type => CHARACTER_TOKEN, data => chr $code, has_reference => 1,        if ($self->{prev_state} == DATA_STATE) {
3309                line => $l, column => $c,          !!!cp (988);
3310               };          $self->{state} = $self->{prev_state};
3311        $self->{state} = $self->{entity_in_attr} ? ENTITY_IN_ATTRIBUTE_VALUE_STATE : ENTITY_DATA_STATE;          ## Reconsume.
3312        redo A;          !!!emit ({type => CHARACTER_TOKEN, data => chr $code,
3313      } else {                    line => $l, column => $c,
3314        !!!cp (1019);                   });
3315        !!!parse-error (type => 'bare nero', line => $l, column => $c);          redo A;
3316        !!!back-next-input-character ($self->{next_char});        } else {
3317        $self->{next_char} = 0x0023; # #          !!!cp (987);
3318        $self->{entity_return} = undef;          $self->{ca}->{value} .= chr $code;
3319        $self->{state} = $self->{entity_in_attr} ? ENTITY_IN_ATTRIBUTE_VALUE_STATE : ENTITY_DATA_STATE;          $self->{ca}->{has_reference} = 1;
3320        redo A;          $self->{state} = $self->{prev_state};
3321      }          ## Reconsume.
3322    } elsif ((0x0041 <= $self->{next_char} and          redo A;
3323              $self->{next_char} <= 0x005A) or        }
3324             (0x0061 <= $self->{next_char} and      } elsif ($self->{state} == ENTITY_NAME_STATE) {
3325              $self->{next_char} <= 0x007A)) {        if (length $self->{s_kwd} < 30 and
3326      my $entity_name = chr $self->{next_char};            ## NOTE: Some number greater than the maximum length of entity name
3327      !!!next-input-character;            ((0x0041 <= $self->{nc} and # a
3328                $self->{nc} <= 0x005A) or # x
3329      my $value = $entity_name;             (0x0061 <= $self->{nc} and # a
3330      my $match = 0;              $self->{nc} <= 0x007A) or # z
3331      require Whatpm::_NamedEntityList;             (0x0030 <= $self->{nc} and # 0
3332      our $EntityChar;              $self->{nc} <= 0x0039) or # 9
3333               $self->{nc} == 0x003B)) { # ;
3334      while (length $entity_name < 30 and          our $EntityChar;
3335             ## NOTE: Some number greater than the maximum length of entity name          $self->{s_kwd} .= chr $self->{nc};
3336             ((0x0041 <= $self->{next_char} and # a          if (defined $EntityChar->{$self->{s_kwd}}) {
3337               $self->{next_char} <= 0x005A) or # x            if ($self->{nc} == 0x003B) { # ;
3338              (0x0061 <= $self->{next_char} and # a              !!!cp (1020);
3339               $self->{next_char} <= 0x007A) or # z              $self->{entity__value} = $EntityChar->{$self->{s_kwd}};
3340              (0x0030 <= $self->{next_char} and # 0              $self->{entity__match} = 1;
3341               $self->{next_char} <= 0x0039) or # 9              !!!next-input-character;
3342              $self->{next_char} == 0x003B)) { # ;              #
3343        $entity_name .= chr $self->{next_char};            } else {
3344        if (defined $EntityChar->{$entity_name}) {              !!!cp (1021);
3345          if ($self->{next_char} == 0x003B) { # ;              $self->{entity__value} = $EntityChar->{$self->{s_kwd}};
3346            !!!cp (1020);              $self->{entity__match} = -1;
3347            $value = $EntityChar->{$entity_name};              ## Stay in the state.
3348            $match = 1;              !!!next-input-character;
3349            !!!next-input-character;              redo A;
3350            last;            }
3351          } else {          } else {
3352            !!!cp (1021);            !!!cp (1022);
3353            $value = $EntityChar->{$entity_name};            $self->{entity__value} .= chr $self->{nc};
3354            $match = -1;            $self->{entity__match} *= 2;
3355              ## Stay in the state.
3356            !!!next-input-character;            !!!next-input-character;
3357              redo A;
3358            }
3359          }
3360    
3361          my $data;
3362          my $has_ref;
3363          if ($self->{entity__match} > 0) {
3364            !!!cp (1023);
3365            $data = $self->{entity__value};
3366            $has_ref = 1;
3367            #
3368          } elsif ($self->{entity__match} < 0) {
3369            !!!parse-error (type => 'no refc');
3370            if ($self->{prev_state} != DATA_STATE and # in attribute
3371                $self->{entity__match} < -1) {
3372              !!!cp (1024);
3373              $data = '&' . $self->{s_kwd};
3374              #
3375            } else {
3376              !!!cp (1025);
3377              $data = $self->{entity__value};
3378              $has_ref = 1;
3379              #
3380          }          }
3381        } else {        } else {
3382          !!!cp (1022);          !!!cp (1026);
3383          $value .= chr $self->{next_char};          !!!parse-error (type => 'bare ero',
3384          $match *= 2;                          line => $self->{line_prev},
3385          !!!next-input-character;                          column => $self->{column_prev} - length $self->{s_kwd});
3386            $data = '&' . $self->{s_kwd};
3387            #
3388        }        }
3389      }    
3390              ## NOTE: In these cases, when a character reference is found,
3391      if ($match > 0) {        ## it is consumed and a character token is returned, or, otherwise,
3392        !!!cp (1023);        ## nothing is consumed and returned, according to the spec algorithm.
3393        $self->{entity_return} = {type => CHARACTER_TOKEN, data => $value, has_reference => 1,        ## In this implementation, anything that has been examined by the
3394                line => $l, column => $c,        ## tokenizer is appended to the parent element or the attribute value
3395               };        ## as string, either literal string when no character reference or
3396        $self->{state} = $self->{entity_in_attr} ? ENTITY_IN_ATTRIBUTE_VALUE_STATE : ENTITY_DATA_STATE;        ## entity-replaced string otherwise, in this stage, since any characters
3397        redo A;        ## that would not be consumed are appended in the data state or in an
3398      } elsif ($match < 0) {        ## appropriate attribute value state anyway.
3399        !!!parse-error (type => 'no refc', line => $l, column => $c);  
3400        if ($in_attr and $match < -1) {        if ($self->{prev_state} == DATA_STATE) {
3401          !!!cp (1024);          !!!cp (986);
3402          $self->{entity_return} = {type => CHARACTER_TOKEN, data => '&'.$entity_name,          $self->{state} = $self->{prev_state};
3403                  line => $l, column => $c,          ## Reconsume.
3404                 };          !!!emit ({type => CHARACTER_TOKEN,
3405          $self->{state} = $self->{entity_in_attr} ? ENTITY_IN_ATTRIBUTE_VALUE_STATE : ENTITY_DATA_STATE;                    data => $data,
3406                      line => $self->{line_prev},
3407                      column => $self->{column_prev} + 1 - length $self->{s_kwd},
3408                     });
3409          redo A;          redo A;
3410        } else {        } else {
3411          !!!cp (1025);          !!!cp (985);
3412          $self->{entity_return} = {type => CHARACTER_TOKEN, data => $value, has_reference => 1,          $self->{ca}->{value} .= $data;
3413                  line => $l, column => $c,          $self->{ca}->{has_reference} = 1 if $has_ref;
3414                 };          $self->{state} = $self->{prev_state};
3415          $self->{state} = $self->{entity_in_attr} ? ENTITY_IN_ATTRIBUTE_VALUE_STATE : ENTITY_DATA_STATE;          ## Reconsume.
3416          redo A;          redo A;
3417        }        }
3418      } else {      } else {
       !!!cp (1026);  
       !!!parse-error (type => 'bare ero', line => $l, column => $c);  
       ## NOTE: "No characters are consumed" in the spec.  
       $self->{entity_return} = {type => CHARACTER_TOKEN, data => '&'.$value,  
               line => $l, column => $c,  
              };  
       $self->{state} = $self->{entity_in_attr} ? ENTITY_IN_ATTRIBUTE_VALUE_STATE : ENTITY_DATA_STATE;  
       redo A;  
     }  
   } else {  
     !!!cp (1027);  
     ## no characters are consumed  
     !!!parse-error (type => 'bare ero', line => $l, column => $c);  
     $self->{entity_return} = undef;  
     $self->{state} = $self->{entity_in_attr} ? ENTITY_IN_ATTRIBUTE_VALUE_STATE : ENTITY_DATA_STATE;  
     redo A;  
   }  
   
     } else {  
3419        die "$0: $self->{state}: Unknown state";        die "$0: $self->{state}: Unknown state";
3420      }      }
3421    } # A      } # A  
# Line 3291  sub _tree_construction_initial ($) { Line 3488  sub _tree_construction_initial ($) {
3488        $doctype_name = '' unless defined $doctype_name;        $doctype_name = '' unless defined $doctype_name;
3489        $doctype_name =~ tr/a-z/A-Z/; # ASCII case-insensitive        $doctype_name =~ tr/a-z/A-Z/; # ASCII case-insensitive
3490        if (not defined $token->{name} or # <!DOCTYPE>        if (not defined $token->{name} or # <!DOCTYPE>
3491            defined $token->{system_identifier}) {            defined $token->{sysid}) {
3492          !!!cp ('t1');          !!!cp ('t1');
3493          !!!parse-error (type => 'not HTML5', token => $token);          !!!parse-error (type => 'not HTML5', token => $token);
3494        } elsif ($doctype_name ne 'HTML') {        } elsif ($doctype_name ne 'HTML') {
3495          !!!cp ('t2');          !!!cp ('t2');
3496          !!!parse-error (type => 'not HTML5', token => $token);          !!!parse-error (type => 'not HTML5', token => $token);
3497        } elsif (defined $token->{public_identifier}) {        } elsif (defined $token->{pubid}) {
3498          if ($token->{public_identifier} eq 'XSLT-compat') {          if ($token->{pubid} eq 'XSLT-compat') {
3499            !!!cp ('t1.2');            !!!cp ('t1.2');
3500            !!!parse-error (type => 'XSLT-compat', token => $token,            !!!parse-error (type => 'XSLT-compat', token => $token,
3501                            level => $self->{level}->{should});                            level => $self->{level}->{should});
# Line 3314  sub _tree_construction_initial ($) { Line 3511  sub _tree_construction_initial ($) {
3511          ($token->{name}); ## ISSUE: If name is missing (e.g. <!DOCTYPE>)?          ($token->{name}); ## ISSUE: If name is missing (e.g. <!DOCTYPE>)?
3512        ## NOTE: Default value for both |public_id| and |system_id| attributes        ## NOTE: Default value for both |public_id| and |system_id| attributes
3513        ## are empty strings, so that we don't set any value in missing cases.        ## are empty strings, so that we don't set any value in missing cases.
3514        $doctype->public_id ($token->{public_identifier})        $doctype->public_id ($token->{pubid}) if defined $token->{pubid};
3515            if defined $token->{public_identifier};        $doctype->system_id ($token->{sysid}) if defined $token->{sysid};
       $doctype->system_id ($token->{system_identifier})  
           if defined $token->{system_identifier};  
3516        ## NOTE: Other DocumentType attributes are null or empty lists.        ## NOTE: Other DocumentType attributes are null or empty lists.
3517        ## ISSUE: internalSubset = null??        ## ISSUE: internalSubset = null??
3518        $self->{document}->append_child ($doctype);        $self->{document}->append_child ($doctype);
# Line 3325  sub _tree_construction_initial ($) { Line 3520  sub _tree_construction_initial ($) {
3520        if ($token->{quirks} or $doctype_name ne 'HTML') {        if ($token->{quirks} or $doctype_name ne 'HTML') {
3521          !!!cp ('t4');          !!!cp ('t4');
3522          $self->{document}->manakai_compat_mode ('quirks');          $self->{document}->manakai_compat_mode ('quirks');
3523        } elsif (defined $token->{public_identifier}) {        } elsif (defined $token->{pubid}) {
3524          my $pubid = $token->{public_identifier};          my $pubid = $token->{pubid};
3525          $pubid =~ tr/a-z/A-z/;          $pubid =~ tr/a-z/A-z/;
3526          my $prefix = [          my $prefix = [
3527            "+//SILMARIL//DTD HTML PRO V0R11 19970101//",            "+//SILMARIL//DTD HTML PRO V0R11 19970101//",
# Line 3400  sub _tree_construction_initial ($) { Line 3595  sub _tree_construction_initial ($) {
3595            $self->{document}->manakai_compat_mode ('quirks');            $self->{document}->manakai_compat_mode ('quirks');
3596          } elsif ($pubid =~ m[^-//W3C//DTD HTML 4.01 FRAMESET//] or          } elsif ($pubid =~ m[^-//W3C//DTD HTML 4.01 FRAMESET//] or
3597                   $pubid =~ m[^-//W3C//DTD HTML 4.01 TRANSITIONAL//]) {                   $pubid =~ m[^-//W3C//DTD HTML 4.01 TRANSITIONAL//]) {
3598            if (defined $token->{system_identifier}) {            if (defined $token->{sysid}) {
3599              !!!cp ('t6');              !!!cp ('t6');
3600              $self->{document}->manakai_compat_mode ('quirks');              $self->{document}->manakai_compat_mode ('quirks');
3601            } else {            } else {
# Line 3417  sub _tree_construction_initial ($) { Line 3612  sub _tree_construction_initial ($) {
3612        } else {        } else {
3613          !!!cp ('t10');          !!!cp ('t10');
3614        }        }
3615        if (defined $token->{system_identifier}) {        if (defined $token->{sysid}) {
3616          my $sysid = $token->{system_identifier};          my $sysid = $token->{sysid};
3617          $sysid =~ tr/A-Z/a-z/;          $sysid =~ tr/A-Z/a-z/;
3618          if ($sysid eq "http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd") {          if ($sysid eq "http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd") {
3619            ## NOTE: Ensure that |PUBLIC "(limited quirks)" "(quirks)"| is            ## NOTE: Ensure that |PUBLIC "(limited quirks)" "(quirks)"| is
# Line 3448  sub _tree_construction_initial ($) { Line 3643  sub _tree_construction_initial ($) {
3643        !!!ack-later;        !!!ack-later;
3644        return;        return;
3645      } elsif ($token->{type} == CHARACTER_TOKEN) {      } elsif ($token->{type} == CHARACTER_TOKEN) {
3646        if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) { # \x0D        if ($token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
3647          ## Ignore the token          ## Ignore the token
3648    
3649          unless (length $token->{data}) {          unless (length $token->{data}) {
# Line 3505  sub _tree_construction_root_element ($) Line 3700  sub _tree_construction_root_element ($)
3700          !!!next-token;          !!!next-token;
3701          redo B;          redo B;
3702        } elsif ($token->{type} == CHARACTER_TOKEN) {        } elsif ($token->{type} == CHARACTER_TOKEN) {
3703          if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) { # \x0D          if ($token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
3704            ## Ignore the token.            ## Ignore the token.
3705    
3706            unless (length $token->{data}) {            unless (length $token->{data}) {
# Line 4319  sub _tree_construction_main ($) { Line 4514  sub _tree_construction_main ($) {
4514    
4515      if ($self->{insertion_mode} & HEAD_IMS) {      if ($self->{insertion_mode} & HEAD_IMS) {
4516        if ($token->{type} == CHARACTER_TOKEN) {        if ($token->{type} == CHARACTER_TOKEN) {
4517          if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {          if ($token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
4518            unless ($self->{insertion_mode} == BEFORE_HEAD_IM) {            unless ($self->{insertion_mode} == BEFORE_HEAD_IM) {
4519              !!!cp ('t88.2');              !!!cp ('t88.2');
4520              $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);              $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);
4521                #
4522            } else {            } else {
4523              !!!cp ('t88.1');              !!!cp ('t88.1');
4524              ## Ignore the token.              ## Ignore the token.
4525              !!!next-token;              #
             next B;  
4526            }            }
4527            unless (length $token->{data}) {            unless (length $token->{data}) {
4528              !!!cp ('t88');              !!!cp ('t88');
4529              !!!next-token;              !!!next-token;
4530              next B;              next B;
4531            }            }
4532    ## TODO: set $token->{column} appropriately
4533          }          }
4534    
4535          if ($self->{insertion_mode} == BEFORE_HEAD_IM) {          if ($self->{insertion_mode} == BEFORE_HEAD_IM) {
# Line 4497  sub _tree_construction_main ($) { Line 4693  sub _tree_construction_main ($) {
4693                  } elsif ($token->{attributes}->{content}) {                  } elsif ($token->{attributes}->{content}) {
4694                    if ($token->{attributes}->{content}->{value}                    if ($token->{attributes}->{content}->{value}
4695                        =~ /[Cc][Hh][Aa][Rr][Ss][Ee][Tt]                        =~ /[Cc][Hh][Aa][Rr][Ss][Ee][Tt]
4696                            [\x09-\x0D\x20]*=                            [\x09\x0A\x0C\x0D\x20]*=
4697                            [\x09-\x0D\x20]*(?>"([^"]*)"|'([^']*)'|                            [\x09\x0A\x0C\x0D\x20]*(?>"([^"]*)"|'([^']*)'|
4698                            ([^"'\x09-\x0D\x20][^\x09-\x0D\x20\x3B]*))/x) {                            ([^"'\x09\x0A\x0C\x0D\x20]
4699                               [^\x09\x0A\x0C\x0D\x20\x3B]*))/x) {
4700                      !!!cp ('t107');                      !!!cp ('t107');
4701                      ## NOTE: Whether the encoding is supported or not is handled                      ## NOTE: Whether the encoding is supported or not is handled
4702                      ## in the {change_encoding} callback.                      ## in the {change_encoding} callback.
# Line 5308  sub _tree_construction_main ($) { Line 5505  sub _tree_construction_main ($) {
5505      } elsif ($self->{insertion_mode} & TABLE_IMS) {      } elsif ($self->{insertion_mode} & TABLE_IMS) {
5506        if ($token->{type} == CHARACTER_TOKEN) {        if ($token->{type} == CHARACTER_TOKEN) {
5507          if (not $open_tables->[-1]->[1] and # tainted          if (not $open_tables->[-1]->[1] and # tainted
5508              $token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {              $token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
5509            $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);            $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);
5510                                
5511            unless (length $token->{data}) {            unless (length $token->{data}) {
# Line 5992  sub _tree_construction_main ($) { Line 6189  sub _tree_construction_main ($) {
6189        }        }
6190      } elsif ($self->{insertion_mode} == IN_COLUMN_GROUP_IM) {      } elsif ($self->{insertion_mode} == IN_COLUMN_GROUP_IM) {
6191            if ($token->{type} == CHARACTER_TOKEN) {            if ($token->{type} == CHARACTER_TOKEN) {
6192              if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {              if ($token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
6193                $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);                $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);
6194                unless (length $token->{data}) {                unless (length $token->{data}) {
6195                  !!!cp ('t260');                  !!!cp ('t260');
# Line 6333  sub _tree_construction_main ($) { Line 6530  sub _tree_construction_main ($) {
6530        }        }
6531      } elsif ($self->{insertion_mode} & BODY_AFTER_IMS) {      } elsif ($self->{insertion_mode} & BODY_AFTER_IMS) {
6532        if ($token->{type} == CHARACTER_TOKEN) {        if ($token->{type} == CHARACTER_TOKEN) {
6533          if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {          if ($token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
6534            my $data = $1;            my $data = $1;
6535            ## As if in body            ## As if in body
6536            $reconstruct_active_formatting_elements->($insert_to_current);            $reconstruct_active_formatting_elements->($insert_to_current);
# Line 6350  sub _tree_construction_main ($) { Line 6547  sub _tree_construction_main ($) {
6547          if ($self->{insertion_mode} == AFTER_HTML_BODY_IM) {          if ($self->{insertion_mode} == AFTER_HTML_BODY_IM) {
6548            !!!cp ('t301');            !!!cp ('t301');
6549            !!!parse-error (type => 'after html:#text', token => $token);            !!!parse-error (type => 'after html:#text', token => $token);
6550              #
           ## Reprocess in the "after body" insertion mode.  
6551          } else {          } else {
6552            !!!cp ('t302');            !!!cp ('t302');
6553              ## "after body" insertion mode
6554              !!!parse-error (type => 'after body:#text', token => $token);
6555              #
6556          }          }
           
         ## "after body" insertion mode  
         !!!parse-error (type => 'after body:#text', token => $token);  
6557    
6558          $self->{insertion_mode} = IN_BODY_IM;          $self->{insertion_mode} = IN_BODY_IM;
6559          ## reprocess          ## reprocess
# Line 6367  sub _tree_construction_main ($) { Line 6563  sub _tree_construction_main ($) {
6563            !!!cp ('t303');            !!!cp ('t303');
6564            !!!parse-error (type => 'after html',            !!!parse-error (type => 'after html',
6565                            text => $token->{tag_name}, token => $token);                            text => $token->{tag_name}, token => $token);
6566                        #
           ## Reprocess in the "after body" insertion mode.  
6567          } else {          } else {
6568            !!!cp ('t304');            !!!cp ('t304');
6569              ## "after body" insertion mode
6570              !!!parse-error (type => 'after body',
6571                              text => $token->{tag_name}, token => $token);
6572              #
6573          }          }
6574    
         ## "after body" insertion mode  
         !!!parse-error (type => 'after body',  
                         text => $token->{tag_name}, token => $token);  
   
6575          $self->{insertion_mode} = IN_BODY_IM;          $self->{insertion_mode} = IN_BODY_IM;
6576          !!!ack-later;          !!!ack-later;
6577          ## reprocess          ## reprocess
# Line 6387  sub _tree_construction_main ($) { Line 6582  sub _tree_construction_main ($) {
6582            !!!parse-error (type => 'after html:/',            !!!parse-error (type => 'after html:/',
6583                            text => $token->{tag_name}, token => $token);                            text => $token->{tag_name}, token => $token);
6584                        
6585            $self->{insertion_mode} = AFTER_BODY_IM;            $self->{insertion_mode} = IN_BODY_IM;
6586            ## Reprocess in the "after body" insertion mode.            ## Reprocess.
6587              next B;
6588          } else {          } else {
6589            !!!cp ('t306');            !!!cp ('t306');
6590          }          }
# Line 6426  sub _tree_construction_main ($) { Line 6622  sub _tree_construction_main ($) {
6622        }        }
6623      } elsif ($self->{insertion_mode} & FRAME_IMS) {      } elsif ($self->{insertion_mode} & FRAME_IMS) {
6624        if ($token->{type} == CHARACTER_TOKEN) {        if ($token->{type} == CHARACTER_TOKEN) {
6625          if ($token->{data} =~ s/^([\x09\x0A\x0B\x0C\x20]+)//) {          if ($token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
6626            $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);            $self->{open_elements}->[-1]->[0]->manakai_append_text ($1);
6627                        
6628            unless (length $token->{data}) {            unless (length $token->{data}) {
# Line 6436  sub _tree_construction_main ($) { Line 6632  sub _tree_construction_main ($) {
6632            }            }
6633          }          }
6634                    
6635          if ($token->{data} =~ s/^[^\x09\x0A\x0B\x0C\x20]+//) {          if ($token->{data} =~ s/^[^\x09\x0A\x0C\x20]+//) {
6636            if ($self->{insertion_mode} == IN_FRAMESET_IM) {            if ($self->{insertion_mode} == IN_FRAMESET_IM) {
6637              !!!cp ('t311');              !!!cp ('t311');
6638              !!!parse-error (type => 'in frameset:#text', token => $token);              !!!parse-error (type => 'in frameset:#text', token => $token);
# Line 6613  sub _tree_construction_main ($) { Line 6809  sub _tree_construction_main ($) {
6809            } elsif ($token->{attributes}->{content}) {            } elsif ($token->{attributes}->{content}) {
6810              if ($token->{attributes}->{content}->{value}              if ($token->{attributes}->{content}->{value}
6811                  =~ /[Cc][Hh][Aa][Rr][Ss][Ee][Tt]                  =~ /[Cc][Hh][Aa][Rr][Ss][Ee][Tt]
6812                      [\x09-\x0D\x20]*=                      [\x09\x0A\x0C\x0D\x20]*=
6813                      [\x09-\x0D\x20]*(?>"([^"]*)"|'([^']*)'|                      [\x09\x0A\x0C\x0D\x20]*(?>"([^"]*)"|'([^']*)'|
6814                      ([^"'\x09-\x0D\x20][^\x09-\x0D\x20\x3B]*))/x) {                      ([^"'\x09\x0A\x0C\x0D\x20][^\x09\x0A\x0C\x0D\x20\x3B]*))
6815                       /x) {
6816                !!!cp ('t336');                !!!cp ('t336');
6817                ## NOTE: Whether the encoding is supported or not is handled                ## NOTE: Whether the encoding is supported or not is handled
6818                ## in the {change_encoding} callback.                ## in the {change_encoding} callback.
# Line 7578  sub _tree_construction_main ($) { Line 7775  sub _tree_construction_main ($) {
7775    ## TODO: script stuffs    ## TODO: script stuffs
7776  } # _tree_construct_main  } # _tree_construct_main
7777    
7778  sub set_inner_html ($$$;$) {  sub set_inner_html ($$$$;$) {
7779    my $class = shift;    my $class = shift;
7780    my $node = shift;    my $node = shift;
7781    my $s = \$_[0];    #my $s = \$_[0];
7782    my $onerror = $_[1];    my $onerror = $_[1];
7783    my $get_wrapper = $_[2] || sub ($) { return $_[0] };    my $get_wrapper = $_[2] || sub ($) { return $_[0] };
7784    
# Line 7602  sub set_inner_html ($$$;$) { Line 7799  sub set_inner_html ($$$;$) {
7799      }      }
7800    
7801      ## Step 3, 4, 5 # MUST      ## Step 3, 4, 5 # MUST
7802      $class->parse_char_string ($$s => $node, $onerror, $get_wrapper);      $class->parse_char_string ($_[0] => $node, $onerror, $get_wrapper);
7803    } elsif ($nt == 1) {    } elsif ($nt == 1) {
7804      ## TODO: If non-html element      ## TODO: If non-html element
7805    
# Line 7621  sub set_inner_html ($$$;$) { Line 7818  sub set_inner_html ($$$;$) {
7818      my $i = 0;      my $i = 0;
7819      $p->{line_prev} = $p->{line} = 1;      $p->{line_prev} = $p->{line} = 1;
7820      $p->{column_prev} = $p->{column} = 0;      $p->{column_prev} = $p->{column} = 0;
7821      $p->{set_next_char} = sub {      require Whatpm::Charset::DecodeHandle;
7822        my $input = Whatpm::Charset::DecodeHandle::CharString->new (\($_[0]));
7823        $input = $get_wrapper->($input);
7824        $p->{set_nc} = sub {
7825        my $self = shift;        my $self = shift;
7826    
7827        pop @{$self->{prev_char}};        my $char = '';
7828        unshift @{$self->{prev_char}}, $self->{next_char};        if (defined $self->{next_nc}) {
7829            $char = $self->{next_nc};
7830        $self->{next_char} = -1 and return if $i >= length $$s;          delete $self->{next_nc};
7831        $self->{next_char} = ord substr $$s, $i++, 1;          $self->{nc} = ord $char;
7832          } else {
7833            $self->{char_buffer} = '';
7834            $self->{char_buffer_pos} = 0;
7835            
7836            my $count = $input->manakai_read_until
7837                ($self->{char_buffer}, qr/[^\x00\x0A\x0D]/,
7838                 $self->{char_buffer_pos});
7839            if ($count) {
7840              $self->{line_prev} = $self->{line};
7841              $self->{column_prev} = $self->{column};
7842              $self->{column}++;
7843              $self->{nc}
7844                  = ord substr ($self->{char_buffer},
7845                                $self->{char_buffer_pos}++, 1);
7846              return;
7847            }
7848            
7849            if ($input->read ($char, 1)) {
7850              $self->{nc} = ord $char;
7851            } else {
7852              $self->{nc} = -1;
7853              return;
7854            }
7855          }
7856    
7857        ($p->{line_prev}, $p->{column_prev}) = ($p->{line}, $p->{column});        ($p->{line_prev}, $p->{column_prev}) = ($p->{line}, $p->{column});
7858        $p->{column}++;        $p->{column}++;
7859    
7860        if ($self->{next_char} == 0x000A) { # LF        if ($self->{nc} == 0x000A) { # LF
7861          $p->{line}++;          $p->{line}++;
7862          $p->{column} = 0;          $p->{column} = 0;
7863          !!!cp ('i1');          !!!cp ('i1');
7864        } elsif ($self->{next_char} == 0x000D) { # CR        } elsif ($self->{nc} == 0x000D) { # CR
7865          $i++ if substr ($$s, $i, 1) eq "\x0A";  ## TODO: support for abort/streaming
7866          $self->{next_char} = 0x000A; # LF # MUST          my $next = '';
7867            if ($input->read ($next, 1) and $next ne "\x0A") {
7868              $self->{next_nc} = $next;
7869            }
7870            $self->{nc} = 0x000A; # LF # MUST
7871          $p->{line}++;          $p->{line}++;
7872          $p->{column} = 0;          $p->{column} = 0;
7873          !!!cp ('i2');          !!!cp ('i2');
7874        } elsif ($self->{next_char} > 0x10FFFF) {        } elsif ($self->{nc} == 0x0000) { # NULL
         $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST  
         !!!cp ('i3');  
       } elsif ($self->{next_char} == 0x0000) { # NULL  
7875          !!!cp ('i4');          !!!cp ('i4');
7876          !!!parse-error (type => 'NULL');          !!!parse-error (type => 'NULL');
7877          $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST          $self->{nc} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
       } elsif ($self->{next_char} <= 0x0008 or  
                (0x000E <= $self->{next_char} and  
                 $self->{next_char} <= 0x001F) or  
                (0x007F <= $self->{next_char} and  
                 $self->{next_char} <= 0x009F) or  
                (0xD800 <= $self->{next_char} and  
                 $self->{next_char} <= 0xDFFF) or  
                (0xFDD0 <= $self->{next_char} and  
                 $self->{next_char} <= 0xFDDF) or  
                {  
                 0xFFFE => 1, 0xFFFF => 1, 0x1FFFE => 1, 0x1FFFF => 1,  
                 0x2FFFE => 1, 0x2FFFF => 1, 0x3FFFE => 1, 0x3FFFF => 1,  
                 0x4FFFE => 1, 0x4FFFF => 1, 0x5FFFE => 1, 0x5FFFF => 1,  
                 0x6FFFE => 1, 0x6FFFF => 1, 0x7FFFE => 1, 0x7FFFF => 1,  
                 0x8FFFE => 1, 0x8FFFF => 1, 0x9FFFE => 1, 0x9FFFF => 1,  
                 0xAFFFE => 1, 0xAFFFF => 1, 0xBFFFE => 1, 0xBFFFF => 1,  
                 0xCFFFE => 1, 0xCFFFF => 1, 0xDFFFE => 1, 0xDFFFF => 1,  
                 0xEFFFE => 1, 0xEFFFF => 1, 0xFFFFE => 1, 0xFFFFF => 1,  
                 0x10FFFE => 1, 0x10FFFF => 1,  
                }->{$self->{next_char}}) {  
         !!!cp ('i4.1');  
         if ($self->{next_char} < 0x10000) {  
           !!!parse-error (type => 'control char',  
                           text => (sprintf 'U+%04X', $self->{next_char}));  
         } else {  
           !!!parse-error (type => 'control char',  
                           text => (sprintf 'U-%08X', $self->{next_char}));  
         }  
7878        }        }
7879      };      };
7880      $p->{prev_char} = [-1, -1, -1];  
7881      $p->{next_char} = -1;      $p->{read_until} = sub {
7882              #my ($scalar, $specials_range, $offset) = @_;
7883          return 0 if defined $p->{next_nc};
7884    
7885          my $pattern = qr/[^$_[1]\x00\x0A\x0D]/;
7886          my $offset = $_[2] || 0;
7887          
7888          if ($p->{char_buffer_pos} < length $p->{char_buffer}) {
7889            pos ($p->{char_buffer}) = $p->{char_buffer_pos};
7890            if ($p->{char_buffer} =~ /\G(?>$pattern)+/) {
7891              substr ($_[0], $offset)
7892                  = substr ($p->{char_buffer}, $-[0], $+[0] - $-[0]);
7893              my $count = $+[0] - $-[0];
7894              if ($count) {
7895                $p->{column} += $count;
7896                $p->{char_buffer_pos} += $count;
7897                $p->{line_prev} = $p->{line};
7898                $p->{column_prev} = $p->{column} - 1;
7899                $p->{nc} = -1;
7900              }
7901              return $count;
7902            } else {
7903              return 0;
7904            }
7905          } else {
7906            my $count = $input->manakai_read_until ($_[0], $pattern, $_[2]);
7907            if ($count) {
7908              $p->{column} += $count;
7909              $p->{column_prev} += $count;
7910              $p->{nc} = -1;
7911            }
7912            return $count;
7913          }
7914        }; # $p->{read_until}
7915    
7916      my $ponerror = $onerror || sub {      my $ponerror = $onerror || sub {
7917        my (%opt) = @_;        my (%opt) = @_;
7918        my $line = $opt{line};        my $line = $opt{line};
# Line 7697  sub set_inner_html ($$$;$) { Line 7927  sub set_inner_html ($$$;$) {
7927        $ponerror->(line => $p->{line}, column => $p->{column}, @_);        $ponerror->(line => $p->{line}, column => $p->{column}, @_);
7928      };      };
7929            
7930        my $char_onerror = sub {
7931          my (undef, $type, %opt) = @_;
7932          $ponerror->(layer => 'encode',
7933                      line => $p->{line}, column => $p->{column} + 1,
7934                      %opt, type => $type);
7935        }; # $char_onerror
7936        $input->onerror ($char_onerror);
7937    
7938      $p->_initialize_tokenizer;      $p->_initialize_tokenizer;
7939      $p->_initialize_tree_constructor;      $p->_initialize_tree_constructor;
7940    
Legend:
Removed from v.1.167  
changed lines
  Added in v.1.191
[email protected]
 ViewVC Help
Powered by ViewVC 1.1.24