/[suikacvs]/markup/html/whatpm/Whatpm/HTML.pm.src

Diff of /markup/html/whatpm/Whatpm/HTML.pm.src

Parent Directory | Revision Log | View Patch Patch

-revision 1.152 by wakaba,
Sun Jun 29 11:15:53 2008 UTC
+revision 1.196 by wakaba,
Sat Oct  4 07:58:58 2008 UTC
 Line 3 
 use strict;
  our $VERSION=do{my @r=(q$Revision$=~/\d+/g);sprintf "%d."."%02d" x $#r,@r};
  use Error qw(:try);
+ ## NOTE: This module don't check all HTML5 parse errors; character
+ ## encoding related parse errors are expected to be handled by relevant
+ ## modules.
+ ## Parse errors for control characters that are not allowed in HTML5
+ ## documents, for surrogate code points, and for noncharacter code
+ ## points, as well as U+FFFD substitions for characters whose code points
+ ## is higher than U+10FFFF may be detected by combining the parser with
+ ## the checker implemented by Whatpm::Charset::UnicodeChecker (for its
+ ## usage example, see |t/HTML-tree.t| in the Whatpm package or the
+ ## WebHACC::Language::HTML module in the WebHACC package).
  ## ISSUE:
  ## var doc = implementation.createDocument (null, null, null);
  ## doc.write ('');
-Line 55 
 sub TABLE_ROWS_EL () {
+Line 66 
 sub TABLE_ROWS_EL () {
  }
  ## NOTE: Used in "generate implied end tags" algorithm.
- ## NOTE: There is a code where a modified version of END_TAG_OPTIONAL_EL
+ ## NOTE: There is a code where a modified version of
- ## is used in "generate implied end tags" implementation (search for the
+ ## END_TAG_OPTIONAL_EL is used in "generate implied end tags"
- ## function mae).
+ ## implementation (search for the algorithm name).
  sub END_TAG_OPTIONAL_EL () {
    DD_EL |
    DT_EL |
    LI_EL |
+   OPTION_EL |
+   OPTGROUP_EL |
    P_EL |
    RUBY_COMPONENT_EL
  }
-Line 130 
 my $el_category = {
+Line 143 
 my $el_category = {
    address => ADDRESS_EL,
    applet => MISC_SCOPING_EL,
    area => MISC_SPECIAL_EL,
+   article => MISC_SPECIAL_EL,
+   aside => MISC_SPECIAL_EL,
    b => FORMATTING_EL,
    base => MISC_SPECIAL_EL,
    basefont => MISC_SPECIAL_EL,
-Line 143 
 my $el_category = {
+Line 158 
 my $el_category = {
    center => MISC_SPECIAL_EL,
    col => MISC_SPECIAL_EL,
    colgroup => MISC_SPECIAL_EL,
+   command => MISC_SPECIAL_EL,
+   datagrid => MISC_SPECIAL_EL,
    dd => DD_EL,
+   details => MISC_SPECIAL_EL,
+   dialog => MISC_SPECIAL_EL,
    dir => MISC_SPECIAL_EL,
    div => DIV_EL,
    dl => MISC_SPECIAL_EL,
    dt => DT_EL,
    em => FORMATTING_EL,
    embed => MISC_SPECIAL_EL,
+   eventsource => MISC_SPECIAL_EL,
    fieldset => MISC_SPECIAL_EL,
+   figure => MISC_SPECIAL_EL,
    font => FORMATTING_EL,
+   footer => MISC_SPECIAL_EL,
    form => FORM_EL,
    frame => MISC_SPECIAL_EL,
    frameset => FRAMESET_EL,
-Line 162 
 my $el_category = {
+Line 184 
 my $el_category = {
    h5 => HEADING_EL,
    h6 => HEADING_EL,
    head => MISC_SPECIAL_EL,
+   header => MISC_SPECIAL_EL,
    hr => MISC_SPECIAL_EL,
    html => HTML_EL,
    i => FORMATTING_EL,
    iframe => MISC_SPECIAL_EL,
    img => MISC_SPECIAL_EL,
+   #image => MISC_SPECIAL_EL, ## NOTE: Commented out in the spec.
    input => MISC_SPECIAL_EL,
    isindex => MISC_SPECIAL_EL,
    li => LI_EL,
-Line 175 
 my $el_category = {
+Line 199 
 my $el_category = {
    marquee => MISC_SCOPING_EL,
    menu => MISC_SPECIAL_EL,
    meta => MISC_SPECIAL_EL,
+   nav => MISC_SPECIAL_EL,
    nobr => NOBR_EL | FORMATTING_EL,
    noembed => MISC_SPECIAL_EL,
    noframes => MISC_SPECIAL_EL,
-Line 193 
 my $el_category = {
+Line 218 
 my $el_category = {
    s => FORMATTING_EL,
    script => MISC_SPECIAL_EL,
    select => SELECT_EL,
+   section => MISC_SPECIAL_EL,
    small => FORMATTING_EL,
    spacer => MISC_SPECIAL_EL,
    strike => FORMATTING_EL,
-Line 312 
 my $foreign_attr_xname = {
+Line 338 
 my $foreign_attr_xname = {
  ## ISSUE: xmlns:xlink="non-xlink-ns" is not an error.
- my $c1_entity_char = {
+ my $charref_map = {
+x0D => 0x000A,
 x80 => 0x20AC,
 x81 => 0xFFFD,
 x82 => 0x201A,
-Line 345 
 my $c1_entity_char = {
+Line 372 
 my $c1_entity_char = {
 x9D => 0xFFFD,
 x9E => 0x017E,
 x9F => 0x0178,
- }; # $c1_entity_char
+ }; # $charref_map
+ $charref_map->{$_} = 0xFFFD
+     for 0x0000..0x0008, 0x000B, 0x000E..0x001F, 0x007F,
+xD800..0xDFFF, 0xFDD0..0xFDDF, ## ISSUE: 0xFDEF
+xFFFE, 0xFFFF, 0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, 0x3FFFF,
+x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF, 0x6FFFE, 0x6FFFF, 0x7FFFE,
+x7FFFF, 0x8FFFE, 0x8FFFF, 0x9FFFE, 0x9FFFF, 0xAFFFE, 0xAFFFF,
+xBFFFE, 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE, 0xDFFFF, 0xEFFFE,
+xEFFFF, 0xFFFFE, 0xFFFFF, 0x10FFFE, 0x10FFFF;
+ ## TODO: Invoke the reset algorithm when a resettable element is
+ ## created (cf. HTML5 revision 2259).
  sub parse_byte_string ($$$$;$) {
    my $self = shift;
-Line 354 
 sub parse_byte_string ($$$$;$) {
+Line 392 
 sub parse_byte_string ($$$$;$) {
    return $self->parse_byte_stream ($charset_name, $input, @_[1..$#_]);
  } # parse_byte_string
- sub parse_byte_stream ($$$$;$) {
+ sub parse_byte_stream ($$$$;$$) {
+   # my ($self, $charset_name, $byte_stream, $doc, $onerror, $get_wrapper) = @_;
    my $self = ref $_[0] ? shift : shift->new;
    my $charset_name = shift;
    my $byte_stream = $_[0];
-Line 365 
 sub parse_byte_stream ($$$$;$) {
+Line 404 
 sub parse_byte_stream ($$$$;$) {
    };
    $self->{parse_error} = $onerror; # updated later by parse_char_string
+   my $get_wrapper = $_[3] || sub ($) {
+     return $_[0]; # $_[0] = byte stream handle, returned = arg to char handle
+   };
    ## HTML5 encoding sniffing algorithm
    require Message::Charset::Info;
    my $charset;
-Line 372 
 sub parse_byte_stream ($$$$;$) {
+Line 415 
 sub parse_byte_stream ($$$$;$) {
    my ($char_stream, $e_status);
    SNIFFING: {
+     ## NOTE: By setting |allow_fallback| option true when the
+     ## |get_decode_handle| method is invoked, we ignore what the HTML5
+     ## spec requires, i.e. unsupported encoding should be ignored.
+       ## TODO: We should not do this unless the parser is invoked
+       ## in the conformance checking mode, in which this behavior
+       ## would be useful.
      ## Step 1
      if (defined $charset_name) {
-       $charset = Message::Charset::Info->get_by_iana_name ($charset_name);
+       $charset = Message::Charset::Info->get_by_html_name ($charset_name);
+           ## TODO: Is this ok?  Transfer protocol's parameter should be
+           ## interpreted in its semantics?
-       ## ISSUE: Unsupported encoding is not ignored according to the spec.
        ($char_stream, $e_status) = $charset->get_decode_handle
            ($byte_stream, allow_error_reporting => 1,
             allow_fallback => 1);
-Line 385 
 sub parse_byte_stream ($$$$;$) {
+Line 435 
 sub parse_byte_stream ($$$$;$) {
          $self->{confident} = 1;
          last SNIFFING;
        } else {
-         ## TODO: unsupported error
+         !!!parse-error (type => 'charset:not supported',
+                         layer => 'encode',
+                         line => 1, column => 1,
+                         value => $charset_name,
+                         level => $self->{level}->{uncertain});
        }
      }
-Line 399 
 sub parse_byte_stream ($$$$;$) {
+Line 453 
 sub parse_byte_stream ($$$$;$) {
      ## Step 3
      if ($byte_buffer =~ /^\xFE\xFF/) {
-       $charset = Message::Charset::Info->get_by_iana_name ('utf-16be');
+       $charset = Message::Charset::Info->get_by_html_name ('utf-16be');
        ($char_stream, $e_status) = $charset->get_decode_handle
            ($byte_stream, allow_error_reporting => 1,
             allow_fallback => 1, byte_buffer => \$byte_buffer);
        $self->{confident} = 1;
        last SNIFFING;
      } elsif ($byte_buffer =~ /^\xFF\xFE/) {
-       $charset = Message::Charset::Info->get_by_iana_name ('utf-16le');
+       $charset = Message::Charset::Info->get_by_html_name ('utf-16le');
        ($char_stream, $e_status) = $charset->get_decode_handle
            ($byte_stream, allow_error_reporting => 1,
             allow_fallback => 1, byte_buffer => \$byte_buffer);
        $self->{confident} = 1;
        last SNIFFING;
      } elsif ($byte_buffer =~ /^\xEF\xBB\xBF/) {
-       $charset = Message::Charset::Info->get_by_iana_name ('utf-8');
+       $charset = Message::Charset::Info->get_by_html_name ('utf-8');
        ($char_stream, $e_status) = $charset->get_decode_handle
            ($byte_stream, allow_error_reporting => 1,
             allow_fallback => 1, byte_buffer => \$byte_buffer);
-Line 432 
 sub parse_byte_stream ($$$$;$) {
+Line 486 
 sub parse_byte_stream ($$$$;$) {
      $charset_name = Whatpm::Charset::UniversalCharDet->detect_byte_string
          ($byte_buffer);
      if (defined $charset_name) {
-       $charset = Message::Charset::Info->get_by_iana_name ($charset_name);
+       $charset = Message::Charset::Info->get_by_html_name ($charset_name);
        ## ISSUE: Unsupported encoding is not ignored according to the spec.
        require Whatpm::Charset::DecodeHandle;
-Line 443 
 sub parse_byte_stream ($$$$;$) {
+Line 497 
 sub parse_byte_stream ($$$$;$) {
             allow_fallback => 1, byte_buffer => \$byte_buffer);
        if ($char_stream) {
          $buffer->{buffer} = $byte_buffer;
-         !!!parse-error (type => 'sniffing:chardet', ## TODO: type name
+         !!!parse-error (type => 'sniffing:chardet',
-                         value => $charset_name,
+                         text => $charset_name,
-                         level => $self->{info_level},
+                         level => $self->{level}->{info},
+                         layer => 'encode',
                          line => 1, column => 1);
          $self->{confident} = 0;
          last SNIFFING;
-Line 454 
 sub parse_byte_stream ($$$$;$) {
+Line 509 
 sub parse_byte_stream ($$$$;$) {
      ## Step 7: default
      ## TODO: Make this configurable.
-     $charset = Message::Charset::Info->get_by_iana_name ('windows-1252');
+     $charset = Message::Charset::Info->get_by_html_name ('windows-1252');
          ## NOTE: We choose |windows-1252| here, since |utf-8| should be
          ## detectable in the step 6.
      require Whatpm::Charset::DecodeHandle;
-Line 466 
 sub parse_byte_stream ($$$$;$) {
+Line 521 
 sub parse_byte_stream ($$$$;$) {
                                         allow_fallback => 1,
                                         byte_buffer => \$byte_buffer);
      $buffer->{buffer} = $byte_buffer;
-     !!!parse-error (type => 'sniffing:default', ## TODO: type name
+     !!!parse-error (type => 'sniffing:default',
-                     value => 'windows-1252',
+                     text => 'windows-1252',
-                     level => $self->{info_level},
+                     level => $self->{level}->{info},
-                     line => 1, column => 1);
+                     line => 1, column => 1,
+                     layer => 'encode');
      $self->{confident} = 0;
    } # SNIFFING
-   $self->{input_encoding} = $charset->get_iana_name;
    if ($e_status & Message::Charset::Info::FALLBACK_ENCODING_IMPL ()) {
-     !!!parse-error (type => 'chardecode:fallback', ## TODO: type name
+     $self->{input_encoding} = $charset->get_iana_name; ## TODO: Should we set actual charset decoder's encoding name?
-                     value => $self->{input_encoding},
+     !!!parse-error (type => 'chardecode:fallback',
-                     level => $self->{unsupported_level},
+                     #text => $self->{input_encoding},
-                     line => 1, column => 1);
+                     level => $self->{level}->{uncertain},
+                     line => 1, column => 1,
+                     layer => 'encode');
    } elsif (not ($e_status &
-                 Message::Charset::Info::ERROR_REPORTING_ENCODING_IMPL())) {
+                 Message::Charset::Info::ERROR_REPORTING_ENCODING_IMPL ())) {
-     !!!parse-error (type => 'chardecode:no error', ## TODO: type name
+     $self->{input_encoding} = $charset->get_iana_name;
-                     value => $self->{input_encoding},
+     !!!parse-error (type => 'chardecode:no error',
-                     level => $self->{unsupported_level},
+                     text => $self->{input_encoding},
-                     line => 1, column => 1);
+                     level => $self->{level}->{uncertain},
+                     line => 1, column => 1,
+                     layer => 'encode');
+   } else {
+     $self->{input_encoding} = $charset->get_iana_name;
    }
    $self->{change_encoding} = sub {
-Line 492 
 sub parse_byte_stream ($$$$;$) {
+Line 553 
 sub parse_byte_stream ($$$$;$) {
      $charset_name = shift;
      my $token = shift;
-     $charset = Message::Charset::Info->get_by_iana_name ($charset_name);
+     $charset = Message::Charset::Info->get_by_html_name ($charset_name);
      ($char_stream, $e_status) = $charset->get_decode_handle
          ($byte_stream, allow_error_reporting => 1, allow_fallback => 1,
           byte_buffer => \ $buffer->{buffer});
-Line 503 
 sub parse_byte_stream ($$$$;$) {
+Line 564 
 sub parse_byte_stream ($$$$;$) {
        ## Step 1
        if ($charset->{category} &
            Message::Charset::Info::CHARSET_CATEGORY_UTF16 ()) {
-         $charset = Message::Charset::Info->get_by_iana_name ('utf-8');
+         $charset = Message::Charset::Info->get_by_html_name ('utf-8');
          ($char_stream, $e_status) = $charset->get_decode_handle
              ($byte_stream,
               byte_buffer => \ $buffer->{buffer});
-Line 513 
 sub parse_byte_stream ($$$$;$) {
+Line 574 
 sub parse_byte_stream ($$$$;$) {
        ## Step 2
        if (defined $self->{input_encoding} and
            $self->{input_encoding} eq $charset_name) {
-         !!!parse-error (type => 'charset label:matching', ## TODO: type
+         !!!parse-error (type => 'charset label:matching',
-                         value => $charset_name,
+                         text => $charset_name,
-                         level => $self->{info_level});
+                         level => $self->{level}->{info});
          $self->{confident} = 1;
          return;
        }
-       !!!parse-error (type => 'charset label detected:'.$self->{input_encoding}.
+       !!!parse-error (type => 'charset label detected',
-           ':'.$charset_name, level => 'w', token => $token);
+                       text => $self->{input_encoding},
+                       value => $charset_name,
+                       level => $self->{level}->{warn},
+                       token => $token);
        ## Step 3
        # if (can) {
-Line 537 
 sub parse_byte_stream ($$$$;$) {
+Line 601 
 sub parse_byte_stream ($$$$;$) {
    my $char_onerror = sub {
      my (undef, $type, %opt) = @_;
-     !!!parse-error (%opt, type => $type,
+     !!!parse-error (layer => 'encode',
-                     line => $self->{line}, column => $self->{column} + 1);
+                     line => $self->{line}, column => $self->{column} + 1,
+                     %opt, type => $type);
      if ($opt{octets}) {
        ${$opt{octets}} = "\x{FFFD}"; # relacement character
      }
    };
-   $char_stream->onerror ($char_onerror);
-   my @args = @_; shift @args; # $s
+   my $wrapped_char_stream = $get_wrapper->($char_stream);
+   $wrapped_char_stream->onerror ($char_onerror);
+   my @args = ($_[1], $_[2]); # $doc, $onerror - $get_wrapper = undef;
    my $return;
    try {
-     $return = $self->parse_char_stream ($char_stream, @args);
+     $return = $self->parse_char_stream ($wrapped_char_stream, @args);
    } catch Whatpm::HTML::RestartParser with {
      ## NOTE: Invoked after {change_encoding}.
-     $self->{input_encoding} = $charset->get_iana_name;
      if ($e_status & Message::Charset::Info::FALLBACK_ENCODING_IMPL ()) {
-       !!!parse-error (type => 'chardecode:fallback', ## TODO: type name
+       $self->{input_encoding} = $charset->get_iana_name; ## TODO: Should we set actual charset decoder's encoding name?
-                       value => $self->{input_encoding},
+       !!!parse-error (type => 'chardecode:fallback',
-                       level => $self->{unsupported_level},
+                       level => $self->{level}->{uncertain},
-                       line => 1, column => 1);
+                       #text => $self->{input_encoding},
+                       line => 1, column => 1,
+                       layer => 'encode');
      } elsif (not ($e_status &
-                   Message::Charset::Info::ERROR_REPORTING_ENCODING_IMPL())) {
+                   Message::Charset::Info::ERROR_REPORTING_ENCODING_IMPL ())) {
-       !!!parse-error (type => 'chardecode:no error', ## TODO: type name
+       $self->{input_encoding} = $charset->get_iana_name;
-                       value => $self->{input_encoding},
+       !!!parse-error (type => 'chardecode:no error',
-                       level => $self->{unsupported_level},
+                       text => $self->{input_encoding},
-                       line => 1, column => 1);
+                       level => $self->{level}->{uncertain},
+                       line => 1, column => 1,
+                       layer => 'encode');
+     } else {
+       $self->{input_encoding} = $charset->get_iana_name;
      }
      $self->{confident} = 1;
-     $char_stream->onerror ($char_onerror);
-     $return = $self->parse_char_stream ($char_stream, @args);
+     $wrapped_char_stream = $get_wrapper->($char_stream);
+     $wrapped_char_stream->onerror ($char_onerror);
+     $return = $self->parse_char_stream ($wrapped_char_stream, @args);
    };
    return $return;
  } # parse_byte_stream
-Line 581 
 sub parse_byte_stream ($$$$;$) {
+Line 656 
 sub parse_byte_stream ($$$$;$) {
  ## such as |parse_byte_string| in this module, must ensure that it does
  ## strip the BOM and never strip any ZWNBSP.
- sub parse_char_string ($$$;$) {
+ sub parse_char_string ($$$;$$) {
+   #my ($self, $s, $doc, $onerror, $get_wrapper) = @_;
    my $self = shift;
-   require utf8;
    my $s = ref $_[0] ? $_[0] : \($_[0]);
-   open my $input, '<' . (utf8::is_utf8 ($$s) ? ':utf8' : ''), $s;
+   require Whatpm::Charset::DecodeHandle;
+   my $input = Whatpm::Charset::DecodeHandle::CharString->new ($s);
    return $self->parse_char_stream ($input, @_[1..$#_]);
  } # parse_char_string
- *parse_string = \&parse_char_string;
+ *parse_string = \&parse_char_string; ## NOTE: Alias for backward compatibility.
- sub parse_char_stream ($$$;$) {
+ sub parse_char_stream ($$$;$$) {
    my $self = ref $_[0] ? shift : shift->new;
    my $input = $_[0];
    $self->{document} = $_[1];
-Line 601 
 sub parse_char_stream ($$$;$) {
+Line 677 
 sub parse_char_stream ($$$;$) {
    $self->{confident} = 1 unless exists $self->{confident};
    $self->{document}->input_encoding ($self->{input_encoding})
        if defined $self->{input_encoding};
+ ## TODO: |{input_encoding}| is needless?
-   my $i = 0;
    $self->{line_prev} = $self->{line} = 1;
-   $self->{column_prev} = $self->{column} = 0;
+   $self->{column_prev} = -1;
-   $self->{set_next_char} = sub {
+   $self->{column} = 0;
+   $self->{set_nc} = sub {
      my $self = shift;
-     pop @{$self->{prev_char}};
+     my $char = '';
-     unshift @{$self->{prev_char}}, $self->{next_char};
+     if (defined $self->{next_nc}) {
+       $char = $self->{next_nc};
-     my $char;
+       delete $self->{next_nc};
-     if (defined $self->{next_next_char}) {
+       $self->{nc} = ord $char;
-       $char = $self->{next_next_char};
-       delete $self->{next_next_char};
      } else {
-       $char = $input->getc;
+       $self->{char_buffer} = '';
+       $self->{char_buffer_pos} = 0;
+       my $count = $input->manakai_read_until
+          ($self->{char_buffer}, qr/[^\x00\x0A\x0D]/, $self->{char_buffer_pos});
+       if ($count) {
+         $self->{line_prev} = $self->{line};
+         $self->{column_prev} = $self->{column};
+         $self->{column}++;
+         $self->{nc}
+             = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
+         return;
+       }
+       if ($input->read ($char, 1)) {
+         $self->{nc} = ord $char;
+       } else {
+         $self->{nc} = -1;
+         return;
+       }
      }
-     $self->{next_char} = -1 and return unless defined $char;
-     $self->{next_char} = ord $char;
      ($self->{line_prev}, $self->{column_prev})
          = ($self->{line}, $self->{column});
      $self->{column}++;
-     if ($self->{next_char} == 0x000A) { # LF
+     if ($self->{nc} == 0x000A) { # LF
        !!!cp ('j1');
        $self->{line}++;
        $self->{column} = 0;
-     } elsif ($self->{next_char} == 0x000D) { # CR
+     } elsif ($self->{nc} == 0x000D) { # CR
        !!!cp ('j2');
-       my $next = $input->getc;
+ ## TODO: support for abort/streaming
-       if (defined $next and $next ne "\x0A") {
+       my $next = '';
-         $self->{next_next_char} = $next;
+       if ($input->read ($next, 1) and $next ne "\x0A") {
+         $self->{next_nc} = $next;
        }
-       $self->{next_char} = 0x000A; # LF # MUST
+       $self->{nc} = 0x000A; # LF # MUST
        $self->{line}++;
        $self->{column} = 0;
-     } elsif ($self->{next_char} > 0x10FFFF) {
+     } elsif ($self->{nc} == 0x0000) { # NULL
-       !!!cp ('j3');
-       $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
-     } elsif ($self->{next_char} == 0x0000) { # NULL
        !!!cp ('j4');
        !!!parse-error (type => 'NULL');
-       $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
+       $self->{nc} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
-     } elsif ($self->{next_char} <= 0x0008 or
-              (0x000E <= $self->{next_char} and $self->{next_char} <= 0x001F) or
-              (0x007F <= $self->{next_char} and $self->{next_char} <= 0x009F) or
-              (0xD800 <= $self->{next_char} and $self->{next_char} <= 0xDFFF) or
-              (0xFDD0 <= $self->{next_char} and $self->{next_char} <= 0xFDDF) or
-              {
-xFFFE => 1, 0xFFFF => 1, 0x1FFFE => 1, 0x1FFFF => 1,
-x2FFFE => 1, 0x2FFFF => 1, 0x3FFFE => 1, 0x3FFFF => 1,
-x4FFFE => 1, 0x4FFFF => 1, 0x5FFFE => 1, 0x5FFFF => 1,
-x6FFFE => 1, 0x6FFFF => 1, 0x7FFFE => 1, 0x7FFFF => 1,
-x8FFFE => 1, 0x8FFFF => 1, 0x9FFFE => 1, 0x9FFFF => 1,
-xAFFFE => 1, 0xAFFFF => 1, 0xBFFFE => 1, 0xBFFFF => 1,
-xCFFFE => 1, 0xCFFFF => 1, 0xDFFFE => 1, 0xDFFFF => 1,
-xEFFFE => 1, 0xEFFFF => 1, 0xFFFFE => 1, 0xFFFFF => 1,
-x10FFFE => 1, 0x10FFFF => 1,
-              }->{$self->{next_char}}) {
-       !!!cp ('j5');
-       !!!parse-error (type => 'control char', level => $self->{must_level});
- ## TODO: error type documentation
      }
    };
-   $self->{prev_char} = [-1, -1, -1];
-   $self->{next_char} = -1;
+   $self->{read_until} = sub {
+     #my ($scalar, $specials_range, $offset) = @_;
+     return 0 if defined $self->{next_nc};
+     my $pattern = qr/[^$_[1]\x00\x0A\x0D]/;
+     my $offset = $_[2] || 0;
+     if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
+       pos ($self->{char_buffer}) = $self->{char_buffer_pos};
+       if ($self->{char_buffer} =~ /\G(?>$pattern)+/) {
+         substr ($_[0], $offset)
+             = substr ($self->{char_buffer}, $-[0], $+[0] - $-[0]);
+         my $count = $+[0] - $-[0];
+         if ($count) {
+           $self->{column} += $count;
+           $self->{char_buffer_pos} += $count;
+           $self->{line_prev} = $self->{line};
+           $self->{column_prev} = $self->{column} - 1;
+           $self->{nc} = -1;
+         }
+         return $count;
+       } else {
+         return 0;
+       }
+     } else {
+       my $count = $input->manakai_read_until ($_[0], $pattern, $_[2]);
+       if ($count) {
+         $self->{column} += $count;
+         $self->{line_prev} = $self->{line};
+         $self->{column_prev} = $self->{column} - 1;
+         $self->{nc} = -1;
+       }
+       return $count;
+     }
+   }; # $self->{read_until}
    my $onerror = $_[2] || sub {
      my (%opt) = @_;
-Line 679 
 sub parse_char_stream ($$$;$) {
+Line 784 
 sub parse_char_stream ($$$;$) {
      $onerror->(line => $self->{line}, column => $self->{column}, @_);
    };
+   my $char_onerror = sub {
+     my (undef, $type, %opt) = @_;
+     !!!parse-error (layer => 'encode',
+                     line => $self->{line}, column => $self->{column} + 1,
+                     %opt, type => $type);
+   }; # $char_onerror
+   if ($_[3]) {
+     $input = $_[3]->($input);
+     $input->onerror ($char_onerror);
+   } else {
+     $input->onerror ($char_onerror) unless defined $input->onerror;
+   }
    $self->_initialize_tokenizer;
    $self->_initialize_tree_constructor;
    $self->_construct_tree;
-Line 692 
 sub parse_char_stream ($$$;$) {
+Line 811 
 sub parse_char_stream ($$$;$) {
  sub new ($) {
    my $class = shift;
    my $self = bless {
-     must_level => 'm',
+     level => {must => 'm',
-     should_level => 's',
+               should => 's',
-     good_level => 'w',
+               warn => 'w',
-     warn_level => 'w',
+               info => 'i',
-     info_level => 'i',
+               uncertain => 'u'},
-     unsupported_level => 'u',
    }, $class;
-   $self->{set_next_char} = sub {
+   $self->{set_nc} = sub {
-     $self->{next_char} = -1;
+     $self->{nc} = -1;
    };
    $self->{parse_error} = sub {
      #
-Line 727 
 sub RCDATA_CONTENT_MODEL () { CM_ENTITY
+Line 845 
 sub RCDATA_CONTENT_MODEL () { CM_ENTITY
  sub PCDATA_CONTENT_MODEL () { CM_ENTITY | CM_FULL_MARKUP }
  sub DATA_STATE () { 0 }
- sub ENTITY_DATA_STATE () { 1 }
+ #sub ENTITY_DATA_STATE () { 1 }
  sub TAG_OPEN_STATE () { 2 }
  sub CLOSE_TAG_OPEN_STATE () { 3 }
  sub TAG_NAME_STATE () { 4 }
-Line 738 
 sub BEFORE_ATTRIBUTE_VALUE_STATE () { 8
+Line 856 
 sub BEFORE_ATTRIBUTE_VALUE_STATE () { 8
  sub ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE () { 9 }
  sub ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE () { 10 }
  sub ATTRIBUTE_VALUE_UNQUOTED_STATE () { 11 }
- sub ENTITY_IN_ATTRIBUTE_VALUE_STATE () { 12 }
+ #sub ENTITY_IN_ATTRIBUTE_VALUE_STATE () { 12 }
  sub MARKUP_DECLARATION_OPEN_STATE () { 13 }
  sub COMMENT_START_STATE () { 14 }
  sub COMMENT_START_DASH_STATE () { 15 }
-Line 761 
 sub AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STAT
+Line 879 
 sub AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STAT
  sub BOGUS_DOCTYPE_STATE () { 32 }
  sub AFTER_ATTRIBUTE_VALUE_QUOTED_STATE () { 33 }
  sub SELF_CLOSING_START_TAG_STATE () { 34 }
- sub CDATA_BLOCK_STATE () { 35 }
+ sub CDATA_SECTION_STATE () { 35 }
+ sub MD_HYPHEN_STATE () { 36 } # "markup declaration open state" in the spec
+ sub MD_DOCTYPE_STATE () { 37 } # "markup declaration open state" in the spec
+ sub MD_CDATA_STATE () { 38 } # "markup declaration open state" in the spec
+ sub CDATA_RCDATA_CLOSE_TAG_STATE () { 39 } # "close tag open state" in the spec
+ sub CDATA_SECTION_MSE1_STATE () { 40 } # "CDATA section state" in the spec
+ sub CDATA_SECTION_MSE2_STATE () { 41 } # "CDATA section state" in the spec
+ sub PUBLIC_STATE () { 42 } # "after DOCTYPE name state" in the spec
+ sub SYSTEM_STATE () { 43 } # "after DOCTYPE name state" in the spec
+ ## NOTE: "Entity data state", "entity in attribute value state", and
+ ## "consume a character reference" algorithm are jointly implemented
+ ## using the following six states:
+ sub ENTITY_STATE () { 44 }
+ sub ENTITY_HASH_STATE () { 45 }
+ sub NCR_NUM_STATE () { 46 }
+ sub HEXREF_X_STATE () { 47 }
+ sub HEXREF_HEX_STATE () { 48 }
+ sub ENTITY_NAME_STATE () { 49 }
+ sub PCDATA_STATE () { 50 } # "data state" in the spec
  sub DOCTYPE_TOKEN () { 1 }
  sub COMMENT_TOKEN () { 2 }
-Line 814 
 sub IN_COLUMN_GROUP_IM () { 0b10 }
+Line 950 
 sub IN_COLUMN_GROUP_IM () { 0b10 }
  sub _initialize_tokenizer ($) {
    my $self = shift;
    $self->{state} = DATA_STATE; # MUST
+   #$self->{s_kwd}; # state keyword - initialized when used
+   #$self->{entity__value}; # initialized when used
+   #$self->{entity__match}; # initialized when used
    $self->{content_model} = PCDATA_CONTENT_MODEL; # be
-   undef $self->{current_token}; # start tag, end tag, comment, or DOCTYPE
+   undef $self->{ct}; # current token
-   undef $self->{current_attribute};
+   undef $self->{ca}; # current attribute
-   undef $self->{last_emitted_start_tag_name};
+   undef $self->{last_stag_name}; # last emitted start tag name
-   undef $self->{last_attribute_value_state};
+   #$self->{prev_state}; # initialized when used
    delete $self->{self_closing};
-   $self->{char} = [];
+   $self->{char_buffer} = '';
-   # $self->{next_char}
+   $self->{char_buffer_pos} = 0;
+   $self->{nc} = -1; # next input character
+   #$self->{next_nc}
    !!!next-input-character;
    $self->{token} = [];
    # $self->{escape}
-Line 832 
 sub _initialize_tokenizer ($) {
+Line 973 
 sub _initialize_tokenizer ($) {
  ##       CHARACTER_TOKEN, or END_OF_FILE_TOKEN
  ##   ->{name} (DOCTYPE_TOKEN)
  ##   ->{tag_name} (START_TAG_TOKEN, END_TAG_TOKEN)
- ##   ->{public_identifier} (DOCTYPE_TOKEN)
+ ##   ->{pubid} (DOCTYPE_TOKEN)
- ##   ->{system_identifier} (DOCTYPE_TOKEN)
+ ##   ->{sysid} (DOCTYPE_TOKEN)
  ##   ->{quirks} == 1 or 0 (DOCTYPE_TOKEN): "force-quirks" flag
  ##   ->{attributes} isa HASH (START_TAG_TOKEN, END_TAG_TOKEN)
  ##        ->{name}
-Line 852 
 sub _initialize_tokenizer ($) {
+Line 993 
 sub _initialize_tokenizer ($) {
  ## has completed loading.  If one has, then it MUST be executed
  ## and removed from the list.
- ## NOTE: HTML5 "Writing HTML documents" section, applied to
+ ## TODO: Polytheistic slash SHOULD NOT be used. (Applied only to atheists.)
- ## documents and not to user agents and conformance checkers,
+ ## (This requirement was dropped from HTML5 spec, unfortunately.)
- ## contains some requirements that are not detected by the
- ## parsing algorithm:
+ my $is_space = {
- ## - Some requirements on character encoding declarations. ## TODO
+x0009 => 1, # CHARACTER TABULATION (HT)
- ## - "Elements MUST NOT contain content that their content model disallows."
+x000A => 1, # LINE FEED (LF)
- ##   ... Some are parse error, some are not (will be reported by c.c.).
+   #0x000B => 0, # LINE TABULATION (VT)
- ## - Polytheistic slash SHOULD NOT be used. (Applied only to atheists.) ## TODO
+x000C => 1, # FORM FEED (FF)
- ## - Text (in elements, attributes, and comments) SHOULD NOT contain
+   #0x000D => 1, # CARRIAGE RETURN (CR)
- ##   control characters other than space characters. ## TODO: (what is control character? C0, C1 and DEL?  Unicode control character?)
+x0020 => 1, # SPACE (SP)
+ };
- ## TODO: HTML5 poses authors two SHOULD-level requirements that cannot
- ## be detected by the HTML5 parsing algorithm:
- ## - Text,
  sub _get_next_token ($) {
    my $self = shift;
    if ($self->{self_closing}) {
-     !!!parse-error (type => 'nestc', token => $self->{current_token});
+     !!!parse-error (type => 'nestc', token => $self->{ct});
      ## NOTE: The |self_closing| flag is only set by start tag token.
      ## In addition, when a start tag token is emitted, it is always set to
-     ## |current_token|.
+     ## |ct|.
      delete $self->{self_closing};
    }
-Line 884 
 sub _get_next_token ($) {
+Line 1022 
 sub _get_next_token ($) {
    }
    A: {
-     if ($self->{state} == DATA_STATE) {
+     if ($self->{state} == PCDATA_STATE) {
-       if ($self->{next_char} == 0x0026) { # &
+       ## NOTE: Same as |DATA_STATE|, but only for |PCDATA| content model.
+       if ($self->{nc} == 0x0026) { # &
+         !!!cp (0.1);
+         ## NOTE: In the spec, the tokenizer is switched to the