/[suikacvs]/markup/html/whatpm/t/HTML-tokenizer.t
Suika

Diff of /markup/html/whatpm/t/HTML-tokenizer.t

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.41 by wakaba, Sun Sep 14 14:35:43 2008 UTC revision 1.43 by wakaba, Sat Sep 20 09:28:32 2008 UTC
# Line 26  BEGIN { Line 26  BEGIN {
26  }  }
27    
28  use Test;  use Test;
29  BEGIN { plan tests => 1073 }  BEGIN { plan tests => 1097 }
30    
31  use Data::Dumper;  use Data::Dumper;
32  $Data::Dumper::Useqq = 1;  $Data::Dumper::Useqq = 1;
# Line 107  for my $file_name (grep {$_} split /\s+/ Line 107  for my $file_name (grep {$_} split /\s+/
107        my $p = Whatpm::HTML->new;        my $p = Whatpm::HTML->new;
108        my $i = 0;        my $i = 0;
109        my @token;        my @token;
110        $p->{set_next_char} = sub {        $p->{set_nc} = sub {
111          my $self = shift;          my $self = shift;
112    
113  #        pop @{$self->{prev_char}};  #        pop @{$self->{prev_char}};
114  #        unshift @{$self->{prev_char}}, $self->{next_char};  #        unshift @{$self->{prev_char}}, $self->{nc};
115    
116          $self->{next_char} = -1 and return if $i >= length $s;          $self->{nc} = -1 and return if $i >= length $s;
117          $self->{next_char} = ord substr $s, $i++, 1;          $self->{nc} = ord substr $s, $i++, 1;
118    
119          if ($self->{next_char} == 0x000D) { # CR          if ($self->{nc} == 0x000D) { # CR
120            $i++ if substr ($s, $i, 1) eq "\x0A";            $i++ if substr ($s, $i, 1) eq "\x0A";
121            $self->{next_char} = 0x000A; # LF # MUST            $self->{nc} = 0x000A; # LF # MUST
122          } elsif ($self->{next_char} > 0x10FFFF) {          } elsif ($self->{nc} > 0x10FFFF) {
123            $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST            $self->{nc} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
124            push @token, 'ParseError';            push @token, 'ParseError';
125          } elsif ($self->{next_char} == 0x0000) { # NULL          } elsif ($self->{nc} == 0x0000) { # NULL
126            $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST            $self->{nc} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
127            push @token, 'ParseError';            push @token, 'ParseError';
128          } elsif ($self->{next_char} <= 0x0008 or          } elsif ($self->{nc} <= 0x0008 or
129                   (0x000E <= $self->{next_char} and                   (0x000E <= $self->{nc} and
130                    $self->{next_char} <= 0x001F) or                    $self->{nc} <= 0x001F) or
131                   (0x007F <= $self->{next_char} and                   (0x007F <= $self->{nc} and
132                    $self->{next_char} <= 0x009F) or                    $self->{nc} <= 0x009F) or
133                   (0xD800 <= $self->{next_char} and                   (0xD800 <= $self->{nc} and
134                    $self->{next_char} <= 0xDFFF) or                    $self->{nc} <= 0xDFFF) or
135                   (0xFDD0 <= $self->{next_char} and                   (0xFDD0 <= $self->{nc} and
136                    $self->{next_char} <= 0xFDDF) or                    $self->{nc} <= 0xFDDF) or
137                   {                   {
138                     0xFFFE => 1, 0xFFFF => 1, 0x1FFFE => 1, 0x1FFFF => 1,                     0xFFFE => 1, 0xFFFF => 1, 0x1FFFE => 1, 0x1FFFF => 1,
139                     0x2FFFE => 1, 0x2FFFF => 1, 0x3FFFE => 1, 0x3FFFF => 1,                     0x2FFFE => 1, 0x2FFFF => 1, 0x3FFFE => 1, 0x3FFFF => 1,
# Line 144  for my $file_name (grep {$_} split /\s+/ Line 144  for my $file_name (grep {$_} split /\s+/
144                     0xCFFFE => 1, 0xCFFFF => 1, 0xDFFFE => 1, 0xDFFFF => 1,                     0xCFFFE => 1, 0xCFFFF => 1, 0xDFFFE => 1, 0xDFFFF => 1,
145                     0xEFFFE => 1, 0xEFFFF => 1, 0xFFFFE => 1, 0xFFFFF => 1,                     0xEFFFE => 1, 0xEFFFF => 1, 0xFFFFE => 1, 0xFFFFF => 1,
146                     0x10FFFE => 1, 0x10FFFF => 1,                     0x10FFFE => 1, 0x10FFFF => 1,
147                    }->{$self->{next_char}}) {                    }->{$self->{nc}}) {
148            push @token, 'ParseError';            push @token, 'ParseError';
149          }          }
150        };        };
# Line 164  for my $file_name (grep {$_} split /\s+/ Line 164  for my $file_name (grep {$_} split /\s+/
164          PCDATA => Whatpm::HTML::PCDATA_CONTENT_MODEL (),          PCDATA => Whatpm::HTML::PCDATA_CONTENT_MODEL (),
165          PLAINTEXT => Whatpm::HTML::PLAINTEXT_CONTENT_MODEL (),          PLAINTEXT => Whatpm::HTML::PLAINTEXT_CONTENT_MODEL (),
166        }->{$cm};        }->{$cm};
167        $p->{last_emitted_start_tag_name} = $last_start_tag;        $p->{last_stag_name} = $last_start_tag;
168    
169        while (1) {        while (1) {
170          my $token = $p->_get_next_token;          my $token = $p->_get_next_token;
# Line 187  for my $file_name (grep {$_} split /\s+/ Line 187  for my $file_name (grep {$_} split /\s+/
187            delete $p->{self_closing};            delete $p->{self_closing};
188          } elsif ($token->{type} == Whatpm::HTML::DOCTYPE_TOKEN ()) {          } elsif ($token->{type} == Whatpm::HTML::DOCTYPE_TOKEN ()) {
189            $test_token->[1] = $token->{name};            $test_token->[1] = $token->{name};
190            $test_token->[2] = $token->{public_identifier};            $test_token->[2] = $token->{pubid};
191            $test_token->[3] = $token->{system_identifier};            $test_token->[3] = $token->{sysid};
192            $test_token->[4] = $token->{quirks} ? 0 : 1;            $test_token->[4] = $token->{quirks} ? 0 : 1;
193          }          }
194    
Legend:
Removed from v.1.41  
changed lines
  Added in v.1.43
[email protected]
 ViewVC Help
Powered by ViewVC 1.1.24