/[suikacvs]/markup/html/whatpm/t/HTML-tokenizer.t
Suika

Diff of /markup/html/whatpm/t/HTML-tokenizer.t

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.24 by wakaba, Sun Mar 2 23:51:00 2008 UTC revision 1.38 by wakaba, Sat Aug 30 13:43:50 2008 UTC
# Line 1  Line 1 
1  #!/usr/bin/perl  #!/usr/bin/perl
2  use strict;  use strict;
3    
4    my $DEBUG = $ENV{DEBUG};
5    
6  my $dir_name;  my $dir_name;
7  my $test_dir_name;  my $test_dir_name;
8  BEGIN {  BEGIN {
# Line 24  BEGIN { Line 26  BEGIN {
26  }  }
27    
28  use Test;  use Test;
29  BEGIN { plan tests => 420 }  BEGIN { plan tests => 1073 }
30    
31  use Data::Dumper;  use Data::Dumper;
32  $Data::Dumper::Useqq = 1;  $Data::Dumper::Useqq = 1;
33    $Data::Dumper::Sortkeys = 1;
34  sub Data::Dumper::qquote {  sub Data::Dumper::qquote {
35    my $s = shift;    my $s = shift;
36    $s =~ s/([^\x20\x21-\x26\x28-\x5B\x5D-\x7E])/sprintf '\x{%02X}', ord $1/ge;    $s =~ s/([^\x20\x21-\x26\x28-\x5B\x5D-\x7E])/sprintf '\x{%02X}', ord $1/ge;
37    return q<qq'> . $s . q<'>;    return q<qq'> . $s . q<'>;
38  } # Data::Dumper::qquote  } # Data::Dumper::qquote
39    
40    if ($DEBUG) {
41      my $not_found = {%{$Whatpm::HTML::Debug::cp or {}}};
42    
43      $Whatpm::HTML::Debug::cp_pass = sub {
44        my $id = shift;
45        delete $not_found->{$id};
46      };
47    
48      END {
49        for my $id (sort {$a <=> $b || $a cmp $b} grep {!/^[ti]/}
50                    keys %$not_found) {
51          print "# checkpoint $id is not reached\n";
52        }
53      }
54    }
55    
56  use Whatpm::HTML;  use Whatpm::HTML;
57    
58  for my $file_name (grep {$_} split /\s+/, qq[  for my $file_name (grep {$_} split /\s+/, qq[
# Line 43  for my $file_name (grep {$_} split /\s+/ Line 62  for my $file_name (grep {$_} split /\s+/
62                        ${dir_name}test4.test                        ${dir_name}test4.test
63                        ${dir_name}contentModelFlags.test                        ${dir_name}contentModelFlags.test
64                        ${dir_name}escapeFlag.test                        ${dir_name}escapeFlag.test
65                          ${dir_name}entities.test
66                          ${dir_name}xmlViolation.test
67                        ${test_dir_name}tokenizer-test-1.test                        ${test_dir_name}tokenizer-test-1.test
68                       ]) {                       ]) {
69    open my $file, '<', $file_name    open my $file, '<', $file_name
# Line 62  for my $file_name (grep {$_} split /\s+/ Line 83  for my $file_name (grep {$_} split /\s+/
83      $c += ((((hex $1) & 0b1111111111) << 10) | ((hex $2) & 0b1111111111));      $c += ((((hex $1) & 0b1111111111) << 10) | ((hex $2) & 0b1111111111));
84      chr $c;      chr $c;
85    }gex;    }gex;
86    my $tests = jsonToObj ($js)->{tests};    my $json = jsonToObj ($js);
87      my $tests = $json->{tests} || $json->{xmlViolationTests};
88    TEST: for my $test (@$tests) {    TEST: for my $test (@$tests) {
89      my $s = $test->{input};      my $s = $test->{input};
90            
# Line 85  for my $file_name (grep {$_} split /\s+/ Line 107  for my $file_name (grep {$_} split /\s+/
107        my $p = Whatpm::HTML->new;        my $p = Whatpm::HTML->new;
108        my $i = 0;        my $i = 0;
109        my @token;        my @token;
110        $p->{set_next_input_character} = sub {        $p->{set_next_char} = sub {
111          my $self = shift;          my $self = shift;
112    
113          pop @{$self->{prev_input_character}};          pop @{$self->{prev_char}};
114          unshift @{$self->{prev_input_character}}, $self->{next_input_character};          unshift @{$self->{prev_char}}, $self->{next_char};
115    
116          $self->{next_input_character} = -1 and return if $i >= length $s;          $self->{next_char} = -1 and return if $i >= length $s;
117          $self->{next_input_character} = ord substr $s, $i++, 1;          $self->{next_char} = ord substr $s, $i++, 1;
118    
119          if ($self->{next_input_character} == 0x000D) { # CR          if ($self->{next_char} == 0x000D) { # CR
120            $i++ if substr ($s, $i, 1) eq "\x0A";            $i++ if substr ($s, $i, 1) eq "\x0A";
121            $self->{next_input_character} = 0x000A; # LF # MUST            $self->{next_char} = 0x000A; # LF # MUST
122          } elsif ($self->{next_input_character} > 0x10FFFF) {          } elsif ($self->{next_char} > 0x10FFFF) {
123            $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST            $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
124              push @token, 'ParseError';
125            } elsif ($self->{next_char} == 0x0000) { # NULL
126              $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
127            push @token, 'ParseError';            push @token, 'ParseError';
128          } elsif ($self->{next_input_character} == 0x0000) { # NULL          } elsif ($self->{next_char} <= 0x0008 or
129            $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST                   (0x000E <= $self->{next_char} and
130                      $self->{next_char} <= 0x001F) or
131                     (0x007F <= $self->{next_char} and
132                      $self->{next_char} <= 0x009F) or
133                     (0xD800 <= $self->{next_char} and
134                      $self->{next_char} <= 0xDFFF) or
135                     (0xFDD0 <= $self->{next_char} and
136                      $self->{next_char} <= 0xFDDF) or
137                     {
138                       0xFFFE => 1, 0xFFFF => 1, 0x1FFFE => 1, 0x1FFFF => 1,
139                       0x2FFFE => 1, 0x2FFFF => 1, 0x3FFFE => 1, 0x3FFFF => 1,
140                       0x4FFFE => 1, 0x4FFFF => 1, 0x5FFFE => 1, 0x5FFFF => 1,
141                       0x6FFFE => 1, 0x6FFFF => 1, 0x7FFFE => 1, 0x7FFFF => 1,
142                       0x8FFFE => 1, 0x8FFFF => 1, 0x9FFFE => 1, 0x9FFFF => 1,
143                       0xAFFFE => 1, 0xAFFFF => 1, 0xBFFFE => 1, 0xBFFFF => 1,
144                       0xCFFFE => 1, 0xCFFFF => 1, 0xDFFFE => 1, 0xDFFFF => 1,
145                       0xEFFFE => 1, 0xEFFFF => 1, 0xFFFFE => 1, 0xFFFFF => 1,
146                       0x10FFFE => 1, 0x10FFFF => 1,
147                      }->{$self->{next_char}}) {
148            push @token, 'ParseError';            push @token, 'ParseError';
149          }          }
150        };        };
151        $p->{prev_input_character} = [-1, -1, -1];        $p->{prev_char} = [-1, -1, -1];
152        $p->{next_input_character} = -1;        $p->{next_char} = -1;
153                
154        $p->{parse_error} = sub {        $p->{parse_error} = sub {
155          push @token, 'ParseError';          push @token, 'ParseError';
# Line 138  for my $file_name (grep {$_} split /\s+/ Line 181  for my $file_name (grep {$_} split /\s+/
181          $test_token->[1] = $token->{data} if defined $token->{data};          $test_token->[1] = $token->{data} if defined $token->{data};
182          if ($token->{type} == Whatpm::HTML::START_TAG_TOKEN ()) {          if ($token->{type} == Whatpm::HTML::START_TAG_TOKEN ()) {
183            $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}};            $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}};
184              $test_token->[3] = 1 if $p->{self_closing};
185              delete $p->{self_closing};
186          } elsif ($token->{type} == Whatpm::HTML::DOCTYPE_TOKEN ()) {          } elsif ($token->{type} == Whatpm::HTML::DOCTYPE_TOKEN ()) {
187            $test_token->[1] = $token->{name};            $test_token->[1] = $token->{name};
188            $test_token->[2] = $token->{public_identifier};            $test_token->[2] = $token->{public_identifier};
189            $test_token->[3] = $token->{system_identifier};            $test_token->[3] = $token->{system_identifier};
190            $test_token->[4] = $token->{correct} ? 1 : 0;            $test_token->[4] = $token->{quirks} ? 0 : 1;
191          }          }
192    
193          if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and          if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
Legend:
Removed from v.1.24  
changed lines
  Added in v.1.38
[email protected]
 ViewVC Help
Powered by ViewVC 1.1.24