/[suikacvs]/markup/html/whatpm/t/HTML-tokenizer.t
Suika

Diff of /markup/html/whatpm/t/HTML-tokenizer.t

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.11 by wakaba, Sat Jun 23 12:21:01 2007 UTC revision 1.17 by wakaba, Sat Jul 21 04:55:20 2007 UTC
# Line 8  BEGIN { Line 8  BEGIN {
8    $dir_name = 't/tokenizer/';    $dir_name = 't/tokenizer/';
9    my $skip = "You don't have JSON module";    my $skip = "You don't have JSON module";
10    eval q{    eval q{
11           use JSON 1.00;           use JSON 1.07;
12           $skip = "You don't have make command";           $skip = "You don't have make command";
13           system ("cd $test_dir_name; make tokenizer-files") == 0 or die           system ("cd $test_dir_name; make tokenizer-files") == 0 or die
14             unless -f $dir_name.'test1.test';             unless -f $dir_name.'test1.test';
# Line 24  BEGIN { Line 24  BEGIN {
24  }  }
25    
26  use Test;  use Test;
27  BEGIN { plan tests => 94 }  BEGIN { plan tests => 337 }
28    
29  use Data::Dumper;  use Data::Dumper;
30  $Data::Dumper::Useqq = 1;  $Data::Dumper::Useqq = 1;
# Line 39  use Whatpm::HTML; Line 39  use Whatpm::HTML;
39  for my $file_name (grep {$_} split /\s+/, qq[  for my $file_name (grep {$_} split /\s+/, qq[
40                        ${dir_name}test1.test                        ${dir_name}test1.test
41                        ${dir_name}test2.test                        ${dir_name}test2.test
42                          ${dir_name}test3.test
43                          ${dir_name}test4.test
44                        ${dir_name}contentModelFlags.test                        ${dir_name}contentModelFlags.test
45                        ${dir_name}escapeFlag.test                        ${dir_name}escapeFlag.test
46                        ${test_dir_name}tokenizer-test-1.test                        ${test_dir_name}tokenizer-test-1.test
# Line 50  for my $file_name (grep {$_} split /\s+/ Line 52  for my $file_name (grep {$_} split /\s+/
52    close $file;    close $file;
53    
54    print "# $file_name\n";    print "# $file_name\n";
55      $js =~ s{\\u[Dd]([89A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])
56          \\u[Dd]([89A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])}{
57        ## NOTE: JSON::Parser does not decode surrogate pair escapes
58        ## NOTE: In older version of JSON::Parser, utf8 string will be broken
59        ## by parsing.  Use latest version!
60        ## NOTE: Encode.pm is broken; it converts e.g. U+10FFFF to U+FFFD.
61        my $c = 0x10000;
62        $c += ((((hex $1) & 0b1111111111) << 10) | ((hex $2) & 0b1111111111));
63        chr $c;
64      }gex;
65    my $tests = jsonToObj ($js)->{tests};    my $tests = jsonToObj ($js)->{tests};
66    TEST: for my $test (@$tests) {    TEST: for my $test (@$tests) {
67      my $s = $test->{input};      my $s = $test->{input};
# Line 83  for my $file_name (grep {$_} split /\s+/ Line 95  for my $file_name (grep {$_} split /\s+/
95          $self->{next_input_character} = ord substr $s, $i++, 1;          $self->{next_input_character} = ord substr $s, $i++, 1;
96    
97          if ($self->{next_input_character} == 0x000D) { # CR          if ($self->{next_input_character} == 0x000D) { # CR
98            if ($i >= length $s) {            $i++ if substr ($s, $i, 1) eq "\x0A";
             #  
           } else {  
             my $next_char = ord substr $s, $i++, 1;  
             if ($next_char == 0x000A) { # LF  
               #  
             } else {  
               push @{$self->{char}}, $next_char;  
             }  
           }  
99            $self->{next_input_character} = 0x000A; # LF # MUST            $self->{next_input_character} = 0x000A; # LF # MUST
100          } elsif ($self->{next_input_character} > 0x10FFFF) {          } elsif ($self->{next_input_character} > 0x10FFFF) {
101            $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST            $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
# Line 148  for my $file_name (grep {$_} split /\s+/ Line 151  for my $file_name (grep {$_} split /\s+/
151        my $expected_dump = Dumper ($test->{output});        my $expected_dump = Dumper ($test->{output});
152        my $parser_dump = Dumper (\@token);        my $parser_dump = Dumper (\@token);
153        ok $parser_dump, $expected_dump,        ok $parser_dump, $expected_dump,
154          $test->{description} . ': ' . $test->{input};          $test->{description} . ': ' . Data::Dumper::qquote ($test->{input});
155      }      }
156    }    }
157  }  }
Legend:
Removed from v.1.11  
changed lines
  Added in v.1.17
[email protected]
 ViewVC Help
Powered by ViewVC 1.1.24