Parent Directory
| 
Revision Log
++ whatpm/Whatpm/ChangeLog 14 Sep 2008 03:06:56 -0000
* HTML.pm.src: Change |{getc_until}| to |{read_until}|
and |manakai_getc_until| to |manakai_read_until| to
reduce the number of string copies.
2008-09-14 Wakaba <wakaba@suika.fam.cx>
++ whatpm/Whatpm/Charset/ChangeLog 14 Sep 2008 03:07:37 -0000
* DecodeHandle.pm, UnicodeChecker.pm: Change |manakai_getc_until|
to |manakai_read_until| to reduce the number of string copies.
2008-09-14 Wakaba <wakaba@suika.fam.cx>
| 1 | wakaba | 1.1 | #!/usr/bin/perl |
| 2 | use strict; | ||
| 3 | |||
| 4 | wakaba | 1.27 | my $DEBUG = $ENV{DEBUG}; |
| 5 | |||
| 6 | wakaba | 1.2 | my $dir_name; |
| 7 | wakaba | 1.4 | my $test_dir_name; |
| 8 | wakaba | 1.1 | BEGIN { |
| 9 | wakaba | 1.4 | $test_dir_name = 't/'; |
| 10 | wakaba | 1.2 | $dir_name = 't/tokenizer/'; |
| 11 | wakaba | 1.1 | my $skip = "You don't have JSON module"; |
| 12 | eval q{ | ||
| 13 | wakaba | 1.15 | use JSON 1.07; |
| 14 | wakaba | 1.1 | $skip = "You don't have make command"; |
| 15 | wakaba | 1.3 | system ("cd $test_dir_name; make tokenizer-files") == 0 or die |
| 16 | wakaba | 1.2 | unless -f $dir_name.'test1.test'; |
| 17 | wakaba | 1.1 | $skip = ''; |
| 18 | }; | ||
| 19 | if ($skip) { | ||
| 20 | print "1..1\n"; | ||
| 21 | print "ok 1 # $skip\n"; | ||
| 22 | exit; | ||
| 23 | } | ||
| 24 | $JSON::UnMapping = 1; | ||
| 25 | wakaba | 1.2 | $JSON::UTF8 = 1; |
| 26 | wakaba | 1.1 | } |
| 27 | |||
| 28 | use Test; | ||
| 29 | wakaba | 1.36 | BEGIN { plan tests => 1073 } |
| 30 | wakaba | 1.2 | |
| 31 | wakaba | 1.1 | use Data::Dumper; |
| 32 | wakaba | 1.2 | $Data::Dumper::Useqq = 1; |
| 33 | wakaba | 1.38 | $Data::Dumper::Sortkeys = 1; |
| 34 | wakaba | 1.2 | sub Data::Dumper::qquote { |
| 35 | my $s = shift; | ||
| 36 | $s =~ s/([^\x20\x21-\x26\x28-\x5B\x5D-\x7E])/sprintf '\x{%02X}', ord $1/ge; | ||
| 37 | return q<qq'> . $s . q<'>; | ||
| 38 | } # Data::Dumper::qquote | ||
| 39 | wakaba | 1.1 | |
| 40 | wakaba | 1.27 | if ($DEBUG) { |
| 41 | wakaba | 1.31 | my $not_found = {%{$Whatpm::HTML::Debug::cp or {}}}; |
| 42 | wakaba | 1.27 | |
| 43 | $Whatpm::HTML::Debug::cp_pass = sub { | ||
| 44 | my $id = shift; | ||
| 45 | delete $not_found->{$id}; | ||
| 46 | }; | ||
| 47 | |||
| 48 | END { | ||
| 49 | wakaba | 1.29 | for my $id (sort {$a <=> $b || $a cmp $b} grep {!/^[ti]/} |
| 50 | keys %$not_found) { | ||
| 51 | wakaba | 1.28 | print "# checkpoint $id is not reached\n"; |
| 52 | wakaba | 1.27 | } |
| 53 | } | ||
| 54 | } | ||
| 55 | |||
| 56 | wakaba | 1.5 | use Whatpm::HTML; |
| 57 | wakaba | 1.1 | |
| 58 | wakaba | 1.4 | for my $file_name (grep {$_} split /\s+/, qq[ |
| 59 | ${dir_name}test1.test | ||
| 60 | ${dir_name}test2.test | ||
| 61 | wakaba | 1.15 | ${dir_name}test3.test |
| 62 | ${dir_name}test4.test | ||
| 63 | wakaba | 1.4 | ${dir_name}contentModelFlags.test |
| 64 | wakaba | 1.8 | ${dir_name}escapeFlag.test |
| 65 | wakaba | 1.34 | ${dir_name}entities.test |
| 66 | ${dir_name}xmlViolation.test | ||
| 67 | wakaba | 1.4 | ${test_dir_name}tokenizer-test-1.test |
| 68 | wakaba | 1.1 | ]) { |
| 69 | wakaba | 1.4 | open my $file, '<', $file_name |
| 70 | or die "$0: $file_name: $!"; | ||
| 71 | wakaba | 1.1 | local $/ = undef; |
| 72 | my $js = <$file>; | ||
| 73 | close $file; | ||
| 74 | wakaba | 1.9 | |
| 75 | print "# $file_name\n"; | ||
| 76 | wakaba | 1.15 | $js =~ s{\\u[Dd]([89A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]) |
| 77 | \\u[Dd]([89A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])}{ | ||
| 78 | ## NOTE: JSON::Parser does not decode surrogate pair escapes | ||
| 79 | ## NOTE: In older version of JSON::Parser, utf8 string will be broken | ||
| 80 | ## by parsing. Use latest version! | ||
| 81 | ## NOTE: Encode.pm is broken; it converts e.g. U+10FFFF to U+FFFD. | ||
| 82 | my $c = 0x10000; | ||
| 83 | $c += ((((hex $1) & 0b1111111111) << 10) | ((hex $2) & 0b1111111111)); | ||
| 84 | chr $c; | ||
| 85 | }gex; | ||
| 86 | wakaba | 1.34 | my $json = jsonToObj ($js); |
| 87 | my $tests = $json->{tests} || $json->{xmlViolationTests}; | ||
| 88 | wakaba | 1.1 | TEST: for my $test (@$tests) { |
| 89 | my $s = $test->{input}; | ||
| 90 | |||
| 91 | my $j = 1; | ||
| 92 | while ($j < @{$test->{output}}) { | ||
| 93 | if (ref $test->{output}->[$j - 1] and | ||
| 94 | $test->{output}->[$j - 1]->[0] eq 'Character' and | ||
| 95 | ref $test->{output}->[$j] and | ||
| 96 | $test->{output}->[$j]->[0] eq 'Character') { | ||
| 97 | $test->{output}->[$j - 1]->[1] | ||
| 98 | .= $test->{output}->[$j]->[1]; | ||
| 99 | splice @{$test->{output}}, $j, 1; | ||
| 100 | } | ||
| 101 | $j++; | ||
| 102 | } | ||
| 103 | |||
| 104 | wakaba | 1.2 | my @cm = @{$test->{contentModelFlags} || ['PCDATA']}; |
| 105 | my $last_start_tag = $test->{lastStartTag}; | ||
| 106 | wakaba | 1.1 | for my $cm (@cm) { |
| 107 | wakaba |