/[suikacvs]/markup/html/whatpm/t/HTML-tokenizer.t
Suika

Diff of /markup/html/whatpm/t/HTML-tokenizer.t

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.1 by wakaba, Mon Apr 30 09:59:35 2007 UTC revision 1.22 by wakaba, Sun Mar 2 14:32:27 2008 UTC
# Line 1  Line 1 
1  #!/usr/bin/perl  #!/usr/bin/perl
2  use strict;  use strict;
3    
4    my $dir_name;
5    my $test_dir_name;
6  BEGIN {  BEGIN {
7      $test_dir_name = 't/';
8      $dir_name = 't/tokenizer/';
9    my $skip = "You don't have JSON module";    my $skip = "You don't have JSON module";
10    eval q{    eval q{
11           use JSON 1.00;           use JSON 1.07;
12           $skip = "You don't have make command";           $skip = "You don't have make command";
13           system ('make', 'tokenizer-files') == 0 or die           system ("cd $test_dir_name; make tokenizer-files") == 0 or die
14             unless -f 'tokenizer/test1.test';             unless -f $dir_name.'test1.test';
15           $skip = '';           $skip = '';
16          };          };
17    if ($skip) {    if ($skip) {
# Line 16  BEGIN { Line 20  BEGIN {
20      exit;      exit;
21    }    }
22    $JSON::UnMapping = 1;    $JSON::UnMapping = 1;
23      $JSON::UTF8 = 1;
24  }  }
25    
26  use Test;  use Test;
27  use Data::Dumper;  BEGIN { plan tests => 396 }
 BEGIN { plan tests => 38 }  
   
 use What::HTML;  
28    
29  for my $file_name (qw[  use Data::Dumper;
30                        tokenizer/test1.test  $Data::Dumper::Useqq = 1;
31                        tokenizer/test2.test  sub Data::Dumper::qquote {
32                        tokenizer/contentModelFlags.test    my $s = shift;
33      $s =~ s/([^\x20\x21-\x26\x28-\x5B\x5D-\x7E])/sprintf '\x{%02X}', ord $1/ge;
34      return q<qq'> . $s . q<'>;
35    } # Data::Dumper::qquote
36    
37    use Whatpm::HTML;
38    
39    for my $file_name (grep {$_} split /\s+/, qq[
40                          ${dir_name}test1.test
41                          ${dir_name}test2.test
42                          ${dir_name}test3.test
43                          ${dir_name}test4.test
44                          ${dir_name}contentModelFlags.test
45                          ${dir_name}escapeFlag.test
46                          ${test_dir_name}tokenizer-test-1.test
47                       ]) {                       ]) {
48    open my $file, '<:utf8', $file_name or die "$0: $file_name: $!";    open my $file, '<', $file_name
49        or die "$0: $file_name: $!";
50    local $/ = undef;    local $/ = undef;
51    my $js = <$file>;    my $js = <$file>;
52    close $file;    close $file;
53      
54      print "# $file_name\n";
55      $js =~ s{\\u[Dd]([89A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])
56          \\u[Dd]([89A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])}{
57        ## NOTE: JSON::Parser does not decode surrogate pair escapes
58        ## NOTE: In older version of JSON::Parser, utf8 string will be broken
59        ## by parsing.  Use latest version!
60        ## NOTE: Encode.pm is broken; it converts e.g. U+10FFFF to U+FFFD.
61        my $c = 0x10000;
62        $c += ((((hex $1) & 0b1111111111) << 10) | ((hex $2) & 0b1111111111));
63        chr $c;
64      }gex;
65    my $tests = jsonToObj ($js)->{tests};    my $tests = jsonToObj ($js)->{tests};
66    TEST: for my $test (@$tests) {    TEST: for my $test (@$tests) {
67      my $s = $test->{input};      my $s = $test->{input};
# Line 51  for my $file_name (qw[ Line 79  for my $file_name (qw[
79        $j++;        $j++;
80      }      }
81    
82      my @cm = @{$test->{content_model_flags} || ['PCDATA']};      my @cm = @{$test->{contentModelFlags} || ['PCDATA']};
83        my $last_start_tag = $test->{lastStartTag};
84      for my $cm (@cm) {      for my $cm (@cm) {
85        my $p = What::HTML->new;        my $p = Whatpm::HTML->new;
86        my $i = 0;        my $i = 0;
87          my @token;
88        $p->{set_next_input_character} = sub {        $p->{set_next_input_character} = sub {
89          my $self = shift;          my $self = shift;
90    
91            pop @{$self->{prev_input_character}};
92            unshift @{$self->{prev_input_character}}, $self->{next_input_character};
93    
94          $self->{next_input_character} = -1 and return if $i >= length $s;          $self->{next_input_character} = -1 and return if $i >= length $s;
95          $self->{next_input_character} = ord substr $s, $i++, 1;          $self->{next_input_character} = ord substr $s, $i++, 1;
96    
97            if ($self->{next_input_character} == 0x000D) { # CR
98              $i++ if substr ($s, $i, 1) eq "\x0A";
99              $self->{next_input_character} = 0x000A; # LF # MUST
100            } elsif ($self->{next_input_character} > 0x10FFFF) {
101              $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
102              push @token, 'ParseError';
103            } elsif ($self->{next_input_character} == 0x0000) { # NULL
104              $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
105              push @token, 'ParseError';
106            }
107        };        };
108          $p->{prev_input_character} = [-1, -1, -1];
109          $p->{next_input_character} = -1;
110                
       my @token;  
111        $p->{parse_error} = sub {        $p->{parse_error} = sub {
112          push @token, 'ParseError';          push @token, 'ParseError';
113        };        };
114                
115        $p->_initialize_tokenizer;        $p->_initialize_tokenizer;
116        $p->{content_model_flag} = $cm;        $p->{content_model} = {
117            CDATA => Whatpm::HTML::CDATA_CONTENT_MODEL (),
118            RCDATA => Whatpm::HTML::RCDATA_CONTENT_MODEL (),
119            PCDATA => Whatpm::HTML::PCDATA_CONTENT_MODEL (),
120            PLAINTEXT => Whatpm::HTML::PLAINTEXT_CONTENT_MODEL (),
121          }->{$cm};
122          $p->{last_emitted_start_tag_name} = $last_start_tag;
123    
124        while (1) {        while (1) {
125          my $token = $p->_get_next_token;          my $token = $p->_get_next_token;
126          last if $token->{type} eq 'end-of-file';          last if $token->{type} == Whatpm::HTML::END_OF_FILE_TOKEN ();
127                    
128          my $test_token = [          my $test_token = [
129           {           {
130            DOCTYPE => 'DOCTYPE',            Whatpm::HTML::DOCTYPE_TOKEN () => 'DOCTYPE',
131            'start tag' => 'StartTag',            Whatpm::HTML::START_TAG_TOKEN () => 'StartTag',
132            'end tag' => 'EndTag',            Whatpm::HTML::END_TAG_TOKEN () => 'EndTag',
133            comment => 'Comment',            Whatpm::HTML::COMMENT_TOKEN () => 'Comment',
134            character => 'Character',            Whatpm::HTML::CHARACTER_TOKEN () => 'Character',
135           }->{$token->{type}} || $token->{type},           }->{$token->{type}} || $token->{type},
136          ];          ];
         $test_token->[1] = $token->{name} if defined $token->{name};  
137          $test_token->[1] = $token->{tag_name} if defined $token->{tag_name};          $test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
138          $test_token->[1] = $token->{data} if defined $token->{data};          $test_token->[1] = $token->{data} if defined $token->{data};
139          $test_token->[2] = $token->{error} ? 1 : 0 if $token->{type} eq 'DOCTYPE';          if ($token->{type} == Whatpm::HTML::START_TAG_TOKEN ()) {
140          $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}}            $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}};
141            if $token->{type} eq 'start tag';          } elsif ($token->{type} == Whatpm::HTML::DOCTYPE_TOKEN ()) {
142                      $test_token->[1] = $token->{name};
143              $test_token->[2] = $token->{public_identifier};
144              $test_token->[3] = $token->{system_identifier};
145              $test_token->[4] = $token->{correct} ? 1 : 0;
146            }
147    
148          if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and          if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
149              $test_token->[0] eq 'Character') {              $test_token->[0] eq 'Character') {
150            $token[-1]->[1] .= $test_token->[1];            $token[-1]->[1] .= $test_token->[1];
# Line 99  for my $file_name (qw[ Line 155  for my $file_name (qw[
155                
156        my $expected_dump = Dumper ($test->{output});        my $expected_dump = Dumper ($test->{output});
157        my $parser_dump = Dumper (\@token);        my $parser_dump = Dumper (\@token);
158        ok $parser_dump, $expected_dump, $test->{description};        ok $parser_dump, $expected_dump,
159            $test->{description} . ': ' . Data::Dumper::qquote ($test->{input});
160      }      }
161    }    }
162  }  }
163    
164    ## License: Public Domain.
165    ## $Date$
Legend:
Removed from v.1.1  
changed lines
  Added in v.1.22
[email protected]
 ViewVC Help
Powered by ViewVC 1.1.24