/[suikacvs]/markup/html/whatpm/t/HTML-tokenizer.t
Suika

Diff of /markup/html/whatpm/t/HTML-tokenizer.t

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.4 by wakaba, Tue May 1 06:22:12 2007 UTC revision 1.34 by wakaba, Sun Apr 13 06:44:27 2008 UTC
# Line 1  Line 1 
1  #!/usr/bin/perl  #!/usr/bin/perl
2  use strict;  use strict;
3    
4    my $DEBUG = $ENV{DEBUG};
5    
6  my $dir_name;  my $dir_name;
7  my $test_dir_name;  my $test_dir_name;
8  BEGIN {  BEGIN {
# Line 8  BEGIN { Line 10  BEGIN {
10    $dir_name = 't/tokenizer/';    $dir_name = 't/tokenizer/';
11    my $skip = "You don't have JSON module";    my $skip = "You don't have JSON module";
12    eval q{    eval q{
13           use JSON 1.00;           use JSON 1.07;
14           $skip = "You don't have make command";           $skip = "You don't have make command";
15           system ("cd $test_dir_name; make tokenizer-files") == 0 or die           system ("cd $test_dir_name; make tokenizer-files") == 0 or die
16             unless -f $dir_name.'test1.test';             unless -f $dir_name.'test1.test';
# Line 24  BEGIN { Line 26  BEGIN {
26  }  }
27    
28  use Test;  use Test;
29  BEGIN { plan tests => 70 }  BEGIN { plan tests => 477 }
30    
31  use Data::Dumper;  use Data::Dumper;
32  $Data::Dumper::Useqq = 1;  $Data::Dumper::Useqq = 1;
# Line 34  sub Data::Dumper::qquote { Line 36  sub Data::Dumper::qquote {
36    return q<qq'> . $s . q<'>;    return q<qq'> . $s . q<'>;
37  } # Data::Dumper::qquote  } # Data::Dumper::qquote
38    
39  use What::HTML;  if ($DEBUG) {
40      my $not_found = {%{$Whatpm::HTML::Debug::cp or {}}};
41    
42      $Whatpm::HTML::Debug::cp_pass = sub {
43        my $id = shift;
44        delete $not_found->{$id};
45      };
46    
47      END {
48        for my $id (sort {$a <=> $b || $a cmp $b} grep {!/^[ti]/}
49                    keys %$not_found) {
50          print "# checkpoint $id is not reached\n";
51        }
52      }
53    }
54    
55    use Whatpm::HTML;
56    
57  for my $file_name (grep {$_} split /\s+/, qq[  for my $file_name (grep {$_} split /\s+/, qq[
58                        ${dir_name}test1.test                        ${dir_name}test1.test
59                        ${dir_name}test2.test                        ${dir_name}test2.test
60                          ${dir_name}test3.test
61                          ${dir_name}test4.test
62                        ${dir_name}contentModelFlags.test                        ${dir_name}contentModelFlags.test
63                          ${dir_name}escapeFlag.test
64                          ${dir_name}entities.test
65                          ${dir_name}xmlViolation.test
66                        ${test_dir_name}tokenizer-test-1.test                        ${test_dir_name}tokenizer-test-1.test
67                       ]) {                       ]) {
68    open my $file, '<', $file_name    open my $file, '<', $file_name
# Line 47  for my $file_name (grep {$_} split /\s+/ Line 70  for my $file_name (grep {$_} split /\s+/
70    local $/ = undef;    local $/ = undef;
71    my $js = <$file>;    my $js = <$file>;
72    close $file;    close $file;
73      
74    my $tests = jsonToObj ($js)->{tests};    print "# $file_name\n";
75      $js =~ s{\\u[Dd]([89A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])
76          \\u[Dd]([89A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])}{
77        ## NOTE: JSON::Parser does not decode surrogate pair escapes
78        ## NOTE: In older version of JSON::Parser, utf8 string will be broken
79        ## by parsing.  Use latest version!
80        ## NOTE: Encode.pm is broken; it converts e.g. U+10FFFF to U+FFFD.
81        my $c = 0x10000;
82        $c += ((((hex $1) & 0b1111111111) << 10) | ((hex $2) & 0b1111111111));
83        chr $c;
84      }gex;
85      my $json = jsonToObj ($js);
86      my $tests = $json->{tests} || $json->{xmlViolationTests};
87    TEST: for my $test (@$tests) {    TEST: for my $test (@$tests) {
88      my $s = $test->{input};      my $s = $test->{input};
89            
# Line 68  for my $file_name (grep {$_} split /\s+/ Line 103  for my $file_name (grep {$_} split /\s+/
103      my @cm = @{$test->{contentModelFlags} || ['PCDATA']};      my @cm = @{$test->{contentModelFlags} || ['PCDATA']};
104      my $last_start_tag = $test->{lastStartTag};      my $last_start_tag = $test->{lastStartTag};
105      for my $cm (@cm) {      for my $cm (@cm) {
106        my $p = What::HTML->new;        my $p = Whatpm::HTML->new;
107        my $i = 0;        my $i = 0;
108        $p->{set_next_input_character} = sub {        my @token;
109          $p->{set_next_char} = sub {
110          my $self = shift;          my $self = shift;
         $self->{next_input_character} = -1 and return if $i >= length $s;  
         $self->{next_input_character} = ord substr $s, $i++, 1;  
111    
112          if ($self->{next_input_character} == 0x000D) { # CR          pop @{$self->{prev_char}};
113            if ($i >= length $s) {          unshift @{$self->{prev_char}}, $self->{next_char};
114              #  
115            } else {          $self->{next_char} = -1 and return if $i >= length $s;
116              my $next_char = ord substr $s, $i++, 1;          $self->{next_char} = ord substr $s, $i++, 1;
117              if ($next_char == 0x000A) { # LF  
118                #          if ($self->{next_char} == 0x000D) { # CR
119              } else {            $i++ if substr ($s, $i, 1) eq "\x0A";
120                push @{$self->{char}}, $next_char;            $self->{next_char} = 0x000A; # LF # MUST
121              }          } elsif ($self->{next_char} > 0x10FFFF) {
122            }            $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
123            $self->{next_input_character} = 0x000A; # LF # MUST            push @token, 'ParseError';
124          } elsif ($self->{next_input_character} > 0x10FFFF) {          } elsif ($self->{next_char} == 0x0000) { # NULL
125            $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST            $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
126          } elsif ($self->{next_input_character} == 0x0000) { # NULL            push @token, 'ParseError';
           $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST  
127          }          }
128        };        };
129          $p->{prev_char} = [-1, -1, -1];
130          $p->{next_char} = -1;
131                
       my @token;  
132        $p->{parse_error} = sub {        $p->{parse_error} = sub {
133          push @token, 'ParseError';          push @token, 'ParseError';
134        };        };
135                
136        $p->_initialize_tokenizer;        $p->_initialize_tokenizer;
137        $p->{content_model_flag} = $cm;        $p->{content_model} = {
138            CDATA => Whatpm::HTML::CDATA_CONTENT_MODEL (),
139            RCDATA => Whatpm::HTML::RCDATA_CONTENT_MODEL (),
140            PCDATA => Whatpm::HTML::PCDATA_CONTENT_MODEL (),
141            PLAINTEXT => Whatpm::HTML::PLAINTEXT_CONTENT_MODEL (),
142          }->{$cm};
143        $p->{last_emitted_start_tag_name} = $last_start_tag;        $p->{last_emitted_start_tag_name} = $last_start_tag;
144    
145        while (1) {        while (1) {
146          my $token = $p->_get_next_token;          my $token = $p->_get_next_token;
147          last if $token->{type} eq 'end-of-file';          last if $token->{type} == Whatpm::HTML::END_OF_FILE_TOKEN ();
148                    
149          my $test_token = [          my $test_token = [
150           {           {
151            DOCTYPE => 'DOCTYPE',            Whatpm::HTML::DOCTYPE_TOKEN () => 'DOCTYPE',
152            'start tag' => 'StartTag',            Whatpm::HTML::START_TAG_TOKEN () => 'StartTag',
153            'end tag' => 'EndTag',            Whatpm::HTML::END_TAG_TOKEN () => 'EndTag',
154            comment => 'Comment',            Whatpm::HTML::COMMENT_TOKEN () => 'Comment',
155            character => 'Character',            Whatpm::HTML::CHARACTER_TOKEN () => 'Character',
156           }->{$token->{type}} || $token->{type},           }->{$token->{type}} || $token->{type},
157          ];          ];
         $test_token->[1] = $token->{name} if defined $token->{name};  
158          $test_token->[1] = $token->{tag_name} if defined $token->{tag_name};          $test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
159          $test_token->[1] = $token->{data} if defined $token->{data};          $test_token->[1] = $token->{data} if defined $token->{data};
160          $test_token->[2] = $token->{error} ? 1 : 0 if $token->{type} eq 'DOCTYPE';          if ($token->{type} == Whatpm::HTML::START_TAG_TOKEN ()) {
161          $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}}            $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}};
162            if $token->{type} eq 'start tag';            if ({
163                           ## NOTE: Permitted slash
164                   br => 1, link => 1,
165                  }->{$token->{tag_name}}) {
166                delete $p->{self_closing};
167              }
168            } elsif ($token->{type} == Whatpm::HTML::DOCTYPE_TOKEN ()) {
169              $test_token->[1] = $token->{name};
170              $test_token->[2] = $token->{public_identifier};
171              $test_token->[3] = $token->{system_identifier};
172              $test_token->[4] = $token->{quirks} ? 0 : 1;
173            }
174    
175          if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and          if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
176              $test_token->[0] eq 'Character') {              $test_token->[0] eq 'Character') {
177            $token[-1]->[1] .= $test_token->[1];            $token[-1]->[1] .= $test_token->[1];
# Line 134  for my $file_name (grep {$_} split /\s+/ Line 183  for my $file_name (grep {$_} split /\s+/
183        my $expected_dump = Dumper ($test->{output});        my $expected_dump = Dumper ($test->{output});
184        my $parser_dump = Dumper (\@token);        my $parser_dump = Dumper (\@token);
185        ok $parser_dump, $expected_dump,        ok $parser_dump, $expected_dump,
186          $test->{description} . ': ' . $test->{input};          $test->{description} . ': ' . Data::Dumper::qquote ($test->{input});
187      }      }
188    }    }
189  }  }
190    
191    ## License: Public Domain.
192  ## $Date$  ## $Date$
Legend:
Removed from v.1.4  
changed lines
  Added in v.1.34
[email protected]
 ViewVC Help
Powered by ViewVC 1.1.24