/[suikacvs]/markup/html/whatpm/t/HTML-tokenizer.t
Suika

Diff of /markup/html/whatpm/t/HTML-tokenizer.t

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.16 by wakaba, Mon Jul 16 07:48:19 2007 UTC revision 1.29 by wakaba, Mon Mar 3 13:15:54 2008 UTC
# Line 1  Line 1 
1  #!/usr/bin/perl  #!/usr/bin/perl
2  use strict;  use strict;
3    
4    my $DEBUG = $ENV{DEBUG};
5    
6  my $dir_name;  my $dir_name;
7  my $test_dir_name;  my $test_dir_name;
8  BEGIN {  BEGIN {
# Line 24  BEGIN { Line 26  BEGIN {
26  }  }
27    
28  use Test;  use Test;
29  BEGIN { plan tests => 337 }  BEGIN { plan tests => 471 }
30    
31  use Data::Dumper;  use Data::Dumper;
32  $Data::Dumper::Useqq = 1;  $Data::Dumper::Useqq = 1;
# Line 34  sub Data::Dumper::qquote { Line 36  sub Data::Dumper::qquote {
36    return q<qq'> . $s . q<'>;    return q<qq'> . $s . q<'>;
37  } # Data::Dumper::qquote  } # Data::Dumper::qquote
38    
39    if ($DEBUG) {
40      my $not_found = {%$Whatpm::HTML::Debug::cp};
41    
42      $Whatpm::HTML::Debug::cp_pass = sub {
43        my $id = shift;
44        delete $not_found->{$id};
45      };
46    
47      END {
48        for my $id (sort {$a <=> $b || $a cmp $b} grep {!/^[ti]/}
49                    keys %$not_found) {
50          print "# checkpoint $id is not reached\n";
51        }
52      }
53    }
54    
55  use Whatpm::HTML;  use Whatpm::HTML;
56    
57  for my $file_name (grep {$_} split /\s+/, qq[  for my $file_name (grep {$_} split /\s+/, qq[
# Line 85  for my $file_name (grep {$_} split /\s+/ Line 103  for my $file_name (grep {$_} split /\s+/
103        my $p = Whatpm::HTML->new;        my $p = Whatpm::HTML->new;
104        my $i = 0;        my $i = 0;
105        my @token;        my @token;
106        $p->{set_next_input_character} = sub {        $p->{set_next_char} = sub {
107          my $self = shift;          my $self = shift;
108    
109          pop @{$self->{prev_input_character}};          pop @{$self->{prev_char}};
110          unshift @{$self->{prev_input_character}}, $self->{next_input_character};          unshift @{$self->{prev_char}}, $self->{next_char};
111    
112          $self->{next_input_character} = -1 and return if $i >= length $s;          $self->{next_char} = -1 and return if $i >= length $s;
113          $self->{next_input_character} = ord substr $s, $i++, 1;          $self->{next_char} = ord substr $s, $i++, 1;
114    
115          if ($self->{next_input_character} == 0x000D) { # CR          if ($self->{next_char} == 0x000D) { # CR
116            $i++ if substr ($s, $i, 1) eq "\x0A";            $i++ if substr ($s, $i, 1) eq "\x0A";
117            $self->{next_input_character} = 0x000A; # LF # MUST            $self->{next_char} = 0x000A; # LF # MUST
118          } elsif ($self->{next_input_character} > 0x10FFFF) {          } elsif ($self->{next_char} > 0x10FFFF) {
119            $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST            $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
120            push @token, 'ParseError';            push @token, 'ParseError';
121          } elsif ($self->{next_input_character} == 0x0000) { # NULL          } elsif ($self->{next_char} == 0x0000) { # NULL
122            $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST            $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
123            push @token, 'ParseError';            push @token, 'ParseError';
124          }          }
125        };        };
126        $p->{prev_input_character} = [-1, -1, -1];        $p->{prev_char} = [-1, -1, -1];
127        $p->{next_input_character} = -1;        $p->{next_char} = -1;
128                
129        $p->{parse_error} = sub {        $p->{parse_error} = sub {
130          push @token, 'ParseError';          push @token, 'ParseError';
131        };        };
132                
133        $p->_initialize_tokenizer;        $p->_initialize_tokenizer;
134        $p->{content_model_flag} = $cm;        $p->{content_model} = {
135            CDATA => Whatpm::HTML::CDATA_CONTENT_MODEL (),
136            RCDATA => Whatpm::HTML::RCDATA_CONTENT_MODEL (),
137            PCDATA => Whatpm::HTML::PCDATA_CONTENT_MODEL (),
138            PLAINTEXT => Whatpm::HTML::PLAINTEXT_CONTENT_MODEL (),
139          }->{$cm};
140        $p->{last_emitted_start_tag_name} = $last_start_tag;        $p->{last_emitted_start_tag_name} = $last_start_tag;
141    
142        while (1) {        while (1) {
143          my $token = $p->_get_next_token;          my $token = $p->_get_next_token;
144          last if $token->{type} eq 'end-of-file';          last if $token->{type} == Whatpm::HTML::END_OF_FILE_TOKEN ();
145                    
146          my $test_token = [          my $test_token = [
147           {           {
148            DOCTYPE => 'DOCTYPE',            Whatpm::HTML::DOCTYPE_TOKEN () => 'DOCTYPE',
149            'start tag' => 'StartTag',            Whatpm::HTML::START_TAG_TOKEN () => 'StartTag',
150            'end tag' => 'EndTag',            Whatpm::HTML::END_TAG_TOKEN () => 'EndTag',
151            comment => 'Comment',            Whatpm::HTML::COMMENT_TOKEN () => 'Comment',
152            character => 'Character',            Whatpm::HTML::CHARACTER_TOKEN () => 'Character',
153           }->{$token->{type}} || $token->{type},           }->{$token->{type}} || $token->{type},
154          ];          ];
155          $test_token->[1] = $token->{tag_name} if defined $token->{tag_name};          $test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
156          $test_token->[1] = $token->{data} if defined $token->{data};          $test_token->[1] = $token->{data} if defined $token->{data};
157          if ($token->{type} eq 'start tag') {          if ($token->{type} == Whatpm::HTML::START_TAG_TOKEN ()) {
158            $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}};            $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}};
159          } elsif ($token->{type} eq 'DOCTYPE') {          } elsif ($token->{type} == Whatpm::HTML::DOCTYPE_TOKEN ()) {
160            $test_token->[1] = $token->{name};            $test_token->[1] = $token->{name};
161            $test_token->[2] = $token->{public_identifier};            $test_token->[2] = $token->{public_identifier};
162            $test_token->[3] = $token->{system_identifier};            $test_token->[3] = $token->{system_identifier};
163            $test_token->[4] = $token->{correct} ? 1 : 0;            $test_token->[4] = $token->{quirks} ? 0 : 1;
164          }          }
165    
166          if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and          if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
# Line 151  for my $file_name (grep {$_} split /\s+/ Line 174  for my $file_name (grep {$_} split /\s+/
174        my $expected_dump = Dumper ($test->{output});        my $expected_dump = Dumper ($test->{output});
175        my $parser_dump = Dumper (\@token);        my $parser_dump = Dumper (\@token);
176        ok $parser_dump, $expected_dump,        ok $parser_dump, $expected_dump,
177          $test->{description} . ': ' . $test->{input};          $test->{description} . ': ' . Data::Dumper::qquote ($test->{input});
178      }      }
179    }    }
180  }  }
181    
182    ## License: Public Domain.
183  ## $Date$  ## $Date$
Legend:
Removed from v.1.16  
changed lines
  Added in v.1.29
[email protected]
 ViewVC Help
Powered by ViewVC 1.1.24