whatpm/t/HTML-tokenizer.t

#!/usr/bin/perl
use strict;

my $DEBUG = $ENV{DEBUG};

my $dir_name;
my $test_dir_name;
BEGIN {
  $test_dir_name = 't/';
  $dir_name = 't/tokenizer/';
  my $skip = "You don't have JSON module";
  eval q{
         use JSON 1.07;
         $skip = "You don't have make command";
         system ("cd $test_dir_name; make tokenizer-files") == 0 or die
           unless -f $dir_name.'test1.test';
         $skip = '';
        };
  if ($skip) {
    print "1..1\n";
    print "ok 1 # $skip\n";
    exit;
  }
  $JSON::UnMapping = 1;
  $JSON::UTF8 = 1;
}

use Test;
BEGIN { plan tests => 477 }

use Data::Dumper;
$Data::Dumper::Useqq = 1;
sub Data::Dumper::qquote {
  my $s = shift;
  $s =~ s/([^\x20\x21-\x26\x28-\x5B\x5D-\x7E])/sprintf '\x{%02X}', ord $1/ge;
  return q<qq'> . $s . q<'>;
} # Data::Dumper::qquote

if ($DEBUG) {
  my $not_found = {%{$Whatpm::HTML::Debug::cp or {}}};

  $Whatpm::HTML::Debug::cp_pass = sub {
    my $id = shift;
    delete $not_found->{$id};
  };

  END {
    for my $id (sort {$a <=> $b || $a cmp $b} grep {!/^[ti]/}
                keys %$not_found) {
      print "# checkpoint $id is not reached\n";
    }
  }
}

use Whatpm::HTML;

for my $file_name (grep {$_} split /\s+/, qq[
                      ${dir_name}test1.test
                      ${dir_name}test2.test
                      ${dir_name}test3.test
                      ${dir_name}test4.test
                      ${dir_name}contentModelFlags.test
                      ${dir_name}escapeFlag.test
                      ${dir_name}entities.test
                      ${dir_name}xmlViolation.test
                      ${test_dir_name}tokenizer-test-1.test
                     ]) {
  open my $file, '<', $file_name
    or die "$0: $file_name: $!";
  local $/ = undef;
  my $js = <$file>;
  close $file;

  print "# $file_name\n";
  $js =~ s{\\u[Dd]([89A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])
      \\u[Dd]([89A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])}{
    ## NOTE: JSON::Parser does not decode surrogate pair escapes
    ## NOTE: In older version of JSON::Parser, utf8 string will be broken
    ## by parsing.  Use latest version!
    ## NOTE: Encode.pm is broken; it converts e.g. U+10FFFF to U+FFFD.
    my $c = 0x10000;
    $c += ((((hex $1) & 0b1111111111) << 10) | ((hex $2) & 0b1111111111));
    chr $c;
  }gex;
  my $json = jsonToObj ($js);
  my $tests = $json->{tests} || $json->{xmlViolationTests};
  TEST: for my $test (@$tests) {
    my $s = $test->{input};
    
    my $j = 1;
    while ($j < @{$test->{output}}) {
      if (ref $test->{output}->[$j - 1] and
          $test->{output}->[$j - 1]->[0] eq 'Character' and
          ref $test->{output}->[$j] and 
          $test->{output}->[$j]->[0] eq 'Character') {
        $test->{output}->[$j - 1]->[1]
          .= $test->{output}->[$j]->[1];
        splice @{$test->{output}}, $j, 1;
      }
      $j++;
    }

    my @cm = @{$test->{contentModelFlags} || ['PCDATA']};
    my $last_start_tag = $test->{lastStartTag};
    for my $cm (@cm) {
      my $p = Whatpm::HTML->new;
      my $i = 0;
      my @token;
      $p->{set_next_char} = sub {
        my $self = shift;

        pop @{$self->{prev_char}};
        unshift @{$self->{prev_char}}, $self->{next_char};

        $self->{next_char} = -1 and return if $i >= length $s;
        $self->{next_char} = ord substr $s, $i++, 1;

        if ($self->{next_char} == 0x000D) { # CR
          $i++ if substr ($s, $i, 1) eq "\x0A";
          $self->{next_char} = 0x000A; # LF # MUST
        } elsif ($self->{next_char} > 0x10FFFF) {
          $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
          push @token, 'ParseError';
        } elsif ($self->{next_char} == 0x0000) { # NULL
          $self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
          push @token, 'ParseError';
        }
      };
      $p->{prev_char} = [-1, -1, -1];
      $p->{next_char} = -1;
      
      $p->{parse_error} = sub {
        push @token, 'ParseError';
      };
      
      $p->_initialize_tokenizer;
      $p->{content_model} = {
        CDATA => Whatpm::HTML::CDATA_CONTENT_MODEL (),
        RCDATA => Whatpm::HTML::RCDATA_CONTENT_MODEL (),
        PCDATA => Whatpm::HTML::PCDATA_CONTENT_MODEL (),
        PLAINTEXT => Whatpm::HTML::PLAINTEXT_CONTENT_MODEL (),
      }->{$cm};
      $p->{last_emitted_start_tag_name} = $last_start_tag;

      while (1) {
        my $token = $p->_get_next_token;
        last if $token->{type} == Whatpm::HTML::END_OF_FILE_TOKEN ();
        
        my $test_token = [
         {
          Whatpm::HTML::DOCTYPE_TOKEN () => 'DOCTYPE',
          Whatpm::HTML::START_TAG_TOKEN () => 'StartTag',
          Whatpm::HTML::END_TAG_TOKEN () => 'EndTag',
          Whatpm::HTML::COMMENT_TOKEN () => 'Comment',
          Whatpm::HTML::CHARACTER_TOKEN () => 'Character',
         }->{$token->{type}} || $token->{type},
        ];
        $test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
        $test_token->[1] = $token->{data} if defined $token->{data};
        if ($token->{type} == Whatpm::HTML::START_TAG_TOKEN ()) {
          $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}};
          if ({
               ## NOTE: Permitted slash
               br => 1, link => 1,
              }->{$token->{tag_name}}) {
            delete $p->{self_closing};
          }
        } elsif ($token->{type} == Whatpm::HTML::DOCTYPE_TOKEN ()) {
          $test_token->[1] = $token->{name};
          $test_token->[2] = $token->{public_identifier};
          $test_token->[3] = $token->{system_identifier};
          $test_token->[4] = $token->{quirks} ? 0 : 1;
        }

        if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
            $test_token->[0] eq 'Character') {
          $token[-1]->[1] .= $test_token->[1];
        } else {
          push @token, $test_token;
        }
      }
      
      my $expected_dump = Dumper ($test->{output});
      my $parser_dump = Dumper (\@token);
      ok $parser_dump, $expected_dump,
        $test->{description} . ': ' . Data::Dumper::qquote ($test->{input});
    }
  }
}

## License: Public Domain.
## $Date: 2008/04/12 10:41:31 $
1	wakaba	1.1	#!/usr/bin/perl
2			use strict;
3
4	wakaba	1.27	my $DEBUG = $ENV{DEBUG};
5
6	wakaba	1.2	my $dir_name;
7	wakaba	1.4	my $test_dir_name;
8	wakaba	1.1	BEGIN {
9	wakaba	1.4	$test_dir_name = 't/';
10	wakaba	1.2	$dir_name = 't/tokenizer/';
11	wakaba	1.1	my $skip = "You don't have JSON module";
12			eval q{
13	wakaba	1.15	use JSON 1.07;
14	wakaba	1.1	$skip = "You don't have make command";
15	wakaba	1.3	system ("cd $test_dir_name; make tokenizer-files") == 0 or die
16	wakaba	1.2	unless -f $dir_name.'test1.test';
17	wakaba	1.1	$skip = '';
18			};
19			if ($skip) {
20			print "1..1\n";
21			print "ok 1 # $skip\n";
22			exit;
23			}
24			$JSON::UnMapping = 1;
25	wakaba	1.2	$JSON::UTF8 = 1;
26	wakaba	1.1	}
27
28			use Test;
29	wakaba	1.32	BEGIN { plan tests => 477 }
30	wakaba	1.2
31	wakaba	1.1	use Data::Dumper;
32	wakaba	1.2	$Data::Dumper::Useqq = 1;
33			sub Data::Dumper::qquote {
34			my $s = shift;
35			$s =~ s/([^\x20\x21-\x26\x28-\x5B\x5D-\x7E])/sprintf '\x{%02X}', ord $1/ge;
36			return q<qq'> . $s . q<'>;
37			} # Data::Dumper::qquote
38	wakaba	1.1
39	wakaba	1.27	if ($DEBUG) {
40	wakaba	1.31	my $not_found = {%{$Whatpm::HTML::Debug::cp or {}}};
41	wakaba	1.27
42			$Whatpm::HTML::Debug::cp_pass = sub {
43			my $id = shift;
44			delete $not_found->{$id};
45			};
46
47			END {
48	wakaba	1.29	for my $id (sort {$a <=> $b \|\| $a cmp $b} grep {!/^[ti]/}
49			keys %$not_found) {
50	wakaba	1.28	print "# checkpoint $id is not reached\n";
51	wakaba	1.27	}
52			}
53			}
54
55	wakaba	1.5	use Whatpm::HTML;
56	wakaba	1.1
57	wakaba	1.4	for my $file_name (grep {$_} split /\s+/, qq[
58			${dir_name}test1.test
59			${dir_name}test2.test
60	wakaba	1.15	${dir_name}test3.test
61			${dir_name}test4.test
62	wakaba	1.4	${dir_name}contentModelFlags.test
63	wakaba	1.8	${dir_name}escapeFlag.test
64	wakaba	1.34	${dir_name}entities.test
65			${dir_name}xmlViolation.test
66	wakaba	1.4	${test_dir_name}tokenizer-test-1.test
67	wakaba	1.1	]) {
68	wakaba	1.4	open my $file, '<', $file_name
69			or die "$0: $file_name: $!";
70	wakaba	1.1	local $/ = undef;
71			my $js = <$file>;
72			close $file;
73	wakaba	1.9
74			print "# $file_name\n";
75	wakaba	1.15	$js =~ s{\\u[Dd]([89A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])
76			\\u[Dd]([89A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])}{
77			## NOTE: JSON::Parser does not decode surrogate pair escapes
78			## NOTE: In older version of JSON::Parser, utf8 string will be broken
79			## by parsing. Use latest version!
80			## NOTE: Encode.pm is broken; it converts e.g. U+10FFFF to U+FFFD.
81			my $c = 0x10000;
82			$c += ((((hex $1) & 0b1111111111) << 10) \| ((hex $2) & 0b1111111111));
83			chr $c;
84			}gex;
85	wakaba	1.34	my $json = jsonToObj ($js);
86			my $tests = $json->{tests} \|\| $json->{xmlViolationTests};
87	wakaba	1.1	TEST: for my $test (@$tests) {
88			my $s = $test->{input};
89
90			my $j = 1;
91			while ($j < @{$test->{output}}) {
92			if (ref $test->{output}->[$j - 1] and
93			$test->{output}->[$j - 1]->[0] eq 'Character' and
94			ref $test->{output}->[$j] and
95			$test->{output}->[$j]->[0] eq 'Character') {
96			$test->{output}->[$j - 1]->[1]
97			.= $test->{output}->[$j]->[1];
98			splice @{$test->{output}}, $j, 1;
99			}
100			$j++;
101			}
102
103	wakaba	1.2	my @cm = @{$test->{contentModelFlags} \|\| ['PCDATA']};
104			my $last_start_tag = $test->{lastStartTag};
105	wakaba	1.1	for my $cm (@cm) {
106	wakaba	1.5	my $p = Whatpm::HTML->new;
107	wakaba	1.1	my $i = 0;
108	wakaba	1.9	my @token;
109	wakaba	1.26	$p->{set_next_char} = sub {
110	wakaba	1.1	my $self = shift;
111	wakaba	1.10
112	wakaba	1.26	pop @{$self->{prev_char}};
113			unshift @{$self->{prev_char}}, $self->{next_char};
114	wakaba	1.10
115	wakaba	1.26	$self->{next_char} = -1 and return if $i >= length $s;
116			$self->{next_char} = ord substr $s, $i++, 1;
117	wakaba	1.2
118	wakaba	1.26	if ($self->{next_char} == 0x000D) { # CR
119	wakaba	1.15	$i++ if substr ($s, $i, 1) eq "\x0A";
120	wakaba	1.26	$self->{next_char} = 0x000A; # LF # MUST
121			} elsif ($self->{next_char} > 0x10FFFF) {
122			$self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
123	wakaba	1.9	push @token, 'ParseError';
124	wakaba	1.26	} elsif ($self->{next_char} == 0x0000) { # NULL
125			$self->{next_char} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
126	wakaba	1.9	push @token, 'ParseError';
127	wakaba	1.2	}
128	wakaba	1.1	};
129	wakaba	1.26	$p->{prev_char} = [-1, -1, -1];
130			$p->{next_char} = -1;
131	wakaba	1.1
132			$p->{parse_error} = sub {
133			push @token, 'ParseError';
134			};
135
136			$p->_initialize_tokenizer;
137	wakaba	1.18	$p->{content_model} = {
138			CDATA => Whatpm::HTML::CDATA_CONTENT_MODEL (),
139			RCDATA => Whatpm::HTML::RCDATA_CONTENT_MODEL (),
140			PCDATA => Whatpm::HTML::PCDATA_CONTENT_MODEL (),
141			PLAINTEXT => Whatpm::HTML::PLAINTEXT_CONTENT_MODEL (),
142			}->{$cm};
143	wakaba	1.2	$p->{last_emitted_start_tag_name} = $last_start_tag;
144	wakaba	1.1
145			while (1) {
146			my $token = $p->_get_next_token;
147	wakaba	1.19	last if $token->{type} == Whatpm::HTML::END_OF_FILE_TOKEN ();
148	wakaba	1.1
149			my $test_token = [
150			{
151	wakaba	1.19	Whatpm::HTML::DOCTYPE_TOKEN () => 'DOCTYPE',
152			Whatpm::HTML::START_TAG_TOKEN () => 'StartTag',
153			Whatpm::HTML::END_TAG_TOKEN () => 'EndTag',
154			Whatpm::HTML::COMMENT_TOKEN () => 'Comment',
155			Whatpm::HTML::CHARACTER_TOKEN () => 'Character',
156	wakaba	1.1	}->{$token->{type}} \|\| $token->{type},
157			];
158			$test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
159			$test_token->[1] = $token->{data} if defined $token->{data};
160	wakaba	1.19	if ($token->{type} == Whatpm::HTML::START_TAG_TOKEN ()) {
161	wakaba	1.11	$test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}};
162	wakaba	1.33	if ({
163			## NOTE: Permitted slash
164			br => 1, link => 1,
165			}->{$token->{tag_name}}) {
166			delete $p->{self_closing};
167			}
168	wakaba	1.19	} elsif ($token->{type} == Whatpm::HTML::DOCTYPE_TOKEN ()) {
169	wakaba	1.11	$test_token->[1] = $token->{name};
170			$test_token->[2] = $token->{public_identifier};
171			$test_token->[3] = $token->{system_identifier};
172	wakaba	1.25	$test_token->[4] = $token->{quirks} ? 0 : 1;
173	wakaba	1.11	}
174
175	wakaba	1.1	if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
176			$test_token->[0] eq 'Character') {
177			$token[-1]->[1] .= $test_token->[1];
178			} else {
179			push @token, $test_token;
180			}
181			}
182
183			my $expected_dump = Dumper ($test->{output});
184			my $parser_dump = Dumper (\@token);
185	wakaba	1.2	ok $parser_dump, $expected_dump,
186	wakaba	1.17	$test->{description} . ': ' . Data::Dumper::qquote ($test->{input});
187	wakaba	1.1	}
188			}
189			}
190
191	wakaba	1.19	## License: Public Domain.
192	wakaba	1.34	## $Date: 2008/04/12 10:41:31 $