whatpm/t/HTML-tokenizer.t

#!/usr/bin/perl
use strict;

my $dir_name;
BEGIN {
  $dir_name = 't/tokenizer/';
  my $skip = "You don't have JSON module";
  eval q{
         use JSON 1.00;
         $skip = "You don't have make command";
         system ('cd $dir_name; cd ..; make tokenizer-files') == 0 or die
           unless -f $dir_name.'test1.test';
         $skip = '';
        };
  if ($skip) {
    print "1..1\n";
    print "ok 1 # $skip\n";
    exit;
  }
  $JSON::UnMapping = 1;
  $JSON::UTF8 = 1;
}

use Test;
BEGIN { plan tests => 38 }

use Data::Dumper;
$Data::Dumper::Useqq = 1;
sub Data::Dumper::qquote {
  my $s = shift;
  $s =~ s/([^\x20\x21-\x26\x28-\x5B\x5D-\x7E])/sprintf '\x{%02X}', ord $1/ge;
  return q<qq'> . $s . q<'>;
} # Data::Dumper::qquote

use What::HTML;
use Encode;

for my $file_name (qw[
                      test1.test
                      test2.test
                      contentModelFlags.test
                     ]) {
  open my $file, '<', $dir_name.$file_name
    or die "$0: $dir_name$file_name: $!";
  local $/ = undef;
  my $js = <$file>;
  close $file;
  
  my $tests = jsonToObj ($js)->{tests};
  TEST: for my $test (@$tests) {
    my $s = $test->{input};
    
    my $j = 1;
    while ($j < @{$test->{output}}) {
      if (ref $test->{output}->[$j - 1] and
          $test->{output}->[$j - 1]->[0] eq 'Character' and
          ref $test->{output}->[$j] and 
          $test->{output}->[$j]->[0] eq 'Character') {
        $test->{output}->[$j - 1]->[1]
          .= $test->{output}->[$j]->[1];
        splice @{$test->{output}}, $j, 1;
      }
      $j++;
    }

    my @cm = @{$test->{contentModelFlags} || ['PCDATA']};
    my $last_start_tag = $test->{lastStartTag};
    for my $cm (@cm) {
      my $p = What::HTML->new;
      my $i = 0;
      $p->{set_next_input_character} = sub {
        my $self = shift;
        $self->{next_input_character} = -1 and return if $i >= length $s;
        $self->{next_input_character} = ord substr $s, $i++, 1;

        if ($self->{next_input_character} == 0x000D) { # CR
          if ($i >= length $s) {
            #
          } else {
            my $next_char = ord substr $s, $i++, 1;
            if ($next_char == 0x000A) { # LF
              #
            } else {
              push @{$self->{char}}, $next_char;
            }
          }
          $self->{next_input_character} = 0x000A; # LF # MUST
        } elsif ($self->{next_input_character} > 0x10FFFF) {
          $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
        } elsif ($self->{next_input_character} == 0x0000) { # NULL
          $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
        }
      };
      
      my @token;
      $p->{parse_error} = sub {
        push @token, 'ParseError';
      };
      
      $p->_initialize_tokenizer;
      $p->{content_model_flag} = $cm;
      $p->{last_emitted_start_tag_name} = $last_start_tag;

      while (1) {
        my $token = $p->_get_next_token;
        last if $token->{type} eq 'end-of-file';
        
        my $test_token = [
         {
          DOCTYPE => 'DOCTYPE',
          'start tag' => 'StartTag',
          'end tag' => 'EndTag',
          comment => 'Comment',
          character => 'Character',
         }->{$token->{type}} || $token->{type},
        ];
        $test_token->[1] = $token->{name} if defined $token->{name};
        $test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
        $test_token->[1] = $token->{data} if defined $token->{data};
        $test_token->[2] = $token->{error} ? 1 : 0 if $token->{type} eq 'DOCTYPE';
        $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}}
          if $token->{type} eq 'start tag';
        
        if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
            $test_token->[0] eq 'Character') {
          $token[-1]->[1] .= $test_token->[1];
        } else {
          push @token, $test_token;
        }
      }
      
      my $expected_dump = Dumper ($test->{output});
      my $parser_dump = Dumper (\@token);
      ok $parser_dump, $expected_dump,
        $test->{description} . ': ' . $test->{input};
    }
  }
}

## $Date:$
1	#!/usr/bin/perl
2	use strict;
3
4	my $dir_name;
5	BEGIN {
6	$dir_name = 't/tokenizer/';
7	my $skip = "You don't have JSON module";
8	eval q{
9	use JSON 1.00;
10	$skip = "You don't have make command";
11	system ('cd $dir_name; cd ..; make tokenizer-files') == 0 or die
12	unless -f $dir_name.'test1.test';
13	$skip = '';
14	};
15	if ($skip) {
16	print "1..1\n";
17	print "ok 1 # $skip\n";
18	exit;
19	}
20	$JSON::UnMapping = 1;
21	$JSON::UTF8 = 1;
22	}
23
24	use Test;
25	BEGIN { plan tests => 38 }
26
27	use Data::Dumper;
28	$Data::Dumper::Useqq = 1;
29	sub Data::Dumper::qquote {
30	my $s = shift;
31	$s =~ s/([^\x20\x21-\x26\x28-\x5B\x5D-\x7E])/sprintf '\x{%02X}', ord $1/ge;
32	return q<qq'> . $s . q<'>;
33	} # Data::Dumper::qquote
34
35	use What::HTML;
36	use Encode;
37
38	for my $file_name (qw[
39	test1.test
40	test2.test
41	contentModelFlags.test
42	]) {
43	open my $file, '<', $dir_name.$file_name
44	or die "$0: $dir_name$file_name: $!";
45	local $/ = undef;
46	my $js = <$file>;
47	close $file;
48
49	my $tests = jsonToObj ($js)->{tests};
50	TEST: for my $test (@$tests) {
51	my $s = $test->{input};
52
53	my $j = 1;
54	while ($j < @{$test->{output}}) {
55	if (ref $test->{output}->[$j - 1] and
56	$test->{output}->[$j - 1]->[0] eq 'Character' and
57	ref $test->{output}->[$j] and
58	$test->{output}->[$j]->[0] eq 'Character') {
59	$test->{output}->[$j - 1]->[1]
60	.= $test->{output}->[$j]->[1];
61	splice @{$test->{output}}, $j, 1;
62	}
63	$j++;
64	}
65
66	my @cm = @{$test->{contentModelFlags} \|\| ['PCDATA']};
67	my $last_start_tag = $test->{lastStartTag};
68	for my $cm (@cm) {
69	my $p = What::HTML->new;
70	my $i = 0;
71	$p->{set_next_input_character} = sub {
72	my $self = shift;
73	$self->{next_input_character} = -1 and return if $i >= length $s;
74	$self->{next_input_character} = ord substr $s, $i++, 1;
75
76	if ($self->{next_input_character} == 0x000D) { # CR
77	if ($i >= length $s) {
78	#
79	} else {
80	my $next_char = ord substr $s, $i++, 1;
81	if ($next_char == 0x000A) { # LF
82	#
83	} else {
84	push @{$self->{char}}, $next_char;
85	}
86	}
87	$self->{next_input_character} = 0x000A; # LF # MUST
88	} elsif ($self->{next_input_character} > 0x10FFFF) {
89	$self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
90	} elsif ($self->{next_input_character} == 0x0000) { # NULL
91	$self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
92	}
93	};
94
95	my @token;
96	$p->{parse_error} = sub {
97	push @token, 'ParseError';
98	};
99
100	$p->_initialize_tokenizer;
101	$p->{content_model_flag} = $cm;
102	$p->{last_emitted_start_tag_name} = $last_start_tag;
103
104	while (1) {
105	my $token = $p->_get_next_token;
106	last if $token->{type} eq 'end-of-file';
107
108	my $test_token = [
109	{
110	DOCTYPE => 'DOCTYPE',
111	'start tag' => 'StartTag',
112	'end tag' => 'EndTag',
113	comment => 'Comment',
114	character => 'Character',
115	}->{$token->{type}} \|\| $token->{type},
116	];
117	$test_token->[1] = $token->{name} if defined $token->{name};
118	$test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
119	$test_token->[1] = $token->{data} if defined $token->{data};
120	$test_token->[2] = $token->{error} ? 1 : 0 if $token->{type} eq 'DOCTYPE';
121	$test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}}
122	if $token->{type} eq 'start tag';
123
124	if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
125	$test_token->[0] eq 'Character') {
126	$token[-1]->[1] .= $test_token->[1];
127	} else {
128	push @token, $test_token;
129	}
130	}
131
132	my $expected_dump = Dumper ($test->{output});
133	my $parser_dump = Dumper (\@token);
134	ok $parser_dump, $expected_dump,
135	$test->{description} . ': ' . $test->{input};
136	}
137	}
138	}
139
140	## $Date:$