whatpm/t/HTML-tokenizer.t

#!/usr/bin/perl
use strict;

BEGIN {
  my $skip = "You don't have JSON module";
  eval q{
         use JSON 1.00;
         $skip = "You don't have make command";
         system ('make', 'tokenizer-files') == 0 or die
           unless -f 'tokenizer/test1.test';
         $skip = '';
        };
  if ($skip) {
    print "1..1\n";
    print "ok 1 # $skip\n";
    exit;
  }
  $JSON::UnMapping = 1;
}

use Test;
use Data::Dumper;
BEGIN { plan tests => 38 }

use What::HTML;

for my $file_name (qw[
                      tokenizer/test1.test
                      tokenizer/test2.test
                      tokenizer/contentModelFlags.test
                     ]) {
  open my $file, '<:utf8', $file_name or die "$0: $file_name: $!";
  local $/ = undef;
  my $js = <$file>;
  close $file;
  
  my $tests = jsonToObj ($js)->{tests};
  TEST: for my $test (@$tests) {
    my $s = $test->{input};
    
    my $j = 1;
    while ($j < @{$test->{output}}) {
      if (ref $test->{output}->[$j - 1] and
          $test->{output}->[$j - 1]->[0] eq 'Character' and
          ref $test->{output}->[$j] and 
          $test->{output}->[$j]->[0] eq 'Character') {
        $test->{output}->[$j - 1]->[1]
          .= $test->{output}->[$j]->[1];
        splice @{$test->{output}}, $j, 1;
      }
      $j++;
    }

    my @cm = @{$test->{content_model_flags} || ['PCDATA']};
    for my $cm (@cm) {
      my $p = What::HTML->new;
      my $i = 0;
      $p->{set_next_input_character} = sub {
        my $self = shift;
        $self->{next_input_character} = -1 and return if $i >= length $s;
        $self->{next_input_character} = ord substr $s, $i++, 1;
      };
      
      my @token;
      $p->{parse_error} = sub {
        push @token, 'ParseError';
      };
      
      $p->_initialize_tokenizer;
      $p->{content_model_flag} = $cm;

      while (1) {
        my $token = $p->_get_next_token;
        last if $token->{type} eq 'end-of-file';
        
        my $test_token = [
         {
          DOCTYPE => 'DOCTYPE',
          'start tag' => 'StartTag',
          'end tag' => 'EndTag',
          comment => 'Comment',
          character => 'Character',
         }->{$token->{type}} || $token->{type},
        ];
        $test_token->[1] = $token->{name} if defined $token->{name};
        $test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
        $test_token->[1] = $token->{data} if defined $token->{data};
        $test_token->[2] = $token->{error} ? 1 : 0 if $token->{type} eq 'DOCTYPE';
        $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}}
          if $token->{type} eq 'start tag';
        
        if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
            $test_token->[0] eq 'Character') {
          $token[-1]->[1] .= $test_token->[1];
        } else {
          push @token, $test_token;
        }
      }
      
      my $expected_dump = Dumper ($test->{output});
      my $parser_dump = Dumper (\@token);
      ok $parser_dump, $expected_dump, $test->{description};
    }
  }
}

1	#!/usr/bin/perl
2	use strict;
3
4	BEGIN {
5	my $skip = "You don't have JSON module";
6	eval q{
7	use JSON 1.00;
8	$skip = "You don't have make command";
9	system ('make', 'tokenizer-files') == 0 or die
10	unless -f 'tokenizer/test1.test';
11	$skip = '';
12	};
13	if ($skip) {
14	print "1..1\n";
15	print "ok 1 # $skip\n";
16	exit;
17	}
18	$JSON::UnMapping = 1;
19	}
20
21	use Test;
22	use Data::Dumper;
23	BEGIN { plan tests => 38 }
24
25	use What::HTML;
26
27	for my $file_name (qw[
28	tokenizer/test1.test
29	tokenizer/test2.test
30	tokenizer/contentModelFlags.test
31	]) {
32	open my $file, '<:utf8', $file_name or die "$0: $file_name: $!";
33	local $/ = undef;
34	my $js = <$file>;
35	close $file;
36
37	my $tests = jsonToObj ($js)->{tests};
38	TEST: for my $test (@$tests) {
39	my $s = $test->{input};
40
41	my $j = 1;
42	while ($j < @{$test->{output}}) {
43	if (ref $test->{output}->[$j - 1] and
44	$test->{output}->[$j - 1]->[0] eq 'Character' and
45	ref $test->{output}->[$j] and
46	$test->{output}->[$j]->[0] eq 'Character') {
47	$test->{output}->[$j - 1]->[1]
48	.= $test->{output}->[$j]->[1];
49	splice @{$test->{output}}, $j, 1;
50	}
51	$j++;
52	}
53
54	my @cm = @{$test->{content_model_flags} \|\| ['PCDATA']};
55	for my $cm (@cm) {
56	my $p = What::HTML->new;
57	my $i = 0;
58	$p->{set_next_input_character} = sub {
59	my $self = shift;
60	$self->{next_input_character} = -1 and return if $i >= length $s;
61	$self->{next_input_character} = ord substr $s, $i++, 1;
62	};
63
64	my @token;
65	$p->{parse_error} = sub {
66	push @token, 'ParseError';
67	};
68
69	$p->_initialize_tokenizer;
70	$p->{content_model_flag} = $cm;
71
72	while (1) {
73	my $token = $p->_get_next_token;
74	last if $token->{type} eq 'end-of-file';
75
76	my $test_token = [
77	{
78	DOCTYPE => 'DOCTYPE',
79	'start tag' => 'StartTag',
80	'end tag' => 'EndTag',
81	comment => 'Comment',
82	character => 'Character',
83	}->{$token->{type}} \|\| $token->{type},
84	];
85	$test_token->[1] = $token->{name} if defined $token->{name};
86	$test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
87	$test_token->[1] = $token->{data} if defined $token->{data};
88	$test_token->[2] = $token->{error} ? 1 : 0 if $token->{type} eq 'DOCTYPE';
89	$test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}}
90	if $token->{type} eq 'start tag';
91
92	if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
93	$test_token->[0] eq 'Character') {
94	$token[-1]->[1] .= $test_token->[1];
95	} else {
96	push @token, $test_token;
97	}
98	}
99
100	my $expected_dump = Dumper ($test->{output});
101	my $parser_dump = Dumper (\@token);
102	ok $parser_dump, $expected_dump, $test->{description};
103	}
104	}
105	}
106