-
Notifications
You must be signed in to change notification settings - Fork 2
/
parse.pl
74 lines (61 loc) · 1.57 KB
/
parse.pl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#! /usr/bin/perl
# Parses files in a single directory with the specified parser.
use strict;
use warnings;
use FindBin;
use lib "$FindBin::Bin/..";
use GoogleDownloader;
use GoogleParser;
use SubparserManager;
my $usage = "Usage parse.pl parser download-path results-path\nparse.pl list will list available parsers\n";
my $subparserManager = SubparserManager->new();
if (scalar(@ARGV) == 0) {
print $usage;
exit(0);
}
if (scalar(@ARGV) != 3) {
print $usage;
if (scalar(@ARGV) > 0 && lc($ARGV[0]) eq "list") {
print "all - All parsers in below list will be run\n";
my %subparsers = $subparserManager->getSubparsers('');
foreach my $key (keys(%subparsers)) {
print "$key\n";
}
}
exit(0);
}
my $parser = lc(shift @ARGV);
my $downloadDir = shift @ARGV;
my $resultDir = shift @ARGV;
my %subparsers = $subparserManager->getSubparsers($resultDir);
my @subparserList = values(%subparsers);
my $googleParser;
my @parsers;
if ($parser eq "all") {
@parsers = @subparserList;
} else {
if (defined($subparsers{$parser})) {
@parsers = ($subparsers{$parser});
} else {
print "No such parser $parser. Use runner.pl list to find available parsers\n";
exit(1);
}
}
foreach my $parserObj (@parsers) {
$parserObj->init();
}
opendir(DIR, $downloadDir);
my @files = readdir(DIR);
closedir(DIR);
my $id = 1;
foreach my $file (@files) {
if ($file !~ m/^\./ && $file =~ m/\.htm/) {
foreach my $parserObj (@parsers) {
$parserObj->parse($id, "$downloadDir/$file");
$id++;
}
}
}
foreach my $parserObj (@parsers) {
$parserObj->teardown();
}