| author | Vincent Tondellier <tonton+hg@team1664.org> |
| Thu, 31 Jul 2014 00:05:00 +0200 | |
| changeset 25 | 1a1cf5c35cfb |
| parent 20 | 169c73eb8881 |
| child 34 | e0d6597078a5 |
| permissions | -rwxr-xr-x |
|
18
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
1 |
#!/usr/bin/env perl |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
2 |
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
3 |
# This program is free software: you can redistribute it and/or modify |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
4 |
# it under the terms of the GNU General Public License as published by |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
5 |
# the Free Software Foundation, either version 3 of the License, or |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
6 |
# (at your option) any later version. |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
7 |
# |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
8 |
# This program is distributed in the hope that it will be useful, |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
11 |
# GNU General Public License for more details. |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
12 |
# |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
13 |
# You should have received a copy of the GNU General Public License |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
14 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
15 |
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
16 |
use Gearman::Worker; |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
17 |
use Mojo::JSON; |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
18 |
use Mojo::Util qw(decode slurp); |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
19 |
use File::Temp; |
|
19
300b902b5461
Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents:
18
diff
changeset
|
20 |
use lib 'lib'; |
|
18
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
21 |
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
22 |
sub load_config {
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
23 |
my ($file) = @_; |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
24 |
my $code = decode('UTF-8', slurp $file);
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
25 |
return eval($code); |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
26 |
} |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
27 |
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
28 |
my $config = load_config($ARGV[0]); |
|
19
300b902b5461
Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents:
18
diff
changeset
|
29 |
|
|
300b902b5461
Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents:
18
diff
changeset
|
30 |
eval "require $config->{Storage}->{Type}";
|
|
300b902b5461
Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents:
18
diff
changeset
|
31 |
my $storage = $config->{Storage}->{Type}->new($config->{Storage});
|
|
300b902b5461
Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents:
18
diff
changeset
|
32 |
|
|
300b902b5461
Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents:
18
diff
changeset
|
33 |
my $worker = Gearman::Worker->new(job_servers => $config->{DecodeQueue}->{GearmanServers});
|
|
300b902b5461
Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents:
18
diff
changeset
|
34 |
|
|
18
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
35 |
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
36 |
$worker->register_function("dump_decode", 60, sub {
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
37 |
my $args = $_[0]->arg; |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
38 |
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
39 |
my $json = Mojo::JSON->new(); |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
40 |
my $jsonin = $json->decode($args); |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
41 |
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
42 |
my $file = File::Temp->new(); |
|
19
300b902b5461
Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents:
18
diff
changeset
|
43 |
binmode($file); |
|
18
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
44 |
print $file $jsonin->{file};
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
45 |
|
|
19
300b902b5461
Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents:
18
diff
changeset
|
46 |
my $cmd = $config->{Dumper}->{JSONStackwalker} . " " . $file->filename . " " . $config->{Dumper}->{SymbolsPath};
|
|
18
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
47 |
my $out = qx($cmd 2>/dev/null) or die $!; |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
48 |
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
49 |
my $pjson = $json->decode($out); |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
50 |
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
51 |
# Create json for the params |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
52 |
$pjson->{client_info} = $jsonin->{params};
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
53 |
|
|
19
300b902b5461
Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents:
18
diff
changeset
|
54 |
my $uuidstr = $jsonin->{uuid};
|
|
20
169c73eb8881
Store file content, not temporary file name
Vincent Tondellier <tonton+hg@team1664.org>
parents:
19
diff
changeset
|
55 |
$storage->store_dump($uuidstr, slurp $file); |
|
19
300b902b5461
Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents:
18
diff
changeset
|
56 |
$storage->store_processed_data($uuidstr, $pjson); |
|
300b902b5461
Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents:
18
diff
changeset
|
57 |
|
|
18
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
58 |
return $json->encode($pjson); |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
59 |
}); |
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
60 |
|
|
60db1090095b
Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff
changeset
|
61 |
$worker->work() while 1; |