bin/gearman_decode_worker.pl
author Vincent Tondellier <tonton+hg@team1664.org>
Wed, 25 Jun 2014 00:16:09 +0200
changeset 21 fab315d8c7a8
parent 20 169c73eb8881
child 34 e0d6597078a5
permissions -rwxr-xr-x
Added tag 0.0.1 for changeset e9c1591a4f54
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
18
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
     1
#!/usr/bin/env perl
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
     2
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
     3
# This program is free software: you can redistribute it and/or modify
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
     4
# it under the terms of the GNU General Public License as published by
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
     5
# the Free Software Foundation, either version 3 of the License, or
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
     6
# (at your option) any later version.
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
     7
#
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
     8
# This program is distributed in the hope that it will be useful,
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
     9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    11
# GNU General Public License for more details.
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    12
#
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    13
# You should have received a copy of the GNU General Public License
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    14
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    15
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    16
use Gearman::Worker;
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    17
use Mojo::JSON;
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    18
use Mojo::Util qw(decode slurp);
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    19
use File::Temp;
19
300b902b5461 Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents: 18
diff changeset
    20
use lib 'lib';
18
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    21
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    22
sub load_config {
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    23
    my ($file) = @_;
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    24
    my $code = decode('UTF-8', slurp $file);
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    25
    return eval($code);
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    26
}
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    27
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    28
my $config = load_config($ARGV[0]);
19
300b902b5461 Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents: 18
diff changeset
    29
300b902b5461 Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents: 18
diff changeset
    30
eval "require $config->{Storage}->{Type}";
300b902b5461 Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents: 18
diff changeset
    31
my $storage = $config->{Storage}->{Type}->new($config->{Storage});
300b902b5461 Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents: 18
diff changeset
    32
300b902b5461 Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents: 18
diff changeset
    33
my $worker = Gearman::Worker->new(job_servers => $config->{DecodeQueue}->{GearmanServers});
300b902b5461 Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents: 18
diff changeset
    34
18
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    35
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    36
$worker->register_function("dump_decode", 60, sub {
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    37
    my $args = $_[0]->arg;
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    38
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    39
    my $json = Mojo::JSON->new();
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    40
    my $jsonin = $json->decode($args);
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    41
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    42
    my $file = File::Temp->new();
19
300b902b5461 Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents: 18
diff changeset
    43
    binmode($file);
18
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    44
    print $file $jsonin->{file};
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    45
19
300b902b5461 Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents: 18
diff changeset
    46
    my $cmd = $config->{Dumper}->{JSONStackwalker} . " " . $file->filename . " " . $config->{Dumper}->{SymbolsPath};
18
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    47
    my $out = qx($cmd 2>/dev/null) or die $!;
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    48
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    49
    my $pjson = $json->decode($out);
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    50
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    51
    # Create json for the params
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    52
    $pjson->{client_info} = $jsonin->{params};
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    53
19
300b902b5461 Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents: 18
diff changeset
    54
    my $uuidstr = $jsonin->{uuid};
20
169c73eb8881 Store file content, not temporary file name
Vincent Tondellier <tonton+hg@team1664.org>
parents: 19
diff changeset
    55
    $storage->store_dump($uuidstr, slurp $file);
19
300b902b5461 Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents: 18
diff changeset
    56
    $storage->store_processed_data($uuidstr, $pjson);
300b902b5461 Reorganize config
Vincent Tondellier <tonton+hg@team1664.org>
parents: 18
diff changeset
    57
18
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    58
    return $json->encode($pjson);
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    59
});
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    60
60db1090095b Add Gearman decode worker
Vincent Tondellier <tonton+hg@team1664.org>
parents:
diff changeset
    61
$worker->work() while 1;