# HG changeset patch # User Vincent Tondellier # Date 1455477169 -3600 # Node ID 31013a09b4838fcb10c21867c570cf79ad5bbd2b # Parent 702e246e5c9f9044696fc3f0c26ac3104212fdbe Factorize search between reports and groups Use Mojo::JSON::MaybeXS, and update deps diff -r 702e246e5c9f -r 31013a09b483 dist.ini --- a/dist.ini Sun Feb 14 19:58:33 2016 +0100 +++ b/dist.ini Sun Feb 14 20:12:49 2016 +0100 @@ -21,12 +21,13 @@ format = %-9v %{yyyy-MM-dd}d [Prereqs] -Mojolicious = 5.81 +Mojolicious = 6.00 Mojolicious::Plugin::BootstrapPagination = 0.12 UUID = 0.0.3 Text::Balanced = 0 Data::Page = 0 -Minion = 2.00 +Minion = 4.00 +Mojo::JSON::MaybeXS = 0 [Prereqs / RuntimeRecommends] ; For search field (only for SQL) diff -r 702e246e5c9f -r 31013a09b483 lib/CrashTest/Command/get_trace.pm --- a/lib/CrashTest/Command/get_trace.pm Sun Feb 14 19:58:33 2016 +0100 +++ b/lib/CrashTest/Command/get_trace.pm Sun Feb 14 20:12:49 2016 +0100 @@ -13,6 +13,7 @@ package CrashTest::Command::get_trace; use Mojo::Base 'Mojolicious::Command'; +use Mojo::JSON::MaybeXS; use Mojo::JSON qw/decode_json/; use Mojo::Util qw/slurp/; use File::Spec::Functions qw/catdir catfile/; diff -r 702e246e5c9f -r 31013a09b483 lib/CrashTest/Command/insert.pm --- a/lib/CrashTest/Command/insert.pm Sun Feb 14 19:58:33 2016 +0100 +++ b/lib/CrashTest/Command/insert.pm Sun Feb 14 20:12:49 2016 +0100 @@ -13,6 +13,7 @@ package CrashTest::Command::insert; use Mojo::Base 'Mojolicious::Command'; +use Mojo::JSON::MaybeXS; use Mojo::JSON qw/decode_json/; use Mojo::Util qw/slurp/; use File::Spec::Functions qw/catdir catfile/; diff -r 702e246e5c9f -r 31013a09b483 lib/CrashTest/Plugin/Storage/Sql/Model/CrashGroup.pm --- a/lib/CrashTest/Plugin/Storage/Sql/Model/CrashGroup.pm Sun Feb 14 19:58:33 2016 +0100 +++ b/lib/CrashTest/Plugin/Storage/Sql/Model/CrashGroup.pm Sun Feb 14 20:12:49 2016 +0100 @@ -14,69 +14,37 @@ package CrashTest::Plugin::Storage::Sql::Model::CrashGroup; use Mojo::Base -base; use Data::Page; -#use DBI::Log; -#$DBI::Log::trace = 0; use Search::QueryParser::SQL; use Storable 'dclone'; use CrashTest::Model::Thread; use CrashTest::Plugin::CrashSignatureExtractor::C_Cpp; - +use CrashTest::Plugin::Storage::Sql::Utils; has [ qw/instance app config/ ]; -has qw/db/; +has [ qw/db sql_utils/ ]; sub new { my $self = shift->SUPER::new(@_); $self->db($self->instance->dbh->db); + $self->sql_utils(CrashTest::Plugin::Storage::Sql::Utils->new(@_)); return $self; } sub _build_query_from_search_string { - my ($self, $search, $extra_columns) = @_; - - my @values; - - # define a callback to collect values safely (without magic markers) - my $cb = sub { - my ($column, $this_op, $value) = @_; - if($column->type eq "fuzzy") { - $this_op = " ILIKE "; - $value = "%$value%"; - } - push @values, $value; - return join('', $column->stringify, $this_op, '?'); - }; - - my $search_fields = { - user_id => { callback => $cb, name => 'crash_user.user_id' }, - product => { callback => $cb, name => 'product.name' }, - version => { callback => $cb, name => 'product.version' }, - channel => { callback => $cb, name => 'product.release_channel' }, - function => { callback => $cb, name => 'crash_group.crash_thread_signature_bt', type => "fuzzy" }, - }; - - foreach my $extra_col(@$extra_columns) { - $search_fields->{$extra_col->{id}} = { callback => $cb, name => $extra_col->{db_column} }; - } - - my $parser = Search::QueryParser::SQL->new( - columns => $search_fields, - default_column => "function", - strict => 1, - ); - - my $query = $parser->parse($search, 1) - or die "Error in query: " . $parser->err; - - # reset before calling dbi - @values = (); - my $dbi = $query->dbi(); - - return [ $dbi->[0], \@values ]; + my ($self, $search_str) = @_; + return $self->sql_utils->build_query_from_search_string( + $search_str, + $self->app->config->{WebInterface}->{ExtraColumns}->{Index}, { + user_id => { name => 'crash_user.user_id' }, + product => { name => 'product.name' }, + version => { name => 'product.version' }, + channel => { name => 'product.release_channel' }, + function => { name => 'crash_group.crash_thread_signature_bt', type => "fuzzy" }, + }); } sub index { @@ -85,7 +53,7 @@ my $where = ""; my @values = (); if(defined($search_str) && $search_str ne "") { - my $q = $self->_build_query_from_search_string($search_str, $self->app->config->{WebInterface}->{ExtraColumns}->{Index}); + my $q = $self->_build_query_from_search_string($search_str); $where = "WHERE " . $q->[0]; @values = @{$q->[1]}; } diff -r 702e246e5c9f -r 31013a09b483 lib/CrashTest/Plugin/Storage/Sql/Model/CrashReport.pm --- a/lib/CrashTest/Plugin/Storage/Sql/Model/CrashReport.pm Sun Feb 14 19:58:33 2016 +0100 +++ b/lib/CrashTest/Plugin/Storage/Sql/Model/CrashReport.pm Sun Feb 14 20:12:49 2016 +0100 @@ -13,7 +13,8 @@ package CrashTest::Plugin::Storage::Sql::Model::CrashReport; use Mojo::Base -base; -use Mojo::JSON qw/j decode_json/; +use Mojo::JSON::MaybeXS; +use Mojo::JSON qw/encode_json decode_json/; use DateTime; use Data::Page; #use DBI::Log; @@ -21,64 +22,34 @@ use Search::QueryParser::SQL; use CrashTest::Plugin::Storage::Sql::Model::CrashGroup; +use CrashTest::Plugin::Storage::Sql::Utils; has [ qw/instance app config/ ]; -has qw/db/; -has qw/crash_groups/; +has [ qw/db sql_utils crash_groups/ ]; sub new { my $self = shift->SUPER::new(@_); $self->db($self->instance->dbh->db); + $self->sql_utils(CrashTest::Plugin::Storage::Sql::Utils->new(@_)); $self->crash_groups(CrashTest::Plugin::Storage::Sql::Model::CrashGroup->new(@_)); return $self; } sub _build_query_from_search_string { - my ($self, $search, $extra_columns) = @_; - - my @values; - - # define a callback to collect values safely (without magic markers) - my $cb = sub { - my ($column, $this_op, $value) = @_; - if($column->type eq "fuzzy") { - $this_op = " ILIKE "; - $value = "%$value%"; - } - push @values, $value; - return join('', $column->stringify, $this_op, '?'); - }; - - my $search_fields = { - user_id => { callback => $cb, name => 'crash_user.user_id' }, - product => { callback => $cb, name => 'product.name' }, - version => { callback => $cb, name => 'product.version' }, - channel => { callback => $cb, name => 'product.release_channel' }, - group_id => { callback => $cb, name => 'crash_reports.crash_group_id' }, - function => { callback => $cb, name => 'crash_group.crash_thread_signature_bt', type => "fuzzy" }, - }; - - foreach my $extra_col(@$extra_columns) { - $search_fields->{$extra_col->{id}} = { callback => $cb, name => $extra_col->{db_column} }; - } - - my $parser = Search::QueryParser::SQL->new( - columns => $search_fields, - default_column => "function", - strict => 1, - ); - - my $query = $parser->parse($search, 1) - or die "Error in query: " . $parser->err; - - # reset before calling dbi - @values = (); - my $dbi = $query->dbi(); - - return [ $dbi->[0], \@values ]; + my ($self, $search_str) = @_; + return $self->sql_utils->build_query_from_search_string( + $search_str, + $self->app->config->{WebInterface}->{ExtraColumns}->{Index}, { + user_id => { name => 'crash_user.user_id' }, + product => { name => 'product.name' }, + version => { name => 'product.version' }, + channel => { name => 'product.release_channel' }, + group_id => { name => 'crash_reports.crash_group_id' }, + function => { name => 'crash_group.crash_thread_signature_bt', type => "fuzzy" }, + }); } sub index { @@ -87,7 +58,7 @@ my $where = ""; my @values = (); if(defined($search_str) && $search_str ne "") { - my $q = $self->_build_query_from_search_string($search_str, $self->app->config->{WebInterface}->{ExtraColumns}->{Index}); + my $q = $self->_build_query_from_search_string($search_str); $where = "WHERE " . $q->[0]; @values = @{$q->[1]}; } @@ -186,7 +157,7 @@ push @user_values, $pjson->{system_info}->{cpu_arch}; push @user_values, $pjson->{system_info}->{cpu_count}; push @user_values, $pjson->{system_info}->{os}; - push @user_values, j($client_info); + push @user_values, encode_json($client_info); $dbuser = $self->db->query( "INSERT INTO crash_users (user_id, cpu_arch, cpu_count, os, extra_info) VALUES (?, ?, ?, ?, ?) RETURNING *", @@ -212,7 +183,7 @@ $self->db->query( "INSERT INTO crash_report_datas (crash_report_id, processed) VALUES (?, ?) RETURNING id", - $dbcrash->{id}, j($pjson) + $dbcrash->{id}, encode_json($pjson) ); $tx->commit; diff -r 702e246e5c9f -r 31013a09b483 lib/CrashTest/Plugin/Storage/Sql/Utils.pm --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/CrashTest/Plugin/Storage/Sql/Utils.pm Sun Feb 14 20:12:49 2016 +0100 @@ -0,0 +1,76 @@ +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +package CrashTest::Plugin::Storage::Sql::Utils; +use Mojo::Base -base; +use Storable 'dclone'; + +has [ qw/instance app config/ ]; + +has qw/db/; + +sub new { + my $self = shift->SUPER::new(@_); + + $self->db($self->instance->dbh->db); + + return $self; +} + +sub build_query_from_search_string { + my ($self, $search, $extra_columns, $extra_search_fields) = @_; + + my @values; + + # define a callback to collect values safely (without magic markers) + my $cb = sub { + my ($column, $this_op, $value) = @_; + if($column->type eq "fuzzy") { + if ( $this_op =~ /!|NOT/ ) { + $this_op = " NOT ILIKE "; + } else { + $this_op = " ILIKE "; + } + $value = "%$value%"; + } + push @values, $value; + return join('', $column->stringify, $this_op, '?'); + }; + + my $search_fields = dclone($extra_search_fields); + + foreach my $k(keys %$search_fields) { + $search_fields->{$k}->{callback} = $cb; + } + + foreach my $extra_col(@$extra_columns) { + $search_fields->{$extra_col->{id}} = { callback => $cb, name => $extra_col->{db_column} }; + } + + my $parser = Search::QueryParser::SQL->new( + columns => $search_fields, + default_column => "function", + strict => 1, + ); + + my $query = $parser->parse($search, 1) + or die "Error in query: " . $parser->err; + + # reset before calling dbi + @values = (); + my $dbi = $query->dbi(); + + return [ $dbi->[0], \@values ]; +} + +1;