Factorize search between reports and groups
Use Mojo::JSON::MaybeXS, and update deps
--- a/dist.ini Sun Feb 14 19:58:33 2016 +0100
+++ b/dist.ini Sun Feb 14 20:12:49 2016 +0100
@@ -21,12 +21,13 @@
format = %-9v %{yyyy-MM-dd}d
[Prereqs]
-Mojolicious = 5.81
+Mojolicious = 6.00
Mojolicious::Plugin::BootstrapPagination = 0.12
UUID = 0.0.3
Text::Balanced = 0
Data::Page = 0
-Minion = 2.00
+Minion = 4.00
+Mojo::JSON::MaybeXS = 0
[Prereqs / RuntimeRecommends]
; For search field (only for SQL)
--- a/lib/CrashTest/Command/get_trace.pm Sun Feb 14 19:58:33 2016 +0100
+++ b/lib/CrashTest/Command/get_trace.pm Sun Feb 14 20:12:49 2016 +0100
@@ -13,6 +13,7 @@
package CrashTest::Command::get_trace;
use Mojo::Base 'Mojolicious::Command';
+use Mojo::JSON::MaybeXS;
use Mojo::JSON qw/decode_json/;
use Mojo::Util qw/slurp/;
use File::Spec::Functions qw/catdir catfile/;
--- a/lib/CrashTest/Command/insert.pm Sun Feb 14 19:58:33 2016 +0100
+++ b/lib/CrashTest/Command/insert.pm Sun Feb 14 20:12:49 2016 +0100
@@ -13,6 +13,7 @@
package CrashTest::Command::insert;
use Mojo::Base 'Mojolicious::Command';
+use Mojo::JSON::MaybeXS;
use Mojo::JSON qw/decode_json/;
use Mojo::Util qw/slurp/;
use File::Spec::Functions qw/catdir catfile/;
--- a/lib/CrashTest/Plugin/Storage/Sql/Model/CrashGroup.pm Sun Feb 14 19:58:33 2016 +0100
+++ b/lib/CrashTest/Plugin/Storage/Sql/Model/CrashGroup.pm Sun Feb 14 20:12:49 2016 +0100
@@ -14,69 +14,37 @@
package CrashTest::Plugin::Storage::Sql::Model::CrashGroup;
use Mojo::Base -base;
use Data::Page;
-#use DBI::Log;
-#$DBI::Log::trace = 0;
use Search::QueryParser::SQL;
use Storable 'dclone';
use CrashTest::Model::Thread;
use CrashTest::Plugin::CrashSignatureExtractor::C_Cpp;
-
+use CrashTest::Plugin::Storage::Sql::Utils;
has [ qw/instance app config/ ];
-has qw/db/;
+has [ qw/db sql_utils/ ];
sub new {
my $self = shift->SUPER::new(@_);
$self->db($self->instance->dbh->db);
+ $self->sql_utils(CrashTest::Plugin::Storage::Sql::Utils->new(@_));
return $self;
}
sub _build_query_from_search_string {
- my ($self, $search, $extra_columns) = @_;
-
- my @values;
-
- # define a callback to collect values safely (without magic markers)
- my $cb = sub {
- my ($column, $this_op, $value) = @_;
- if($column->type eq "fuzzy") {
- $this_op = " ILIKE ";
- $value = "%$value%";
- }
- push @values, $value;
- return join('', $column->stringify, $this_op, '?');
- };
-
- my $search_fields = {
- user_id => { callback => $cb, name => 'crash_user.user_id' },
- product => { callback => $cb, name => 'product.name' },
- version => { callback => $cb, name => 'product.version' },
- channel => { callback => $cb, name => 'product.release_channel' },
- function => { callback => $cb, name => 'crash_group.crash_thread_signature_bt', type => "fuzzy" },
- };
-
- foreach my $extra_col(@$extra_columns) {
- $search_fields->{$extra_col->{id}} = { callback => $cb, name => $extra_col->{db_column} };
- }
-
- my $parser = Search::QueryParser::SQL->new(
- columns => $search_fields,
- default_column => "function",
- strict => 1,
- );
-
- my $query = $parser->parse($search, 1)
- or die "Error in query: " . $parser->err;
-
- # reset before calling dbi
- @values = ();
- my $dbi = $query->dbi();
-
- return [ $dbi->[0], \@values ];
+ my ($self, $search_str) = @_;
+ return $self->sql_utils->build_query_from_search_string(
+ $search_str,
+ $self->app->config->{WebInterface}->{ExtraColumns}->{Index}, {
+ user_id => { name => 'crash_user.user_id' },
+ product => { name => 'product.name' },
+ version => { name => 'product.version' },
+ channel => { name => 'product.release_channel' },
+ function => { name => 'crash_group.crash_thread_signature_bt', type => "fuzzy" },
+ });
}
sub index {
@@ -85,7 +53,7 @@
my $where = "";
my @values = ();
if(defined($search_str) && $search_str ne "") {
- my $q = $self->_build_query_from_search_string($search_str, $self->app->config->{WebInterface}->{ExtraColumns}->{Index});
+ my $q = $self->_build_query_from_search_string($search_str);
$where = "WHERE " . $q->[0];
@values = @{$q->[1]};
}
--- a/lib/CrashTest/Plugin/Storage/Sql/Model/CrashReport.pm Sun Feb 14 19:58:33 2016 +0100
+++ b/lib/CrashTest/Plugin/Storage/Sql/Model/CrashReport.pm Sun Feb 14 20:12:49 2016 +0100
@@ -13,7 +13,8 @@
package CrashTest::Plugin::Storage::Sql::Model::CrashReport;
use Mojo::Base -base;
-use Mojo::JSON qw/j decode_json/;
+use Mojo::JSON::MaybeXS;
+use Mojo::JSON qw/encode_json decode_json/;
use DateTime;
use Data::Page;
#use DBI::Log;
@@ -21,64 +22,34 @@
use Search::QueryParser::SQL;
use CrashTest::Plugin::Storage::Sql::Model::CrashGroup;
+use CrashTest::Plugin::Storage::Sql::Utils;
has [ qw/instance app config/ ];
-has qw/db/;
-has qw/crash_groups/;
+has [ qw/db sql_utils crash_groups/ ];
sub new {
my $self = shift->SUPER::new(@_);
$self->db($self->instance->dbh->db);
+ $self->sql_utils(CrashTest::Plugin::Storage::Sql::Utils->new(@_));
$self->crash_groups(CrashTest::Plugin::Storage::Sql::Model::CrashGroup->new(@_));
return $self;
}
sub _build_query_from_search_string {
- my ($self, $search, $extra_columns) = @_;
-
- my @values;
-
- # define a callback to collect values safely (without magic markers)
- my $cb = sub {
- my ($column, $this_op, $value) = @_;
- if($column->type eq "fuzzy") {
- $this_op = " ILIKE ";
- $value = "%$value%";
- }
- push @values, $value;
- return join('', $column->stringify, $this_op, '?');
- };
-
- my $search_fields = {
- user_id => { callback => $cb, name => 'crash_user.user_id' },
- product => { callback => $cb, name => 'product.name' },
- version => { callback => $cb, name => 'product.version' },
- channel => { callback => $cb, name => 'product.release_channel' },
- group_id => { callback => $cb, name => 'crash_reports.crash_group_id' },
- function => { callback => $cb, name => 'crash_group.crash_thread_signature_bt', type => "fuzzy" },
- };
-
- foreach my $extra_col(@$extra_columns) {
- $search_fields->{$extra_col->{id}} = { callback => $cb, name => $extra_col->{db_column} };
- }
-
- my $parser = Search::QueryParser::SQL->new(
- columns => $search_fields,
- default_column => "function",
- strict => 1,
- );
-
- my $query = $parser->parse($search, 1)
- or die "Error in query: " . $parser->err;
-
- # reset before calling dbi
- @values = ();
- my $dbi = $query->dbi();
-
- return [ $dbi->[0], \@values ];
+ my ($self, $search_str) = @_;
+ return $self->sql_utils->build_query_from_search_string(
+ $search_str,
+ $self->app->config->{WebInterface}->{ExtraColumns}->{Index}, {
+ user_id => { name => 'crash_user.user_id' },
+ product => { name => 'product.name' },
+ version => { name => 'product.version' },
+ channel => { name => 'product.release_channel' },
+ group_id => { name => 'crash_reports.crash_group_id' },
+ function => { name => 'crash_group.crash_thread_signature_bt', type => "fuzzy" },
+ });
}
sub index {
@@ -87,7 +58,7 @@
my $where = "";
my @values = ();
if(defined($search_str) && $search_str ne "") {
- my $q = $self->_build_query_from_search_string($search_str, $self->app->config->{WebInterface}->{ExtraColumns}->{Index});
+ my $q = $self->_build_query_from_search_string($search_str);
$where = "WHERE " . $q->[0];
@values = @{$q->[1]};
}
@@ -186,7 +157,7 @@
push @user_values, $pjson->{system_info}->{cpu_arch};
push @user_values, $pjson->{system_info}->{cpu_count};
push @user_values, $pjson->{system_info}->{os};
- push @user_values, j($client_info);
+ push @user_values, encode_json($client_info);
$dbuser = $self->db->query(
"INSERT INTO crash_users (user_id, cpu_arch, cpu_count, os, extra_info) VALUES (?, ?, ?, ?, ?) RETURNING *",
@@ -212,7 +183,7 @@
$self->db->query(
"INSERT INTO crash_report_datas (crash_report_id, processed) VALUES (?, ?) RETURNING id",
- $dbcrash->{id}, j($pjson)
+ $dbcrash->{id}, encode_json($pjson)
);
$tx->commit;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/CrashTest/Plugin/Storage/Sql/Utils.pm Sun Feb 14 20:12:49 2016 +0100
@@ -0,0 +1,76 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package CrashTest::Plugin::Storage::Sql::Utils;
+use Mojo::Base -base;
+use Storable 'dclone';
+
+has [ qw/instance app config/ ];
+
+has qw/db/;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+
+ $self->db($self->instance->dbh->db);
+
+ return $self;
+}
+
+sub build_query_from_search_string {
+ my ($self, $search, $extra_columns, $extra_search_fields) = @_;
+
+ my @values;
+
+ # define a callback to collect values safely (without magic markers)
+ my $cb = sub {
+ my ($column, $this_op, $value) = @_;
+ if($column->type eq "fuzzy") {
+ if ( $this_op =~ /!|NOT/ ) {
+ $this_op = " NOT ILIKE ";
+ } else {
+ $this_op = " ILIKE ";
+ }
+ $value = "%$value%";
+ }
+ push @values, $value;
+ return join('', $column->stringify, $this_op, '?');
+ };
+
+ my $search_fields = dclone($extra_search_fields);
+
+ foreach my $k(keys %$search_fields) {
+ $search_fields->{$k}->{callback} = $cb;
+ }
+
+ foreach my $extra_col(@$extra_columns) {
+ $search_fields->{$extra_col->{id}} = { callback => $cb, name => $extra_col->{db_column} };
+ }
+
+ my $parser = Search::QueryParser::SQL->new(
+ columns => $search_fields,
+ default_column => "function",
+ strict => 1,
+ );
+
+ my $query = $parser->parse($search, 1)
+ or die "Error in query: " . $parser->err;
+
+ # reset before calling dbi
+ @values = ();
+ my $dbi = $query->dbi();
+
+ return [ $dbi->[0], \@values ];
+}
+
+1;