From ddbd22bd5c71e5b0f2e5ca7660a6b94c053d1f49 Mon Sep 17 00:00:00 2001 From: Samir Gondzetovic Date: Wed, 27 Jun 2018 17:45:15 +0100 Subject: [PATCH 01/10] project search update --- src/routes/projects/list.js | 114 ++++++++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 26 deletions(-) diff --git a/src/routes/projects/list.js b/src/routes/projects/list.js index b0b31169..a8e55cba 100755 --- a/src/routes/projects/list.js +++ b/src/routes/projects/list.js @@ -10,6 +10,11 @@ import util from '../../util'; const ES_PROJECT_INDEX = config.get('elasticsearchConfig.indexName'); const ES_PROJECT_TYPE = config.get('elasticsearchConfig.docType'); + +const MATCH_TYPE_EXACT_PHRASE = 1; +const MATCH_TYPE_WILDCARD = 2; +const MATCH_TYPE_SINGLE_FIELD = 3; + /** * API to handle retrieving projects * @@ -40,6 +45,59 @@ const PROJECT_PHASE_PRODUCTS_ATTRIBUTES = _.without( const escapeEsKeyword = keyword => keyword.replace(/[+-=> { + let should = [ + { + query_string: { + query: (matchType === MATCH_TYPE_EXACT_PHRASE) ? keyword : `*${keyword}*`, + analyze_wildcard: (matchType === MATCH_TYPE_WILDCARD), + fields: ['name^5', 'description^3', 'type^2'], + }, + }, + { + nested: { + path: 'details', + query: { + nested: { + path: 'details.utm', + query: { + query_string: { + query: (matchType === MATCH_TYPE_EXACT_PHRASE) ? keyword :`*${keyword}*`, + analyze_wildcard: (matchType === MATCH_TYPE_WILDCARD), + fields: ['details.utm.code^4'], + } + } + } + } + } + }, + { + nested: { + path: 'members', + query: { + query_string: { + query: (matchType === MATCH_TYPE_EXACT_PHRASE) ? keyword : `*${keyword}*`, + analyze_wildcard: (matchType === MATCH_TYPE_WILDCARD), + fields: ['members.email', 'members.handle', 'members.firstName', 'members.lastName'], + }, + }, + }, + }, + ]; + + if (matchType === MATCH_TYPE_SINGLE_FIELD && singleFieldName === 'ref') { + // only need to match the second item in the should array + should = should.slice(1, 2); + } + + return { + bool: { + should, + }, + }; +}; + /** * Parse the ES search criteria and prepare search request body * @@ -126,32 +184,36 @@ const parseElasticSearchCriteria = (criteria, fields, order) => { if (_.has(criteria, 'filters.keyword')) { // keyword is a full text search // escape special fields from keyword search - const keyword = escapeEsKeyword(criteria.filters.keyword); - fullTextQuery = { - bool: { - should: [ - { - query_string: { - query: `*${keyword}*`, - analyze_wildcard: true, - fields: ['name^3', 'description', 'type'], // boost name field - }, - }, - { - nested: { - path: 'members', - query: { - query_string: { - query: `*${keyword}*`, - analyze_wildcard: true, - fields: ['members.email', 'members.handle', 'members.firstName', 'members.lastName'], - }, - }, - }, - }, - ], - }, - }; + const keywordCriterion = criteria.filters.keyword; + let keyword, matchType, singleFieldName; + // check exact phrase match first (starts and ends with double quotes) + if (keywordCriterion.startsWith('"') && keywordCriterion.endsWith('"')) { + keyword = keywordCriterion; + matchType = MATCH_TYPE_EXACT_PHRASE; + } + + if (keywordCriterion.indexOf(' ') > -1 || keywordCriterion.indexOf(':') > -1) { + const parts = keywordCriterion.split(' '); + if (parts.length > 0) { + for (let i = 0; i < parts.length; i++) { + const part = parts[i].trim(); + if (part.length > 4 && part.startsWith('ref:')) { + keyword = escapeEsKeyword(part.substring(4)); + matchType = MATCH_TYPE_SINGLE_FIELD; + singleFieldName = part.substring(0, 3); + break; + } + } + } + } + + if (!keyword) { + // Not a specific field search nor an exact phrase search, do a wildcard match + keyword = escapeEsKeyword(criteria.filters.keyword); + matchType = MATCH_TYPE_WILDCARD; + } + + fullTextQuery = buildEsFullTextQuery(keyword, matchType, singleFieldName); } const body = { query: { } }; if (boolQuery.length > 0) { From 7b0f803e61588a3aa8eb3855b2de7ad36f0c911a Mon Sep 17 00:00:00 2001 From: Samir Gondzetovic Date: Wed, 27 Jun 2018 17:49:11 +0100 Subject: [PATCH 02/10] lint fix --- src/routes/projects/list.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/routes/projects/list.js b/src/routes/projects/list.js index a8e55cba..af3dd6d1 100755 --- a/src/routes/projects/list.js +++ b/src/routes/projects/list.js @@ -63,14 +63,14 @@ const buildEsFullTextQuery = (keyword, matchType, singleFieldName) => { path: 'details.utm', query: { query_string: { - query: (matchType === MATCH_TYPE_EXACT_PHRASE) ? keyword :`*${keyword}*`, + query: (matchType === MATCH_TYPE_EXACT_PHRASE) ? keyword : `*${keyword}*`, analyze_wildcard: (matchType === MATCH_TYPE_WILDCARD), fields: ['details.utm.code^4'], - } - } - } - } - } + }, + }, + }, + }, + }, }, { nested: { @@ -185,7 +185,9 @@ const parseElasticSearchCriteria = (criteria, fields, order) => { // keyword is a full text search // escape special fields from keyword search const keywordCriterion = criteria.filters.keyword; - let keyword, matchType, singleFieldName; + let keyword; + let matchType; + let singleFieldName; // check exact phrase match first (starts and ends with double quotes) if (keywordCriterion.startsWith('"') && keywordCriterion.endsWith('"')) { keyword = keywordCriterion; @@ -195,7 +197,7 @@ const parseElasticSearchCriteria = (criteria, fields, order) => { if (keywordCriterion.indexOf(' ') > -1 || keywordCriterion.indexOf(':') > -1) { const parts = keywordCriterion.split(' '); if (parts.length > 0) { - for (let i = 0; i < parts.length; i++) { + for (let i = 0; i < parts.length; i += 1) { const part = parts[i].trim(); if (part.length > 4 && part.startsWith('ref:')) { keyword = escapeEsKeyword(part.substring(4)); From 5a76e3c85b6b834c70d968283296ada6d022560b Mon Sep 17 00:00:00 2001 From: Samir Gondzetovic Date: Wed, 27 Jun 2018 17:50:26 +0100 Subject: [PATCH 03/10] deploy feature branch --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c3db4502..e58c7fc6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,7 +76,7 @@ workflows: - test filters: branches: - only: ['dev', 'feature/dev-challenges'] + only: ['dev', 'feature/dev-challenges', 'feature/projectSearchUpdate'] - deployProd: requires: - test From b7f1465c13970976a9a4a5c4a0a4b5f605994aa5 Mon Sep 17 00:00:00 2001 From: Samir Gondzetovic Date: Wed, 27 Jun 2018 19:05:31 +0100 Subject: [PATCH 04/10] use filter instead of must in bool query --- src/routes/projects/list.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/routes/projects/list.js b/src/routes/projects/list.js index af3dd6d1..dc181332 100755 --- a/src/routes/projects/list.js +++ b/src/routes/projects/list.js @@ -220,7 +220,7 @@ const parseElasticSearchCriteria = (criteria, fields, order) => { const body = { query: { } }; if (boolQuery.length > 0) { body.query.bool = { - must: boolQuery, + filter: boolQuery, }; } if (fullTextQuery) { @@ -252,6 +252,7 @@ const retrieveProjects = (req, criteria, sort, ffields) => { } const searchCriteria = parseElasticSearchCriteria(criteria, fields, order); + req.log.info(searchCriteria); return new Promise((accept, reject) => { const es = util.getElasticSearchClient(); From 6da7a0780bcc100c351c0e5d761cee2ee76fa3bb Mon Sep 17 00:00:00 2001 From: Samir Gondzetovic Date: Wed, 27 Jun 2018 20:03:31 +0100 Subject: [PATCH 05/10] explain ES queries --- src/routes/projects/list.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/projects/list.js b/src/routes/projects/list.js index dc181332..a74721f1 100755 --- a/src/routes/projects/list.js +++ b/src/routes/projects/list.js @@ -257,7 +257,7 @@ const retrieveProjects = (req, criteria, sort, ffields) => { return new Promise((accept, reject) => { const es = util.getElasticSearchClient(); es.search(searchCriteria).then((docs) => { - const rows = _.map(docs.hits.hits, single => single._source); // eslint-disable-line no-underscore-dangle + const rows = docs; // eslint-disable-line no-underscore-dangle accept({ rows, count: docs.hits.total }); }).catch(reject); }); From 732d8b62ec85dacd23ea0b830970521f54f54e07 Mon Sep 17 00:00:00 2001 From: Samir Gondzetovic Date: Wed, 27 Jun 2018 20:09:51 +0100 Subject: [PATCH 06/10] ES query explain --- src/routes/projects/list.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/routes/projects/list.js b/src/routes/projects/list.js index a74721f1..c2f6333c 100755 --- a/src/routes/projects/list.js +++ b/src/routes/projects/list.js @@ -252,12 +252,13 @@ const retrieveProjects = (req, criteria, sort, ffields) => { } const searchCriteria = parseElasticSearchCriteria(criteria, fields, order); + searchCriteria.explain = true; req.log.info(searchCriteria); return new Promise((accept, reject) => { const es = util.getElasticSearchClient(); es.search(searchCriteria).then((docs) => { - const rows = docs; // eslint-disable-line no-underscore-dangle + const rows = _.map(docs.hits.hits, (single) => { const r = single._source; r.all = docs; return r; }); // eslint-disable-line no-underscore-dangle accept({ rows, count: docs.hits.total }); }).catch(reject); }); From c410f071ac1d03c49d4f9c7760c1843e32d28350 Mon Sep 17 00:00:00 2001 From: Samir Gondzetovic Date: Wed, 27 Jun 2018 20:17:11 +0100 Subject: [PATCH 07/10] ES query explain --- src/routes/projects/list.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/routes/projects/list.js b/src/routes/projects/list.js index c2f6333c..71af0835 100755 --- a/src/routes/projects/list.js +++ b/src/routes/projects/list.js @@ -258,7 +258,8 @@ const retrieveProjects = (req, criteria, sort, ffields) => { return new Promise((accept, reject) => { const es = util.getElasticSearchClient(); es.search(searchCriteria).then((docs) => { - const rows = _.map(docs.hits.hits, (single) => { const r = single._source; r.all = docs; return r; }); // eslint-disable-line no-underscore-dangle + const jres = JSON.stringify(docs); + const rows = _.map(docs.hits.hits, (single) => { const r = single._source; r.all = jres; return r; }); // eslint-disable-line no-underscore-dangle accept({ rows, count: docs.hits.total }); }).catch(reject); }); From 53c64635e94b955f4bf53035ffdcbb0426ad005b Mon Sep 17 00:00:00 2001 From: Samir Gondzetovic Date: Wed, 27 Jun 2018 21:02:19 +0100 Subject: [PATCH 08/10] ES query explain --- src/routes/projects/list.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/routes/projects/list.js b/src/routes/projects/list.js index 71af0835..2696c165 100755 --- a/src/routes/projects/list.js +++ b/src/routes/projects/list.js @@ -225,6 +225,9 @@ const parseElasticSearchCriteria = (criteria, fields, order) => { } if (fullTextQuery) { body.query = _.merge(body.query, fullTextQuery); + if (body.query.bool) { + body.query.bool.minimum_should_match = 1; + } } if (fullTextQuery || boolQuery.length > 0) { From 9effb26858fb7ef1207865209f375dc94a58871e Mon Sep 17 00:00:00 2001 From: Samir Gondzetovic Date: Wed, 27 Jun 2018 21:21:42 +0100 Subject: [PATCH 09/10] remove ES query explain --- src/routes/projects/list.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/routes/projects/list.js b/src/routes/projects/list.js index 2696c165..82f98080 100755 --- a/src/routes/projects/list.js +++ b/src/routes/projects/list.js @@ -255,14 +255,11 @@ const retrieveProjects = (req, criteria, sort, ffields) => { } const searchCriteria = parseElasticSearchCriteria(criteria, fields, order); - searchCriteria.explain = true; - req.log.info(searchCriteria); return new Promise((accept, reject) => { const es = util.getElasticSearchClient(); es.search(searchCriteria).then((docs) => { - const jres = JSON.stringify(docs); - const rows = _.map(docs.hits.hits, (single) => { const r = single._source; r.all = jres; return r; }); // eslint-disable-line no-underscore-dangle + const rows = _.map(docs.hits.hits, single => single._source); // eslint-disable-line no-underscore-dangle accept({ rows, count: docs.hits.total }); }).catch(reject); }); From 1db13423e845fd7515dc88768752cb2f4d0a3018 Mon Sep 17 00:00:00 2001 From: Samir Gondzetovic Date: Wed, 27 Jun 2018 21:23:43 +0100 Subject: [PATCH 10/10] remove feature branch from deployment --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e58c7fc6..c3db4502 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,7 +76,7 @@ workflows: - test filters: branches: - only: ['dev', 'feature/dev-challenges', 'feature/projectSearchUpdate'] + only: ['dev', 'feature/dev-challenges'] - deployProd: requires: - test