SCMOD-10180: Updated to ignore subfield issues (#16)

Author: kusumaghoshdastidar

elastic-mapping-updater/pom.xml

@@ -50,6 +50,10 @@
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpasyncclient</artifactId>

elastic-mapping-updater/src/main/java/com/github/cafdataprocessing/elastic/tools/ElasticMappingUpdater.java

@@ -17,6 +17,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -27,6 +28,7 @@
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
 import org.elasticsearch.client.indices.GetIndexResponse;
 import org.elasticsearch.client.indices.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -53,6 +55,9 @@ public final class ElasticMappingUpdater
     private static final String MAPPING_DYNAMIC_TEMPLATES_KEY = "dynamic_templates";
     private static final String MAPPING_TYPE_KEY = "type";
 
+    private static final Set<String> UNSUPPORTED_PARAMS = Collections.unmodifiableSet(
+            new HashSet<>(Arrays.asList("doc_values", "store")));
+
     private final ObjectMapper objectMapper;
     private final ElasticRequestHandler elasticRequestHandler;
     private final boolean dryRun;
@@ -212,43 +217,83 @@ private void updateIndexesForTemplate(final IndexTemplateMetaData template)
     private static boolean isMappingChangeSafe(
         final Map<String, Object> templateMapping,
         final Map<String, Object> indexMapping,
-        final Set<String> allowedFieldDifferences
+        final Set<String> allowedFieldDifferences,
+        final Set<String> unSupportedFieldDifferences
     )
         throws JsonProcessingException
     {
         final Map<String, Object> ftemplateMapping = FlatMapUtil.flatten(templateMapping);
         final Map<String, Object> findexMapping = FlatMapUtil.flatten(indexMapping);
         final MapDifference<String, Object> diff = Maps.difference(ftemplateMapping, findexMapping);
         final Map<String, ValueDifference<Object>> entriesDiffering = diff.entriesDiffering();
+        boolean safeChangesOnly;
         if (entriesDiffering.isEmpty()) {
-            return true;
+            safeChangesOnly = true;
         } else {
             // Elasticsearch would throw IllegalArgumentException if any such
             // change is included in the index mapping updates
             entriesDiffering.forEach((key, value) -> {
                 LOGGER.warn("Unsupported mapping change : {} - current: {} target: {}",
                             key, value.rightValue(), value.leftValue());
-                allowedFieldDifferences.remove(getFieldName(key));
+                if(key.contains(MAPPING_PROPS_KEY))
+                {
+                    // nested field
+                    unSupportedFieldDifferences.add(key);
+                }
+                else
+                {
+                    allowedFieldDifferences.remove(getFieldName(key));
+                }
             });
-            return false;
+            safeChangesOnly = false;
+        }
+        final Set<String> unsupportedParamChanges = new HashSet<>();
+        final Map<String, Object> entriesOnlyInIndex = diff.entriesOnlyOnRight();
+        // Field parameters that are currently set on a field in the index are now being removed
+        entriesOnlyInIndex.entrySet().stream()
+                .filter(e -> isUnsupportedParam(e.getKey()))
+                .forEach(e -> {
+                        LOGGER.warn("Unsupported mapping change : field parameter being removed : {}:{}", e.getKey(), e.getValue());
+                        unsupportedParamChanges.add(e.getKey());
+                    }
+                );
+        if(!unsupportedParamChanges.isEmpty())
+        {
+            unSupportedFieldDifferences.addAll(unsupportedParamChanges);
+            safeChangesOnly = false;
         }
+        return safeChangesOnly;
+    }
+
+    private static boolean isUnsupportedParam(final String fieldPath)
+    {
+        final String paramName = getParamName(fieldPath);
+        return UNSUPPORTED_PARAMS.contains(paramName);
     }
 
     private static String getFieldName(final String key)
     {
         return key.split(Pattern.quote("/"))[1];
     }
 
+    private static String getParamName(final String key)
+    {
+        final String[] path = key.split(Pattern.quote("/"));
+        return path[path.length - 1];
+    }
+
     private Map<String, Object> getMappingChanges(final Map<String, Object> templateMapping, final Map<String, Object> indexMapping)
         throws JsonProcessingException
     {
         final Map<String, Object> mappingsChanges = new HashMap<>();
         final MapDifference<String, Object> diff = Maps.difference(templateMapping, indexMapping);
         final Map<String, ValueDifference<Object>> entriesDiffering = diff.entriesDiffering();
         final Set<String> allowedFieldDifferences = new HashSet<>(entriesDiffering.keySet());
+        final Set<String> unSupportedFieldDifferences = new HashSet<>();
 
         boolean unsupportedObjectChanges = false;
         if (!entriesDiffering.isEmpty()) {
+            // Template has mapping changes to existing properties
             LOGGER.info("--Differences between template and index mapping--");
             entriesDiffering.forEach((key, value) -> LOGGER.info("  {} - current: {} target: {}",
                                                                  key, value.rightValue(), value.leftValue()));
@@ -286,7 +331,8 @@ private Map<String, Object> getMappingChanges(final Map<String, Object> template
             }
         }
 
-        if (!isMappingChangeSafe(templateMapping, indexMapping, allowedFieldDifferences) || unsupportedObjectChanges) {
+        if (!isMappingChangeSafe(templateMapping, indexMapping, allowedFieldDifferences, unSupportedFieldDifferences)
+                || unsupportedObjectChanges) {
             LOGGER.warn("Unsupported mapping changes will not be applied to the index.");
         }
 
@@ -297,6 +343,15 @@ private Map<String, Object> getMappingChanges(final Map<String, Object> template
             mappingsChanges.put(field, ((ValueDifference<?>) entriesDiffering.get(field)).leftValue());
         }
 
+        // Remove any unsupportedMappings
+        LOGGER.info("{}", unSupportedFieldDifferences.isEmpty()
+                ? "No unsupported field changes."
+                : "Unsupported field changes that will not be included in the update: " + unSupportedFieldDifferences);
+        for (final String field : unSupportedFieldDifferences) {
+            removeUnsupportedFieldChange(mappingsChanges, field);
+        }
+
+        // Add new properties defined in the template
         final Map<String, Object> entriesOnlyInTemplate = diff.entriesOnlyOnLeft();
         final Set<String> newProperties = entriesOnlyInTemplate.keySet();
         LOGGER.info("{}", newProperties.isEmpty()
@@ -314,4 +369,31 @@ private boolean hasDynamicTemplateChanged(List<Object> dynamicTemplatesInTemplat
         return dynamicTemplatesInTemplate.retainAll(dynamicTemplatesInIndex);
     }
 
+    @SuppressWarnings("unchecked")
+    private void removeUnsupportedFieldChange(final Map<String, Object> mappingsChanges, final String fieldPath) {
+        final List<String> path = Arrays.asList(StringUtils.split(fieldPath.trim(), "/"));
+        final int size = path.size();
+        int index = 0;
+        if(size == 2)
+        {
+            // for a field path like, /LANGUAGE_CODES/properties/CODE/type, the 'fieldName' to be removed here is 'CODE'
+            // remove property with unsupported mapping change
+            final String fieldName = path.get(0);
+            mappingsChanges.remove(fieldName);
+        }
+        else
+        {
+            while (index != size - 2) {
+                final int i = index++;
+                final String currentFieldName = path.get(i);
+                final Object field = mappingsChanges.get(path.get(i));
+                if (field instanceof Map<?, ?>) {
+                    final Map<String, Object> currentField = (Map<String, Object>) field;
+                    final String subPath = fieldPath.substring(fieldPath.indexOf(currentFieldName) + currentFieldName.length());
+                    removeUnsupportedFieldChange(currentField, subPath);
+                }
+            }
+        }
+    }
+
 }

elastic-mapping-updater/src/test/java/com/github/cafdataprocessing/elastic/tools/test/ElasticMappingUpdaterIT.java

@@ -15,6 +15,7 @@
  */
 package com.github.cafdataprocessing.elastic.tools.test;
 
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -439,14 +440,31 @@ public void testUpdateIndexesOfUnSupportedChangesInTemplate() throws IOException
         LOGGER.info("idPropMapping {} ", idPropMapping);
         final Object idPropValue = idPropMapping.get("ignore_malformed");
         // Verify property mapping parameter was not removed (although mapping update was to remove the ignore_malformed parameter)
-        assertNotNull("testUpdateUnsupportedChanges", idPropValue);
+        assertNotNull("testUpdateIndexesOfUnSupportedChangesInTemplate", idPropValue);
 
         // Verify index mapping of unsupported field changes has not changed
         @SuppressWarnings("unchecked")
         final Map<String, Object> langPropMapping = (Map<String, Object>) props.get("LANGUAGE_CODES");
         final String propValue = (String) langPropMapping.get("type");
         // Verify property mapping value is same as before
-        assertTrue("testUpdateUnsupportedChanges", propValue.equals("nested"));
+        assertTrue("testUpdateIndexesOfUnSupportedChangesInTemplate", propValue.equals("nested"));
+
+        // Verify index mapping of unsupported field changes has been updated with allowed changes
+        @SuppressWarnings("unchecked")
+        final Map<String, Object> targetRefPropMapping = (Map<String, Object>) props.get("TARGET_REFERENCES");
+        @SuppressWarnings("unchecked")
+        final Map<String, Object> targetRefProps = (Map<String, Object>) targetRefPropMapping.get("properties");
+        // Verify new property is added
+        @SuppressWarnings("unchecked")
+        final Map<String, Object> tRefMapping = (Map<String, Object>) targetRefProps.get("TARGET_REFERENCE");
+        assertNotNull("testUpdateIndexesOfUnSupportedChangesInTemplate", tRefMapping);
+        // Verify unsupported change to nested property is not applied
+        final Boolean propDocValuesValue = (Boolean) tRefMapping.get("doc_values");
+        assertFalse("testUpdateIndexesOfUnSupportedChangesInTemplate", propDocValuesValue);
+        // Verify new nested property is added
+        @SuppressWarnings("unchecked")
+        final Map<String, Object> destMapping = (Map<String, Object>) targetRefProps.get("DESTINATION_ID");
+        assertNotNull("testUpdateIndexesOfUnSupportedChangesInTemplate", destMapping);
 
         // Index more data
         request = new IndexRequest(indexName);
@@ -475,6 +493,95 @@ public void testUpdateIndexesOfUnSupportedChangesInTemplate() throws IOException
         verifyIndexData(indexName, QueryBuilders.matchAllQuery(), 2);
     }
 
+    @Test
+    public void testUpdateIndexesWithNestedFieldChanges() throws IOException, GetIndexException, InterruptedException
+    {
+        LOGGER.info("Running test 'testUpdateIndexesWithNestedFieldChanges'...");
+        final String templateName = "sample-template";
+        final String origTemplateSourceFile = "/template10.json";
+        // 'store' param removed from nested property LANGUAGE_CODES/CODE
+        // nested property type='nested' is removed
+        final String updatedTemplateSourceFile = "/template11.json";
+        final String indexName = "jan_blue-000001";
+
+        final String origTemplateSource = readFile(origTemplateSourceFile);
+        LOGGER.info("testUpdateIndexesWithNestedFieldChanges - Creating initial template {}", templateName);
+
+        // Create a template
+        final PutIndexTemplateRequest trequest = new PutIndexTemplateRequest(templateName);
+        trequest.source(origTemplateSource, XContentType.JSON);
+        final AcknowledgedResponse putTemplateResponse = client.indices().putTemplate(trequest, RequestOptions.DEFAULT);
+        if (!putTemplateResponse.isAcknowledged()) {
+            fail();
+        }
+        LOGGER.info("testUpdateIndexesWithNestedFieldChanges - Creating index matching template {}", templateName);
+        // Create an index with some data
+        IndexRequest request = new IndexRequest(indexName);
+        request.id("1");
+        request.routing("1");
+        String jsonString = "{" + "'TITLE':'doc1'," + "'DATE_PROCESSED\":'2020-02-11'," + "'CONTENT_PRIMARY':'just a test',"
+            + "'IS_HEAD_OF_FAMILY':true," + "'PERSON':{ 'NAME':'person1' }" + "}";
+        jsonString = jsonString.replaceAll("'", "\"");
+        request.source(jsonString, XContentType.JSON);
+        request.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        final boolean needsRetries = indexDocumentWithRetry(request);
+        if (needsRetries) {
+            // Indexing has failed after multiple retries
+            fail();
+        }
+
+        verifyIndexData(indexName, QueryBuilders.matchAllQuery(), 1);
+
+        LOGGER.info("testUpdateIndexesWithNestedFieldChanges - Updating template {}", templateName);
+        final String updatedTemplateSource = readFile(updatedTemplateSourceFile);
+        // Create a template
+        final PutIndexTemplateRequest utrequest = new PutIndexTemplateRequest(templateName);
+        utrequest.source(updatedTemplateSource, XContentType.JSON);
+        final AcknowledgedResponse updateTemplateResponse = client.indices().putTemplate(utrequest, RequestOptions.DEFAULT);
+        if (!updateTemplateResponse.isAcknowledged()) {
+            fail();
+        }
+
+        LOGGER.info("testUpdateIndexesWithNestedFieldChanges - Updating indexes matching template {}", templateName);
+        updateIndex("testUpdateIndexesWithNestedFieldChanges", templateName);
+
+        // Verify index mapping has new properties
+        final Map<String, Object> indexTypeMappings = getIndexMapping(indexName);
+        @SuppressWarnings("unchecked")
+        final Map<String, Object> props = (Map<String, Object>) indexTypeMappings.get("properties");
+        @SuppressWarnings("unchecked")
+        final Map<String, Object> propMapping = (Map<String, Object>) props.get("DATE_DISPOSED");
+        assertNotNull("testUpdateIndexesWithNestedFieldChanges", propMapping);
+        @SuppressWarnings("unchecked")
+        final Map<String, Object> entPropMapping = (Map<String, Object>) props.get("ENTITIES");
+        LOGGER.info("entitiesPropMapping {} ", entPropMapping);
+
+        // Index more data
+        request = new IndexRequest(indexName);
+        request.id("2");
+        request.routing("1");
+        jsonString = "{"
+            + "'TITLE':'doc2',"
+            + "'DATE_PROCESSED':'2020-02-11',"
+            + "'CONTENT_PRIMARY':'just a test',"
+            + "'IS_HEAD_OF_FAMILY':true,"
+            + "'PERSON':{ 'NAME':'person2', 'AGE':5 },"
+            + "'HOLD_DETAILS': {'FIRST_HELD_DATE':'2020-02-11', 'HOLD_HISTORY': '2020-02-11', 'HOLD_ID': '12'}"
+            + "}";
+        jsonString = jsonString.replaceAll("'", "\"");
+        request.source(jsonString, XContentType.JSON);
+        request.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+
+        try {
+            final IndexResponse response = client.index(request, RequestOptions.DEFAULT);
+            assertTrue(response.status() == RestStatus.CREATED);
+        } catch (final ElasticsearchException e) {
+            fail();
+        }
+
+        verifyIndexData(indexName, QueryBuilders.matchAllQuery(), 2);
+    }
+
     private void updateIndex(final String testName, final String templateName)
     {
         LOGGER.info("{}: {}", testName, templateName);