US397427: Stop using existing OpenSearch Java High Level REST Client (#39)

Author: markmccallion28

elastic-mapping-updater/pom.xml

@@ -50,6 +50,10 @@
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
         </dependency>
+        <dependency>
+            <groupId>jakarta.json</groupId>
+            <artifactId>jakarta.json-api</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
@@ -66,21 +70,13 @@
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpcore</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.opensearch</groupId>
-            <artifactId>opensearch</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.opensearch</groupId>
-            <artifactId>opensearch-x-content</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.opensearch.client</groupId>
             <artifactId>opensearch-rest-client</artifactId>
         </dependency>
         <dependency>
             <groupId>org.opensearch.client</groupId>
-            <artifactId>opensearch-rest-high-level-client</artifactId>
+            <artifactId>opensearch-java</artifactId>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>

elastic-mapping-updater/src/main/java/com/github/cafdataprocessing/elastic/tools/ElasticMappingUpdater.java

@@ -20,24 +20,31 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.io.StringWriter;
 
 import org.apache.commons.lang3.StringUtils;
-import org.opensearch.client.indices.GetIndexResponse;
-import org.opensearch.client.indices.IndexTemplateMetadata;
-import org.opensearch.cluster.metadata.MappingMetadata;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.opensearch.client.json.JsonpSerializable;
+import org.opensearch.client.json.jackson.JacksonJsonpGenerator;
+import org.opensearch.client.json.jackson.JacksonJsonpMapper;
+import org.opensearch.client.opensearch._types.mapping.DynamicTemplate;
+import org.opensearch.client.opensearch._types.mapping.TypeMapping;
+import org.opensearch.client.opensearch.indices.TemplateMapping;
 
 import com.fasterxml.jackson.annotation.PropertyAccessor;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.cafdataprocessing.elastic.tools.exceptions.GetIndexException;
 import com.github.cafdataprocessing.elastic.tools.exceptions.GetTemplatesException;
@@ -55,6 +62,8 @@ public final class ElasticMappingUpdater
     private static final String MAPPING_DYNAMIC_TEMPLATES_KEY = "dynamic_templates";
     private static final String MAPPING_TYPE_KEY = "type";
 
+    private final ObjectMapper mapper = new ObjectMapper();
+
     private static final Set<String> MODIFIABLE_PROPERTIES = Collections.unmodifiableSet(
             new HashSet<>(Arrays.asList(
                     "boost",
@@ -135,66 +144,53 @@ private ElasticMappingUpdater(
     private void updateIndexes()
         throws IOException, GetIndexException, GetTemplatesException, UnexpectedResponseException
     {
-        final List<IndexTemplateMetadata> templates = elasticRequestHandler.getTemplates();
-        LOGGER.info("Templates found in Elasticsearch: {}",
-                    templates.stream().map(template -> template.name()).collect(Collectors.toList()));
-        for (final IndexTemplateMetadata template : templates) {
-            updateIndexesForTemplate(template);
+        final Map<String, TemplateMapping> templates = elasticRequestHandler.getTemplates();
+        LOGGER.info("Templates found in Elasticsearch: {}", templates.keySet());
+        for (final Entry<String, TemplateMapping> template : templates.entrySet()) {
+            updateIndexesForTemplate(template.getKey(), template.getValue());
         }
     }
 
-    private void updateIndexesForTemplate(final IndexTemplateMetadata template)
-        throws IOException, GetIndexException, GetTemplatesException, UnexpectedResponseException
+    private void updateIndexesForTemplate(final String templateName, final TemplateMapping template)
+        throws IOException, GetIndexException, UnexpectedResponseException
     {
-        final String templateName = template.name();
         LOGGER.info("---- Analyzing indexes matching template '{}' ----", templateName);
 
-        final List<String> patterns = template.patterns();
+        final List<String> patterns = template.indexPatterns();
 
-        final MappingMetadata mapping = template.mappings();
+        final TypeMapping mapping = template.mappings();
         if (mapping == null) {
             LOGGER.info("No mappings in template '{}'. Indexes for this template will not be updated.", templateName);
             return;
         }
 
-        final Map<String, Object> templateTypeMappings = mapping.getSourceAsMap();
-
-        final Object templateProperties = Optional
-            .ofNullable(templateTypeMappings.get(MAPPING_PROPS_KEY))
-            .orElseGet(Collections::emptyMap);
+        final Map<String, Object> templateProperties = getObjectAsHashMap(mapping.properties());
 
         // Find all indices that match template patterns
         final List<String> indexes = elasticRequestHandler.getIndexNames(patterns);
         LOGGER.info("Found {} index(es) that match template '{}'", indexes.size(), templateName);
         for (final String indexName : indexes) {
-            GetIndexResponse getIndexResponse = elasticRequestHandler.getIndex(indexName);
-            MappingMetadata indexMappings = getIndexResponse.getMappings().get(indexName);
-            Map<String, Object> indexTypeMappings = indexMappings.getSourceAsMap();
+            final TypeMapping indexMappings = elasticRequestHandler.getIndexMapping(indexName);
+            final Map<String, Object> indexProperties = getObjectAsHashMap(indexMappings.properties());
 
             LOGGER.info("Comparing index mapping for '{}'", indexName);
 
-            final Object indexProperties = Optional
-                .ofNullable(indexTypeMappings.get(MAPPING_PROPS_KEY))
-                .orElseGet(Collections::emptyMap);
-
-            @SuppressWarnings("unchecked")
-            final Map<String, Object> mappingsChanges = getMappingChanges(
-                (Map<String, Object>) templateProperties,
-                (Map<String, Object>) indexProperties);
+            final Map<String, Object> mappingsChanges = getMappingChanges(templateProperties, indexProperties);
 
             final Map<String, Object> mappingsRequest = new HashMap<>();
             mappingsRequest.put(MAPPING_PROPS_KEY, mappingsChanges);
 
             // Add all dynamic_templates in template to index mapping
-            @SuppressWarnings("unchecked")
-            final List<Object> dynamicTemplatesInTemplate = (List<Object>) Optional
-                .ofNullable(templateTypeMappings.get(MAPPING_DYNAMIC_TEMPLATES_KEY))
-                .orElseGet(Collections::emptyList); // Empty list will clear all existing dynamic_templates in index mapping
+            // Empty list will clear all existing dynamic_templates in index mapping
+            final List<Object> dynamicTemplatesInTemplate = new ArrayList<>();
+            for (final Map<String, DynamicTemplate> t : mapping.dynamicTemplates()) {
+                dynamicTemplatesInTemplate.add(getObjectAsHashMap(t));
+            }
 
-            @SuppressWarnings("unchecked")
-            final List<Object> dynamicTemplatesInIndex = (List<Object>) Optional
-                .ofNullable(indexTypeMappings.get(MAPPING_DYNAMIC_TEMPLATES_KEY))
-                .orElseGet(Collections::emptyList);
+            final List<Object> dynamicTemplatesInIndex = new ArrayList<>();
+            for (final Map<String, DynamicTemplate> t : indexMappings.dynamicTemplates()) {
+                dynamicTemplatesInIndex.add(getObjectAsHashMap(t));
+            }
 
             final List<Object> dynamicTemplatesUpdates = new ArrayList<>(dynamicTemplatesInTemplate);
 
@@ -353,8 +349,8 @@ private Map<String, Object> getMappingChanges(final Map<String, Object> template
             // Check if 'type' has changed for object/nested properties
             final Map<String, ValueDifference<Object>> typeDifferences = entriesDiffering.entrySet().stream()
                 .filter(e -> ((Map<?, ?>) (e.getValue().leftValue())).containsKey(MAPPING_PROPS_KEY)
-                && (((Map<?, ?>) (e.getValue().leftValue())).containsKey(MAPPING_TYPE_KEY)
-                || ((Map<?, ?>) (e.getValue().rightValue())).containsKey(MAPPING_TYPE_KEY)))
+                && (!((Map<?, ?>) (e.getValue().leftValue())).get(MAPPING_TYPE_KEY).equals("object")
+                || !((Map<?, ?>) (e.getValue().rightValue())).get(MAPPING_TYPE_KEY).equals("object")))
                 .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue()));
 
             if (!typeDifferences.isEmpty()) {
@@ -406,7 +402,7 @@ private Map<String, Object> getMappingChanges(final Map<String, Object> template
         return mappingsChanges;
     }
 
-    private boolean hasDynamicTemplateChanged(List<Object> dynamicTemplatesInTemplate, List<Object> dynamicTemplatesInIndex)
+    private boolean hasDynamicTemplateChanged(final List<Object> dynamicTemplatesInTemplate, final List<Object> dynamicTemplatesInIndex)
     {
         if (dynamicTemplatesInTemplate.size() != dynamicTemplatesInIndex.size()) {
             return true;
@@ -451,4 +447,22 @@ private void removeUnsupportedFieldChange(final Map<String, Object> mappingsChan
         }
     }
 
+    private Map<String,Object> getObjectAsHashMap(final Map<String, ? extends JsonpSerializable> obj) throws JsonProcessingException, IOException
+    {
+        final Map<String, Object> mapFromString = new LinkedHashMap<>();
+        for (final Entry<String, ? extends JsonpSerializable> val : obj.entrySet()) {
+            final String result = "{\"" + val.getKey() + "\":" + getStringFromObject(val.getValue()) + "}";
+            mapFromString.putAll(mapper.readValue(result, new TypeReference<Map<String, Object>>(){}));
+        }
+        return mapFromString;
+    }
+
+    private String getStringFromObject(final JsonpSerializable value) throws IOException
+    {
+        final StringWriter writer = new StringWriter();
+        try (final JacksonJsonpGenerator generator = new JacksonJsonpGenerator(new JsonFactory().createGenerator(writer))) {
+            value.serialize(generator, new JacksonJsonpMapper());
+        }
+        return writer.toString();
+    }
 }

elastic-mapping-updater/src/main/java/com/github/cafdataprocessing/elastic/tools/ElasticRequestHandler.java

@@ -17,6 +17,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StringReader;
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.List;
@@ -31,13 +32,6 @@
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
 import org.opensearch.client.RestClient;
-import org.opensearch.client.indices.GetIndexResponse;
-import org.opensearch.client.indices.GetIndexTemplatesResponse;
-import org.opensearch.client.indices.IndexTemplateMetadata;
-import org.opensearch.common.xcontent.NamedXContentRegistry;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.common.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -48,6 +42,16 @@
 import com.github.cafdataprocessing.elastic.tools.exceptions.UnexpectedResponseException;
 import com.google.common.net.UrlEscapers;
 
+import jakarta.json.Json;
+import jakarta.json.JsonObject;
+import jakarta.json.JsonReader;
+import jakarta.json.stream.JsonParser;
+import org.opensearch.client.json.jackson.JacksonJsonpMapper;
+import org.opensearch.client.opensearch._types.mapping.TypeMapping;
+import org.opensearch.client.opensearch.indices.GetTemplateResponse;
+import org.opensearch.client.opensearch.indices.TemplateMapping;
+import org.opensearch.client.util.ApiTypeHelper;
+
 final class ElasticRequestHandler
 {
     private static final Logger LOGGER = LoggerFactory.getLogger(ElasticRequestHandler.class);
@@ -60,7 +64,7 @@ public ElasticRequestHandler(final ElasticMappingUpdaterConfiguration schemaUpda
         this.elasticClient = ElasticProvider.getClient(schemaUpdaterConfig.getElasticSettings());
     }
 
-    List<IndexTemplateMetadata> getTemplates()
+    Map<String, TemplateMapping> getTemplates()
         throws IOException, GetTemplatesException
     {
         LOGGER.debug("Getting templates...");
@@ -70,11 +74,13 @@ List<IndexTemplateMetadata> getTemplates()
         final int statusCode = response.getStatusLine().getStatusCode();
         if (statusCode == 200) {
             try (final InputStream resultJsonStream = response.getEntity().getContent();
-                 final XContentParser parser
-                 = XContentFactory.xContent(XContentType.JSON).createParser(NamedXContentRegistry.EMPTY, null, resultJsonStream)) {
-                final GetIndexTemplatesResponse getTemplatesResponse = GetIndexTemplatesResponse.fromXContent(parser);
-                final List<IndexTemplateMetadata> templates = getTemplatesResponse.getIndexTemplates();
-                return templates;
+                final JsonParser jsonValueParser = Json.createParser(resultJsonStream)) {
+                ApiTypeHelper.DANGEROUS_disableRequiredPropertiesCheck(true);
+                final GetTemplateResponse getTemplatesResponse =
+                    GetTemplateResponse._DESERIALIZER.deserialize(jsonValueParser, new JacksonJsonpMapper());
+                final Map<String, TemplateMapping> indexTemplates = getTemplatesResponse.result();
+                ApiTypeHelper.DANGEROUS_disableRequiredPropertiesCheck(false);
+                return indexTemplates;
             }
         } else {
             throw new GetTemplatesException(String.format("Error getting templates. Status code: %s, response: %s",
@@ -110,19 +116,22 @@ List<String> getIndexNames(final List<String> indexNamePatterns) throws Unexpect
         }
     }
 
-    public GetIndexResponse getIndex(final String indexName) throws IOException, GetIndexException
+    public TypeMapping getIndexMapping(final String indexName) throws IOException, GetIndexException
     {
-        LOGGER.debug("Getting index {}...", indexName);
+        LOGGER.debug("Getting index mapping for: {}...", indexName);
         final Request request = new Request("GET", "/" + UrlEscapers.urlPathSegmentEscaper().escape(indexName));
         final Response response = elasticClient.performRequest(request);
 
         final int statusCode = response.getStatusLine().getStatusCode();
         if (statusCode == 200) {
-            try (final InputStream resultJsonStream = response.getEntity().getContent();
-                 final XContentParser parser
-                 = XContentFactory.xContent(XContentType.JSON).createParser(NamedXContentRegistry.EMPTY, null, resultJsonStream)) {
-                return GetIndexResponse.fromXContent(parser);
-            }
+            final JsonReader jsonReader = Json.createReader(new StringReader(EntityUtils.toString(response.getEntity())));
+            final JsonObject mappings = jsonReader.readObject().getJsonObject(indexName).getJsonObject("mappings");
+
+            final JsonParser jsonMappingParser = Json.createParser(new StringReader(mappings.toString()));
+            ApiTypeHelper.DANGEROUS_disableRequiredPropertiesCheck(true);
+            final TypeMapping mapping = TypeMapping._DESERIALIZER.deserialize(jsonMappingParser, new JacksonJsonpMapper());
+            ApiTypeHelper.DANGEROUS_disableRequiredPropertiesCheck(false);
+            return mapping;
         } else {
             throw new GetIndexException(String.format("Error getting index '%s'. Status code: %s, response: %s",
                                                       indexName, statusCode, EntityUtils.toString(response.getEntity())));