Merge branch 'master' into bug/test-build-failures

CHANGES.md

@@ -1,5 +1,6 @@
-### Apache MetaModel [WIP]
+### Apache MetaModel 5.3.3
 
+ * [METAMODEL-1233] - Elasticsearch: "term" query is used for text search.
  * [METAMODEL-1229] - FixedWidth works with UrlResource/HttpInputStream.
  * [METAMODEL-1228] - Better handling of fieldnames with dots in Elasticsearch
  * [METAMODEL-1227] - Better handling of nested objects in Elasticsearch data

arff/pom.xml

@@ -21,7 +21,7 @@ under the License.
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-arff</artifactId>

cassandra/pom.xml

@@ -13,7 +13,7 @@
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-cassandra</artifactId>

core/pom.xml

@@ -21,7 +21,7 @@ under the License.
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-core</artifactId>

couchdb/pom.xml

@@ -21,7 +21,7 @@ under the License.
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-couchdb</artifactId>

csv/pom.xml

@@ -21,7 +21,7 @@ under the License.
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-csv</artifactId>

dynamodb/pom.xml

@@ -21,7 +21,7 @@ under the License.
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-dynamodb</artifactId>

elasticsearch/common/pom.xml

@@ -13,7 +13,7 @@
 	<parent>
 		<artifactId>MetaModel-elasticsearch</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 

elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java

@@ -170,57 +170,18 @@ private static String getType(Column column) {
      * @return a {@link QueryBuilder} if one was produced, or null if the items
      *         could not be pushed down to an ElasticSearch query
      */
-    public static QueryBuilder createQueryBuilderForSimpleWhere(List<FilterItem> whereItems,
-            LogicalOperator logicalOperator) {
+    public static QueryBuilder createQueryBuilderForSimpleWhere(final List<FilterItem> whereItems,
+            final LogicalOperator logicalOperator) {
         if (whereItems.isEmpty()) {
             return QueryBuilders.matchAllQuery();
         }
 
-        List<QueryBuilder> children = new ArrayList<>(whereItems.size());
-        for (FilterItem item : whereItems) {
-            final QueryBuilder itemQueryBuilder;
-
-            if (item.isCompoundFilter()) {
-                final List<FilterItem> childItems = Arrays.asList(item.getChildItems());
-                itemQueryBuilder = createQueryBuilderForSimpleWhere(childItems, item.getLogicalOperator());
-                if (itemQueryBuilder == null) {
-                    // something was not supported, so we have to forfeit here
-                    // too.
-                    return null;
-                }
-            } else {
-                final Column column = item.getSelectItem().getColumn();
-                if (column == null) {
-                    // unsupported type of where item - must have a column
-                    // reference
-                    return null;
-                }
-                final String fieldName = column.getName();
-                final Object operand = item.getOperand();
-                final OperatorType operator = item.getOperator();
-
-                if (OperatorType.EQUALS_TO.equals(operator)) {
-                    if (operand == null) {
-                        itemQueryBuilder = getMissingQuery(fieldName);
-                    } else {
-                        itemQueryBuilder = QueryBuilders.termQuery(fieldName, operand);
-                    }
-                } else if (OperatorType.DIFFERENT_FROM.equals(operator)) {
-                    if (operand == null) {
-                        itemQueryBuilder = getExistsQuery(fieldName);
-                    } else {
-                        itemQueryBuilder = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(fieldName,
-                                operand));
-                    }
-                } else if (OperatorType.IN.equals(operator)) {
-                    final List<?> operands = CollectionUtils.toList(operand);
-                    itemQueryBuilder = QueryBuilders.termsQuery(fieldName, operands);
-                } else {
-                    // not (yet) support operator types
-                    return null;
-                }
+        final List<QueryBuilder> children = new ArrayList<>(whereItems.size());
+        for (final FilterItem item : whereItems) {
+            final QueryBuilder itemQueryBuilder = createFilterItemQueryBuilder(item);
+            if (itemQueryBuilder == null) {
+                return null;
             }
-
             children.add(itemQueryBuilder);
         }
 
@@ -243,6 +204,65 @@ public static QueryBuilder createQueryBuilderForSimpleWhere(List<FilterItem> whe
         return result;
     }
 
+    private static QueryBuilder createFilterItemQueryBuilder(final FilterItem filterItem) {
+        final QueryBuilder itemQueryBuilder;
+        if (filterItem.isCompoundFilter()) {
+            final List<FilterItem> childItems = Arrays.asList(filterItem.getChildItems());
+            itemQueryBuilder = createQueryBuilderForSimpleWhere(childItems, filterItem.getLogicalOperator());
+        } else {
+            final Column column = filterItem.getSelectItem().getColumn();
+            if (column == null) {
+                // unsupported type of where item - must have a column reference
+                return null;
+            }
+            itemQueryBuilder = createQueryBuilderForOperator(filterItem, column);
+        }
+
+        return itemQueryBuilder;
+    }
+
+    private static QueryBuilder createQueryBuilderForOperator(final FilterItem filterItem, final Column column) {
+        if (OperatorType.EQUALS_TO.equals(filterItem.getOperator())) {
+            if (filterItem.getOperand() == null) {
+                return getMissingQuery(column.getName());
+            } else {
+                return matchOrTermQuery(column, filterItem.getOperand());
+            }
+        } else if (OperatorType.DIFFERENT_FROM.equals(filterItem.getOperator())) {
+            if (filterItem.getOperand() == null) {
+                return getExistsQuery(column.getName());
+            } else {
+                return QueryBuilders.boolQuery().mustNot(matchOrTermQuery(column, filterItem.getOperand()));
+            }
+        } else if (OperatorType.IN.equals(filterItem.getOperator())) {
+            final List<?> operands = CollectionUtils.toList(filterItem.getOperand());
+            if (column.getType().isLiteral()) {
+                return createMultipleValuesQueryBuilder(column.getName(), operands);
+            } else {
+                return QueryBuilders.termsQuery(column.getName(), operands);
+            }
+        } else {
+            // not (yet) supported operator types
+            return null;
+        }
+    }
+
+    private static QueryBuilder matchOrTermQuery(final Column column, final Object operand) {
+        if (column.getType().isLiteral()) {
+            return QueryBuilders.matchQuery(column.getName(), operand);
+        } else {
+            return QueryBuilders.termQuery(column.getName(), operand);
+        }
+    }
+
+    private static QueryBuilder createMultipleValuesQueryBuilder(final String columnName, final List<?> operands) {
+        final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+        for (final Object value : operands) {
+            boolQueryBuilder.should(QueryBuilders.matchQuery(columnName, value.toString()));
+        }
+        return boolQueryBuilder;
+    }
+
     public static ColumnType getColumnTypeFromElasticSearchType(final String metaDataFieldType) {
         final ColumnType columnType;
         if (metaDataFieldType.startsWith("date")) {

elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java

@@ -18,15 +18,26 @@
  */
 package org.apache.metamodel.elasticsearch.common;
 
-import junit.framework.TestCase;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.metamodel.data.DataSetHeader;
 import org.apache.metamodel.data.Row;
 import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.OperatorType;
 import org.apache.metamodel.query.SelectItem;
 import org.apache.metamodel.schema.ColumnType;
 import org.apache.metamodel.schema.MutableColumn;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryBuilder;
 
-import java.util.*;
+import junit.framework.TestCase;
 
 public class ElasticSearchUtilsTest extends TestCase {
 
@@ -60,4 +71,63 @@ public void testCreateRowWithParsableDates() throws Exception {
         assertTrue(stringValue instanceof String);
         assertTrue(dateValue instanceof Date);
     }
+
+    /**
+     * For text-based conditions a 'match' query is recommended (instead of 'term' query).
+     */
+    public void testMatchQueryIsCreatedForTextEqualTo() {
+        final SelectItem selectItem = new SelectItem(new MutableColumn("column_name", ColumnType.STRING));
+        final FilterItem filterItem = new FilterItem(selectItem, OperatorType.EQUALS_TO, "text-value");
+        final QueryBuilder queryBuilder =
+                ElasticSearchUtils.createQueryBuilderForSimpleWhere(Collections.singletonList(filterItem), null);
+        assertNotNull(queryBuilder);
+        assertEquals("match", queryBuilder.getName());
+    }
+
+    /**
+     * For text-based conditions a 'match' query is recommended (instead of 'term' query).
+     * In case of 'DIFFERENT_FROM', we need a 'bool' query with 'must not' and 'match' query.
+     */
+    @SuppressWarnings("unchecked")
+    public void testBoolQueryIsCreatedForTextDifferentFrom() {
+        final SelectItem selectItem = new SelectItem(new MutableColumn("column_name", ColumnType.STRING));
+        final FilterItem filterItem = new FilterItem(selectItem, OperatorType.DIFFERENT_FROM, "text-value");
+        final QueryBuilder queryBuilder =
+                ElasticSearchUtils.createQueryBuilderForSimpleWhere(Collections.singletonList(filterItem), null);
+        assertNotNull(queryBuilder);
+        assertEquals("bool", queryBuilder.getName());
+        final Map<String, Object> queryMap =
+                XContentHelper.convertToMap(XContentType.JSON.xContent(), queryBuilder.toString(), false);
+        final Map<String, Object> boolMap = (Map<String, Object>) queryMap.get("bool");
+        assertNotNull(boolMap);
+        assertTrue(boolMap.containsKey("must_not"));
+        final List<Object> mustNotList = (List<Object>) boolMap.get("must_not");
+        final Map<String, Object> mustNotMap = (Map<String, Object>) mustNotList.get(0);
+        assertTrue(mustNotMap.containsKey("match"));
+    }
+
+    /**
+     * For text-based conditions a 'match' query is recommended (instead of 'term' query).
+     * To simulate 'IN' operator, we need a 'bool' query with multiple 'match' queries combined with 'OR'.
+     */
+    @SuppressWarnings("unchecked")
+    public void testBoolQueryIsCreatedForTextIn() {
+        final SelectItem selectItem = new SelectItem(new MutableColumn("column_name", ColumnType.STRING));
+        final FilterItem filterItem =
+                new FilterItem(selectItem, OperatorType.IN, Arrays.asList("text-value-a", "text-value-b"));
+        final QueryBuilder queryBuilder =
+                ElasticSearchUtils.createQueryBuilderForSimpleWhere(Collections.singletonList(filterItem), null);
+        assertNotNull(queryBuilder);
+        assertEquals("bool", queryBuilder.getName());
+        final Map<String, Object> queryMap =
+                XContentHelper.convertToMap(XContentType.JSON.xContent(), queryBuilder.toString(), false);
+        final Map<String, Object> matchMap = (Map<String, Object>) queryMap.get("bool");
+        assertNotNull(matchMap);
+        assertTrue(matchMap.containsKey("should"));
+        final List<Object> shouldList = (List<Object>) matchMap.get("should");
+        assertNotNull(shouldList);
+        assertEquals(2, shouldList.size());
+        assertTrue(((Map<String, Object>)shouldList.get(0)).containsKey("match"));
+        assertTrue(((Map<String, Object>)shouldList.get(1)).containsKey("match"));
+    }
 }

elasticsearch/native/pom.xml

@@ -13,7 +13,7 @@
 	<parent>
 		<artifactId>MetaModel-elasticsearch</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 

elasticsearch/pom.xml

@@ -13,7 +13,7 @@
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-elasticsearch</artifactId>

elasticsearch/rest/pom.xml

@@ -21,7 +21,7 @@ under the License.
 	<parent>
 		<artifactId>MetaModel-elasticsearch</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 
 	<modelVersion>4.0.0</modelVersion>

excel/pom.xml

@@ -21,7 +21,7 @@ under the License.
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-excel</artifactId>

fixedwidth/pom.xml

@@ -21,7 +21,7 @@ under the License.
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-fixedwidth</artifactId>

full/pom.xml

@@ -21,7 +21,7 @@ under the License.
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-full</artifactId>

hadoop/pom.xml

@@ -13,7 +13,7 @@
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-hadoop</artifactId>

hbase/pom.xml

@@ -13,7 +13,7 @@
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-hbase</artifactId>

jdbc/pom.xml

@@ -13,7 +13,7 @@
 	<parent>
 		<artifactId>MetaModel</artifactId>
 		<groupId>org.apache.metamodel</groupId>
-		<version>5.3.3-SNAPSHOT</version>
+		<version>5.3.4-SNAPSHOT</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>MetaModel-jdbc</artifactId>