test(spark): tail-sweep coverage for low-coverage datasource classes

hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestCatalystExpressionOrderPreserving.scala

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi
+
+import org.apache.spark.sql.catalyst.expressions.{Add, AttributeReference, BitwiseOr, Cast, DateAdd, DateSub, Divide, Exp, Expression, Literal, Log, Lower, Multiply, ShiftLeft, Sqrt, Upper}
+import org.apache.spark.sql.types.{DateType, DoubleType, IntegerType, LongType, StringType}
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+/**
+ * Branch coverage for the order-preserving transformation matcher in
+ * [[org.apache.spark.sql.BaseHoodieCatalystExpressionUtils]]. This is what lets data skipping map a
+ * transformed column reference back to its source attribute, so each case is pinned to the exact
+ * source [[AttributeReference]] it must recover, and non-order-preserving shapes must not match.
+ */
+class TestCatalystExpressionOrderPreserving extends SparkAdapterSupport {
+
+  private val intAttr = AttributeReference("i", IntegerType)()
+  private val strAttr = AttributeReference("s", StringType)()
+  private val dblAttr = AttributeReference("d", DoubleType)()
+  private val dateAttr = AttributeReference("dt", DateType)()
+
+  private def matched(expr: Expression): Option[AttributeReference] =
+    sparkAdapter.getCatalystExpressionUtils.tryMatchAttributeOrderingPreservingTransformation(expr)
+
+  @Test
+  def testIdentityAttributeMatches(): Unit = {
+    assertEquals(Some(intAttr), matched(intAttr))
+  }
+
+  @Test
+  def testArithmeticTransformationsPreserveOrdering(): Unit = {
+    assertEquals(Some(intAttr), matched(Add(intAttr, Literal(1))))
+    assertEquals(Some(intAttr), matched(Add(Literal(1), intAttr)))
+    assertEquals(Some(intAttr), matched(Multiply(intAttr, Literal(2))))
+    assertEquals(Some(intAttr), matched(Multiply(Literal(2), intAttr)))
+    assertEquals(Some(intAttr), matched(Divide(intAttr, Literal(2))))
+    assertEquals(Some(intAttr), matched(BitwiseOr(intAttr, Literal(1))))
+    assertEquals(Some(intAttr), matched(BitwiseOr(Literal(1), intAttr)))
+    assertEquals(Some(intAttr), matched(ShiftLeft(intAttr, Literal(1))))
+  }
+
+  @Test
+  def testUnaryMathAndStringTransformationsPreserveOrdering(): Unit = {
+    assertEquals(Some(dblAttr), matched(Exp(dblAttr)))
+    assertEquals(Some(dblAttr), matched(Log(dblAttr)))
+    assertEquals(Some(strAttr), matched(Upper(strAttr)))
+    assertEquals(Some(strAttr), matched(Lower(strAttr)))
+  }
+
+  @Test
+  def testDateTransformationsPreserveOrdering(): Unit = {
+    assertEquals(Some(dateAttr), matched(DateAdd(dateAttr, Literal(1))))
+    assertEquals(Some(dateAttr), matched(DateSub(dateAttr, Literal(1))))
+  }
+
+  @Test
+  def testUpCastPreservesOrderingButNumericToStringDoesNot(): Unit = {
+    // Widening a numeric column preserves ordering, so the source attribute is recovered.
+    assertEquals(Some(intAttr), matched(Cast(intAttr, LongType)))
+    // Casting a numeric column to string can reorder values, so it must not match.
+    assertEquals(None, matched(Cast(intAttr, StringType)))
+  }
+
+  @Test
+  def testNestedComposition(): Unit = {
+    assertEquals(Some(intAttr), matched(Add(Multiply(intAttr, Literal(2)), Literal(3))))
+  }
+
+  @Test
+  def testNonOrderPreservingExpressionsDoNotMatch(): Unit = {
+    // A bare literal carries no attribute.
+    assertEquals(None, matched(Literal(5)))
+    // No attribute on either operand.
+    assertEquals(None, matched(Add(Literal(1), Literal(2))))
+    // Sqrt is not one of the whitelisted order-preserving transformations.
+    assertEquals(None, matched(Sqrt(dblAttr)))
+  }
+}

hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/TestFileFormatUtilsForFileGroupReader.scala

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, Contains, EndsWith, EqualNullSafe, EqualTo, Expression, GreaterThan, In, IsNotNull, IsNull, LessThanOrEqual, Literal, Not, Or, StartsWith}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LocalRelation}
+import org.apache.spark.sql.types.{BooleanType, IntegerType, StringType, StructType}
+import org.junit.jupiter.api.Assertions.{assertEquals, assertSame}
+import org.junit.jupiter.api.Test
+
+/**
+ * Coverage for [[FileFormatUtilsForFileGroupReader.applyFiltersToPlan]], which lowers pushed-down
+ * data-source [[org.apache.spark.sql.sources.Filter]]s into a Catalyst [[Filter]] on the plan. Each
+ * case pins the exact translated Catalyst expression so a wrong mapping would fail, and the
+ * no-filter case must return the input plan untouched.
+ */
+class TestFileFormatUtilsForFileGroupReader {
+
+  private val a: Attribute = AttributeReference("a", IntegerType)()
+  private val b: Attribute = AttributeReference("b", StringType)()
+  private val tableSchema: StructType = new StructType().add("a", IntegerType).add("b", StringType)
+  private val resolved: Seq[Attribute] = Seq(a, b)
+  private val plan: LocalRelation = LocalRelation(a, b)
+
+  private def condOf(filters: Seq[sources.Filter]): Expression =
+    FileFormatUtilsForFileGroupReader.applyFiltersToPlan(plan, tableSchema, resolved, filters) match {
+      case Filter(cond, child) =>
+        assertSame(plan, child)
+        cond
+      case other => throw new AssertionError(s"expected a Filter, got $other")
+    }
+
+  @Test
+  def testComparisonAndNullFilters(): Unit = {
+    assertEquals(EqualTo(a, Literal(1)), condOf(Seq(sources.EqualTo("a", 1))))
+    assertEquals(EqualNullSafe(a, Literal(1)), condOf(Seq(sources.EqualNullSafe("a", 1))))
+    assertEquals(GreaterThan(a, Literal(5)), condOf(Seq(sources.GreaterThan("a", 5))))
+    assertEquals(LessThanOrEqual(a, Literal(5)), condOf(Seq(sources.LessThanOrEqual("a", 5))))
+    assertEquals(IsNull(b), condOf(Seq(sources.IsNull("b"))))
+    assertEquals(IsNotNull(b), condOf(Seq(sources.IsNotNull("b"))))
+  }
+
+  @Test
+  def testStringAndInFilters(): Unit = {
+    assertEquals(Contains(b, Literal("x")), condOf(Seq(sources.StringContains("b", "x"))))
+    assertEquals(StartsWith(b, Literal("x")), condOf(Seq(sources.StringStartsWith("b", "x"))))
+    assertEquals(EndsWith(b, Literal("x")), condOf(Seq(sources.StringEndsWith("b", "x"))))
+    assertEquals(
+      In(a, Seq(Literal(1), Literal(2), Literal(3))),
+      condOf(Seq(sources.In("a", Array[Any](1, 2, 3)))))
+  }
+
+  @Test
+  def testConstantFilters(): Unit = {
+    assertEquals(Literal(true, BooleanType), condOf(Seq(sources.AlwaysTrue())))
+    assertEquals(Literal(false, BooleanType), condOf(Seq(sources.AlwaysFalse())))
+  }
+
+  @Test
+  def testCompositeFilters(): Unit = {
+    // A nested and/or/not tree is lowered structurally.
+    assertEquals(
+      Or(EqualTo(a, Literal(1)), Not(IsNull(b))),
+      condOf(Seq(sources.Or(sources.EqualTo("a", 1), sources.Not(sources.IsNull("b"))))))
+  }
+
+  @Test
+  def testMultipleFiltersAreAndedInOrder(): Unit = {
+    // Several top-level filters combine left-to-right via And.
+    assertEquals(
+      And(GreaterThan(a, Literal(5)), IsNotNull(b)),
+      condOf(Seq(sources.GreaterThan("a", 5), sources.IsNotNull("b"))))
+  }
+
+  @Test
+  def testNoFiltersReturnsPlanUnchanged(): Unit = {
+    val result = FileFormatUtilsForFileGroupReader.applyFiltersToPlan(
+      plan, tableSchema, resolved, Seq.empty)
+    assertSame(plan, result)
+  }
+}

hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestAvroUtils.scala

@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.avro
+
+import org.apache.avro.Schema
+import org.apache.spark.sql.types._
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertThrows, assertTrue}
+import org.junit.jupiter.api.Test
+
+/**
+ * Direct coverage for [[AvroUtils]] and its [[AvroUtils.AvroSchemaHelper]], which are otherwise only
+ * exercised indirectly through the Avro serializers. Focuses on the type-support predicate and the
+ * schema-matching/validation error paths (extra Catalyst fields, extra required Avro fields,
+ * positional vs by-name matching, ambiguous by-name lookup), pinning the raised exception messages.
+ */
+class TestAvroUtils {
+
+  private def parse(json: String): Schema = new Schema.Parser().parse(json)
+
+  @Test
+  def testSupportsDataType(): Unit = {
+    assertTrue(AvroUtils.supportsDataType(IntegerType))
+    assertTrue(AvroUtils.supportsDataType(StringType))
+    assertTrue(AvroUtils.supportsDataType(NullType))
+    assertTrue(AvroUtils.supportsDataType(ArrayType(LongType)))
+    assertTrue(AvroUtils.supportsDataType(MapType(StringType, IntegerType)))
+    assertTrue(AvroUtils.supportsDataType(
+      new StructType().add("a", IntegerType).add("b", ArrayType(StringType))))
+    // CalendarInterval is not representable in Avro, so every wrapper around it is unsupported too.
+    assertFalse(AvroUtils.supportsDataType(CalendarIntervalType))
+    assertFalse(AvroUtils.supportsDataType(ArrayType(CalendarIntervalType)))
+    assertFalse(AvroUtils.supportsDataType(MapType(StringType, CalendarIntervalType)))
+    assertFalse(AvroUtils.supportsDataType(new StructType().add("a", CalendarIntervalType)))
+  }
+
+  @Test
+  def testToFieldStr(): Unit = {
+    assertEquals("top-level record", AvroUtils.toFieldStr(Seq.empty))
+    assertEquals("field 'foo'", AvroUtils.toFieldStr(Seq("foo")))
+    assertEquals("field 'foo.bar'", AvroUtils.toFieldStr(Seq("foo", "bar")))
+  }
+
+  @Test
+  def testIsNullable(): Unit = {
+    val record = parse(
+      """{"type":"record","name":"r","fields":[
+        |  {"name":"req","type":"int"},
+        |  {"name":"opt","type":["null","int"]},
+        |  {"name":"uni","type":["int","long"]}
+        |]}""".stripMargin)
+    assertFalse(AvroUtils.isNullable(record.getField("req")))
+    assertTrue(AvroUtils.isNullable(record.getField("opt")))
+    // A union without a NULL branch is not nullable.
+    assertFalse(AvroUtils.isNullable(record.getField("uni")))
+  }
+
+  @Test
+  def testSchemaHelperRejectsNonRecordSchema(): Unit = {
+    val ex = assertThrows(classOf[IncompatibleSchemaException],
+      () => new AvroUtils.AvroSchemaHelper(
+        Schema.create(Schema.Type.INT), new StructType(), Seq.empty, Seq.empty, false))
+    assertTrue(ex.getMessage.contains("as a RECORD"))
+  }
+
+  @Test
+  def testMatchedFieldsAndGetAvroFieldByName(): Unit = {
+    val avro = parse(
+      """{"type":"record","name":"r","fields":[
+        |  {"name":"id","type":"int"},
+        |  {"name":"name","type":["null","string"]}
+        |]}""".stripMargin)
+    val catalyst = new StructType().add("id", IntegerType).add("name", StringType)
+    val helper = new AvroUtils.AvroSchemaHelper(avro, catalyst, Seq.empty, Seq.empty, false)
+
+    assertEquals(2, helper.matchedFields.size)
+    assertEquals("id", helper.getAvroField("id", 0).get.name())
+    assertTrue(helper.getAvroField("missing", 5).isEmpty)
+  }
+
+  @Test
+  def testGetAvroFieldPositional(): Unit = {
+    val avro = parse(
+      """{"type":"record","name":"r","fields":[{"name":"only","type":"int"}]}""")
+    val helper = new AvroUtils.AvroSchemaHelper(avro, new StructType(), Seq.empty, Seq.empty, true)
+    // Positional matching ignores the name and selects by index.
+    assertEquals("only", helper.getAvroField("anything", 0).get.name())
+    assertTrue(helper.getAvroField("anything", 1).isEmpty)
+  }
+
+  @Test
+  def testValidateNoExtraCatalystFieldsByName(): Unit = {
+    val avro = parse(
+      """{"type":"record","name":"r","fields":[{"name":"id","type":"int"}]}""")
+    // A nullable Catalyst field with no Avro counterpart.
+    val catalyst = new StructType().add("id", IntegerType).add("extra", StringType, nullable = true)
+    val helper = new AvroUtils.AvroSchemaHelper(avro, catalyst, Seq.empty, Seq.empty, false)
+
+    val ex = assertThrows(classOf[IncompatibleSchemaException],
+      () => helper.validateNoExtraCatalystFields(ignoreNullable = false))
+    assertTrue(ex.getMessage.contains("Cannot find field 'extra' in Avro schema"))
+
+    // When nullable Catalyst fields are ignored, the same extra field is tolerated.
+    helper.validateNoExtraCatalystFields(ignoreNullable = true)
+  }
+
+  @Test
+  def testValidateNoExtraCatalystFieldsPositional(): Unit = {
+    val avro = parse(
+      """{"type":"record","name":"r","fields":[{"name":"id","type":"int"}]}""")
+    val catalyst = new StructType().add("id", IntegerType).add("second", IntegerType)
+    val helper = new AvroUtils.AvroSchemaHelper(avro, catalyst, Seq.empty, Seq.empty, true)
+
+    val ex = assertThrows(classOf[IncompatibleSchemaException],
+      () => helper.validateNoExtraCatalystFields(ignoreNullable = false))
+    assertTrue(ex.getMessage.contains("Cannot find field at position 1"))
+  }
+
+  @Test
+  def testValidateNoExtraRequiredAvroFields(): Unit = {
+    val avroWithRequiredGhost = parse(
+      """{"type":"record","name":"r","fields":[
+        |  {"name":"id","type":"int"},
+        |  {"name":"ghost","type":"int"}
+        |]}""".stripMargin)
+    val catalyst = new StructType().add("id", IntegerType)
+    val helper = new AvroUtils.AvroSchemaHelper(
+      avroWithRequiredGhost, catalyst, Seq.empty, Seq.empty, false)
+    val ex = assertThrows(classOf[IncompatibleSchemaException],
+      () => helper.validateNoExtraRequiredAvroFields())
+    assertTrue(ex.getMessage.contains("Found field 'ghost'"))
+
+    // A nullable extra Avro field is not required, so it is tolerated.
+    val avroWithOptionalGhost = parse(
+      """{"type":"record","name":"r","fields":[
+        |  {"name":"id","type":"int"},
+        |  {"name":"ghost","type":["null","int"]}
+        |]}""".stripMargin)
+    val helper2 = new AvroUtils.AvroSchemaHelper(
+      avroWithOptionalGhost, catalyst, Seq.empty, Seq.empty, false)
+    helper2.validateNoExtraRequiredAvroFields()
+  }
+
+  @Test
+  def testGetFieldByNameAmbiguousMatch(): Unit = {
+    // Two fields differing only by case collide under the default case-insensitive resolver.
+    val avro = parse(
+      """{"type":"record","name":"r","fields":[
+        |  {"name":"ID","type":"int"},
+        |  {"name":"id","type":"int"}
+        |]}""".stripMargin)
+    val helper = new AvroUtils.AvroSchemaHelper(avro, new StructType(), Seq.empty, Seq.empty, false)
+    val ex = assertThrows(classOf[IncompatibleSchemaException],
+      () => helper.getFieldByName("id"))
+    assertTrue(ex.getMessage.contains("gave 2 matches"))
+  }
+}