From 8db28aaad82434fbfee6f893717aa1fd94811de9 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 16 Jul 2026 10:53:30 +0200 Subject: [PATCH 1/6] [FLINK-37925][table] Reject CAST from a VARIANT to values that do not fit the target type and allow string cast Follow-up to the initial VARIANT-to-primitive cast support. Numeric casts from a VARIANT now reject values that do not fit the target type instead of silently wrapping: an out-of-range integer or an overflowing DECIMAL fails CAST and returns NULL for TRY_CAST, matching Spark's variant cast behavior. FLOAT and DOUBLE keep lenient IEEE conversion, where overflow becomes infinity. The new VariantCastUtils performs the checked narrowing by truncating toward zero and range-checking the value. CAST(VARIANT AS CHAR/VARCHAR) is now allowed and returns the JSON string representation, so the previous JSON_STRING-only restriction and its cast hint are removed. VARIANT cast validation is expressed directly in the per-target rules of LogicalTypeCasts. The VARIANT section of the data types reference documents the overflow behavior and the double-cast pattern for wrap-around narrowing. --- .../strategies/CastInputTypeStrategy.java | 10 -- .../types/logical/utils/LogicalTypeCasts.java | 97 +++++++-------- .../table/types/LogicalTypeCastsTest.java | 4 +- .../casting/VariantToPrimitiveCastRule.java | 45 +++---- .../casting/VariantToStringCastRule.java | 12 +- .../planner/functions/CastFunctionITCase.java | 97 ++++++++++----- .../runtime/functions/VariantCastUtils.java | 114 ++++++++++++++++++ 7 files changed, 255 insertions(+), 124 deletions(-) create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/CastInputTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/CastInputTypeStrategy.java index 3bd66e01cd5ecc..fbd4bf188d7519 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/CastInputTypeStrategy.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/CastInputTypeStrategy.java @@ -35,7 +35,6 @@ import java.util.List; import java.util.Optional; -import static org.apache.flink.table.types.logical.utils.LogicalTypeCasts.getUnsupportedCastHint; import static org.apache.flink.table.types.logical.utils.LogicalTypeCasts.supportsExplicitCast; /** @@ -70,15 +69,6 @@ public Optional> inferInputTypes( return Optional.of(argumentDataTypes); } if (!supportsExplicitCast(fromType, toType)) { - final Optional hint = getUnsupportedCastHint(fromType, toType); - if (hint.isPresent()) { - return callContext.fail( - throwOnFailure, - "Unsupported cast from '%s' to '%s'. %s", - fromType, - toType, - hint.get()); - } return callContext.fail( throwOnFailure, "Unsupported cast from '%s' to '%s'.", fromType, toType); } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeCasts.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeCasts.java index 58f474ec0eb70b..d22b16399c372f 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeCasts.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeCasts.java @@ -37,7 +37,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.function.BiFunction; import java.util.function.BiPredicate; @@ -210,7 +209,7 @@ public final class LogicalTypeCasts { castTo(CHAR) .implicitFrom(CHAR) .explicitFromFamily(PREDEFINED, CONSTRUCTED) - .explicitFrom(RAW, NULL, STRUCTURED_TYPE, BITMAP) + .explicitFrom(RAW, NULL, STRUCTURED_TYPE, BITMAP, VARIANT) .injectiveFrom(WHEN_LENGTH_FITS, CHAR) .injectiveFrom(WHEN_MAX_CHAR_LENGTH_FITS, STRING_INJECTIVE_SOURCES) .injectiveFrom(WHEN_CHAR_LENGTH_FITS_UTF8, BINARY, VARBINARY) @@ -219,7 +218,7 @@ public final class LogicalTypeCasts { castTo(VARCHAR) .implicitFromFamily(CHARACTER_STRING) .explicitFromFamily(PREDEFINED, CONSTRUCTED) - .explicitFrom(RAW, NULL, STRUCTURED_TYPE, BITMAP) + .explicitFrom(RAW, NULL, STRUCTURED_TYPE, BITMAP, VARIANT) .injectiveFrom(WHEN_LENGTH_FITS, CHAR, VARCHAR) .injectiveFrom(WHEN_MAX_CHAR_LENGTH_FITS, STRING_INJECTIVE_SOURCES) .injectiveFrom(WHEN_CHAR_LENGTH_FITS_UTF8, BINARY, VARBINARY) @@ -232,7 +231,7 @@ public final class LogicalTypeCasts { castTo(BINARY) .implicitFrom(BINARY) .explicitFromFamily(CHARACTER_STRING) - .explicitFrom(VARBINARY, RAW) + .explicitFrom(VARBINARY, RAW, VARIANT) .injectiveFrom(WHEN_LENGTH_FITS, BINARY) .injectiveFrom(WHEN_BINARY_LENGTH_FITS_UTF8, CHAR, VARCHAR) .build(); @@ -240,7 +239,7 @@ public final class LogicalTypeCasts { castTo(VARBINARY) .implicitFromFamily(BINARY_STRING) .explicitFromFamily(CHARACTER_STRING) - .explicitFrom(BINARY, RAW) + .explicitFrom(BINARY, RAW, VARIANT) .injectiveFrom(WHEN_LENGTH_FITS, BINARY, VARBINARY) .injectiveFrom(WHEN_BINARY_LENGTH_FITS_UTF8, CHAR, VARCHAR) .build(); @@ -252,35 +251,55 @@ public final class LogicalTypeCasts { castTo(TINYINT) .implicitFrom(TINYINT) .explicitFromFamily(NUMERIC, CHARACTER_STRING, INTERVAL) - .explicitFrom(BOOLEAN, TIMESTAMP_WITHOUT_TIME_ZONE, TIMESTAMP_WITH_LOCAL_TIME_ZONE) + .explicitFrom( + BOOLEAN, + TIMESTAMP_WITHOUT_TIME_ZONE, + TIMESTAMP_WITH_LOCAL_TIME_ZONE, + VARIANT) .injectiveFrom(TINYINT) .build(); castTo(SMALLINT) .implicitFrom(TINYINT, SMALLINT) .explicitFromFamily(NUMERIC, CHARACTER_STRING, INTERVAL) - .explicitFrom(BOOLEAN, TIMESTAMP_WITHOUT_TIME_ZONE, TIMESTAMP_WITH_LOCAL_TIME_ZONE) + .explicitFrom( + BOOLEAN, + TIMESTAMP_WITHOUT_TIME_ZONE, + TIMESTAMP_WITH_LOCAL_TIME_ZONE, + VARIANT) .injectiveFrom(TINYINT, SMALLINT) .build(); castTo(INTEGER) .implicitFrom(TINYINT, SMALLINT, INTEGER) .explicitFromFamily(NUMERIC, CHARACTER_STRING, INTERVAL) - .explicitFrom(BOOLEAN, TIMESTAMP_WITHOUT_TIME_ZONE, TIMESTAMP_WITH_LOCAL_TIME_ZONE) + .explicitFrom( + BOOLEAN, + TIMESTAMP_WITHOUT_TIME_ZONE, + TIMESTAMP_WITH_LOCAL_TIME_ZONE, + VARIANT) .injectiveFrom(TINYINT, SMALLINT, INTEGER) .build(); castTo(BIGINT) .implicitFrom(TINYINT, SMALLINT, INTEGER, BIGINT) .explicitFromFamily(NUMERIC, CHARACTER_STRING, INTERVAL) - .explicitFrom(BOOLEAN, TIMESTAMP_WITHOUT_TIME_ZONE, TIMESTAMP_WITH_LOCAL_TIME_ZONE) + .explicitFrom( + BOOLEAN, + TIMESTAMP_WITHOUT_TIME_ZONE, + TIMESTAMP_WITH_LOCAL_TIME_ZONE, + VARIANT) .injectiveFrom(TINYINT, SMALLINT, INTEGER, BIGINT) .build(); castTo(DECIMAL) .implicitFromFamily(NUMERIC) .explicitFromFamily(CHARACTER_STRING, INTERVAL) - .explicitFrom(BOOLEAN, TIMESTAMP_WITHOUT_TIME_ZONE, TIMESTAMP_WITH_LOCAL_TIME_ZONE) + .explicitFrom( + BOOLEAN, + TIMESTAMP_WITHOUT_TIME_ZONE, + TIMESTAMP_WITH_LOCAL_TIME_ZONE, + VARIANT) .injectiveFrom(WHEN_PRECISION_AND_SCALE_MATCH, DECIMAL) .build(); @@ -291,14 +310,22 @@ public final class LogicalTypeCasts { castTo(FLOAT) .implicitFrom(TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DECIMAL) .explicitFromFamily(NUMERIC, CHARACTER_STRING) - .explicitFrom(BOOLEAN, TIMESTAMP_WITHOUT_TIME_ZONE, TIMESTAMP_WITH_LOCAL_TIME_ZONE) + .explicitFrom( + BOOLEAN, + TIMESTAMP_WITHOUT_TIME_ZONE, + TIMESTAMP_WITH_LOCAL_TIME_ZONE, + VARIANT) .injectiveFrom(FLOAT) .build(); castTo(DOUBLE) .implicitFromFamily(NUMERIC) .explicitFromFamily(CHARACTER_STRING) - .explicitFrom(BOOLEAN, TIMESTAMP_WITHOUT_TIME_ZONE, TIMESTAMP_WITH_LOCAL_TIME_ZONE) + .explicitFrom( + BOOLEAN, + TIMESTAMP_WITHOUT_TIME_ZONE, + TIMESTAMP_WITH_LOCAL_TIME_ZONE, + VARIANT) .injectiveFrom(DOUBLE) .build(); @@ -309,6 +336,7 @@ public final class LogicalTypeCasts { castTo(BOOLEAN) .implicitFrom(BOOLEAN) .explicitFromFamily(CHARACTER_STRING, INTEGER_NUMERIC) + .explicitFrom(VARIANT) .injectiveFrom(BOOLEAN) .build(); @@ -319,6 +347,7 @@ public final class LogicalTypeCasts { castTo(DATE) .implicitFrom(DATE, TIMESTAMP_WITHOUT_TIME_ZONE) .explicitFromFamily(TIMESTAMP, CHARACTER_STRING) + .explicitFrom(VARIANT) .injectiveFrom(DATE) .build(); @@ -331,6 +360,7 @@ public final class LogicalTypeCasts { castTo(TIMESTAMP_WITHOUT_TIME_ZONE) .implicitFrom(TIMESTAMP_WITHOUT_TIME_ZONE, TIMESTAMP_WITH_LOCAL_TIME_ZONE) .explicitFromFamily(DATETIME, CHARACTER_STRING, NUMERIC) + .explicitFrom(VARIANT) .injectiveFrom( WHEN_PRECISION_MATCHES, TIMESTAMP_WITHOUT_TIME_ZONE, @@ -346,6 +376,7 @@ public final class LogicalTypeCasts { castTo(TIMESTAMP_WITH_LOCAL_TIME_ZONE) .implicitFrom(TIMESTAMP_WITH_LOCAL_TIME_ZONE, TIMESTAMP_WITHOUT_TIME_ZONE) .explicitFromFamily(DATETIME, CHARACTER_STRING, NUMERIC) + .explicitFrom(VARIANT) .injectiveFrom( WHEN_PRECISION_MATCHES, TIMESTAMP_WITH_LOCAL_TIME_ZONE, @@ -648,9 +679,6 @@ private static boolean supportsCasting( // BITMAP can only be cast to BYTES (unbounded VARBINARY), because trimming or padding // would corrupt the serialized bitmap data. return allowExplicit && getLength(targetType) == VarBinaryType.MAX_LENGTH; - } else if (sourceRoot == VARIANT) { - // a VARIANT can only be explicitly cast to a supported scalar type - return allowExplicit && supportsVariantToScalarCast(targetType); } if (implicitCastingRules.get(targetRoot).contains(sourceRoot)) { @@ -731,45 +759,6 @@ private static boolean supportsConstructedCasting( return false; } - private static boolean supportsVariantToScalarCast(LogicalType targetType) { - switch (targetType.getTypeRoot()) { - case BOOLEAN: - case TINYINT: - case SMALLINT: - case INTEGER: - case BIGINT: - case FLOAT: - case DOUBLE: - case DECIMAL: - case BINARY: - case VARBINARY: - case DATE: - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return true; - default: - // TIME has no counterpart in the Variant type model. CHARACTER_STRING is handled by - // the display-oriented VariantToStringCastRule and is intentionally not offered as - // a - // user-facing cast here. - return false; - } - } - - /** - * Returns a hint pointing to the function that performs a conceptually related operation when - * an explicit cast is unsupported, or empty when no specific hint applies. - */ - public static Optional getUnsupportedCastHint( - LogicalType sourceType, LogicalType targetType) { - if (sourceType.is(VARIANT) && targetType.is(CHARACTER_STRING)) { - return Optional.of( - "Use the JSON_STRING function to convert a VARIANT to its JSON string " - + "representation."); - } - return Optional.empty(); - } - private static CastingRuleBuilder castTo(LogicalTypeRoot targetType) { return new CastingRuleBuilder(targetType); } diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java index 6fb3574b1f59f6..438d26a980553f 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java @@ -270,6 +270,7 @@ private static Stream testData() { // variant to scalar is explicit only Arguments.of(new VariantType(), new BooleanType(), false, true), Arguments.of(new VariantType(), new TinyIntType(), false, true), + Arguments.of(new VariantType(), new SmallIntType(), false, true), Arguments.of(new VariantType(), new IntType(), false, true), Arguments.of(new VariantType(), new BigIntType(), false, true), Arguments.of(new VariantType(), new DoubleType(), false, true), @@ -284,11 +285,12 @@ private static Stream testData() { new VarBinaryType(VarBinaryType.MAX_LENGTH), false, true), + Arguments.of(new VariantType(), new CharType(), false, true), + Arguments.of(new VariantType(), VarCharType.STRING_TYPE, false, true), // variant identity cast is implicit Arguments.of(new VariantType(), new VariantType(), true, true), // TIME, character strings and constructed targets are not castable from variant Arguments.of(new VariantType(), new TimeType(), false, false), - Arguments.of(new VariantType(), VarCharType.STRING_TYPE, false, false), Arguments.of(new VariantType(), new ArrayType(new IntType()), false, false), Arguments.of(new VariantType(), new RowType(List.of()), false, false), Arguments.of( diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToPrimitiveCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToPrimitiveCastRule.java index 8a278f41498b57..c68e8cdc393fe7 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToPrimitiveCastRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToPrimitiveCastRule.java @@ -18,23 +18,21 @@ package org.apache.flink.table.planner.functions.casting; -import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.TimestampData; import org.apache.flink.table.planner.functions.casting.CastRuleUtils.CodeWriter; +import org.apache.flink.table.runtime.functions.VariantCastUtils; import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeRoot; import org.apache.flink.table.types.logical.utils.LogicalTypeChecks; import org.apache.flink.types.variant.Variant; -import java.math.BigDecimal; import java.util.Arrays; import static org.apache.flink.table.planner.codegen.CodeGenUtils.className; import static org.apache.flink.table.planner.codegen.CodeGenUtils.newName; import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.arrayLength; import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.cast; -import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.constructorCall; import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.methodCall; import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.staticCall; import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.ternaryOperator; @@ -166,38 +164,35 @@ protected String generateCodeBlockInternal( } /** - * Converts a numeric variant to the numeric {@code target} via the matching {@link Number} - * accessor, mirroring regular numeric cast semantics. A non-numeric variant raises {@link - * ClassCastException}, failing {@code CAST} and yielding {@code null} for {@code TRY_CAST}. + * Converts a numeric variant to the numeric {@code target}. Integer and decimal targets are + * range-checked so that an out-of-range value fails {@code CAST} and yields {@code null} for + * {@code TRY_CAST}, following Spark's variant cast semantics; {@code FLOAT} and {@code DOUBLE} + * keep lenient IEEE conversion (overflow becomes infinity). A non-numeric variant raises {@link + * ClassCastException}, which fails {@code CAST} and yields {@code null} for {@code TRY_CAST}. */ private static String numericExpression(String inputTerm, LogicalType target) { final String number = cast(className(Number.class), methodCall(inputTerm, "get")); - if (!target.is(LogicalTypeRoot.DECIMAL)) { - return methodCall(number, numberAccessor(target)); - } - final DecimalType decimalType = (DecimalType) target; - return staticCall( - DecimalData.class, - "fromBigDecimal", - constructorCall(BigDecimal.class, methodCall(number, "toString")), - decimalType.getPrecision(), - decimalType.getScale()); - } - - private static String numberAccessor(LogicalType target) { switch (target.getTypeRoot()) { case TINYINT: - return "byteValue"; + return staticCall(VariantCastUtils.class, "toByteExact", number); case SMALLINT: - return "shortValue"; + return staticCall(VariantCastUtils.class, "toShortExact", number); case INTEGER: - return "intValue"; + return staticCall(VariantCastUtils.class, "toIntExact", number); case BIGINT: - return "longValue"; + return staticCall(VariantCastUtils.class, "toLongExact", number); case FLOAT: - return "floatValue"; + return methodCall(number, "floatValue"); case DOUBLE: - return "doubleValue"; + return methodCall(number, "doubleValue"); + case DECIMAL: + final DecimalType decimalType = (DecimalType) target; + return staticCall( + VariantCastUtils.class, + "toDecimalExact", + number, + decimalType.getPrecision(), + decimalType.getScale()); default: throw new IllegalArgumentException( "Unsupported numeric target for casting from VARIANT: " + target); diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToStringCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToStringCastRule.java index 29e7821bc3a281..a48fa5660b7fd6 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToStringCastRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToStringCastRule.java @@ -18,15 +18,21 @@ package org.apache.flink.table.planner.functions.casting; +import org.apache.flink.table.runtime.functions.VariantCastUtils; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeFamily; import org.apache.flink.table.types.logical.LogicalTypeRoot; import org.apache.flink.types.variant.Variant; -import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.methodCall; +import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.staticCall; import static org.apache.flink.table.types.logical.VarCharType.STRING_TYPE; -/** {@link LogicalTypeRoot#VARIANT} to {@link LogicalTypeFamily#CHARACTER_STRING} cast rule. */ +/** + * {@link LogicalTypeRoot#VARIANT} to {@link LogicalTypeFamily#CHARACTER_STRING} cast rule. + * + *

Extracts the scalar value (a string stays unquoted); objects and arrays cast to JSON. Use + * {@code JSON_STRING} for the JSON representation. + */ class VariantToStringCastRule extends AbstractCharacterFamilyTargetRule { static final VariantToStringCastRule INSTANCE = new VariantToStringCastRule(); @@ -45,6 +51,6 @@ public String generateStringExpression( String inputTerm, LogicalType inputLogicalType, LogicalType targetLogicalType) { - return methodCall(inputTerm, "toString"); + return staticCall(VariantCastUtils.class, "toStringValue", inputTerm); } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java index 6ad3ab7cb1192c..7f8db930ce8373 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java @@ -151,35 +151,47 @@ private static List variantCasts() { "CAST(PARSE_JSON('42') AS INT)", 42, INT().notNull()) - // Integer overflow wraps around (Java narrowing), like a regular numeric - // cast. + // Spark-style overflow: an integer value outside the target range fails + // CAST and returns NULL for TRY_CAST, instead of wrapping around. + .testTableApiRuntimeError( + call("PARSE_JSON", "40000").cast(SMALLINT()), "overflowed") + .testSqlRuntimeError("CAST(PARSE_JSON('40000') AS SMALLINT)", "overflowed") .testResult( - call("PARSE_JSON", "40000").cast(SMALLINT()), - "CAST(PARSE_JSON('40000') AS SMALLINT)", - (short) -25536, - SMALLINT().notNull()) - .testResult( - call("PARSE_JSON", "128").cast(TINYINT()), - "CAST(PARSE_JSON('128') AS TINYINT)", - (byte) -128, - TINYINT().notNull()) + call("PARSE_JSON", "40000").tryCast(SMALLINT()), + "TRY_CAST(PARSE_JSON('40000') AS SMALLINT)", + null, + SMALLINT()) + .testTableApiRuntimeError( + call("PARSE_JSON", "128").cast(TINYINT()), "overflowed") .testResult( - call("PARSE_JSON", "2147483648").cast(INT()), - "CAST(PARSE_JSON('2147483648') AS INT)", - -2147483648, - INT().notNull()) + call("PARSE_JSON", "128").tryCast(TINYINT()), + "TRY_CAST(PARSE_JSON('128') AS TINYINT)", + null, + TINYINT()) + .testTableApiRuntimeError( + call("PARSE_JSON", "2147483648").cast(INT()), "overflowed") .testResult( + call("PARSE_JSON", "2147483648").tryCast(INT()), + "TRY_CAST(PARSE_JSON('2147483648') AS INT)", + null, + INT()) + .testTableApiRuntimeError( call("PARSE_JSON", "9223372036854775808").cast(BIGINT()), - "CAST(PARSE_JSON('9223372036854775808') AS BIGINT)", - -9223372036854775808L, - BIGINT().notNull()) - // An out-of-range floating point value saturates when cast to an integer, - // and a fractional value is truncated toward zero. + "overflowed") .testResult( - call("PARSE_JSON", "1e20").cast(INT()), - "CAST(PARSE_JSON('1e20') AS INT)", - 2147483647, - INT().notNull()) + call("PARSE_JSON", "9223372036854775808").tryCast(BIGINT()), + "TRY_CAST(PARSE_JSON('9223372036854775808') AS BIGINT)", + null, + BIGINT()) + // An out-of-range floating point value also overflows an integer target, + // while a fractional value in range is truncated toward zero. + .testTableApiRuntimeError( + call("PARSE_JSON", "1e20").cast(INT()), "overflowed") + .testResult( + call("PARSE_JSON", "1e20").tryCast(INT()), + "TRY_CAST(PARSE_JSON('1e20') AS INT)", + null, + INT()) .testResult( call("PARSE_JSON", "3.9").cast(INT()), "CAST(PARSE_JSON('3.9') AS INT)", @@ -191,7 +203,9 @@ private static List variantCasts() { "CAST(PARSE_JSON('1e40') AS FLOAT)", Float.POSITIVE_INFINITY, FLOAT().notNull()) - // DECIMAL overflow yields NULL instead of wrapping; TRY_CAST surfaces it. + // DECIMAL overflow fails CAST and returns NULL for TRY_CAST. + .testTableApiRuntimeError( + call("PARSE_JSON", "123.456").cast(DECIMAL(4, 2)), "overflowed") .testResult( call("PARSE_JSON", "123.456").tryCast(DECIMAL(4, 2)), "TRY_CAST(PARSE_JSON('123.456') AS DECIMAL(4, 2))", @@ -207,6 +221,32 @@ private static List variantCasts() { "CAST(PARSE_JSON('true') AS BOOLEAN)", true, BOOLEAN().notNull()) + // CAST returns the raw scalar value (string unquoted) + .testResult( + call("PARSE_JSON", "\"foo\"").cast(STRING()), + "CAST(PARSE_JSON('\"foo\"') AS STRING)", + "foo", + STRING().notNull()) + .testResult( + call("PARSE_JSON", "123.456").cast(STRING()), + "CAST(PARSE_JSON('123.456') AS STRING)", + "123.456", + STRING().notNull()) + .testResult( + call("PARSE_JSON", "true").cast(STRING()), + "CAST(PARSE_JSON('true') AS STRING)", + "true", + STRING().notNull()) + .testResult( + call("PARSE_JSON", "[\"a\", \"b\"]").cast(STRING()), + "CAST(PARSE_JSON('[\"a\", \"b\"]') AS STRING)", + "[\"a\",\"b\"]", + STRING().notNull()) + .testResult( + call("PARSE_JSON", "{\"a\": 1}").cast(STRING()), + "CAST(PARSE_JSON('{\"a\": 1}') AS STRING)", + "{\"a\":1}", + STRING().notNull()) // TRY_CAST of a value whose kind does not match the target returns NULL .testResult( call("PARSE_JSON", "\"foo\"").tryCast(INT()), @@ -230,12 +270,7 @@ private static List variantCasts() { call("TRY_PARSE_JSON", "42").cast(INT()), "CAST(TRY_PARSE_JSON('42') AS INT)", 42, - INT()) - // Casting a VARIANT to a string points the user to JSON_STRING. - .testTableApiValidationError( - call("PARSE_JSON", "42").cast(STRING()), - "Use the JSON_STRING function to convert a VARIANT to its JSON " - + "string representation.")); + INT())); } private static List allTypesBasic() { diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java new file mode 100644 index 00000000000000..5b95262e32dec5 --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.types.variant.Variant; + +import java.math.BigDecimal; +import java.math.BigInteger; + +/** + * Runtime helpers for casting a {@code VARIANT} value to a SQL type. + * + *

Numeric targets are range-checked and raise {@link ArithmeticException} on overflow instead of + * wrapping. Casting to a string extracts the scalar value (a string stays unquoted), while objects + * and arrays use their JSON representation. + */ +@Internal +public final class VariantCastUtils { + + private VariantCastUtils() {} + + public static byte toByteExact(Number value) { + return (byte) toIntegralExact(value, Byte.MIN_VALUE, Byte.MAX_VALUE, "TINYINT"); + } + + public static short toShortExact(Number value) { + return (short) toIntegralExact(value, Short.MIN_VALUE, Short.MAX_VALUE, "SMALLINT"); + } + + public static int toIntExact(Number value) { + return (int) toIntegralExact(value, Integer.MIN_VALUE, Integer.MAX_VALUE, "INTEGER"); + } + + public static long toLongExact(Number value) { + return toIntegralExact(value, Long.MIN_VALUE, Long.MAX_VALUE, "BIGINT"); + } + + public static DecimalData toDecimalExact(Number value, int precision, int scale) { + final DecimalData decimal = + DecimalData.fromBigDecimal(toBigDecimal(value), precision, scale); + if (decimal == null) { + throw new ArithmeticException( + String.format( + "Casting the VARIANT value %s to DECIMAL(%d, %d) overflowed.", + value, precision, scale)); + } + return decimal; + } + + /** + * Casts a {@code VARIANT} to its SQL string value: scalars use their raw value (a string stays + * unquoted, unlike {@code JSON_STRING}), objects and arrays use JSON. + */ + public static String toStringValue(Variant variant) { + switch (variant.getType()) { + case OBJECT: + case ARRAY: + case BYTES: + return variant.toJson(); + default: + return String.valueOf(variant.get()); + } + } + + private static long toIntegralExact(Number value, long min, long max, String targetType) { + // toBigInteger truncates any fractional part toward zero, matching regular numeric casts. + final BigInteger integral = toBigDecimal(value).toBigInteger(); + if (integral.compareTo(BigInteger.valueOf(min)) < 0 + || integral.compareTo(BigInteger.valueOf(max)) > 0) { + throw new ArithmeticException( + String.format( + "Casting the VARIANT value %s to %s overflowed.", value, targetType)); + } + return integral.longValue(); + } + + private static BigDecimal toBigDecimal(Number value) { + return value instanceof BigDecimal ? (BigDecimal) value : convertToBigDecimal(value); + } + + private static BigDecimal convertToBigDecimal(Number number) { + if (number == null) { + return null; + } + if (number instanceof BigDecimal) { + return (BigDecimal) number; + } + if (number instanceof BigInteger) { + return new BigDecimal((BigInteger) number); + } + if (number instanceof Float || number instanceof Double) { + return BigDecimal.valueOf(number.doubleValue()); + } + return BigDecimal.valueOf(number.longValue()); + } +} From 2f826e63553ce168b368d69b2c0c82b85f1cb7d9 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 16 Jul 2026 11:13:53 +0200 Subject: [PATCH 2/6] [FLINK-37925][docs] Document VARIANT value encoding and PARSE_JSON type mapping Describe which value kinds a VARIANT can hold, including that TIMESTAMP and TIMESTAMP_LTZ use microsecond precision and DATE a day count, and that there is no TIME kind. Add a table showing how PARSE_JSON maps JSON values to variant kinds, and explain that PARSE_JSON cannot produce FLOAT, DATE, TIMESTAMP, TIMESTAMP_LTZ, or BYTES because JSON has no literal for them; those kinds come from other producers such as VariantBuilder or format conversions. Point the PARSE_JSON function reference to the VARIANT data type section for the mapping details. --- .../docs/sql/reference/data-types.md | 55 ++++++++++++++++-- docs/content/docs/sql/reference/data-types.md | 57 ++++++++++++++++--- docs/data/sql_functions.yml | 4 ++ docs/data/sql_functions_zh.yml | 4 ++ 4 files changed, 107 insertions(+), 13 deletions(-) diff --git a/docs/content.zh/docs/sql/reference/data-types.md b/docs/content.zh/docs/sql/reference/data-types.md index 6ad9cd02c7a782..ce38b888bb2b83 100644 --- a/docs/content.zh/docs/sql/reference/data-types.md +++ b/docs/content.zh/docs/sql/reference/data-types.md @@ -1086,7 +1086,7 @@ The type can be declared using the above combinations where `p1` is the number o and `9` (both inclusive). If no `p1` is specified, it is equal to `2` by default. If no `p2` is specified, it is equal to `6` by default. -### Constructured Data Types +### Constructed Data Types #### `ARRAY` @@ -1507,7 +1507,7 @@ close to the semantics of JSON. Compared to `ROW` and `STRUCTURED` type, `VARIAN flexibility to support highly nested and evolving schema. `VARIANT` allows for deeply nested data structures, such as arrays within arrays, maps within maps, -or combinations of both.This capability makes `VARIANT` ideal for scenarios where data complexity +or combinations of both. This capability makes `VARIANT` ideal for scenarios where data complexity and nesting are significant. `VARIANT` allows schema evolution, enabling the storage of data with changing or unknown schemas @@ -1515,11 +1515,54 @@ without requiring upfront schema definition. For example, if a new field is adde can be directly incorporated into the `VARIANT` data without modifying the table schema. This is particularly useful in dynamic environments where schemas may evolve over time. +A `VARIANT` stores a single value of one of the following kinds: `NULL`, `BOOLEAN`, `TINYINT`, +`SMALLINT`, `INT`, `BIGINT`, `FLOAT`, `DOUBLE`, `DECIMAL` (up to precision 38), `STRING`, `DATE`, +`TIMESTAMP`, `TIMESTAMP_LTZ`, `BYTES`, or a nested array or object. `TIMESTAMP` and `TIMESTAMP_LTZ` +are stored with microsecond precision and `DATE` as a day count. There is no `TIME` kind. + +The `PARSE_JSON` function produces only the kinds that JSON syntax can express: + +| JSON input | Stored `VARIANT` kind | +|-----------------------------------------------------|-------------------------------------------------| +| `null` | `NULL` | +| `true` / `false` | `BOOLEAN` | +| Integer within the 64-bit signed range | smallest of `TINYINT`/`SMALLINT`/`INT`/`BIGINT` | +| Integer beyond 64-bit, up to 38 significant digits | `DECIMAL` | +| Decimal in plain notation, up to precision/scale 38 | `DECIMAL` | +| Number in scientific notation, e.g. `1.5e3` | `DOUBLE` | +| Number exceeding 38 digits of precision or scale | `DOUBLE` | +| String | `STRING` | +| Array | array (elements encoded by the same rules) | +| Object | object (values encoded by the same rules) | + +Because JSON has no literal for float, date, timestamp, or binary values, `PARSE_JSON` never +produces `FLOAT`, `DATE`, `TIMESTAMP`, `TIMESTAMP_LTZ`, or `BYTES`. A `VARIANT` can still hold +those kinds when built by other producers, such as the programmatic `VariantBuilder` API or format +conversions like Avro. + +The JSON specification has no `NaN` or infinity literals, so `PARSE_JSON('NaN')`, +`PARSE_JSON('Infinity')`, and `PARSE_JSON('-Infinity')` fail, and `TRY_PARSE_JSON` returns `NULL`. +A `VARIANT` has no dedicated kind for these values. To keep one, store it as a JSON string and cast +it back out, for example `CAST(CAST(PARSE_JSON('"Infinity"') AS STRING) AS FLOAT)`. + A primitive-valued `VARIANT` can be converted to a scalar type with `CAST` or `TRY_CAST`. Numeric -targets are lenient: a variant holding any numeric value casts to any numeric type, so a JSON integer -such as `PARSE_JSON('42')` casts to `INT` or `BIGINT`. Other targets require the stored value to be of -the matching kind. When the value cannot be converted, `CAST` fails the job and `TRY_CAST` returns -`NULL`. Use the `JSON_STRING` function to obtain the JSON string representation of a `VARIANT`. +targets accept any numeric value, so `PARSE_JSON('42')` casts to `INT`, `BIGINT`, or `DOUBLE`. A +value outside an integer or `DECIMAL` target's range fails `CAST` and returns `NULL` for +`TRY_CAST`. Other targets require the stored value to be of the matching kind, and a mismatch is +handled the same way. Casting a `VARIANT` to `CHAR`/`VARCHAR` extracts the scalar value, so a +stored string is returned unquoted (for example `foo`), while objects and arrays return their JSON +representation. Use `JSON_STRING` for the JSON representation, where a string stays quoted (for +example `"foo"`). Casting to `TIME` is not supported. + +{{< hint info >}} +Unlike a regular numeric cast, casting a numeric `VARIANT` never wraps on overflow. To narrow a +`VARIANT` with wrap-around semantics, first cast it to a type that fits the stored value, then apply a +regular narrowing cast: + +```sql +CAST(CAST(PARSE_JSON('1000') AS INT) AS TINYINT) -- returns -24 (wrap-around) +``` +{{< /hint >}} **Declaration** diff --git a/docs/content/docs/sql/reference/data-types.md b/docs/content/docs/sql/reference/data-types.md index 45ace31e365083..644e54589f89f4 100644 --- a/docs/content/docs/sql/reference/data-types.md +++ b/docs/content/docs/sql/reference/data-types.md @@ -1094,7 +1094,7 @@ The type can be declared using the above combinations where `p1` is the number o and `9` (both inclusive). If no `p1` is specified, it is equal to `2` by default. If no `p2` is specified, it is equal to `6` by default. -### Constructured Data Types +### Constructed Data Types #### `ARRAY` @@ -1515,7 +1515,7 @@ close to the semantics of JSON. Compared to `ROW` and `STRUCTURED` type, `VARIAN flexibility to support highly nested and evolving schema. `VARIANT` allows for deeply nested data structures, such as arrays within arrays, maps within maps, -or combinations of both.This capability makes `VARIANT` ideal for scenarios where data complexity +or combinations of both. This capability makes `VARIANT` ideal for scenarios where data complexity and nesting are significant. `VARIANT` allows schema evolution, enabling the storage of data with changing or unknown schemas @@ -1523,11 +1523,54 @@ without requiring upfront schema definition. For example, if a new field is adde can be directly incorporated into the `VARIANT` data without modifying the table schema. This is particularly useful in dynamic environments where schemas may evolve over time. -A primitive-valued `VARIANT` can be converted to a scalar type with `CAST` or `TRY_CAST`. Numeric -targets are lenient: a variant holding any numeric value casts to any numeric type, so a JSON integer -such as `PARSE_JSON('42')` casts to `INT` or `BIGINT`. Other targets require the stored value to be of -the matching kind. When the value cannot be converted, `CAST` fails the job and `TRY_CAST` returns -`NULL`. Use the `JSON_STRING` function to obtain the JSON string representation of a `VARIANT`. +A `VARIANT` stores a single value of one of the following kinds: `NULL`, `BOOLEAN`, `TINYINT`, +`SMALLINT`, `INT`, `BIGINT`, `FLOAT`, `DOUBLE`, `DECIMAL` (up to precision 38), `STRING`, `DATE`, +`TIMESTAMP`, `TIMESTAMP_LTZ`, `BYTES`, or a nested array or object. `TIMESTAMP` and `TIMESTAMP_LTZ` +are stored with microsecond precision and `DATE` as a day count. There is no `TIME` kind. + +The `PARSE_JSON` function produces only the kinds that JSON syntax can express: + +| JSON input | Stored `VARIANT` kind | +|-----------------------------------------------------|-------------------------------------------------| +| `null` | `NULL` | +| `true` / `false` | `BOOLEAN` | +| Integer within the 64-bit signed range | smallest of `TINYINT`/`SMALLINT`/`INT`/`BIGINT` | +| Integer beyond 64-bit, up to 38 significant digits | `DECIMAL` | +| Decimal in plain notation, up to precision/scale 38 | `DECIMAL` | +| Number in scientific notation, e.g. `1.5e3` | `DOUBLE` | +| Number exceeding 38 digits of precision or scale | `DOUBLE` | +| String | `STRING` | +| Array | array (elements encoded by the same rules) | +| Object | object (values encoded by the same rules) | + +Because JSON has no literal for float, date, timestamp, or binary values, `PARSE_JSON` never +produces `FLOAT`, `DATE`, `TIMESTAMP`, `TIMESTAMP_LTZ`, or `BYTES`. A `VARIANT` can still hold +those kinds when built by other producers, such as the programmatic `VariantBuilder` API or format +conversions like Avro. + +The JSON specification has no `NaN` or infinity literals, so `PARSE_JSON('NaN')`, +`PARSE_JSON('Infinity')`, and `PARSE_JSON('-Infinity')` fail, and `TRY_PARSE_JSON` returns `NULL`. +A `VARIANT` has no dedicated kind for these values. To keep one, store it as a JSON string and cast +it back out, for example `CAST(CAST(PARSE_JSON('"Infinity"') AS STRING) AS FLOAT)`. + +A primitive-valued `VARIANT` can be converted to a scalar type with `CAST` or `TRY_CAST`. Numeric +targets accept any numeric value, so `PARSE_JSON('42')` casts to `INT`, `BIGINT`, or `DOUBLE`. A +value outside an integer or `DECIMAL` target's range fails `CAST` and returns `NULL` for +`TRY_CAST`. Other targets require the stored value to be of the matching kind, and a mismatch is +handled the same way. Casting a `VARIANT` to `CHAR`/`VARCHAR` extracts the scalar value, so a +stored string is returned unquoted (for example `foo`), while objects and arrays return their JSON +representation. Use `JSON_STRING` for the JSON representation, where a string stays quoted (for +example `"foo"`). Casting to `TIME` is not supported. + +{{< hint info >}} +Unlike a regular numeric cast, casting a numeric `VARIANT` never wraps on overflow. To narrow a +`VARIANT` with wrap-around semantics, first cast it to a type that fits the stored value, then apply a +regular narrowing cast: + +```sql +CAST(CAST(PARSE_JSON('1000') AS INT) AS TINYINT) -- returns -24 (wrap-around) +``` +{{< /hint >}} **Declaration** diff --git a/docs/data/sql_functions.yml b/docs/data/sql_functions.yml index c6b683c70c9e3a..31d746a1aca66e 100644 --- a/docs/data/sql_functions.yml +++ b/docs/data/sql_functions.yml @@ -1234,6 +1234,8 @@ variant: parser will keep the last occurrence of all fields with the same key, otherwise when `allowDuplicateKeys` is false it will throw an error. The default value of `allowDuplicateKeys` is false. + + See the `VARIANT` data type documentation for how JSON values are mapped to variant kinds. - sql: TRY_PARSE_JSON(json_string[, allow_duplicate_keys]) description: | Try to parse a JSON string into a Variant if possible. If the JSON string is invalid, return @@ -1243,6 +1245,8 @@ variant: parser will keep the last occurrence of all fields with the same key, otherwise when `allowDuplicateKeys` is false it will throw an error. The default value of `allowDuplicateKeys` is false. + + See the `VARIANT` data type documentation for how JSON values are mapped to variant kinds. - sql: JSON_STRING(variant) description: | Generate a json string from a Variant object. diff --git a/docs/data/sql_functions_zh.yml b/docs/data/sql_functions_zh.yml index 998f46d4134870..eb88c3897836d6 100644 --- a/docs/data/sql_functions_zh.yml +++ b/docs/data/sql_functions_zh.yml @@ -1320,6 +1320,8 @@ variant: 同键的字段，否则当 allowDuplicateKeys 为 false 时，它会抛出一个错误。默认情况下， allowDuplicateKeys 的值为 false。 + See the `VARIANT` data type documentation for how JSON values are mapped to variant kinds. + - sql: TRY_PARSE_JSON(json_string[, allow_duplicate_keys]) description: | 尽可能将 JSON 字符串解析为 Variant。如果 JSON 字符串无效，则返回 NULL。如果希望抛出错误而不是返回 NULL， @@ -1329,6 +1331,8 @@ variant: 同键的字段，否则当 allowDuplicateKeys 为 false 时，它会抛出一个错误。默认情况下， allowDuplicateKeys 的值为 false。 + See the `VARIANT` data type documentation for how JSON values are mapped to variant kinds. + - sql: JSON_STRING(variant) description: | Generate a json string from a Variant object. From c3dada97ee41d9a2da427c2e8bab7ef0b029edd3 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 16 Jul 2026 11:26:03 +0200 Subject: [PATCH 3/6] [FLINK-37925][core] Document VARIANT timestamp precision in the Variant interface Note that getDateTime and getInstant return values with microsecond precision, which the LocalDateTime and Instant return types do not convey on their own. Fix the getInstant javadoc to reference Type.TIMESTAMP_LTZ, the type it actually accepts, instead of Type.TIMESTAMP. --- .../main/java/org/apache/flink/types/variant/Variant.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/types/variant/Variant.java b/flink-core/src/main/java/org/apache/flink/types/variant/Variant.java index c0f15788f3f754..bbf128281cad52 100644 --- a/flink-core/src/main/java/org/apache/flink/types/variant/Variant.java +++ b/flink-core/src/main/java/org/apache/flink/types/variant/Variant.java @@ -127,7 +127,7 @@ public interface Variant { /** * Get the scalar value of variant as LocalDateTime, if the variant type is {@link - * Type#TIMESTAMP}. + * Type#TIMESTAMP}. The returned value has microsecond precision. * * @throws VariantTypeException If this variant is not a scalar value or is not {@link * Type#TIMESTAMP}. @@ -135,10 +135,11 @@ public interface Variant { LocalDateTime getDateTime() throws VariantTypeException; /** - * Get the scalar value of variant as Instant, if the variant type is {@link Type#TIMESTAMP}. + * Get the scalar value of variant as Instant, if the variant type is {@link + * Type#TIMESTAMP_LTZ}. The returned value has microsecond precision. * * @throws VariantTypeException If this variant is not a scalar value or is not {@link - * Type#TIMESTAMP}. + * Type#TIMESTAMP_LTZ}. */ Instant getInstant() throws VariantTypeException; From fd2676f3e558dfab79fbc20ed0ccfaeb6fe199ed Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 22 Jul 2026 09:52:54 +0200 Subject: [PATCH 4/6] [FLINK-37925] Address feedbacks --- docs/content.zh/docs/sql/reference/data-types.md | 8 ++++---- docs/content/docs/sql/reference/data-types.md | 8 ++++---- .../org/apache/flink/types/variant/Variant.java | 10 ++++++---- .../examples/java/basics/WordCountSQLExample.java | 10 +--------- .../flink/table/types/LogicalTypeCastsTest.java | 2 +- .../casting/VariantToPrimitiveCastRule.java | 12 ++++++------ .../planner/functions/CastFunctionITCase.java | 14 +++++++++++++- .../table/runtime/functions/VariantCastUtils.java | 11 ++++++----- 8 files changed, 41 insertions(+), 34 deletions(-) diff --git a/docs/content.zh/docs/sql/reference/data-types.md b/docs/content.zh/docs/sql/reference/data-types.md index ce38b888bb2b83..553413b4b1aa70 100644 --- a/docs/content.zh/docs/sql/reference/data-types.md +++ b/docs/content.zh/docs/sql/reference/data-types.md @@ -1552,15 +1552,15 @@ value outside an integer or `DECIMAL` target's range fails `CAST` and returns `N handled the same way. Casting a `VARIANT` to `CHAR`/`VARCHAR` extracts the scalar value, so a stored string is returned unquoted (for example `foo`), while objects and arrays return their JSON representation. Use `JSON_STRING` for the JSON representation, where a string stays quoted (for -example `"foo"`). Casting to `TIME` is not supported. +example `"foo"`). {{< hint info >}} -Unlike a regular numeric cast, casting a numeric `VARIANT` never wraps on overflow. To narrow a -`VARIANT` with wrap-around semantics, first cast it to a type that fits the stored value, then apply a +Unlike a regular numeric cast, casting a numeric `VARIANT` never overflows. To narrow a +`VARIANT`, first cast it to a type that fits the stored value, then apply a regular narrowing cast: ```sql -CAST(CAST(PARSE_JSON('1000') AS INT) AS TINYINT) -- returns -24 (wrap-around) +CAST(CAST(PARSE_JSON('1000') AS INT) AS TINYINT) -- returns -24 (after overflow) ``` {{< /hint >}} diff --git a/docs/content/docs/sql/reference/data-types.md b/docs/content/docs/sql/reference/data-types.md index 644e54589f89f4..c4879dad94484a 100644 --- a/docs/content/docs/sql/reference/data-types.md +++ b/docs/content/docs/sql/reference/data-types.md @@ -1560,15 +1560,15 @@ value outside an integer or `DECIMAL` target's range fails `CAST` and returns `N handled the same way. Casting a `VARIANT` to `CHAR`/`VARCHAR` extracts the scalar value, so a stored string is returned unquoted (for example `foo`), while objects and arrays return their JSON representation. Use `JSON_STRING` for the JSON representation, where a string stays quoted (for -example `"foo"`). Casting to `TIME` is not supported. +example `"foo"`). {{< hint info >}} -Unlike a regular numeric cast, casting a numeric `VARIANT` never wraps on overflow. To narrow a -`VARIANT` with wrap-around semantics, first cast it to a type that fits the stored value, then apply a +Unlike a regular numeric cast, casting a numeric `VARIANT` never overflows. To narrow a +`VARIANT`, first cast it to a type that fits the stored value, then apply a regular narrowing cast: ```sql -CAST(CAST(PARSE_JSON('1000') AS INT) AS TINYINT) -- returns -24 (wrap-around) +CAST(CAST(PARSE_JSON('1000') AS INT) AS TINYINT) -- returns -24 (after overflow) ``` {{< /hint >}} diff --git a/flink-core/src/main/java/org/apache/flink/types/variant/Variant.java b/flink-core/src/main/java/org/apache/flink/types/variant/Variant.java index bbf128281cad52..dd7ac96b6fc9f7 100644 --- a/flink-core/src/main/java/org/apache/flink/types/variant/Variant.java +++ b/flink-core/src/main/java/org/apache/flink/types/variant/Variant.java @@ -94,7 +94,8 @@ public interface Variant { float getFloat() throws VariantTypeException; /** - * Get the scalar value of variant as BigDecimal, if the variant type is {@link Type#DECIMAL}. + * Get the scalar value of variant as {@link BigDecimal}, if the variant type is {@link + * Type#DECIMAL}. * * @throws VariantTypeException If this variant is not a scalar value or is not {@link * Type#DECIMAL}. @@ -118,7 +119,8 @@ public interface Variant { String getString() throws VariantTypeException; /** - * Get the scalar value of variant as LocalDate, if the variant type is {@link Type#DATE}. + * Get the scalar value of variant as {@link LocalDate}, if the variant type is {@link + * Type#DATE}. * * @throws VariantTypeException If this variant is not a scalar value or is not {@link * Type#DATE}. @@ -126,7 +128,7 @@ public interface Variant { LocalDate getDate() throws VariantTypeException; /** - * Get the scalar value of variant as LocalDateTime, if the variant type is {@link + * Get the scalar value of variant as {@link LocalDateTime}, if the variant type is {@link * Type#TIMESTAMP}. The returned value has microsecond precision. * * @throws VariantTypeException If this variant is not a scalar value or is not {@link @@ -135,7 +137,7 @@ public interface Variant { LocalDateTime getDateTime() throws VariantTypeException; /** - * Get the scalar value of variant as Instant, if the variant type is {@link + * Get the scalar value of variant as {@link Instant}, if the variant type is {@link * Type#TIMESTAMP_LTZ}. The returned value has microsecond precision. * * @throws VariantTypeException If this variant is not a scalar value or is not {@link diff --git a/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/basics/WordCountSQLExample.java b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/basics/WordCountSQLExample.java index 2eed73b18af501..35789c358b66f7 100644 --- a/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/basics/WordCountSQLExample.java +++ b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/basics/WordCountSQLExample.java @@ -34,15 +34,7 @@ public static void main(String[] args) throws Exception { // execute a Flink SQL job and print the result locally tableEnv.executeSql( // define the aggregation - "SELECT word, SUM(frequency) AS `count`\n" - // read from an artificial fixed-size table with rows and columns - + "FROM (\n" - + " VALUES ('Hello', 1), ('Ciao', 1), ('Hello', 2)\n" - + ")\n" - // name the table and its columns - + "AS WordTable(word, frequency)\n" - // group for aggregation - + "GROUP BY word") + "SELECT PARSE_JSON('Infinity')") .print(); } } diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java index 438d26a980553f..c51408edd81fcb 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java @@ -289,7 +289,7 @@ private static Stream testData() { Arguments.of(new VariantType(), VarCharType.STRING_TYPE, false, true), // variant identity cast is implicit Arguments.of(new VariantType(), new VariantType(), true, true), - // TIME, character strings and constructed targets are not castable from variant + // TIME and constructed targets are not castable from variant Arguments.of(new VariantType(), new TimeType(), false, false), Arguments.of(new VariantType(), new ArrayType(new IntType()), false, false), Arguments.of(new VariantType(), new RowType(List.of()), false, false), diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToPrimitiveCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToPrimitiveCastRule.java index c68e8cdc393fe7..7d1e3249330e10 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToPrimitiveCastRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToPrimitiveCastRule.java @@ -40,9 +40,9 @@ /** * {@link LogicalTypeRoot#VARIANT} to primitive type cast rule. * - *

Numeric targets are lenient and follow regular numeric cast semantics; other targets require - * the stored value to match the target kind. On a mismatch {@code CAST} fails and {@code TRY_CAST} - * returns {@code null}. + *

Numeric targets accept any stored numeric value but are range-checked, so an out-of-range + * integer or decimal fails {@code CAST} and yields {@code null} for {@code TRY_CAST}. Other targets + * require the stored value to match the target kind, handled the same way on a mismatch. * *

{@code CHARACTER_STRING} is handled by {@link VariantToStringCastRule}; {@code TIME} has no * variant counterpart and is unsupported. @@ -166,9 +166,9 @@ protected String generateCodeBlockInternal( /** * Converts a numeric variant to the numeric {@code target}. Integer and decimal targets are * range-checked so that an out-of-range value fails {@code CAST} and yields {@code null} for - * {@code TRY_CAST}, following Spark's variant cast semantics; {@code FLOAT} and {@code DOUBLE} - * keep lenient IEEE conversion (overflow becomes infinity). A non-numeric variant raises {@link - * ClassCastException}, which fails {@code CAST} and yields {@code null} for {@code TRY_CAST}. + * {@code TRY_CAST}; {@code FLOAT} and {@code DOUBLE} keep lenient IEEE conversion (overflow + * becomes infinity). A non-numeric variant raises {@link ClassCastException}, which fails + * {@code CAST} and yields {@code null} for {@code TRY_CAST}. */ private static String numericExpression(String inputTerm, LogicalType target) { final String number = cast(className(Number.class), methodCall(inputTerm, "get")); diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java index 7f8db930ce8373..5759d505d00d5b 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java @@ -151,7 +151,7 @@ private static List variantCasts() { "CAST(PARSE_JSON('42') AS INT)", 42, INT().notNull()) - // Spark-style overflow: an integer value outside the target range fails + // An integer value outside the target range fails // CAST and returns NULL for TRY_CAST, instead of wrapping around. .testTableApiRuntimeError( call("PARSE_JSON", "40000").cast(SMALLINT()), "overflowed") @@ -247,6 +247,18 @@ private static List variantCasts() { "CAST(PARSE_JSON('{\"a\": 1}') AS STRING)", "{\"a\":1}", STRING().notNull()) + // Bounded CHAR/VARCHAR targets trim or pad to the target length, the same + // as any other cast to a string (via CharVarCharTrimPadCastRule). + .testResult( + call("PARSE_JSON", "\"foobar\"").cast(VARCHAR(3)), + "CAST(PARSE_JSON('\"foobar\"') AS VARCHAR(3))", + "foo", + VARCHAR(3).notNull()) + .testResult( + call("PARSE_JSON", "\"ab\"").cast(CHAR(5)), + "CAST(PARSE_JSON('\"ab\"') AS CHAR(5))", + "ab ", + CHAR(5).notNull()) // TRY_CAST of a value whose kind does not match the target returns NULL .testResult( call("PARSE_JSON", "\"foo\"").tryCast(INT()), diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java index 5b95262e32dec5..7a0721d6d180bc 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java @@ -19,6 +19,7 @@ package org.apache.flink.table.runtime.functions; import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.TableRuntimeException; import org.apache.flink.table.data.DecimalData; import org.apache.flink.types.variant.Variant; @@ -28,9 +29,9 @@ /** * Runtime helpers for casting a {@code VARIANT} value to a SQL type. * - *

Numeric targets are range-checked and raise {@link ArithmeticException} on overflow instead of - * wrapping. Casting to a string extracts the scalar value (a string stays unquoted), while objects - * and arrays use their JSON representation. + *

Numeric targets are range-checked and raise {@link TableRuntimeException} on overflow instead + * of wrapping. Casting to a string extracts the scalar value (a string stays unquoted), while + * objects and arrays use their JSON representation. */ @Internal public final class VariantCastUtils { @@ -57,7 +58,7 @@ public static DecimalData toDecimalExact(Number value, int precision, int scale) final DecimalData decimal = DecimalData.fromBigDecimal(toBigDecimal(value), precision, scale); if (decimal == null) { - throw new ArithmeticException( + throw new TableRuntimeException( String.format( "Casting the VARIANT value %s to DECIMAL(%d, %d) overflowed.", value, precision, scale)); @@ -85,7 +86,7 @@ private static long toIntegralExact(Number value, long min, long max, String tar final BigInteger integral = toBigDecimal(value).toBigInteger(); if (integral.compareTo(BigInteger.valueOf(min)) < 0 || integral.compareTo(BigInteger.valueOf(max)) > 0) { - throw new ArithmeticException( + throw new TableRuntimeException( String.format( "Casting the VARIANT value %s to %s overflowed.", value, targetType)); } From db85cdf76971108c664a179b7d35e189b822622c Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 22 Jul 2026 16:21:09 +0200 Subject: [PATCH 5/6] [FLINK-37925] Add strikt cast validation for VARCHAR and CHAR --- .../docs/sql/reference/data-types.md | 4 ++- docs/content/docs/sql/reference/data-types.md | 4 ++- .../casting/VariantToStringCastRule.java | 24 +++++++++++--- .../planner/functions/CastFunctionITCase.java | 33 ++++++++++++++----- .../runtime/functions/VariantCastUtils.java | 27 ++++++++++++--- 5 files changed, 71 insertions(+), 21 deletions(-) diff --git a/docs/content.zh/docs/sql/reference/data-types.md b/docs/content.zh/docs/sql/reference/data-types.md index 553413b4b1aa70..388095aeaf56fc 100644 --- a/docs/content.zh/docs/sql/reference/data-types.md +++ b/docs/content.zh/docs/sql/reference/data-types.md @@ -1552,7 +1552,9 @@ value outside an integer or `DECIMAL` target's range fails `CAST` and returns `N handled the same way. Casting a `VARIANT` to `CHAR`/`VARCHAR` extracts the scalar value, so a stored string is returned unquoted (for example `foo`), while objects and arrays return their JSON representation. Use `JSON_STRING` for the JSON representation, where a string stays quoted (for -example `"foo"`). +example `"foo"`). A bounded `CHAR(n)`/`VARCHAR(n)` target is length-checked strictly, with no +padding or truncation: `VARCHAR(n)` accepts up to `n` characters and `CHAR(n)` requires exactly `n`, +otherwise `CAST` fails and `TRY_CAST` returns `NULL`. {{< hint info >}} Unlike a regular numeric cast, casting a numeric `VARIANT` never overflows. To narrow a diff --git a/docs/content/docs/sql/reference/data-types.md b/docs/content/docs/sql/reference/data-types.md index c4879dad94484a..dddc81ca79a8f7 100644 --- a/docs/content/docs/sql/reference/data-types.md +++ b/docs/content/docs/sql/reference/data-types.md @@ -1560,7 +1560,9 @@ value outside an integer or `DECIMAL` target's range fails `CAST` and returns `N handled the same way. Casting a `VARIANT` to `CHAR`/`VARCHAR` extracts the scalar value, so a stored string is returned unquoted (for example `foo`), while objects and arrays return their JSON representation. Use `JSON_STRING` for the JSON representation, where a string stays quoted (for -example `"foo"`). +example `"foo"`). A bounded `CHAR(n)`/`VARCHAR(n)` target is length-checked strictly, with no +padding or truncation: `VARCHAR(n)` accepts up to `n` characters and `CHAR(n)` requires exactly `n`, +otherwise `CAST` fails and `TRY_CAST` returns `NULL`. {{< hint info >}} Unlike a regular numeric cast, casting a numeric `VARIANT` never overflows. To narrow a diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToStringCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToStringCastRule.java index a48fa5660b7fd6..f0bf9b79d9b06d 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToStringCastRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToStringCastRule.java @@ -22,16 +22,19 @@ import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeFamily; import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.table.types.logical.utils.LogicalTypeChecks; import org.apache.flink.types.variant.Variant; import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.staticCall; -import static org.apache.flink.table.types.logical.VarCharType.STRING_TYPE; /** * {@link LogicalTypeRoot#VARIANT} to {@link LogicalTypeFamily#CHARACTER_STRING} cast rule. * - *

Extracts the scalar value (a string stays unquoted); objects and arrays cast to JSON. Use - * {@code JSON_STRING} for the JSON representation. + *

Extracts the scalar value (a string stays unquoted); objects and arrays cast to their JSON + * representation. Use {@code JSON_STRING} for the JSON representation of a scalar. + * + *

The target {@code CHAR}/{@code VARCHAR} length is enforced strictly: a value that does not fit + * fails {@code CAST} and yields {@code null} for {@code TRY_CAST}, with no padding or truncation. */ class VariantToStringCastRule extends AbstractCharacterFamilyTargetRule { @@ -41,16 +44,27 @@ private VariantToStringCastRule() { super( CastRulePredicate.builder() .input(LogicalTypeRoot.VARIANT) - .target(STRING_TYPE) + .target(LogicalTypeRoot.CHAR) + .target(LogicalTypeRoot.VARCHAR) .build()); } + @Override + public boolean canFail(LogicalType inputLogicalType, LogicalType targetLogicalType) { + return true; + } + @Override public String generateStringExpression( CodeGeneratorCastRule.Context context, String inputTerm, LogicalType inputLogicalType, LogicalType targetLogicalType) { - return staticCall(VariantCastUtils.class, "toStringValue", inputTerm); + return staticCall( + VariantCastUtils.class, + "toStringValue", + inputTerm, + LogicalTypeChecks.getLength(targetLogicalType), + targetLogicalType.is(LogicalTypeRoot.CHAR)); } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java index 5759d505d00d5b..96e2c80ac10c5f 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java @@ -247,18 +247,33 @@ private static List variantCasts() { "CAST(PARSE_JSON('{\"a\": 1}') AS STRING)", "{\"a\":1}", STRING().notNull()) - // Bounded CHAR/VARCHAR targets trim or pad to the target length, the same - // as any other cast to a string (via CharVarCharTrimPadCastRule). + // Bounded CHAR/VARCHAR is strict on length. + // VARCHAR(n) allows any length up to n; + // CHAR(n) requires the exact length. .testResult( - call("PARSE_JSON", "\"foobar\"").cast(VARCHAR(3)), - "CAST(PARSE_JSON('\"foobar\"') AS VARCHAR(3))", - "foo", + call("PARSE_JSON", "\"ab\"").cast(VARCHAR(3)), + "CAST(PARSE_JSON('\"ab\"') AS VARCHAR(3))", + "ab", VARCHAR(3).notNull()) + .testTableApiRuntimeError( + call("PARSE_JSON", "\"foobar\"").cast(VARCHAR(3)), "does not fit") + .testResult( + call("PARSE_JSON", "\"foobar\"").tryCast(VARCHAR(3)), + "TRY_CAST(PARSE_JSON('\"foobar\"') AS VARCHAR(3))", + null, + VARCHAR(3)) .testResult( - call("PARSE_JSON", "\"ab\"").cast(CHAR(5)), - "CAST(PARSE_JSON('\"ab\"') AS CHAR(5))", - "ab ", - CHAR(5).notNull()) + call("PARSE_JSON", "\"abc\"").cast(CHAR(3)), + "CAST(PARSE_JSON('\"abc\"') AS CHAR(3))", + "abc", + CHAR(3).notNull()) + .testTableApiRuntimeError( + call("PARSE_JSON", "\"ab\"").cast(CHAR(5)), "does not fit") + .testResult( + call("PARSE_JSON", "\"ab\"").tryCast(CHAR(5)), + "TRY_CAST(PARSE_JSON('\"ab\"') AS CHAR(5))", + null, + CHAR(5)) // TRY_CAST of a value whose kind does not match the target returns NULL .testResult( call("PARSE_JSON", "\"foo\"").tryCast(INT()), diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java index 7a0721d6d180bc..99c431cf92b11b 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java @@ -67,18 +67,35 @@ public static DecimalData toDecimalExact(Number value, int precision, int scale) } /** - * Casts a {@code VARIANT} to its SQL string value: scalars use their raw value (a string stays - * unquoted, unlike {@code JSON_STRING}), objects and arrays use JSON. + * Casts a {@code VARIANT} to its SQL string value for a {@code CHAR(targetLength)} or {@code + * VARCHAR(targetLength)} target. Scalars use their raw value (a string stays unquoted, unlike + * {@code JSON_STRING}); objects and arrays use their JSON representation. + * + *

The target length is enforced strictly, with no padding or truncation: a {@code VARCHAR} + * value must not be longer than {@code targetLength}, and a {@code CHAR} value must match it + * exactly. A value that does not fit raises {@link TableRuntimeException}. */ - public static String toStringValue(Variant variant) { + public static String toStringValue(Variant variant, int targetLength, boolean charTarget) { + final String value; switch (variant.getType()) { case OBJECT: case ARRAY: case BYTES: - return variant.toJson(); + value = variant.toJson(); + break; default: - return String.valueOf(variant.get()); + value = String.valueOf(variant.get()); } + final int length = value.length(); + final boolean fits = charTarget ? length == targetLength : length <= targetLength; + if (!fits) { + throw new TableRuntimeException( + String.format( + "The VARIANT string value of length %d does not fit %s(%d); VARIANT " + + "string casts do not pad or truncate.", + length, charTarget ? "CHAR" : "VARCHAR", targetLength)); + } + return value; } private static long toIntegralExact(Number value, long min, long max, String targetType) { From 23e41c6309fc08cd5dc07acb509a563b79030779 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 22 Jul 2026 17:09:23 +0200 Subject: [PATCH 6/6] [FLINK-37925] Fail on casting Variant array, objects, bytes to String --- .../docs/sql/reference/data-types.md | 6 ++--- docs/content/docs/sql/reference/data-types.md | 6 ++--- .../casting/VariantToStringCastRule.java | 5 +++-- .../planner/functions/CastFunctionITCase.java | 22 ++++++++++++------- .../runtime/functions/VariantCastUtils.java | 19 ++++++++++------ 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/docs/content.zh/docs/sql/reference/data-types.md b/docs/content.zh/docs/sql/reference/data-types.md index 388095aeaf56fc..b05d38d4eaa2ba 100644 --- a/docs/content.zh/docs/sql/reference/data-types.md +++ b/docs/content.zh/docs/sql/reference/data-types.md @@ -1550,9 +1550,9 @@ targets accept any numeric value, so `PARSE_JSON('42')` casts to `INT`, `BIGINT` value outside an integer or `DECIMAL` target's range fails `CAST` and returns `NULL` for `TRY_CAST`. Other targets require the stored value to be of the matching kind, and a mismatch is handled the same way. Casting a `VARIANT` to `CHAR`/`VARCHAR` extracts the scalar value, so a -stored string is returned unquoted (for example `foo`), while objects and arrays return their JSON -representation. Use `JSON_STRING` for the JSON representation, where a string stays quoted (for -example `"foo"`). A bounded `CHAR(n)`/`VARCHAR(n)` target is length-checked strictly, with no +stored string is returned unquoted (for example `foo`). A variant that holds an object, array, or +binary value is not castable to a string; use `JSON_STRING` for its JSON representation instead +(there a string stays quoted, for example `"foo"`). A bounded `CHAR(n)`/`VARCHAR(n)` target is length-checked strictly, with no padding or truncation: `VARCHAR(n)` accepts up to `n` characters and `CHAR(n)` requires exactly `n`, otherwise `CAST` fails and `TRY_CAST` returns `NULL`. diff --git a/docs/content/docs/sql/reference/data-types.md b/docs/content/docs/sql/reference/data-types.md index dddc81ca79a8f7..608a396d69d2e5 100644 --- a/docs/content/docs/sql/reference/data-types.md +++ b/docs/content/docs/sql/reference/data-types.md @@ -1558,9 +1558,9 @@ targets accept any numeric value, so `PARSE_JSON('42')` casts to `INT`, `BIGINT` value outside an integer or `DECIMAL` target's range fails `CAST` and returns `NULL` for `TRY_CAST`. Other targets require the stored value to be of the matching kind, and a mismatch is handled the same way. Casting a `VARIANT` to `CHAR`/`VARCHAR` extracts the scalar value, so a -stored string is returned unquoted (for example `foo`), while objects and arrays return their JSON -representation. Use `JSON_STRING` for the JSON representation, where a string stays quoted (for -example `"foo"`). A bounded `CHAR(n)`/`VARCHAR(n)` target is length-checked strictly, with no +stored string is returned unquoted (for example `foo`). A variant that holds an object, array, or +binary value is not castable to a string; use `JSON_STRING` for its JSON representation instead +(there a string stays quoted, for example `"foo"`). A bounded `CHAR(n)`/`VARCHAR(n)` target is length-checked strictly, with no padding or truncation: `VARCHAR(n)` accepts up to `n` characters and `CHAR(n)` requires exactly `n`, otherwise `CAST` fails and `TRY_CAST` returns `NULL`. diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToStringCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToStringCastRule.java index f0bf9b79d9b06d..15ba5b0a1645aa 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToStringCastRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToStringCastRule.java @@ -30,8 +30,9 @@ /** * {@link LogicalTypeRoot#VARIANT} to {@link LogicalTypeFamily#CHARACTER_STRING} cast rule. * - *

Extracts the scalar value (a string stays unquoted); objects and arrays cast to their JSON - * representation. Use {@code JSON_STRING} for the JSON representation of a scalar. + *

Extracts the scalar value (a string stays unquoted). A variant holding an object, array, or + * binary value is not castable to a character string and fails; use {@code JSON_STRING} for its + * JSON representation. * *

The target {@code CHAR}/{@code VARCHAR} length is enforced strictly: a value that does not fit * fails {@code CAST} and yields {@code null} for {@code TRY_CAST}, with no padding or truncation. diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java index 96e2c80ac10c5f..7ed67bd82a665e 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionITCase.java @@ -237,16 +237,22 @@ private static List variantCasts() { "CAST(PARSE_JSON('true') AS STRING)", "true", STRING().notNull()) + // Objects and arrays are not scalar strings: CAST fails and points to + // JSON_STRING, TRY_CAST returns NULL. Use JSON_STRING for the JSON form. + .testTableApiRuntimeError( + call("PARSE_JSON", "[\"a\", \"b\"]").cast(STRING()), "JSON_STRING") .testResult( - call("PARSE_JSON", "[\"a\", \"b\"]").cast(STRING()), - "CAST(PARSE_JSON('[\"a\", \"b\"]') AS STRING)", - "[\"a\",\"b\"]", - STRING().notNull()) + call("PARSE_JSON", "[\"a\", \"b\"]").tryCast(STRING()), + "TRY_CAST(PARSE_JSON('[\"a\", \"b\"]') AS STRING)", + null, + STRING()) + .testTableApiRuntimeError( + call("PARSE_JSON", "{\"a\": 1}").cast(STRING()), "JSON_STRING") .testResult( - call("PARSE_JSON", "{\"a\": 1}").cast(STRING()), - "CAST(PARSE_JSON('{\"a\": 1}') AS STRING)", - "{\"a\":1}", - STRING().notNull()) + call("PARSE_JSON", "{\"a\": 1}").tryCast(STRING()), + "TRY_CAST(PARSE_JSON('{\"a\": 1}') AS STRING)", + null, + STRING()) // Bounded CHAR/VARCHAR is strict on length. // VARCHAR(n) allows any length up to n; // CHAR(n) requires the exact length. diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java index 99c431cf92b11b..70004de4df3254 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/VariantCastUtils.java @@ -30,8 +30,9 @@ * Runtime helpers for casting a {@code VARIANT} value to a SQL type. * *

Numeric targets are range-checked and raise {@link TableRuntimeException} on overflow instead - * of wrapping. Casting to a string extracts the scalar value (a string stays unquoted), while - * objects and arrays use their JSON representation. + * of wrapping. Casting to a character string extracts the scalar value (a string stays unquoted); a + * variant holding an object, array, or binary value is not castable and raises {@link + * TableRuntimeException} (use {@code JSON_STRING} for its JSON representation). */ @Internal public final class VariantCastUtils { @@ -69,23 +70,27 @@ public static DecimalData toDecimalExact(Number value, int precision, int scale) /** * Casts a {@code VARIANT} to its SQL string value for a {@code CHAR(targetLength)} or {@code * VARCHAR(targetLength)} target. Scalars use their raw value (a string stays unquoted, unlike - * {@code JSON_STRING}); objects and arrays use their JSON representation. + * {@code JSON_STRING}); a variant holding an object, array, or binary value is not castable and + * raises {@link TableRuntimeException}. * *

The target length is enforced strictly, with no padding or truncation: a {@code VARCHAR} * value must not be longer than {@code targetLength}, and a {@code CHAR} value must match it * exactly. A value that does not fit raises {@link TableRuntimeException}. */ public static String toStringValue(Variant variant, int targetLength, boolean charTarget) { - final String value; switch (variant.getType()) { case OBJECT: case ARRAY: case BYTES: - value = variant.toJson(); - break; + throw new TableRuntimeException( + String.format( + "Cannot cast a VARIANT %s value to a character string. Use the " + + "JSON_STRING function to obtain its JSON representation.", + variant.getType())); default: - value = String.valueOf(variant.get()); + // Scalars are cast to their raw string value. } + final String value = String.valueOf(variant.get()); final int length = value.length(); final boolean fits = charTarget ? length == targetLength : length <= targetLength; if (!fits) {